From 02c92e55a2c2f24e18b8d7d5a1a261404ea9e7cb Mon Sep 17 00:00:00 2001 From: Vishnu Ks Date: Thu, 4 Apr 2019 16:46:02 +0530 Subject: [PATCH] import: Add tool for importing teams from mattermost. --- templates/zerver/create_realm.html | 4 +- templates/zerver/features.html | 6 +- .../zerver/help/import-from-mattermost.md | 104 +++ .../zerver/help/include/sidebar_index.md | 1 + zerver/data_import/mattermost.py | 705 ++++++++++++++++++ zerver/data_import/mattermost_user.py | 25 + .../commands/convert_mattermost_data.py | 68 ++ .../fixtures/mattermost_fixtures/export.json | 35 + .../7u7x8ytgp78q8jir81o9ejwwnr/image.png | Bin 0 -> 6315 bytes .../h15ni7kf1bnj7jeua4qhmctsdo/image.png | Bin 0 -> 34268 bytes zerver/tests/test_management_commands.py | 15 + zerver/tests/test_mattermost_importer.py | 507 +++++++++++++ 12 files changed, 1465 insertions(+), 5 deletions(-) create mode 100644 templates/zerver/help/import-from-mattermost.md create mode 100644 zerver/data_import/mattermost.py create mode 100644 zerver/data_import/mattermost_user.py create mode 100644 zerver/management/commands/convert_mattermost_data.py create mode 100644 zerver/tests/fixtures/mattermost_fixtures/export.json create mode 100644 zerver/tests/fixtures/mattermost_fixtures/exported_emoji/7u7x8ytgp78q8jir81o9ejwwnr/image.png create mode 100644 zerver/tests/fixtures/mattermost_fixtures/exported_emoji/h15ni7kf1bnj7jeua4qhmctsdo/image.png create mode 100644 zerver/tests/test_mattermost_importer.py diff --git a/templates/zerver/create_realm.html b/templates/zerver/create_realm.html index aff2dbad09..fd15af3540 100644 --- a/templates/zerver/create_realm.html +++ b/templates/zerver/create_realm.html @@ -36,8 +36,8 @@ page can be easily identified in it's respective JavaScript file --> {% endif %}
Or import - from Slack, HipChat or - Gitter. + from Slack, Mattermost, + HipChat or Gitter.
diff --git a/templates/zerver/features.html b/templates/zerver/features.html index a37167f757..89fb3a13f4 100644 --- a/templates/zerver/features.html +++ b/templates/zerver/features.html @@ -242,10 +242,10 @@

-

SLACK/HIPCHAT/GITTER IMPORT

+

DATA IMPORT

- Import an existing Slack, HipChat, Stride, or Gitter workspace into - Zulip. + Import an existing Slack, Mattermost, HipChat, Stride, + or Gitter workspace into Zulip.

diff --git a/templates/zerver/help/import-from-mattermost.md b/templates/zerver/help/import-from-mattermost.md new file mode 100644 index 0000000000..09cbf65502 --- /dev/null +++ b/templates/zerver/help/import-from-mattermost.md @@ -0,0 +1,104 @@ +# Import from Mattermost + +Starting with Zulip 2.1, Zulip supports importing data from Mattermost, +including users, channels, messages, and custom emoji. + + +**Note:** You can only import a Mattermost team as a new Zulip +organization. In particular, you cannot use this tool to import data +into an existing Zulip organization. + +## Import from Mattermost + +First, export your data. The following instructions assume you're +running Mattermost inside a Docker container: + +1. SSH into your Mattermost app server + +2. Run the following command to export the data. + `docker exec -it mattermost-docker_app_1 mattermost export bulk export.json --all-teams` + +3. This will generate `export.json` and possibly an `exported_emoji` + directory inside the **mattermost-docker_app_1** container. The + `exported_emoji` folder will only be created if your users had + uploaded custom emoji to the Mattermost server. + +4. SSH into to **mattermost-docker_app_1** container by running the following command. + `docker exec -it mattermost-docker_app_1 sh` + +4. Tar the exported files by running the following command. + `tar --transform 's|^|mattermost/|' -czf export.tar.gz exported_emoji/ export.json` + +5. Now download the `export.tar.gz` file from the Docker container to your local computer. + +### Import into zulipchat.com + +Email support@zulipchat.com with your exported archive and your desired Zulip +subdomain. Your imported organization will be hosted at +`.zulipchat.com`. + +If you've already created a test organization at +`.zulipchat.com`, let us know, and we can rename the old +organization first. + +### Import into a self-hosted Zulip server + +First +[install a new Zulip server](https://zulip.readthedocs.io/en/stable/production/install.html), +skipping "Step 3: Create a Zulip organization, and log in" (you'll +create your Zulip organization via the data import tool instead). + +Use [upgrade-zulip-from-git][upgrade-zulip-from-git] to +upgrade your Zulip server to the latest `master` branch. + +Log in to a shell on your Zulip server as the `zulip` user. + +Extract the `export.tar.gz` to `/home/zulip/mattermost` as follows. + +```bash +cd /home/zulip +tar -xzvf export.tar.gz +``` + +To import with the most common configuration, run the following commands +replacing `` with the name of the team you want to import from +Mattermost export. + +``` +cd /home/zulip/deployments/current +./manage.py convert_mattermost_data /home/zulip/mattermost --output /home/zulip/converted_mattermost_data +./manage.py import "" /home/zulip/converted_mattermost_data/ +``` + +This could take several minutes to run, depending on how much data you're +importing. + +**Import options** + +The commands above create an imported organization on the root domain +(`EXTERNAL_HOST`) of the Zulip installation. You can also import into a +custom subdomain, e.g. if you already have an existing organization on the +root domain. Replace the last line above with the following, after replacing +`` with the desired subdomain. + +``` +./manage.py import /home/zulip/converted_mattermost_data/ +``` + +{!import-login.md!} + +[upgrade-zulip-from-git]: https://zulip.readthedocs.io/en/latest/production/maintain-secure-upgrade.html#upgrading-from-a-git-repository + +## Limitations + +Mattermost's export tool is incomplete and does not support exporting +the following data: + +* private messages and group private messages between users +* user avatars +* uploaded files and message attachments. + +We expect to add support for importing these data from Mattermost once +Mattermost's export tool includes them. + +[upgrade-zulip-from-git]: https://zulip.readthedocs.io/en/latest/production/maintain-secure-upgrade.html#upgrading-from-a-git-repository diff --git a/templates/zerver/help/include/sidebar_index.md b/templates/zerver/help/include/sidebar_index.md index 7f1e21ee77..253b7901d1 100644 --- a/templates/zerver/help/include/sidebar_index.md +++ b/templates/zerver/help/include/sidebar_index.md @@ -105,6 +105,7 @@ * [Create your organization profile](/help/create-your-organization-profile) * [Link to your Zulip from the web](/help/join-zulip-chat-badge) * [Import from HipChat/Stride](/help/import-from-hipchat) +* [Import from Mattermost](/help/import-from-mattermost) * [Import from Slack](/help/import-from-slack) * [Import from Gitter](/help/import-from-gitter) * [Roles and permissions](/help/roles-and-permissions) diff --git a/zerver/data_import/mattermost.py b/zerver/data_import/mattermost.py new file mode 100644 index 0000000000..0180177513 --- /dev/null +++ b/zerver/data_import/mattermost.py @@ -0,0 +1,705 @@ +""" +spec: +https://docs.mattermost.com/administration/bulk-export.html +""" +import os +import logging +import subprocess +import ujson +import re +import shutil + +from typing import Any, Callable, Dict, List, Set + +from django.conf import settings +from django.utils.timezone import now as timezone_now +from django.forms.models import model_to_dict + +from zerver.models import Recipient, RealmEmoji, Reaction +from zerver.lib.utils import ( + process_list_in_batches, +) +from zerver.lib.emoji import NAME_TO_CODEPOINT_PATH +from zerver.data_import.import_util import ZerverFieldsT, build_zerver_realm, \ + build_stream, build_realm, build_message, create_converted_data_files, \ + make_subscriber_map, build_recipients, build_user_profile, \ + build_stream_subscriptions, build_personal_subscriptions, SubscriberHandler, \ + build_realm_emoji, make_user_messages + +from zerver.data_import.mattermost_user import UserHandler +from zerver.data_import.sequencer import NEXT_ID, IdMapper + +def make_realm(realm_id: int, team: Dict[str, Any]) -> ZerverFieldsT: + # set correct realm details + NOW = float(timezone_now().timestamp()) + domain_name = settings.EXTERNAL_HOST + realm_subdomain = team["name"] + + zerver_realm = build_zerver_realm(realm_id, realm_subdomain, NOW, 'Mattermost') + realm = build_realm(zerver_realm, realm_id, domain_name) + + # We may override these later. + realm['zerver_defaultstream'] = [] + + return realm + +def process_user(user_dict: Dict[str, Any], realm_id: int, team_name: str, + user_id_mapper: IdMapper) -> ZerverFieldsT: + def is_team_admin(user_dict: Dict[str, Any]) -> bool: + for team in user_dict["teams"]: + if team["name"] == team_name and "team_admin" in team["roles"]: + return True + return False + + def get_full_name(user_dict: Dict[str, Any]) -> str: + full_name = "{} {}".format(user_dict["first_name"], user_dict["last_name"]) + if full_name.strip(): + return full_name + return user_dict['username'] + + avatar_source = 'G' + full_name = get_full_name(user_dict) + id = user_id_mapper.get(user_dict['username']) + delivery_email = user_dict['email'] + email = user_dict['email'] + is_realm_admin = is_team_admin(user_dict) + is_guest = False + short_name = user_dict['username'] + date_joined = int(timezone_now().timestamp()) + timezone = 'UTC' + + if user_dict["is_mirror_dummy"]: + is_active = False + is_mirror_dummy = True + else: + is_active = True + is_mirror_dummy = False + + return build_user_profile( + avatar_source=avatar_source, + date_joined=date_joined, + delivery_email=delivery_email, + email=email, + full_name=full_name, + id=id, + is_active=is_active, + is_realm_admin=is_realm_admin, + is_guest=is_guest, + is_mirror_dummy=is_mirror_dummy, + realm_id=realm_id, + short_name=short_name, + timezone=timezone, + ) + +def convert_user_data(user_handler: UserHandler, + user_id_mapper: IdMapper, + user_data_map: Dict[str, Dict[str, Any]], + realm_id: int, + team_name: str) -> None: + + user_data_list = [] + for username in user_data_map: + user = user_data_map[username] + if check_user_in_team(user, team_name) or user["is_mirror_dummy"]: + user_data_list.append(user) + + for raw_item in user_data_list: + user = process_user(raw_item, realm_id, team_name, user_id_mapper) + user_handler.add_user(user) + +def convert_channel_data(channel_data: List[ZerverFieldsT], + user_data_map: Dict[str, Dict[str, Any]], + subscriber_handler: SubscriberHandler, + stream_id_mapper: IdMapper, + user_id_mapper: IdMapper, + realm_id: int, + team_name: str) -> List[ZerverFieldsT]: + channel_data_list = [ + d + for d in channel_data + if d['team'] == team_name + ] + + channel_members_map = {} # type: Dict[str, List[str]] + channel_admins_map = {} # type: Dict[str, List[str]] + + def initialize_stream_membership_dicts() -> None: + for channel in channel_data: + channel_name = channel["name"] + channel_members_map[channel_name] = [] + channel_admins_map[channel_name] = [] + + for username in user_data_map: + user_dict = user_data_map[username] + teams = user_dict["teams"] + for team in teams: + if team["name"] != team_name: + continue + for channel in team["channels"]: + channel_roles = channel["roles"] + channel_name = channel["name"] + if "channel_admin" in channel_roles: + channel_admins_map[channel_name].append(username) + elif "channel_user" in channel_roles: + channel_members_map[channel_name].append(username) + + def get_invite_only_value_from_channel_type(channel_type: str) -> bool: + # Channel can have two types in Mattermost + # "O" for a public channel. + # "P" for a private channel. + if channel_type == 'O': + return False + elif channel_type == 'P': + return True + else: # nocoverage + raise Exception('unexpected value') + + streams = [] + initialize_stream_membership_dicts() + + for channel_dict in channel_data_list: + now = int(timezone_now().timestamp()) + stream_id = stream_id_mapper.get(channel_dict['name']) + stream_name = channel_dict["name"] + invite_only = get_invite_only_value_from_channel_type(channel_dict['type']) + + stream = build_stream( + date_created=now, + realm_id=realm_id, + name=channel_dict['display_name'], + # Purpose describes how the channel should be used. It is similar to + # stream description and is shown in channel list to help others decide + # whether to join. + # Header text always appears right next to channel name in channel header. + # Can be used for advertising the purpose of stream, making announcements as + # well as including frequently used links. So probably not a bad idea to use + # this as description if the channel purpose is empty. + description=channel_dict["purpose"] or channel_dict['header'], + stream_id=stream_id, + # Mattermost export don't include data of archived(~ deactivated) channels. + deactivated=False, + invite_only=invite_only, + ) + + channel_users = set() + for username in channel_admins_map[stream_name]: + channel_users.add(user_id_mapper.get(username)) + + for username in channel_members_map[stream_name]: + channel_users.add(user_id_mapper.get(username)) + + if channel_users: + subscriber_handler.set_info( + stream_id=stream_id, + users=channel_users, + ) + streams.append(stream) + + return streams + +def get_name_to_codepoint_dict() -> Dict[str, str]: + with open(NAME_TO_CODEPOINT_PATH) as fp: + return ujson.load(fp) + +def build_reactions(realm_id: int, total_reactions: List[ZerverFieldsT], reactions: List[ZerverFieldsT], + message_id: int, name_to_codepoint: ZerverFieldsT, + user_id_mapper: IdMapper, zerver_realmemoji: List[ZerverFieldsT]) -> None: + realmemoji = {} + for realm_emoji in zerver_realmemoji: + realmemoji[realm_emoji['name']] = realm_emoji['id'] + + # For the unicode emoji codes, we use equivalent of + # function 'emoji_name_to_emoji_code' in 'zerver/lib/emoji' here + for mattermost_reaction in reactions: + emoji_name = mattermost_reaction['emoji_name'] + username = mattermost_reaction["user"] + # Check in unicode emoji + if emoji_name in name_to_codepoint: + emoji_code = name_to_codepoint[emoji_name] + reaction_type = Reaction.UNICODE_EMOJI + # Check in realm emoji + elif emoji_name in realmemoji: + emoji_code = realmemoji[emoji_name] + reaction_type = Reaction.REALM_EMOJI + else: # nocoverage + continue + + if not user_id_mapper.has(username): + continue + + reaction_id = NEXT_ID('reaction') + reaction = Reaction( + id=reaction_id, + emoji_code=emoji_code, + emoji_name=emoji_name, + reaction_type=reaction_type) + + reaction_dict = model_to_dict(reaction, exclude=['message', 'user_profile']) + reaction_dict['message'] = message_id + reaction_dict['user_profile'] = user_id_mapper.get(username) + total_reactions.append(reaction_dict) + +def get_mentioned_user_ids(raw_message: Dict[str, Any], user_id_mapper: IdMapper) -> Set[int]: + user_ids = set() + content = raw_message["content"] + + # usernames can be of the form user.name, user_name, username., username_, user.name_ etc + matches = re.findall("(?<=^|(?<=[^a-zA-Z0-9-_.]))@(([A-Za-z0-9]+[_.]?)+)", content) + + for match in matches: + possible_username = match[0] + if user_id_mapper.has(possible_username): + user_ids.add(user_id_mapper.get(possible_username)) + return user_ids + +def process_raw_message_batch(realm_id: int, + raw_messages: List[Dict[str, Any]], + subscriber_map: Dict[int, Set[int]], + user_id_mapper: IdMapper, + user_handler: UserHandler, + get_recipient_id: Callable[[ZerverFieldsT], int], + is_pm_data: bool, + output_dir: str, + zerver_realmemoji: List[Dict[str, Any]], + total_reactions: List[Dict[str, Any]], + ) -> None: + + def fix_mentions(content: str, mention_user_ids: Set[int]) -> str: + for user_id in mention_user_ids: + user = user_handler.get_user(user_id=user_id) + mattermost_mention = '@{short_name}'.format(**user) + zulip_mention = '@**{full_name}**'.format(**user) + content = content.replace(mattermost_mention, zulip_mention) + + content = content.replace('@channel', '@**all**') + content = content.replace('@all', '@**all**') + # We don't have an equivalent for Mattermost's @here mention which mentions all users + # online in the channel. + content = content.replace('@here', '@**all**') + return content + + mention_map = dict() # type: Dict[int, Set[int]] + zerver_message = [] + + import html2text + h = html2text.HTML2Text() + + name_to_codepoint = get_name_to_codepoint_dict() + + for raw_message in raw_messages: + message_id = NEXT_ID('message') + mention_user_ids = get_mentioned_user_ids(raw_message, user_id_mapper) + mention_map[message_id] = mention_user_ids + + content = fix_mentions( + content=raw_message['content'], + mention_user_ids=mention_user_ids, + ) + content = h.handle(content) + + if len(content) > 10000: # nocoverage + logging.info('skipping too-long message of length %s' % (len(content),)) + continue + + pub_date = raw_message['pub_date'] + try: + recipient_id = get_recipient_id(raw_message) + except KeyError: + logging.debug("Could not find recipient_id for a message, skipping.") + continue + + rendered_content = None + + topic_name = 'imported from mattermost' + user_id = raw_message['sender_id'] + + message = build_message( + content=content, + message_id=message_id, + pub_date=pub_date, + recipient_id=recipient_id, + rendered_content=rendered_content, + topic_name=topic_name, + user_id=user_id, + has_attachment=False, + ) + zerver_message.append(message) + build_reactions(realm_id, total_reactions, raw_message["reactions"], message_id, + name_to_codepoint, user_id_mapper, zerver_realmemoji) + + zerver_usermessage = make_user_messages( + zerver_message=zerver_message, + subscriber_map=subscriber_map, + is_pm_data=is_pm_data, + mention_map=mention_map, + ) + + message_json = dict( + zerver_message=zerver_message, + zerver_usermessage=zerver_usermessage, + ) + + dump_file_id = NEXT_ID('dump_file_id' + str(realm_id)) + message_file = "/messages-%06d.json" % (dump_file_id,) + create_converted_data_files(message_json, output_dir, message_file) + +def process_posts(team_name: str, + realm_id: int, + post_data: List[Dict[str, Any]], + get_recipient_id: Callable[[ZerverFieldsT], int], + subscriber_map: Dict[int, Set[int]], + output_dir: str, + is_pm_data: bool, + masking_content: bool, + user_id_mapper: IdMapper, + user_handler: UserHandler, + username_to_user: Dict[str, Dict[str, Any]], + zerver_realmemoji: List[Dict[str, Any]], + total_reactions: List[Dict[str, Any]]) -> None: + + post_data_list = [ + d + for d in post_data + if d["team"] == team_name + ] + + def message_to_dict(post_dict: Dict[str, Any]) -> Dict[str, Any]: + sender_id = user_id_mapper.get(post_dict["user"]) + content = post_dict['message'] + + if masking_content: + content = re.sub('[a-z]', 'x', content) + content = re.sub('[A-Z]', 'X', content) + + if "reactions" in post_dict: + reactions = post_dict["reactions"] or [] + else: + reactions = [] + + return dict( + sender_id=sender_id, + receiver_id=post_dict["channel"], + content=content, + pub_date=int(post_dict['create_at'] / 1000), + reactions=reactions + ) + + raw_messages = [] + for post_dict in post_data_list: + raw_messages.append(message_to_dict(post_dict)) + message_replies = post_dict["replies"] + # Replies to a message in Mattermost are stored in the main message object. + # For now, we just append the replies immediately after the original message. + if message_replies is not None: + for reply in message_replies: + reply["channel"] = post_dict["channel"] + raw_messages.append(message_to_dict(reply)) + + def process_batch(lst: List[Dict[str, Any]]) -> None: + process_raw_message_batch( + realm_id=realm_id, + raw_messages=lst, + subscriber_map=subscriber_map, + user_id_mapper=user_id_mapper, + user_handler=user_handler, + get_recipient_id=get_recipient_id, + is_pm_data=is_pm_data, + output_dir=output_dir, + zerver_realmemoji=zerver_realmemoji, + total_reactions=total_reactions, + ) + + chunk_size = 1000 + + process_list_in_batches( + lst=raw_messages, + chunk_size=chunk_size, + process_batch=process_batch, + ) + +def write_message_data(team_name: str, + realm_id: int, + post_data: List[Dict[str, Any]], + zerver_recipient: List[ZerverFieldsT], + subscriber_map: Dict[int, Set[int]], + output_dir: str, + masking_content: bool, + stream_id_mapper: IdMapper, + user_id_mapper: IdMapper, + user_handler: UserHandler, + username_to_user: Dict[str, Dict[str, Any]], + zerver_realmemoji: List[Dict[str, Any]], + total_reactions: List[Dict[str, Any]]) -> None: + + stream_id_to_recipient_id = { + d['type_id']: d['id'] + for d in zerver_recipient + if d['type'] == Recipient.STREAM + } + + def get_stream_recipient_id(raw_message: ZerverFieldsT) -> int: + receiver_id = raw_message['receiver_id'] + stream_id = stream_id_mapper.get(receiver_id) + recipient_id = stream_id_to_recipient_id[stream_id] + return recipient_id + + process_posts( + team_name=team_name, + realm_id=realm_id, + post_data=post_data, + get_recipient_id=get_stream_recipient_id, + subscriber_map=subscriber_map, + output_dir=output_dir, + is_pm_data=False, + masking_content=masking_content, + user_id_mapper=user_id_mapper, + user_handler=user_handler, + username_to_user=username_to_user, + zerver_realmemoji=zerver_realmemoji, + total_reactions=total_reactions, + ) + +def write_emoticon_data(realm_id: int, + custom_emoji_data: List[Dict[str, Any]], + data_dir: str, + output_dir: str) -> List[ZerverFieldsT]: + ''' + This function does most of the work for processing emoticons, the bulk + of which is copying files. We also write a json file with metadata. + Finally, we return a list of RealmEmoji dicts to our caller. + + In our data_dir we have a pretty simple setup: + + The exported JSON file will have emoji rows if it contains any custom emoji + { + "type": "emoji", + "emoji": {"name": "peerdium", "image": "exported_emoji/h15ni7kf1bnj7jeua4qhmctsdo/image"} + } + { + "type": "emoji", + "emoji": {"name": "tick", "image": "exported_emoji/7u7x8ytgp78q8jir81o9ejwwnr/image"} + } + + exported_emoji/ - contains a bunch of image files: + exported_emoji/7u7x8ytgp78q8jir81o9ejwwnr/image + exported_emoji/h15ni7kf1bnj7jeua4qhmctsdo/image + + We move all the relevant files to Zulip's more nested + directory structure. + ''' + + logging.info('Starting to process emoticons') + + flat_data = [ + dict( + path=d['image'], + name=d['name'], + ) + for d in custom_emoji_data + ] + + emoji_folder = os.path.join(output_dir, 'emoji') + os.makedirs(emoji_folder, exist_ok=True) + + def process(data: ZerverFieldsT) -> ZerverFieldsT: + source_sub_path = data['path'] + source_path = os.path.join(data_dir, source_sub_path) + + target_fn = data["name"] + target_sub_path = RealmEmoji.PATH_ID_TEMPLATE.format( + realm_id=realm_id, + emoji_file_name=target_fn, + ) + target_path = os.path.join(emoji_folder, target_sub_path) + + os.makedirs(os.path.dirname(target_path), exist_ok=True) + + source_path = os.path.abspath(source_path) + target_path = os.path.abspath(target_path) + + shutil.copyfile(source_path, target_path) + + return dict( + path=target_path, + s3_path=target_path, + file_name=target_fn, + realm_id=realm_id, + name=data['name'], + ) + + emoji_records = list(map(process, flat_data)) + create_converted_data_files(emoji_records, output_dir, '/emoji/records.json') + + realmemoji = [ + build_realm_emoji( + realm_id=realm_id, + name=rec['name'], + id=NEXT_ID('realmemoji'), + file_name=rec['file_name'], + ) + for rec in emoji_records + ] + logging.info('Done processing emoticons') + + return realmemoji + +def create_username_to_user_mapping(user_data_list: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]: + username_to_user = {} + for user in user_data_list: + username_to_user[user["username"]] = user + return username_to_user + +def check_user_in_team(user: Dict[str, Any], team_name: str) -> bool: + for team in user["teams"]: + if team["name"] == team_name: + return True + return False + +def label_mirror_dummy_users(team_name: str, mattermost_data: Dict[str, List[Dict[str, Any]]], + username_to_user: Dict[str, Dict[str, Any]]) -> None: + # This function might looks like a great place to label admin users. But + # that won't be fully correct since we are iterating only though posts and + # it covers only users that has sent atleast one message. + for post in mattermost_data["post"]: + if post["team"] == team_name: + user = username_to_user[post["user"]] + if not check_user_in_team(user, team_name): + user["is_mirror_dummy"] = True + +def reset_mirror_dummy_users(username_to_user: Dict[str, Dict[str, Any]]) -> None: + for username in username_to_user: + user = username_to_user[username] + user["is_mirror_dummy"] = False + +def mattermost_data_file_to_dict(mattermost_data_file: str) -> Dict[str, List[Dict[str, Any]]]: + mattermost_data = {} # type: Dict[str, List[Dict[str, Any]]] + mattermost_data["version"] = [] + mattermost_data["team"] = [] + mattermost_data["channel"] = [] + mattermost_data["user"] = [] + mattermost_data["post"] = [] + mattermost_data["emoji"] = [] + + with open(mattermost_data_file, "r") as fp: + for line in fp: + row = ujson.loads(line.rstrip("\n")) + data_type = row["type"] + mattermost_data[data_type].append(row[data_type]) + return mattermost_data + +def do_convert_data(mattermost_data_dir: str, output_dir: str, masking_content: bool) -> None: + username_to_user = {} # type: Dict[str, Dict[str, Any]] + + os.makedirs(output_dir, exist_ok=True) + if os.listdir(output_dir): # nocoverage + raise Exception("Output directory should be empty!") + + mattermost_data_file = os.path.join(mattermost_data_dir, "export.json") + mattermost_data = mattermost_data_file_to_dict(mattermost_data_file) + + username_to_user = create_username_to_user_mapping(mattermost_data["user"]) + + for team in mattermost_data["team"]: + realm_id = NEXT_ID("realm_id") + team_name = team["name"] + + user_handler = UserHandler() + subscriber_handler = SubscriberHandler() + user_id_mapper = IdMapper() + stream_id_mapper = IdMapper() + + print("Generating data for", team_name) + realm = make_realm(realm_id, team) + realm_output_dir = os.path.join(output_dir, team_name) + + reset_mirror_dummy_users(username_to_user) + label_mirror_dummy_users(team_name, mattermost_data, username_to_user) + + convert_user_data( + user_handler=user_handler, + user_id_mapper=user_id_mapper, + user_data_map=username_to_user, + realm_id=realm_id, + team_name=team_name, + ) + + zerver_stream = convert_channel_data( + channel_data=mattermost_data["channel"], + user_data_map=username_to_user, + subscriber_handler=subscriber_handler, + stream_id_mapper=stream_id_mapper, + user_id_mapper=user_id_mapper, + realm_id=realm_id, + team_name=team_name, + ) + + realm['zerver_stream'] = zerver_stream + + all_users = user_handler.get_all_users() + + zerver_recipient = build_recipients( + zerver_userprofile=all_users, + zerver_stream=zerver_stream, + ) + realm['zerver_recipient'] = zerver_recipient + + stream_subscriptions = build_stream_subscriptions( + get_users=subscriber_handler.get_users, + zerver_recipient=zerver_recipient, + zerver_stream=zerver_stream, + ) + + personal_subscriptions = build_personal_subscriptions( + zerver_recipient=zerver_recipient, + ) + + # Mattermost currently supports only exporting messages from channels. + # Personal messages and huddles are not exported. + zerver_subscription = personal_subscriptions + stream_subscriptions + realm['zerver_subscription'] = zerver_subscription + + zerver_realmemoji = write_emoticon_data( + realm_id=realm_id, + custom_emoji_data=mattermost_data["emoji"], + data_dir=mattermost_data_dir, + output_dir=realm_output_dir, + ) + realm['zerver_realmemoji'] = zerver_realmemoji + + subscriber_map = make_subscriber_map( + zerver_subscription=zerver_subscription, + ) + + total_reactions = [] # type: List[Dict[str, Any]] + write_message_data( + team_name=team_name, + realm_id=realm_id, + post_data=mattermost_data["post"], + zerver_recipient=zerver_recipient, + subscriber_map=subscriber_map, + output_dir=realm_output_dir, + masking_content=masking_content, + stream_id_mapper=stream_id_mapper, + user_id_mapper=user_id_mapper, + user_handler=user_handler, + username_to_user=username_to_user, + zerver_realmemoji=zerver_realmemoji, + total_reactions=total_reactions, + ) + realm['zerver_reaction'] = total_reactions + realm['zerver_userprofile'] = user_handler.get_all_users() + realm['sort_by_date'] = True + + create_converted_data_files(realm, realm_output_dir, '/realm.json') + # Mattermost currently doesn't support exporting avatars + create_converted_data_files([], realm_output_dir, '/avatars/records.json') + # Mattermost currently doesn't support exporting uploads + create_converted_data_files([], realm_output_dir, '/uploads/records.json') + + # Mattermost currently doesn't support exporting attachments + attachment = {"zerver_attachment": []} # type: Dict[str, List[Any]] + create_converted_data_files(attachment, realm_output_dir, '/attachment.json') + + logging.info('Start making tarball') + subprocess.check_call(["tar", "-czf", realm_output_dir + '.tar.gz', realm_output_dir, '-P']) + logging.info('Done making tarball') diff --git a/zerver/data_import/mattermost_user.py b/zerver/data_import/mattermost_user.py new file mode 100644 index 0000000000..5361dae570 --- /dev/null +++ b/zerver/data_import/mattermost_user.py @@ -0,0 +1,25 @@ +from typing import Any, Dict, List + +class UserHandler: + ''' + Our UserHandler class is a glorified wrapper + around the data that eventually goes into + zerver_userprofile. + + The class helps us do things like map ids + to names for mentions. + ''' + def __init__(self) -> None: + self.id_to_user_map = dict() # type: Dict[int, Dict[str, Any]] + + def add_user(self, user: Dict[str, Any]) -> None: + user_id = user['id'] + self.id_to_user_map[user_id] = user + + def get_user(self, user_id: int) -> Dict[str, Any]: + user = self.id_to_user_map[user_id] + return user + + def get_all_users(self) -> List[Dict[str, Any]]: + users = list(self.id_to_user_map.values()) + return users diff --git a/zerver/management/commands/convert_mattermost_data.py b/zerver/management/commands/convert_mattermost_data.py new file mode 100644 index 0000000000..82d67b51ad --- /dev/null +++ b/zerver/management/commands/convert_mattermost_data.py @@ -0,0 +1,68 @@ +import argparse +import os +from typing import Any + +''' +Example usage for testing purposes. For testing data see the mattermost_fixtures +in zerver/tests/. + + ./manage.py convert_mattermost_data mattermost_fixtures --output mm_export + ./manage.py import --destroy-rebuild-database mattermost mm_export/gryffindor + +Test out the realm: + ./tools/run-dev.py + go to browser and use your dev url +''' + +from django.core.management.base import BaseCommand, CommandParser + +from zerver.data_import.mattermost import do_convert_data + +class Command(BaseCommand): + help = """Convert the mattermost data into Zulip data format.""" + + def add_arguments(self, parser: CommandParser) -> None: + dir_help = "Directory containing exported JSON file and exported_emoji (optional) directory." + parser.add_argument('mattermost_data_dir', + metavar='', + help=dir_help) + + parser.add_argument('--output', dest='output_dir', + action="store", + help='Directory to write converted data to.') + + parser.add_argument('--mask', dest='masking_content', + action="store_true", + help='Mask the content for privacy during QA.') + + parser.formatter_class = argparse.RawTextHelpFormatter + + def handle(self, *args: Any, **options: Any) -> None: + output_dir = options["output_dir"] + if output_dir is None: + print("You need to specify --output ") + exit(1) + + if os.path.exists(output_dir) and not os.path.isdir(output_dir): + print(output_dir + " is not a directory") + exit(1) + + os.makedirs(output_dir, exist_ok=True) + + if os.listdir(output_dir): + print('Output directory should be empty!') + exit(1) + output_dir = os.path.realpath(output_dir) + + data_dir = options['mattermost_data_dir'] + if not os.path.exists(data_dir): + print("Directory not found: '%s'" % (data_dir,)) + exit(1) + data_dir = os.path.realpath(data_dir) + + print("Converting Data ...") + do_convert_data( + mattermost_data_dir=data_dir, + output_dir=output_dir, + masking_content=options.get('masking_content', False), + ) diff --git a/zerver/tests/fixtures/mattermost_fixtures/export.json b/zerver/tests/fixtures/mattermost_fixtures/export.json new file mode 100644 index 0000000000..70d3f800b3 --- /dev/null +++ b/zerver/tests/fixtures/mattermost_fixtures/export.json @@ -0,0 +1,35 @@ +{"type":"version","version":1} +{"type":"team","team":{"name":"gryffindor","display_name":"Iago Realm","type":"O","description":"","allow_open_invite":true}} +{"type":"team","team":{"name":"slytherin","display_name":"Othello Team","type":"O","description":"","allow_open_invite":true}} +{"type":"channel","channel":{"team":"gryffindor","name":"gryffindor-common-room","display_name":"Gryffindor common room","type":"O","header":"","purpose":"A place for talking about Gryffindor common room"}} +{"type":"channel","channel":{"team":"gryffindor","name":"gryffindor-quidditch-team","display_name":"Gryffindor quidditch team","type":"O","header":"","purpose":"A place for talking about Gryffindor quidditch team"}} +{"type":"channel","channel":{"team":"slytherin","name":"slytherin-common-room","display_name":"Slytherin common room","type":"O","header":"","purpose":""}} +{"type":"channel","channel":{"team":"gryffindor","name":"dumbledores-army","display_name":"Dumbledores army","type":"P","header":"https//:github.com/zulip/zulip","purpose":"A place for talking about Dumbledores army"}} +{"type":"channel","channel":{"team":"slytherin","name":"slytherin-quidditch-team","display_name":"Slytherin quidditch team","type":"O","header":"","purpose":""}} +{"type":"user","user":{"username":"ron","email":"ron@zulip.com","auth_service":"","nickname":"","first_name":"Ron","last_name":"Weasley","position":"","roles":"system_user","locale":"en","teams":[{"name":"gryffindor","roles":"team_user","channels":[{"name":"gryffindor-quidditch-team","roles":"channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"gryffindor-common-room","roles":"channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"dumbledores-army","roles":"channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false}]}],"notify_props":{"desktop":"mention","desktop_sound":"true","email":"true","mobile":"mention","mobile_push_status":"away","channel":"true","comments":"never","mention_keys":"ron,@ron"}}} +{"type":"user","user":{"username":"harry","email":"harry@zulip.com","auth_service":"","nickname":"","first_name":"Harry","last_name":"Potter","position":"","roles":"system_admin system_user","locale":"en","teams":[{"name":"gryffindor","roles":"team_admin team_user","channels":[{"name":"dumbledores-army","roles":"channel_admin channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"gryffindor-common-room","roles":"channel_admin channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"gryffindor-quidditch-team","roles":"channel_admin channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false}]}],"notify_props":{"desktop":"mention","desktop_sound":"true","email":"true","mobile":"mention","mobile_push_status":"away","channel":"true","comments":"never","mention_keys":"harry,@harry"}}} +{"type":"user","user":{"username":"malfoy","email":"malfoy@zulip.com","auth_service":"","nickname":"","first_name":"","last_name":"","position":"","roles":"system_user","locale":"en","teams":[{"name":"slytherin","roles":"team_admin team_user","channels":[{"name":"slytherin-common-room","roles":"channel_admin channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"slytherin-quidditch-team","roles":"channel_admin channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false}]}],"notify_props":{"desktop":"mention","desktop_sound":"true","email":"true","mobile":"mention","mobile_push_status":"away","channel":"true","comments":"never","mention_keys":"malfoy,@malfoy"}}} +{"type":"user","user":{"username":"pansy","email":"pansy@zulip.com","auth_service":"","nickname":"","first_name":"","last_name":"","position":"","roles":"system_user","locale":"en","teams":[{"name":"slytherin","roles":"team_admin team_user","channels":[{"name":"slytherin-common-room","roles":"channel_admin channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"slytherin-quidditch-team","roles":"channel_admin channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false}]}],"notify_props":{"desktop":"mention","desktop_sound":"true","email":"true","mobile":"mention","mobile_push_status":"away","channel":"true","comments":"never","mention_keys":"malfoy,@malfoy"}}} +{"type":"user","user":{"username":"snape","email":"snape@zulip.com","auth_service":"","nickname":"","first_name":"Severus","last_name":"Snape","position":"","roles":"system_user","locale":"en","teams":[{"name":"slytherin","roles":"team_user","channels":[{"name":"slytherin-common-room","roles":"channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false}]}],"notify_props":{"desktop":"mention","desktop_sound":"true","email":"true","mobile":"mention","mobile_push_status":"away","channel":"true","comments":"never","mention_keys":"snape,@snape"}}} +{"type":"post","post":{"team":"gryffindor","channel":"dumbledores-army","user":"harry","message":"harry joined the channel.","create_at":1553166657086,"reactions":null,"replies":[{"user":"ron","message":"The weather is so hot!","create_at":1553166584976}]}} +{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-common-room","user":"ron","message":"ron joined the channel.","create_at":1553166512493,"reactions":null,"replies":null}} +{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-quidditch-team","user":"harry","message":"harry joined the team.","create_at":1553165141670,"reactions":null,"replies":null}} +{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-quidditch-team","user":"harry","message":"Awesome!","create_at":1553166557928,"reactions":[{"user":"malfoy","create_at":1553166812156,"emoji_name":"tick"}],"replies":null}} +{"type":"post","post":{"team":"slytherin","channel":"slytherin-quidditch-team","user":"malfoy","message":"malfoy joined the team.","create_at":1553166852598,"reactions":null,"replies":null}} +{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-quidditch-team","user":"ron","message":"ron joined the team.","create_at":1553166512482,"reactions":null,"replies":null}} +{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-quidditch-team","user":"ron","message":"Hey folks","create_at":1553166519720,"reactions":null,"replies":null}} +{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-quidditch-team","user":"harry","message":"@ron Welcome mate!","create_at":1553166519726,"reactions":null,"replies":null}} +{"type":"post","post":{"team":"gryffindor","channel":"dumbledores-army","user":"harry","message":"ron added to the channel by harry.","create_at":1553166681045,"reactions":null,"replies":null}} +{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-quidditch-team","user":"harry","message":"Hello world","create_at":1553165193242,"reactions":[{"user":"harry","create_at":1553165521410,"emoji_name":"tick"},{"user":"ron","create_at":1553166530805,"emoji_name":"smile"},{"user":"ron","create_at":1553166540953,"emoji_name":"world_map"}],"replies":null}} +{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-common-room","user":"harry","message":"Looks like this channel is empty","create_at":1553166567370,"reactions":[{"user":"ron","create_at":1553166584976,"emoji_name":"rocket"}],"replies":null}} +{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-quidditch-team","user":"ron","message":"How is everything going","create_at":1553166525124,"reactions":[{"user":"harry","create_at":1553166552827,"emoji_name":"apple"}],"replies":null}} +{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-common-room","user":"ron","message":"Not really","create_at":1553166593455,"reactions":null,"replies":null}} +{"type":"post","post":{"team":"gryffindor","channel":"dumbledores-army","user":"ron","message":"hello","create_at":1553166686344,"reactions":null,"replies":null}} +{"type":"post","post":{"team":"gryffindor","channel":"dumbledores-army","user":"harry","message":"hey everyone","create_at":1553166668668,"reactions":[{"user":"ron","create_at":1553166695260,"emoji_name":"grin"}],"replies":null}} +{"type":"post","post":{"team":"slytherin","channel":"slytherin-common-room","user":"malfoy","message":"malfoy joined the channel.","create_at":1553166852612,"reactions":null,"replies":null}} +{"type":"post","post":{"team":"slytherin","channel":"slytherin-quidditch-team","user":"malfoy","message":":rofl: 4","create_at":1553166916448,"reactions":[{"user":"harry","create_at":1553167016056,"emoji_name":"peerdium"}],"replies":null}} +{"type":"post","post":{"team":"slytherin","channel":"slytherin-quidditch-team","user":"malfoy","message":"Hello folks","create_at":1553166858280,"reactions":[{"user":"harry","create_at":1553166903980,"emoji_name":"joy"}],"replies":null}} +{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-common-room","user":"harry","message":"harry joined the channel.","create_at":1553165141689,"reactions":null,"replies":null}} +{"type":"post","post":{"team":"gryffindor","channel":"slytherin-quidditch-team","user":"snape","message":"Hey folks! I was always in your team. Time to go now.","create_at":1553166740759,"reactions":null,"replies":null}} +{"type":"emoji","emoji":{"name":"peerdium","image":"exported_emoji/h15ni7kf1bnj7jeua4qhmctsdo/image.png"}} +{"type":"emoji","emoji":{"name":"tick","image":"exported_emoji/7u7x8ytgp78q8jir81o9ejwwnr/image.png"}} diff --git a/zerver/tests/fixtures/mattermost_fixtures/exported_emoji/7u7x8ytgp78q8jir81o9ejwwnr/image.png b/zerver/tests/fixtures/mattermost_fixtures/exported_emoji/7u7x8ytgp78q8jir81o9ejwwnr/image.png new file mode 100644 index 0000000000000000000000000000000000000000..60ed449cda0ec562b6b33efc22d5813a0e5c328f GIT binary patch literal 6315 zcmV;c7*ywpP)r~F017Z^LqkwWLqi}? za&Km7Y-Iodc$}S9)Gc>Uwq5=^` zLdiiLn+!5)wxpu}JlQBip_vQ~8E<-M1e-ydgvYoERMJ!kKI*17T^B0GzjoyKE}SbXLTb{bpEJtE$kCFF=0@fUGX7MGJP;#(rtOckbaMf_GAo5o>g z0)Qfk?E(%fNyMfiCh@~U+(f(-030dtD~|t)1)Lm#_)>1^8M%CJVv>Na%hIEp+1fJb z-kj`IjzC}(#AKx~`E0sddRhjPmkYq+oj*%PTwA)R$kt}I*49Sm#%5m?>c4LOO^JKE zNUwrF_Y9)-eX;$OUwSIQ_o0dBB}pL2 zuro2q&dxUGa#+UVg8rfZ>F_u7)%T3W>Ha7W-JO%b6s8L3;<~ZYQ`3cfdS(Wb#i1Mh zd5HgU;9sA^Focu9;d6MRh;Y%Aae0ZNcJtU=0XLmT=koqj6aQh@pR_pFB2gMX0cxx< zkQ$%@4-!K8&?cw^Du=3}I;aWy9y$eGfUZJ=&^>4rnu30Z-opq?f~l}F ztPPvM4A=$sgTvsJa3Z`K&Vvi#?Qj)b47|-H2{OUqatTkE7pUFc=y}2V;Zr#zbL~ zF>5fTnEjYm%z4ZpW(+fn#bOn(23QAdAeM<0V2iMOvB$9IutV5!>{}cWr;0PjdE%mR zJX`^;5_c4L7B_^Oz|G^O@LG5~d?22U&&8MF8}MED0sJ_Ao*+%oAvh4i2+4$vgepP{ z;S%8?;T4fcR43XJgNaN`iSTS4i zfZ`>^=_S-9_DfhxikF;Na$gBn(pL&mTBCGGsZVKESw-1PIYW7`@Nb*ob80 zVw7dnY&2?2Gxj$wFzzsZVWMdgZL-s(*W{C_m1(MJgXse^88ctA0<$i&-_7;SS>`q7 zw=BpOo)+sZIxSvW8d!2H4_Mx{qF4o3ZL#XM`efGt8hef*7TY zE4FA`SKIZrr)}TaS=$NhPT2isZ)Bfhf7E_*sm@Z)(uSpD4(bj}hdPH5N4jI2<3Yy} zCp9OgQ@zs@XANhzbEETwi=Ioe%Q2T1uBNVh*EZKVH#@hrZs*+*cQ5y1_kIr`84^=_}cic_3iN^`Gxvb`#tg3_via} z1;7Em0lNYoF4J1ZTh0(}B^1wIPW30fWWV=yK-D7Ys0X^2@!en@X9B{VklXy}_T z*RZm%2g`Mr3zv6?ONPgUH-*ndxJQ&nj6|A5u8q7Nr5MGH>Ws!lhetO?&#v%Tv3tdM zj8#lg%$=1wD|1#}U8T4xb=8?z$yjFW$vAXeMBLH156nPjJ##kRCw^c249ktRhxMH8 z%&uThaU3}1oQVX7gz|*RM2Ey(iBm~VNtH>{TsLkt_hqtoa&7WlN?^+2l!erY)Yddy zT3p&Go(wOA*ORW2o|8V9VUSUjF|yij_3qU(d_R6;CX~4{vr|A7{Y>=tT>!5Y<>$=x#tS?+Y zzQJq5k&T3nDI0$(FfAxAc)clNQ&*vK;fBJo&0d?EizJHpMZ;U{x72P$ZRKw5-)6CG z@3v3H?BZ)BrX`gnA4*xJ*S<0Prs|u8?Frla%dE=|?7-~c?YOhkY3Gr0>GHhtv0VYX z+AHW4#TBo2$L_vbX<1pjhp~-lqcg5k#>8o~EPhDeN>$q-xy}i$> zuk9zRpW6DZ``ZU>20Cxp-sl=!I(T--Y3RaD_nVh*`P{mGd)e*5JIn9f9gZ0uxy!yg zc`x0kG~(0%d4Z_dB<%|y24AzHDGx!quM4#Eu?|T)efX3Wk_RVTV4rcNJ2t(X{Sk8NdCz;!d7kqw;b#z@=LrB} zXlMukaP;U=u2d>qC=~QXeX$FHQmG_Q^_k~+M*B57zpry)Q7V;WE|-fu&+8u;82A8y z%e2+$jN@nqroXCVRz05da9PEHyCcK7!7j?B)^Mnoh4M3!Y4K@hwJV0*P% z{o>fz*qff`8HGYYiHrO0D~IQK5&#hqGxNhlbbGVegb+faA7Zpe%=>Cm}ohGwAP{HI1+&HeSe!$ zYTwbLN9zEvsv}^-h7F=nC@3LBH#2|QvaEGM5QIVq11;MK0EkF1GX+7w0Md@*P*M_C ztphyI6Rzuq)6>(2QtBoEx0zJ{cI~=WYyF3*RB9bF za}-63S30>r?CI$t0P_GI5JDVcW^wS~!3bETeLxca6B84+rPJw~!!Qh)*#y9Cw`$wA zMGyqXiRgLPb*BNKudh#*%jF1IWt|{13nJ10uxHPn4+tS9y1TpIH#<8U##z4|VLF{Q ztJUg@hGG11Z*TAG*=$w=t-JuNvIv-*oRk1`u~tFb{oJ($8m0NHk(Q*g_!ItfSDuPwvA@988GwnmSz3B>$(a6P9=t1(hi9C zEf)#}PNh;G(OTcCwFUqgZ~OKtrc$XCgb*(i(ZjCm);-Uo#0i~n$yWi-^Td@`UYQBQ zaHPAt`?^}K77&rub{hlWR4QfG>-GNt@RzRZ&b58s5tnQqP%IWr*L6bx*t2KPqf*NI zEX(R@G#VYt9sp&UCe>=SN7k-g``C#SCph{0)8Ufb0?bT(eSK6c7R_C|c72eD?z1eb zt5&NeOmFeMT1rW_ZJWODzbK{ryKFWa_Vx8GO8~tj9WXvVE_1nD5Jk~18;0=^04EGX z7D9+tiGLeJ#6k%8zTcEmetvXx^pzbuc38PwE?UmBE=dQhU%#F`&l5!STefZgju66# ziB5aX4;{x5N~t*_`m_1@`5yqlb=O_DEaQPo%mH!Mr(7-GajU?+46`_I6imt$KQT z%peGULPTHKxN+lg&-28=g9jO>$@=ZMWE@Z~mqnpa&_u*q>w8Vpe1ElCJr&D?lu~M~ z=cSa-=kxg&lU_fexy!{R(Feqz70X?AWoxA|ihF*=K)2YrRWK`R+!e0cP$*bHU6i2!hw6DEixcK7Y7a zEGAL1?9fke(HkMp^F*xgT^2>r-G*U&PD+U&2(%DlVS$JUQ530EDrG385Cp*}fakBh z_S*TOp&^|JC1;9LMgTLD=Xs)3D#=o*B$Et%HV-)+hK7cO>$+hSMIW|pdz6{AQi`Qq zTJe}!Tb3oX)=DY0zgDY#X>@e-#DN0``2hw)5r0dW(gdV71#6DLmmgX1_8 zvC)(WoGZb?Y9`#9h`3lR-fo)acuYbzODRVIJQzjM!;_Pf*XMG%!1FvhU8I~2mtTH4 z5s|*)iYtDRnLlM&*4kRF7IqqHNGaJgO|#Kxyex!x>aM%)`rqWB!%9~lcsmDB9FjP8 z?AUvR5Z_s|X3e|i=H?cJ1SzFKLFM}X&fGMCFmb8~aOQp&rC zsISpzB=;Bq45d^QGqGEk`LXTWw_i>~oQ$ZJi~jz82>@E_8;R&)A%xj%Hk0QphBgdC zDy6hiYT7i-?=9-`&WZ8B+gm^YfKsXlz^?=73d4}Gv`PrAwYDtFA)>5p+dC&GC$Dl{ zH+<}nU4w~KGoCH^W*91X(OM{ zpHkjB8?=jx0FbpFil2Xdswbl>r-Me?tb=|OBF6&N-F}}vG>xPzPecZO~ zTL2gxC~oo`4M6+8|5CMDolF|BLZNU*L}xh=05Cp29s$7U=;#R|`U-&m6hg3+(%^Op zrsW5zwT`sb5fObtDfQUI#KbxPT&YyJ&9H@^i^bw)MD#!^m3nWzUJp7591H-KWm(l~ z^<^pL*YCgo{^lZe=bWxS5IcZ~IB9TP*ZnU5e^RT}zMsit1b~jhA`wYuCLsiv`DP;e z%2Q80^{zsppr3i>8GDh@MmC$(d-v|W9Khqud`+X#=rqS?wE7TcW=JWg)~s3ct(ln_ z37lHjUnx4R6nma0)~;Rq+qP|g#&Mhv)$8?8DJ7(oa&bV!9U2>kA(T=8@Ha&C*?c}< z*|%?>bLh~aU}$Ja%*@PS?b@|B3n9L0SymXScrfCYLl+ar5HcfMU)22;F zle2&4!+2n6=}A_A;yw7CojZ3*0Mc=s8-pO=AP7{vEm|IH}^&rD2A{F3Xs z;l6$Q9A^H5bUOVZt+fur5NJEYqqWwyZ5u%lG>B+VS6A2TrBX=_3=Ev}J>QOw#3q-^ zMT3Kb%*?c9%a-p-DMy>l<^jiX_%zg001Kn(+k_BbD3waRK@j{N5q)faex3m=ruc9F zVe6U}Lj0rax)0rP#~mjsl?p9qu(48{lG!azIwK<^|6&-%{f$QBU%R@x3}y~GR0p6H z!Y~X&F!N2bv$HP%cvMRH&L~>g#wX45y1Tn&7=|-Kh=&$kzaS^Oe=4<&^?4o{8ToD) zhL8BZKhxFKWwkU(txl%xKomt#6U|~yi;`mxhNtR_%wOZX_n&uao z`EVTe3u4Ln#Ia;bAp6ILH97dGqEOBHHTv{wtYGhM2h{LFr(40_8Xk zqA03rt?%2kY107!*tTukc`bRJa7OZ0p-^DNyy(c0BmbneeyGuC{BuuFk4Z!!GwV~^ zyZ~*QCN-PQx3t!O-QC?igI4opoNJ=99GvN3gaJ@YOf-ORZ{NP%lu~{=l}dfk_k9k- za7nY&4z0Cv9LJiSo&BMd@`*d|ytD2&j;vHF7v=jrXRQjyvXm|q3VQeM-8Ts#9&;S$ zx+sc}u*Jpq$66zsrb*2Fe_h=1x5aOFeq$`z5;-w3|_YiP}e_u>0KEX#rr0<~K0zr!$eGnvfHp+kpsp-?z$Y5iPb#na&=b7!;Jgtor8 zbLUP4Ag`3VMr)m6W?w1wIy1kpe*OCW!^6WWpU-zX-F?AvPDf+REXK#jQ79DjbI(23 zSFKhzXszF&wLY3krT*=nd+zzs#ZGz`8WeAli#~002ovPDHLkV1g`7Oke;2 literal 0 HcmV?d00001 diff --git a/zerver/tests/fixtures/mattermost_fixtures/exported_emoji/h15ni7kf1bnj7jeua4qhmctsdo/image.png b/zerver/tests/fixtures/mattermost_fixtures/exported_emoji/h15ni7kf1bnj7jeua4qhmctsdo/image.png new file mode 100644 index 0000000000000000000000000000000000000000..633c9137a4b339f6721c0d11f58f8146b5578001 GIT binary patch literal 34268 zcmeFYg;P|0-#@-ZHx`Y6fJk?Pii*;LbV@Dl(ha+!pi(Nr5=&U1(#?_w5s+58SwyZ&v|#g&PzQVbxIaW2n0g&;Qk$b2n1$A{6|g-{-xeA zc^m?{3VCqn)}sKcl{%6#VNKtP(~~NTzaJXYuW~hVQBcrdt9o{oV(l&c`0k&AMy~s? zYjTLEAx^Jt8c!L=pVoA!W@rizui#4k{R`L1ztv2#%)|r}e2j-e91GVfa~-=5&)-~z zanZxZcsI_WXY!0Fd%`CGk1mBHNxKm^Qn@m=eQiQ1Fb(jd>7eWIddd^F1Bb&>%UrKr zdU8xQ#etqC6$bpKePLt7TZ+JRSI-i0AW3)%Fq6^FB>;+OS!f|yJ?#n8K^t)9Wk&w& zCOI`9Jw}?zQimEt&vc(K478yIVa7K;EM12mk1m&0LPSY2$U@);jfkQ9SdcO<)J+(J z310m6RfNo4Lh@Y`fo~qPxOyQfoxo5ml`4%AN5Es#dndo&UD28%UVarro;OYgelxfr zRCdb&NH`}+ePvhU-%;?i4qkdpxy4jV3H`7P<^mK;az-mRRY6udb3|Hq?w`^X zbT_5zn^OR@2Kh69k^GL0aAR~a7h;Lq1~_uYp%T!|=qIW92+7(Zl|G4{jd-9Z;04l|3_hB zM1{$M3LE4i^b&Y6{Lmb$oR6=%%U_p$9%qOy>bFB!a*?5&pN4xZ(HtU4G4R3m8NYZ4#XBP>_T%evw1-M7y({;}OU zkgnbDi4kg75q|VyyiJ0b8R2*Bm707<=diGHJyts-P$zdl!STV;Jk+wFux#n7o$sq5 z;wVGH6{ zN7%hV#z+}o04(GpInQr?L}*=i0rp?AmQKa2&6B7MCr^mWf{!5DJ%#|fn3M)V2B^c# z$T*(&bTGc}_~=YjN|7k|dnm&!8dM}Z;tc;uST4x!k=9Vw>d=$UwPaQpm)L8NhnpyQ zVFb%Ho0EZX*gRBdYg}J4;h0Rr<&|}XeSjMc$SG(mu1uAL&Tl zRZtypBU&0v3xyI3>hkjId-5ktV%kmatpCRaGzYAN1aIDt>#l|&)QS2yZDM&;RkqD?@ROHPWL?AFO?pofBnEy)y9Mm}SJ3W|i_6bmf^wy7|=(vJR} z#0&0&>bhEmy9|qEtP(_FNfTnvYKKpY?wxY{!y{-`OB6y;Q6P^(2X4akgMN&u_B$U0 zcE%4TIJecZ$Uq)BioBf2pF#K`lZLPf^PsR08aa}U(s4M^tbRSesY&-!M9E7F^szHU zhH>vDHivVyE8ai)G}&94nJ%hJF>5AOi?N#4-f-lAx>gc8WPYUY>BZR`ob_~c?4S7}c#!GeV*>90?| z1bPS;C=H6~>}7fLB!v2BFQGF%WjYFw0l9Wh2rFQRb#M{%r|}vdd8%n& zO`pKlNSVTSw^K%6KL!bTbQ^pcqq;p4R}loZcH`s%;Wr?UbnFR6*T4#OZ_TF!iZsz+ zG{CbAmy)Rk5sn%+-#BdjK89aOc}df4F~J((`eQ&#Dd}+}Kn?nK7w~}Dmb^0tEpCbw zF4i*64_Ud4l9^jldx`%xZoq5>{sab+PPU}!NFavXL}$K*H?zMDZI&VH=y0z8;O&t; zAy?8i6%6|1ozHFx)77RB+Qo<=j=styh1Y9w!^!-j8KvO1S=|-6ymtVzjgAflvBWq9 z30&>Dt_}u;JYB*L5DURl>%!T5Pep=vHzWz(z_x~dzwW+*B2oCyZt4=F3}$u%ytAC# zf~iUvWkvYGN)Milg-q}skP~8EV^#S7xl|pzl>OFh5qRlua<&UqLMUNECz{t!N8(TK zdri!%Z%<&EkTVjoWMzE*J@3Z~lwb~s2YC#8wna>M1`Mdm_iYc(tH=vxU{SE7$<6gB zpV|#N^!hv1DOf#CEn|S<##&tHU*ZGknG(S}zsU8Of)2xi&@t>)l*@HkOz(rWiiG2K z749t_w*v8y=ZWDeR3RmuMxlrUZi3LiImj&*ymj!`{CDu7$Mj$}R5s7U_OIBLHC5Odpx0`#^dLi6x{iQ&?_1Rnft{7vKk94v$Z*=4L{V*MW;zH!rI z3=d^kCDI9xAWk{0Lr$FIR6m3rzh zW`CU@p<|h`&3~h3^PHrfJf!}j;=X)CP|NX@(!5}8EO*h8*DKWmyLYN*Kf7sgWCu-K z>JQ6}k6AW62@yar`dmcyXC_3IG|UK#XVxb7;U+}2{BGwwU!!Uc9yuOu$axOn$l1!3 zX5uMuB}(%>h8dgT`YK=cOXhQdV>I16WNrBV$hzdDDp-6{62(D45KPe4H;|+vq$eb( zGls}H&HIhJnZR;;{H8guzirEc@u*HtMcXb_0nQjI*cDiuwG|&HY?3_W;=#juLXO|v zBkKh8{ubA)e0cX3u0aA-B7JO`e&#_apOrtSAS?rP>Pw0f)L?9trXRnbBX2%Sx6%vj zqPJuF*D=+bxpU#@d6+2tgyBY6@9nI6u>; zzlwXqz&adDc19{^7iId*_e&i-;PS>&$HoSFq7*zW(4TomeGJk!zko$$?;-*yVT}D$55AJ!T9P8 zRR~VUZ)9Va*QshOMg9h&g_jUhP-+~%G?Fj++Vx%G&JWw&XV@lnuB@u+0DO3%m`41J zpKWJ8XJ5Bd-IasGs)?M+3D9G*8~Dd}ecBjZXGN}KRbAZI?gT1rzzD(Rv=TFBQcx^{ zlYHC-o!Cu#uHO5G`YP67keA?LjLbiglUuJ&*AA#x^q`^Gv>UaIno8L)>+)yENUsW` z^Z{nUq3Macf4aYnj^(t+_i$$64SR3fBp}93gN+FsaFL)Ci3@3>+t3ZKb{iPK!1pCS zUik$=oQ%(h2D-j|C5i)Cpw`U!0?2$S#AUHJC3hK8DWLcvyQcR*IZdsZupi8CJ$QiX zo5?_&kmJcWKD>qJ3W^xKwy5~T*~a+T*hSvM5*9{(fZO32MV?utHZbIdE3pJ9ye8~Z zC|x0DZbt7}ujgZ($A~2z$<)Vn3cKRZOjW9*dJyK-d=PtKH5gL`v(`t!JP3-MkHeJwFOX14NL{Ix{)O{+9Hbc=~)~qLvq{62kllo_= z&B6E`ulHM9dxOa~-#j{34*~e7wy?zf+}ihf2%W%y%AA{~pt zbb8E~s<~`*M{Gk`kxG5biP)oP%3irH`=x_b(}lEdsu-ar(pE66uZl5jM7sKARR3K& zoO*ljhg{>upn|C=P1g72;+Fc`85X^YI}MnzL@8R$?Gd9D)84M)OK3$A7x%{CB>NB2 z_D-A$QHnYi+_f4H{>nK{Eeay4={HOr+>-Wf|KwHEcZeI%7iDA#xOJDKKQ~T<#I#C` z9iMj4HVGU^4#zqf(>%EJ`7EI;?N9vrz%HFM57?KHV;)i@ zBfECEkj&*nM+%Y!n-!+{CR5@mjb@qC#pq>dym>?JQt(Ff+~?18Y4VSHiK!-w1eS!7 z;L3T-s7VEte|_#jp#(=oJsu^_A>fUxa$SJA>we1dx95%m>~OC9_J6xjVR}6i<}~}Z z(#xZ1P|fpe$~`QTaex`zilVyC>1%j}mF~9unD7>9;G~L!&)iH|j1*Yaia}+Km9nl` zs@S7Mw-tWJUK`w#qHvZ;q~5V4_F}VA7ha}rFLf{bO4gK4=_OGZT=VfO=s**@6Gl00 z3aB_~?q^5d=jspvRq0>LMDh~*bWGZPaN%_xj{bX5n)H8N9c8zu<%oIrh@r?LLDD4K z$?0p|$mM=HaBFlLwsy9RQM#vlTbnH)w_rd2Cc~P}++(1YRfM_9DK@xuQuww_TXr6G`oEJC zLNb6MP7C%*z+a5nKDZ)OL8Uh2Z}Q1t%0L-CQv#3)r9+(gbm}1choU=H(mS3zD}$+M z7f~^TYA8~rIGNlKaSY8aM(#s0R9yun4p=B~Wq*@u^jaI`BOEAldi<1BqMqM0hK19N zS7KIH$!4Y_`S=;?`!AWq9Seh{&a{718>=8r2;Wz?*+o_; zG>Oh#p;ZmtLslzKLhPbd!Aiyik$33W*^wXQckoQ2ZzDEi9xSBgn4oshiJ$hVkx65T zXk&Z7s&iqm3(p)`iZPecCj^S#&UVqWIdzh2!7;-c)NW!N+^nZkem9}jVN(=6;EV*_ z@sy9r{gh{`g;!WtUwWsun7b$lHs)C+pJnmZ2`^COL3>ie2-ZZuFQpIyNb0vG5dEGv8vF&())xZ z>}aLOr+T;wYu+>YmPwBlYV9^8#9Y=jS8wbcI2w>Y4{2MlO5>kb=9r47%&YGh*seG_ zkDIixnN!p<{g#mt3-;1(t)d*-rhzNV(i@YcZ*#*;Ag8qT{ zk?(}PPOI5SmNfLA(+qUS^ZW`)*m>W#&E;V*BjYBycxCUqlH4@@p;fYEbcCcM9%G{m zdd2j0Toio5j{L`>t&Z!z|#G;TqJRVG`<;&oQkFp(N+o?|4>F+tL4HiP@vcGgR$2dp0EWz4Qe3V*(7?KD1QL80sYdin^Hc4hdb+93H z&wTMJPtl-%r>7?OkKp_%!(C+)j(J~R<)-;uPIpo!V)r&fU!;;)J{(x-jN{wj2 zo$_pS@v35Y26M4& zY_KK@6TNYqUeW_|FYlo&&*t^^wZ`3XHcUB3!@x*VmuKi?7VufduFxXBf%kP*>*pZ6 zN=;0l_|IW4_A~z+;7Dr(HBH6!JY#u#qW7M?G}P73$|3$J*}L-4{O%nU$+o*g{tk%2 zBWh#igN<+@)aGV&UN*l!ty%D|AK#1&!NH3*JxX9M9y10Vh|v6 zdAqEeMw4|(h0wJ>=G4o?D>zL*c&1f=My>Rx_4@L5w!2(kz=YB7zIY9VS7eS4vMIyM z#9r7XZe6Q{b<>&1Pupm}@l7cCv>APw@S`Q4p@?+VvP0`^vqI*Tvxr-m8uq5O8RZvU z@5IQUSg_O@;o~^1dAqwmJmXhaJ}DGEvjTKsMBUvl;!&EQ1E=(FW@5a3rN&3>tZ6CQ zSILEydto$+tjN zP}jYly^*s!gvV|E((L=>d5gmZK{6Ysk-U{Je5qWYh)}Uu;8Jo&mCMIBndMFj(ehE% zGLki!WCwSKE`W2|nEQa7P{-$qz#l3(OH0wfX-B8It!vH)r5WEDiZ@z=<>ouSLIOlqq9pBQ|#^aiIijoVkdHtIn~W!H**#YDs_{*{eC$w5B#$<*Xw?uSQS z(H`|q+1VLqHS}P6AgQl=ox1}>0Q2EX-!ab8k$Gu+JycC%;DT=Sw?JBq9yzWR=`& z4sn%;xUWn&e8~;iacI|`)5?yJGCsiOTjU!)^Fuigdg0goPKIi3da~g zWQ7A~bt+u_a`HB3dD>Fjxa6<<+PtirikQqNf4<}IkyBr@Nnz_tm-|>=s`S!&cCsN8 zuFs9UI*yZJhGs6VJu9z{+I&`E=C&k%p1_@lD-Y6g%;($Q%wSiyagRFg&OLLWc)CA3 zt)teb7&C$6-Xq%|D2cI!6uWMW@XE<7?pz^8x&}=^O1Xt#Rrlx!<>&hC@2AT*Wlx`S za_Ev-T|l}Kr*M12>$N4tyvC3-HaVm~17Rr6pra*&+PNkB;CbN-eG`1mQmn&V`{%}g zMz2ok==GbK#jiM;q;bN&rd^i(XUvj(yN`h99d3gu9c1NymJeLrLbjSl=4*2qRpy1A z=a!zCj+|DO34^opQSKACh1vUAI-NHT+G-JhP9NSVJ7G|$nPq&5NytT8BXh!xI7rvB zN}O_3A@Qo{${gQv?1IC)KMLAPo?vN&zi~j3jm{~b5 z$lY1eXbCHp0sY6nHXL+BOl(oh1);$*wWA3SHxa0W_=q{e~dFeImF45 zZrTB4(N$Y1igPAIK$-zh(NaCx_|h^qv2la+I2HXnK$v96#s8JKVN%Iv)ZQ#(o`=>e z=PNP{=CcPvAxkS`BpNu(qthTS&5iJ+6v?|*VF$ibPna#q%WeavxeX-as83Z2zuu;E zM)e4mo2V*GZ)N?M68kolQC2P=mgJ{qlD z0!2HSHPA{f3*0S59#q#>*48?iOWgV{FoQAnDH8^(V{qj74pCsf#aJ!rTR7+T+gqc> z8(A(hLgdf7%gTLg6hBB#Y^~A5PFK3Wn3Dc{wZ*>x0Bq*6(8=G^!)%O169MP1%-tru z*u`q8nY2z$hg3Lr0i0)D0E=v$WUpT$uoer;YI`9+vl>@#(q%78x^7S6s8S3H&9n@a zOktG@Yh_%PYJA;0%LrEZ!oPF-#_S>4c+0h+ZT%Jorbnq--<2DMS5Qb*<1{WA=hWgG zq|ll~<$O!nzK}!X5xp44GAY1%gnMTO?f9jtz#bK1t(G=2z1FF(g%@1zfjqBL zw2J-hy?l71M~}u)pd=>8Q%1AcL7rj8Ym%~7xYRbjR zM$7w8w{hiy`TZc)8lRjZ@zAqWbWZ_4Et&x%4KeG;xnonlayng+_M!1}y?T;;pk;87 zg#dbV_khh<6(fkfGC0zqHBj*i8~;G#mP0p}H+9`7w>hyMi%|)3xsNWexl?_qm9jzm zjJ2m>+YXIE;N)0{+~U(g7;jz--ptmU z;&<@g2oO=?Q^j;8&1$ZVJ-^tdq;z*(WHXpux8rgCg%^O!DXmFQ*o=z@`FJ4bo84Z< zLWk_bvcL;Psc3mjUb=HgqJW~A(;8Zu{tSb@)$6>?N7&j`h3rF4OqBEtEClm7EkoA% zUY5}>fE(mo$!}}{^e~UgO8paI$jlQzl{{*;u&@!&SYPr>ri$C!_`aQ~j=V(aGa>oxb#NPe1+-zSDz6O{M2LxL!Y}s& zl}}a{uax=k2TWGReHbY`9eYRE4Ky!3TS#w1<-8AkBb50TyMClqITGO+aRuIONZHq* z{MEiclZ?1AO$V|cm5WT2UJB5;+gew@;{+9!4Ud-D+fYb5qj7S7amrWJ-D0;SGcGS^zx3bvB(VpT%2%(ms7G*0`%VzB-q1Gz4L>U~wtU-Zm;P~}?);5Dm>0|9*QlEzt z0xkBYXKlg1V6$p}N@J93EWbJyI8Y(#(O zW`g(Ve_%8n#f>Q!n+GN^eSTscq9)dB_2k=(%a9dFRF#sSA%TH+xJFlWja+K-C*Wb1 zm(-tUFz+_lg>i+neA#suz0pySxfE<0cTR(eh8c^h3=49mHJqQrINo+1;e%bcz=cvY zzXUDC$t_bOygnV9wWu?R)$k0C?Z_M^5~0je6{36nRqioF7gj&8Juc6_wzZRhp`v3d zQ{*qCJ0W{5w0;V$ZrXkcG(%Mpd#cIDTyF1K{vmKpqDl6S{i%V#=k3I>g0|ldj=RNZ z&1aTrtioE7eBeq49Fx$pZerRa5kGTqs=v zXOM(v#V@UOVG>kk=??>6)9T5WeU^nS-ljUO!P5DPP}Ni+smYvFbWHji!;`ddVPQlC z5!m6n;5`keud+yKRM4bTXnCB%LSR(xZhPOr2kvJQ9k7nZd?IbtH3kpI-r9&A*NBE7 zA05dwf9Z%(T?n?FX{BQ@;r2HD&5oP11E&P};C^wG<;dayf;cO(Z!~i9b#Jj`BJ)`$ z63RoKyu9?*f!{IH(#s#gG32zy{1`VaQw>)zRtCVZW{ zc31xwkz#beC(t!JSu<)ow}IART!LD7dfxBqz(KZ^QEIk&N|fA2ZUMjWf*i8=j48lS zpuBHDY6dX@h+}``TCS!Z$vtm49OxdhyOQ%#eVtP2eKGUk*U&d-WW)vzBAlb80SJ%m zlXZ5LS$hZm8iD$=SCm)S(xhu-&@3=VcK5GOR@3?A z0LQ3jg^peg#tZ8sh$tonR5riLs5!jU~i^| zT5SAN?>JgPm!OfostpK@Y`Hkk6B=JsQ8aZ>`(?_PuFz_9Tvx9D3ul!WTfxg- zFOE=5s6Wy|8U139@VJJG^qrt97!0`>%nz<(r5T2OYGj~y2d!a4g1h9Q*UuF5 zcEJm?lD$a3uTGH)=lJpzy>e~qSRVSmG|gKRAR8p#o}w6SUDJ7TwpHAZeU)bIVND4f z|K6?J64I+R<(^LqRqWj465|C$Ca{>jE>6J&d9t^`W!;eLd7vjVlr$@4X3l8QdZnB3 zx?g(vg9x>bImiXGY+rst=#CCyU9)j-4SBZqru(M3Xz%SFaP>%>xVl&-V6<-ne|~5C zi1Ad~3g)vysjirlAGuG^TAw^ja>pCtv6r8${3Jaq`tc#fsQLD^GgpxbIP{P9b*EN@ zVA7I!Ul~N|h{8CpOYcthE$7X0xKY2}-L$H+xWHe&@{#vav@aZNo{K>}TwWe!@`J$W zZj4lQK@$hl0oIO^?x>o~k^`P|>VuwUbNN_Ix27 z(ZNY%$6Vo7DV_(98u~7W`XB~_)d9C6|V`8ykJ8$pw@xsB-+oZ=kaaSha z?^1RhWVjtR_!6&{Jv|pJ{g7!=n1*AA?Sie#w;@^PkpqylNO5WWm+~DmU5V-s^WNJZ zMhxXw9B&}++T=eMC$)puojkRWG;|I5NM*i_Q3paA0KGO-SW_wru? zqu19g+*rZWDsCwcjb8?(JN>td)A7Slf3GG7INSdHq$dVoiLgETk_y(U18Z%BdqmhL z+k$&Xw`eJ{a#Cx1Gak5PPF5Uw3PR(6j<4H9gbtmm!@C)x2X=4XMRPBgHr1GPrVMgF zx_T{h6otJ9ZkVx>ND$8A$`{uzrVa*gX||b4bO(0y$?`iv7L}(8-Aa6@RsAZtMU}|o zYv9no1}A?3)S#s2qeyv5kjh=L$qND08|mPl_rLA+pau3aI4j~{cVtvNsz#KeuW}N1Uh^E67XHJdjj+Y9-5TF1jheu@07{i8m ziSN>2w~|jGmeT`bJq82T-p=}BGUJ9E%9Yc}1C&v5F6wd(n7cMzJxFHH|AJvm!HSn< zD0Hh9N+_9TOzUYict^cAZ}9G}Ai)H~Tda8B zFC6S^BL)w&`s^G)iQVwNA1NsSYQlNr?AyCv^3(x>8+Qp&eHJdgR$cDCHwZTXZA}3j zk9JVApJF`VGPK`i!hsox$+f!VgX06N!QT3rZ|+dC$ist(#=K%;Vsr6q`5V-^6vW*HIM?4EX#3E0@dLdIPYRkgu#$3;QtO1$22sdB5dGROU^n2$z^?T55N zFxUF}j-QG|Y$*ASDB-7|J#&xC;Thr~9JC=<`{6w82`#_{h5fhiebC&>Ip3cguYl&(HThipKd@33#T{dS48sar zS3$3;!wR5E6+-NDBlt-gM&bejrwoS$-tPqAhJTzu!@Z}5{}Ol6I0{|~wSu3VTpeU5 zIq)WUf=-95=#2zxO_mm6WY>QovH^==_ z>&YX;;7*M9b73$fj+Vf^4r7V)z3XeH7AuNC#4DSvb<0%l<$a7I%or9oHT#KeTM;C} zOHC*P6vrP+Hn>0NRqd-q3BW#x1f_$kNJ6>QUPC*Ex2%Ra1El~^|WS@wiA0fXV zM>bAT)j=Lw-g_5eB$(kb+)bVrnLd@Z$Y4UF5E|9?DpwauN0OoPwhBz>o&+o4aI|Ht zdBmZGQ7-0=))-Np@(gyjXGN#So z0V9`ukgy$x1}N|A_`}ToRSDPK>jZC}WV>&XrLY!DOA;{(XET5BzzNgYpDDzc@^waH zJ|0u=EfD2H`+X*+AE`M3+ScFId+wlZ^>4(BD3v)h`oC@(hED0+F&5-ILu{H+rS{F^8H@J=_T>&!{&?deq z3o*x{U)5*LjYg#n8}Ut`46IhREVgQUiUiEQ!_%Ly2iEY>Wvvp8Y~6BH2X@KE)HOc zcL@4M%oCsE+KTvyMG?iIi^A!3)S*6YEBmu$naub>rrPiy1J&2hdkz^)KJsWsl@qa_ zT|CS^??8&PKh;dsHyBBLYjFR>9?@-d>OLYKu(Fz3@iSN5Mz}X$+=ZW*r|5*in^FhI zx>-)?4~zuZy)6hPeO6m9CL72sI0NK25*P}QxB(J05{86q;n<){1tQ=M3z^Cx$n%;^ z-GR0Z#YY5!3rzAn8%V%3p{E2#x(g}T!f(IXvLs-DgjYr8L${Fy;Ev@oYEOu!AbL9s zRD^b<4Eqof!F*ZQo-K`7B(l$PLe?fFnL-j-rtJY53@aoD5vQ;ZIllwujddU&ExJT=JC)mvV~DtgruInN)~l+ z-!YxCN?qW20<4?omhydB29v%m?e$B*p-ThOP*MPJO)jV?%T{{c+e^NPw|8Yqs4Vv>@CT}?&YpaF4zGP=qr$x(EPFS zS*NwU8tA-9Whrhd1vDDuBY9K-;z(tD&IQ@R4g=kjWnD#Vz5i6BuA#oJ8nG|yFt^;P z+};yHd!SCpL_gW7TY$!?Qlg3_Sm##E+6gg`a=R=40;ms(&Rw`Jv2_O0b0n$wiCykIqll_b6Q?ssY+u(dLSouuTE9{(=guj(7V%8?L?f=G3-OVZW?}|zGI^h zX`W;v=1-UHPK@kxDf6#b!B(aGF#$hr1bSY6{J9!0#r%1})WG_V4Nz11pc1NMdDeW9 zeqC5n`w5c(d(%setv++C+cr7Vc1jMY_g#`}_3B|tICyq8VN7f|A@%y6S`n9ld zY63iH^?Njgj8nIHG50TB6Vy?=-|DugVgYw%k+4mCelX=8iePGo7Ow|FE|nwK|+YM#7s%IK@ycV zw~O{os{4gwT7E$fK}5rY-qZ-^F=9Se-qWlp8Q?mxk%&s?#rI$4Cq2n3!3}Ddht!9Z znYw@f711{K4yJi)Hlp|Hpf^@je}+8-P5FZ?W9-ue%c?$_0xpby|3x1~L1+T;&Hk`+ znhoQ%4$QT#BlS2D0@}yWc-D`-n(QW!li<@f9-c9Dhy&BL7m_JdM@5# zn__u7eX5&&Le2Rjo~+=qr?mXC$`$ZewcH82@ItpR{9{wnM_Tet%GrKz)?9Ww4NKJ_ z=Q5;Q;-t=-M$1oA7)E#-w53UgYTEbQ!}IL=uZ{H&d#A604@ixxvBv1a*Tj2ECYgA4 zdDbrjJp2Jy1;qfOiD^+3Kl5BLKSIc$R}5a z3I$!i)jg)JLv^RofabYqIjgmmh8|128jv$7I}uWF7$afT*H7oXOH4&Q{9|ly8cg}! zapfi7pR)dSaZ>4-v|1`SLr(dKhav`D1~m)=$JpB-uIM6`h#Kll{iGWzi6eQb76#2G z+(k0;r2cc`xrq3^YbJ-~fL$Su&nPlyL72C5{~k6XZdhoep6oEvGaZ@DluV%W;jqY$;?)_|2H@ z-W((9c<@D@J3gy=`JRpYB(I~L?7?ZKrbd(ib}hZwi+{#YrgqwK`_X)OGkMaZ0}xrkkV2O8gYvg6?y zo$x-Qp+qi+)g8?u@9ptJGlk4Q>?`K1H<(}gQvk$jlFrx9&mVMwVi^@$XDM`45mc@1 zHku*qF(t4rUNpU&Wo*yxV>b~7>u?IId~oJ|w{ayj^s&DF1+LU7?tJ3Am}A=weT$+( z6si-iB^)gVJv9^}6`@fcJwGh)|Gq1BIpB-u+qsh3Us~?ot!4O*28&lu2 z?9bJqtK!DThd-rgD-mb@4e;&tXkxIn+`myWsm~amDjpakA-CpIA9P4x*1Z_U89T=N zI=2u^x8=d2RqxhIvf9Gu$LmQ=!j@l>^<5?lW@6X>!=%Mh_3qpDs8@r0u%w-J8W?y- z$<-GL%6ILVPu9Ap-Z!pizbF>MHn?J$kCOhdG$;E=5Zi>SZl$ViOdiI~TFrWzuX22| z&geF75(0UZ``34TuVNe?opWA!?kwK?xl9x~BoJ7VWNWappbl0doP`4MGMAU}4(f5% zPu1=WW@6=DsHXp(OA|=x{<;pv;TrClCh48&4Jm;G()=&lvzoqqf- zB^UCbi>#{S!jPc35Z2z@ZISr?Eo+W8yTe9J$37j&3F8V+NjKFv$$=(eIE@8WAjtwc z&~Lr&?2l8Tl0#uk?j?DX{pfMi1=5VD#Pr?<{GK{{ig7!<4-l#Jh z5|>Iq;(q_nfCgIZ@X75x&BJ@2ajlH*G1_EcGBcc?c>l8mbsUN@UX{EfJ0(}M7IAdV zH#Om*FIb%oKzpCk79+at(4|6hM!m4#gUAZLEe7`-z$JvW|1z_$ z7dUy_DPKD1*1>%KcXKE*(wh^L73=GcaS$jtFBl9KH1;C%b;`G+5oXTQHAhh<}eU9r)`HAk$~@H^de zgoQJgGxJ^qa@DSbQXzWxbb#@_!MbRsr_sP^jLNTuT(Lr$WJ85~8X|8SrQ97Og8ru} z6+(jJSu0Eyy=xx>1ME^C-m}qeDzC80mtJ@>Tq2h*Wmg_TM7C*ra`le{O ztuUBi<{r5bBr>bzKI0-k=DOQ zz8R{=mbx_|N$VjM)#Fl$p>fFaSF_TgksaQ7j_G>-bZ65h?A>27_n9Scj2pl+(|-f4 zY~L1HYN0JJn#`JCvQ>&nJLjg%I2>Oh)bOqkn%w@8qAtsi``90+u8Z7!i96@s2Hz_?I63`7%@9VM zX7VzKY@T1nHiswjVSyY6La5uoL2VP}$vb3v&zN{>k5T4KIOK%c#>;%{7*okoYmqT) zN5Sd7ZR#PwybL*Z+m$~OAneWtO|hCf<+~|4w+YPe#%lTv6|B>q?}XN`IX<~5ca^iMSmBUEQnZ(whwpHTK6me`*-c^a zuxRk=wr&ai3h%c)b}U|gb9X%7d6L2U(b>Thqd&WG<+*ClzqzjVAcX!b8xJXTF-tTA zjlbkFfL}9a726p}L#t!|q^bl|SN!KiXf4gGcwnokovv*R?ughot#L`>^{o8*6=Lyn zW5zYz5ntsEl@$max@e}D*5ie(vqBt?Yh%BJhVYk_Fd)307<{=?|s!$;O zwd~Zq;}*%2Y~zON$TPi*EMc=s`Ws2OE<&V4ouBdnr^4A6 zmDx(Qbmea?Jrnk(F=nFN9;V7kxr)Uzxb z9>78%n|tk86|;G}o^FDq!gw2Rmr+qjgZ~xheK9VK+#GUf)1#0%%Lm<0S*Wpd0@Z2c zt7^8oyb|nVzxb^t5qwvkyJLXNI5MyHmrG3@K!a-V$VEWH7luUpibBPI_AKhUez7c^ zX``eM*R$r7n{^fD&-FWZh@&8E>vO-S40PPHN%(AWn+)>MuhZ%c}k&gBL3W={3R0k#ZG=~>4>m10RX72I(*7>5x#>KX_LtNIR z=XzVOaCuEO>AZa+4cs_GmR>FTeMg&p@I9;}uNgEQM^4Id%k}F*EZuE0V^PW7u~w0LPy;c zpw(j<98tG!rO~o0K1HWD#if>t=g;Ts-JKcNqw5$sq+J+0u_+(kq6*Gfcn0vI_jjm| zQD1kVz+sTplLt}ZCYV_tyP=Vnyvz+H&CFLC0%Dq%#&CbivmbNbB1o$k4IWLz_U6!} z`1z*%VwTBCiYnv!a3{m6^a-$NojBXmx6Aa!VCcz8va7=jVdcoTY9Lr%fpLMms6e05 z<`}oyNByu!qt!SV=ktx*%HqvJp9RcY`op?g-Oz>-a;iqgScy8of282aHv5yh$7*s% zGRRTf;Zo^Em&bl;$fM4rU3TNu*6>@H&wtjxWx2{a{X768cDD`l{YhEY6aM!8Ek49x zrAEnVGw0?)lxn z;U|Ay{o7S2=sMqfrL7r|e4sc#{HZ$1-beAHd?IZ@1o5t8v>fum^}O07IkRPtlnVFF z$mdJl*_O$C-o>mIvFMJt)$vC-isv^+_)p;545oX1V-M`7o3{=_q4Gt2cXprX{HNZ# zf*t~Dv)JtI+oiM12IZ95qn3h=2T?9KsF1RZ%Y5gH4|$d?B6PuO;&b5)eT`+Tr>(m3 zMH>6yyWbFHjt5d9T)hLT-I>vSx#h*Z~qK^ z<)bZJQDuYs=IKXG%cmpbU-9XhABPsP_0I*rz8|^c_t?6NDS6%`B-5hO-D2@$V_%U| zPQ!yJL60flHk$us>eMVN#qpCMg__>Hy{3G>j1C{S5{0BFmgTi=c$EzrJI-nQ$mTVO z!L#%ySf4Dt%_0`zc&H|xzRvU-YsagV<(tqiUfJ{Ao|{J+Y?EGb=f#NyKPDbur-E`t zGYt3fzejsMu}Pid4uwg7w*7j@wskcA#EYiazWYz>W>njOY1kL5VLHZdp)4+VQL?Ll zGDIP6{fC1?vlf>kZ#oa!YiwBCj!seQ*HJ9UnjCzvz2OEkVQx;)$E zIGXxdnf>!l@nx2%!SDL!dYjo)27k*UZ~M3RU4{ofWr~bh>V18%bByh5L@H%CXB$n( z)GDm?)HT>875!@>ndyf^(%yO5bqe9;bUAl#lYl!E(U*Zeqx9Kqyp%r4ih-%M!QV5p z;pMBf*0%5Zq-USJm@Bdg&zm-kRf*RvvkM86@2- z;S-O`h=Et!_gh;=X6%u6Cd*sBT4&gNkfVN+u-*(C11+dN}6&_1g>Alp(e@UGN0bImCy z-?@3d20^*KIka^O|K&V@b@{H!zSxc0Oh=I4T-XPZe!ESVWc=oYcFxBwf)d}oXIU(u z8fE=P1r@Ub{;d2`-_gKoS2C|(RD;(>iTh^e-1v#k&5Oly?2-QMdE5_i2BxQ?kImA3 zzl?286(Me3_Ug;X<&+-nNL2Tm$(P^y{n_`!o3cjBn&xsXbYoSz@b|5BnR&<4Yo7LD z#riU>lucjLqP-o*MtaHfy=JqWkX6zl109bz@CHbwM!}(gHtN*&s zf!h_`%g0DZ$HC@o8QY7>O1cAlKdK$Z-ttR$^>@1>{9Ddp;`UauPTa$GeT{3RMGTt3gl73+7e*ZGX`&}{A3Eb31y!~3Iym_b)% z#p=QBLm`Huns@bTM=?U(_KS~qM4U<*Ji!20VwCTAZxc@Wo?7J|tfOyo>#UX~(A0onavV zhc%k#&qF2d!F_*vfLg*0*l%jnV)E1A!vOSix}4%&_TBS$V`GhD=ik@11{_a^*6ycn zIc0`5Kj&wfmNFVqkSUX-3J^?>vCrW|2#~C8syg?RTuY^>5EdeO-1vLMysG4BlcD9J zhJ^yyrmmbz(+q^8bOD%SOH2b+&(>>5Rp9cdwBhF_DcC=7Xq8EFU{7Ve`#5an*gE_2 zb;y9cIbG1dgBEX0dLLi7RO(($gl7X6pKIyIkJ&9nI&PNzL-Ok0xN8|LazBYpwVcDj zHjg1RCixH~z>5}CZKH;MtBhCeB-D9EZH*1T5dF?#1rvDEVn*Bj=6dPPrSjQ^2N4I= zl9)Wgk2O;>O#b{Vr(`;$L0q+57%RuG%kBJzCEN1Cg(P_^A<@$zrU zDR3>9|AKqzAc>e1HJ1hWg# z3bizJF3g&w7^(f;m&&!qG8*=1J-6PQ_wdCu6sK-;~1+w!M2P+HJj);3L9*?YP@aJ2xgY^3tZO? z)PmfeX%%~x>o@j`xwYM>xLDxw5}!HIC%W0_J>*hf7E3CaCL{I1vAGSL!6Y{UYBDg_ zMlV<-B?t{9OmChE5RocZLg!00#HMf5?`E|hgf&?QChLcr4qw+wu~c&g ziD}~?h1kyjq}B{#HgBT@HQsdamRCi$_G_Tsy4UsuBI+#QG9sD@v$*wU-3N`)?K+;? zj^!mdk90^i1@dYzIh0#DU5m0{3}YLqysGx}4cyqf0`jI0ws^}GjTRhGQiXHx=WHee z;&Ff+$<2v2;!!zT67L0nJHwka!@t_(*QmocAiWv(cxLA5@E$L9*ZEr`=C0ioTU%zv zUow0&-9eJ~cpteV{;95kn?zMM?lSF0vQ%bGix&D~;DpTKZG=!MS7@yMHLsX^d(6M) zG_kHITwalUCJ+EwLCeQG#kAPvf)RBeN*+i?g$_(dB0hQzSoXY5Q~3J6O>^Fb3OTmA z*A8REX}99ExPxjcey|(&d+s4}TL;n`9@XxZK6GKqopIW-Pl4vTVZxCklfd<&?Jm;Z z(^zHfb1!XDXT`{>CQ5ozQOM>C9-i=HUi%Lw;MgxHwoLB8AZa;wcR9!V+{x^qdVbM_ z)b3lTfsdnr#xHk2!`W6a3XQYjP^r?LE5%= z&lwwZA)1t?LL>^M7Yc1nx;ZsY^2_pnlOd!-_bcMeaXjVnOOZA6>ARfg&>+l#@K1Q9o`N5Ul^o{7jOlU zzY58pm%Aiu_t zU$#i8(c$G-dE^3u?;C=}JcMsVpYf(-?{vn@XpULyx?;S(bWMZf){g<|_BbwTR zI}=;$Hu|AG&Ado-ehooMet=K?EgGcc{COj1vVJtBv=fmO#V!2(y*K7y#j-incqhy_ zg_uOQ*q;B>h_AzI-^W9;O0OC=eTZ)Q4h=)cyEOTztMESv#-fsu=h%)G*^gBPsxEG! zUwD`kNF!}$-%WQ!tv}}+8_LQTdp<)b;*J~G*$9f8ukQ$@E~Sp*G!SEfj@@yvNUP_*p<|@V8(Xjo`d^rucU*OpHfcDgM{ghz=#4PF~$w3 zt&eO)EK4e*3EpCuvzoxMF0$r&d6mf;#8R<^9c{S4m04wc%o~vz5pPzkJ)E;p10*JJ z_4_PAy!&qMW56p#d+`SuDa5mMsIzB>=PYx)Ihi7znO|Fx7sLCgHooUar;4Q+y|Sxl zVJUw_b72U<@e<@zjbA|T@0Q0nCr&RuLujtlVGqc<9pi8Bv}w2%r>4R(YpOfxAkN1; ze-Dc=-p?2uPm7W+I5?}TGN;@!QK=4;Xf$b6y_X$9yi1}-U-ozW$I zpCs#DUTo#~yP702UVfVa-H+U(G|y>e0;^asZ6c9l9ukkG2lmoc6@ z8v{-5IqgGhNjN{IV8st|nrUT6A%6Cg4&$~6%y_z0fWR+0SkTuBjp{g403Iz1*TzZ(%- zZU@ie<@UZ}6T!f(^*jrzu497CdUDj`|D(gI@IfAuqdQRyNOv7v(LXwO(7NbFR6fF6 z$19>yHX2pb%R=7|1)b$WWC#^=Rv01=L(mPNGVVUk<(`eBWvJmU#o4vy!5;oqCj#ImgHcAnhi6D($J{{W!gtu@vpx%QhMA#KO(BuO9Y4E zo0DmW7#wK>gE@P}UF2aPaP;TqUdtV>hYj0eKmH&!g+~E~FG5z*K?T0T{3sCx{8b%N z6>|J|pjbvciyf08BFUQIr=0GHk*$q}2;>KlwY2 zKHpPpDhWz{2@LfRl;FxraQ*E~DyBD%8W&?DGrk+|1=}P&=sX&6&FO)M`|vdn13lcK zFa>T~6mTYq@6Y+oHLCT-63r6Tb9#*8nuA{dFOLl;vItZ$3gh&>N|p?RthP34b-VhN zoo0F~5&h4Zf_V+j{byoRS{dyuR#jZ9&rErb@!&7x;j!1G9B4;-5uGxz_$&~YdV z+>{tS!xs9X)+;;(ixlF-u@L6E(!n1~SRVxC@XARo{oiSZ^T&c0m*)F;CcwI@2Cq2_ zWac!fP5D)W-`V*{DW-I0->6vij67phQ1GBZ&!Tu&MYn72fx*(WX366NA$srYf0d@e z81*OV{G4lP09QRe`fTKC9O$>}94up#Zy{DF3Ir6bE*>(b16*AF3s5D_0zfb`pPSMT)4L$Rjw>Bp)o{aO z|6z=(fh`rRWX^A4T;wcE4sID+iak#dFm$5KF*eijL{1{(<>iRd2dD2FpgNYM+)19yTY-6_>(m z3?n=l3$*G#*k0hr-hEwCeF$PfX_v6$Acky%I4g-Q22;m$D@hT{8gCZ+MBkQu>jZTJ z>6|qEY%9KR3OGZ28Vo4>r%()14)bo9+{HD2O2GSih&mqMtK4LcvCOVR7SGGRq%GpA2jIk*_j80womx z6E?WCK%{8q;qYXOj5#FOZbKB~`-nhe=KEeQQUAc%g!Vp>AAL){VN33Ds$zEpAF-ppkXX021FXP^HV;(qiDQGV!QK}1 zIssMxzeD)HQusfhNT6KqDT)OrjDV5hgc_8R2%@%-?i2RL_b4I%QXLOSp*m!MVN(H} zgo;Pp$KJ!X#tSaZCI1iSYg!sA6tb0u5=0bZf56iL6rKbnEL8D_4#0f}aWgwNk|F;= zm3!40*gc2=MZ7Ok9Qy@C!N0{l!h*Z&x+QFsK~oD~<*3n265ad2?E zCP;0-?$0%UCXc<$+%N zJ8%|A5UgYi=Lx=!i9*5A=4Kb7ZEMD^LFCxd71*28FSV~#>YdMk}DiWtnq6-+|wN9tC;2+Fff< z18l|UsIkw8wp-s0P)I}wte$_r53I57v+3SKwm`>~5t+ZQnTEn|fsD?fu_HnoNY<%m zu1fUiA>ax9bxijYgM!%NmlQX>g;*T|h1sYyNbiGINWBq^HqB#lLl)Wy7OtBEe5!6S z2su1zQ`&V;b=+|)vGB2QQ$XXrLr!M_Zy~7XL~Hq(6BuGjPza^$t~C+R1w9x#JCyxm zQ>r6D`q@5WV)wQfeVJ#|DMPiI}X7K zqD9lOFwLwN4NS0sr5(qTLG(Qh}-!>6!&WaAeVXLJCk-z?kL z!*d5+2V~5cQD6U8g^DhB8F%vr_}gy&)!1%7VUlMuRctG3=;%iA{(7~ zR_rYb^VDqtAi6g#n|}}3dO?9E39=00?CWd$Nyi`?`-+G$dsuoznyV!SWcZxrQ#L+^ zg2;4u&U3>hT*(GGhL>#Gzyp4X^m2@v!XS0r;;$8AQn^f_-*~t%G_`EAK2}U%0yZrZ zuqD72vskaI&dv{NtT`V8kSPuL7@?D!`kt`m{?E#PUohWA-s*UxJm9^1ObZDh#tR0> z)dEQ|6Z{vG67}#7c#pC}dtwaZk_I^Gb#uXc*L{OE9Y9oF-kH3POb%lGZ&Fn3H4FqW zZ2~DP6HKmBHq}lEQyudERDyk|8`IjllyUk@r?Tk_SjbIF-X2ip_biy0!39b;X>G0PhesmKs9&Qu%YEm3G7~^zC_(OMa z83TF><%_U!^R#Te(!rIdtB1TDGQZW-N+6HxkGDT27$cgREegg;FIWEboyE)3>wxTk zWLGr14%-B~wQ@!DT*<_4M`B!%Of;1Y*-3dqx-X4(v#1_@ve1vjhjh)^G73X?VZnBh zw*C2zm~DJPOfh^?A;}GU*14->%=+`0h=(ZZPU(yjHzeEO>!&mGxp7mkxoeN&5#lp@ zcy1&oKWd^^gZ3&>ZBLp{e)Xs>clGoIca3!s z{2s7>Tyek69qLBZTsNpLQ~U1@iLk=MU8-iH{o`A59;=%jD2u>ssm>o^q_cD)N>`5z zELlSUBRRNk;Df!%d>hu)CwtEoe{0+e`}+&QH1*i~XBZB|=Rhq$ZH|Q26S=BSCKEeT zs@f1|2_KyK*ol!CMrOcN2Np(BKOGyHTH|w{gFpf?!MARWCtqcc#BBQrq}*m-E+ui^ zsgq)9=mJ+PWD%V_iuK0O>w_yO)X@uw;=}Es{Ps4S%i^^FndS#TXGrA2v`4HSo_AG# z8Lufm##!4A^oaROYz-r*A3d(l{-aQp=YQ%m_{+=xGe)QVE;9;PSn3tvsC9MO-}a2p zEvn{jC;hwSh27&R#S7oo7#xueP?$iQ683G2_A8>6Ny-B!hh zgFp&$@hexBL(WQvBQ^4{RGUm};_rv8pjzU;c+f8Aai^i(tCu1urDJW0oSj_{1&4rI z1{{Z%D$`NGjgIXIgu!!nutRz-7mUi_Uift&F^yLiu~W!Z7Pi#v=Aw`LQPiWd8WAemPSKh`Bzz?}yGuy}-J< z;?0<_9`ihuald1Cwe@R(?6G2Yn65(JA0?yp)2w6qs}({&d!2SW@{oJ*v6Xu`U#ay% zjZWA4P_KF^W?{(06ef54o;#P?D8&V=R^y&wo<|CLJ@RyNB4f01OA7pO%akhf;%fXH z;R#8lQ8AgBH2dP-&-DGekWk%|#ZPhdEY$OBykFZn2*gL=OXtO%ri)#Zy`9>|=j zqaGe!KG`NSib!lqH5v3zl;AH69zV~jGcLNCo12Z;F{Lt^zY5jmuM8fLGwwY-X$xaM z9{*Q&DSgA29({qD>WxYkSH46)m+Xvsb?cAlS?;gfKS=2u6AK7OeKv-fo~!bPAF;uo z0$M9N4HRwv3Nn%knV%+=eZTz@4=u40AzgiyMGGMm%&|~-K`lB!bLzP5Ko`OCjutv# znElHkSxUQaXxK`@hWeSpn)Y(hw?NfysT8ufPq!wk+6YRrQC!lq8Kx6LO>JrSKP1dF1?NK%>ks18!GNt>84_32n&C3sL7;YgO>l(TvfPfU)~?rS+;NkXdxHVcC6=ZRPk!Uq^d}&7K267+isFb!|N&cJh3DijPuE4rj|{ ztf1&#N#$Cm;aCzCi>34yUq4ukG+SJFd4An2SRtDGxZ!&9Nj@F%p27)}KV?Q#Y_ z8`^!*5A{ZN{Iw77Ec@bsgS{C?oe>E%D)lh?=F533zwoiT?^)l)IKE^>5DKO$jYwi} z^2%>}?X#F`;e{5@NSqiOcNJC(+YBC>&8{4mebI`3B^g4(iWc7x-~?0&RaTilBMOK- zZB@((Tp|&fB7Nod^8#CCyYogZ(Vq0N&V7r;$QP+h@iTO)-V;%%>8xIo(Uu^ya4EWG zt|J|8IU)}_cq^ZGTV4c8lA!*X7bW`BE#ch(-c;N9#JQqd-7EW_2u>`ePAdoXBE||* zaGIiQ;_VrCyEK^0mOMtdxO7J}LmwK{jkjn%l>+p!`gEfr&b}gaLggn9n^^YeO38a8 zEV$saJ%4)l@`YbGV-Mc1-D#clx*&jNH3TYsG^sb~LldN$7ZartDB`C{AO1Y0GmBhl z>g5+$FH zC5{>pDteSRXHzX>Y+fLG1?sSvzgJP(!lFZDnt^ka$J{Q|mps^0Tr$#~GT@89n~TK1 zpY|+JvtZ0~3e=ZhpksGs?FHv&jlEq%=Vs*;TN@hPQ0gq zv?7wXQL)H#NL|kNzA-nU9A3@L!M@?MlR7O--8!`iE&eV}){~1}srg*Zh}PzUqR%y+ z;a=^VFzIvW*LaSI?-WAhY zi5B`I%S?$JBRA~&Y#Av<9p0o?HuN)Vn#_CcRa4cx$+?nvs|vYDY+PGz@h*8=VOk|= z=EJ@osM&z^sGx3*Xc(lqe)Lso;9DmAwX%+&6b+V-^fTnsp)XY6JF0~XZ5C1EX~j)P za;2NOA+OBb+S3IhBLU}R0YSa`vIA-%iG96TOuxWRM8w3?{!+5jsEFL9hrc0aabdYH zwM>Bevuw7gM668Ztcn{}@bo|4HPc3qTbKGPNO!hDP6-?9)^D>4?gJ%&&4da&)kaA7 z=}W<9yAw5@#;Z*S#WQb@1Z>=U7#dK3Dv@XktmY^s?+m`OSb^AGE*s~I`k z^_?P_*f1Ka^-w*&iPv!toL-y1;=M6(=U#qBlI5EIZf?qZOWTJc&+eAyMe2TiOVAM2 zLugAZ%8^jW>oZ~4cUCqPwzantvTzkJ;4QbdJ!rjU*5bZ5f4;ron6oC|Ruv{iT>?`Q zAdTr(U-=0DIGfgOAn3b(u*)<^d^lpH6lSAPhzEmg&D-sj|cFW>^TcWKu+*h$- zD|p`(uk@bKple7H51YC~6rKFe(`A818hg^=VngBkZ^<;lVe z_7bqt67WVcl!EP22wvBdcYkP{0V_Pi+^w0=SkuGa^u2~xQsajI$Zt2FgZwX3x7b=F zVQNYcWv=}weQ%E~HlhZ`DIw?)PH?`?lUK}IUsPNL(?gf`)9V;AF=Jt) z{MKui5Lu>Z$Ci3yrXN*?>byr;I}<=%;x{YC_lk_N_8fU+@Y~N$%rVE>D=dAgrEX@u z156($0f)xLr2b?Bjp9tF+Ts(xjP$vgko$%e)1JMBV~8+3r(bR6tSLSu!qCuR=65$Z zMJSLQzx!in4{te;ob1-6A3g zm9B%{G|Mi+m8UMD8S05^)>bVp|2Zj5aPT=rDl@iPbzKcs5~tMhBN4*6njVzAH`Vcr zZ{zmEf|y@BKTsbe9{6)>ze2TgZmjA$lLhqZ|VZ9ICSB6faR{bzqZ zwvR!3)2pTQIDtoLf?zq{UX|5-Fd*K{EHWXPvMeiaeqQ3X<^4SvBVltvR`!u~S_*`t zp(y^DAo?S`24VbM{`!f=`HvPB`n98Di10=W2jP$eL_Nq0FK*8?@u>5?oz-++RS3sW zoKh&uf~k%=11Y^-T^H7s8^@3JpyHYu>7#G@AdjHpQ@s;+M*BFhzyxS#D*WTSniI3^SLz7ffP2kEh86dq zwS_@ckxSELEMLCymwm9o`g zukIV79Z0zQ!xc5AxRDkNF`qJ_8P{GeW*f5B6fD7VFUHM;K6KAjYO(y&6Zs400TM4x zkEqZ`JMOHiWpSwK{3DB`DF2&Gj}-_+5xhi!6odlP{ji8=;i@A|p4e zXLSBri&MWom%|b;mcV7Q-te`)zgnxtI+(vaug+V<4H0E1E40#k9g{;iJEZc!(x4nB zPw5}>k!Q&Hwz<${Q2y&X)`hM&D0JJelTW_i?Y_yBcvr)JbW(`#M=$LtPqWzlH%dBX zwSrvhJZ8=UYp*|sr$5|cfnGXp@*hAXDAa^%rEp@4^!(8-tU9|BhJ0714}mdtr9K?& zj!y^-mZ%+l8f3lVFns+uBt>j(dGj4gA%E3TV>K^-(!1|zcXG1J9-m3oY;t2+a^fiT z{-4hGn!I8ZTox8%F{D_ zSY%tEd~girg`oh*G0r^6_WWtGjvulcn})yBolDO3R@Fy%blrq2(MZgbvC|kGo?xfe z&W5YwUskV8QZe80g}i#Ctz^)X%c*KXA{bg5gk@8W2t400Zg(1Vd$1e=QcEs%OM~No z=qawV{DSKnPI9nf!$y@@^S{5hyH$_(09lSDjFqrAG@gLKc=O2ToQi|DQ9)mGihhYN zm#&tumvgGiwpj`3LMYO%A(wAy4b-$C^VaDD<)G5Ade8NzItB{VCEY`F6#l{$Gj;`A z+X9OG?h>+fPy8Y+(?Ab7A@{N8jo){D@Oh=9b#8`ER(hK5VtWnf7S8YQyKEH|H;h?b-fcu|Ba8-2ZlZq|!Z2^o`+us<}=P zO{>q^M@!$w&P5iQ9rFfS@)5qOk8^}2#LE8$K#|*!kLy4ESpzB)>PnqI20p~=du5bV z9LdE~XHbd&4#vejdQ{+IOJC&Xj}|Kt`U3O7J3fPWp>Qhj7lLv>Zb$(A6!{7C8fDN( z3Zp`o)41E~-cSSfI7h%TtS{@D6Ei9p#j;a}6i3FUzkZ-0*Ov9zsKTKu@P~$H=mH+N zY!i!BwfEN}_2CnHG%>6DRX3Ymq7p5^my%FZnQo_pKQAB4Nr<&Q_){(9Vsq&HZO@5f z>C)(D`y8&RdyHYiODQ)NT4#!^9ic*af6DP^@~-q(c^b3*@uzHi-1eaKfF z>Xr90p^bM~$?IM8A?V;EBqczC%oEQ=c!Us z$=!Kovp&IIgGzlXw9RX;s;&}?YHTG~xy4e~__{6ijVz&9#E+JJ$sHojGqaPnHP zb&;z##ueeO_cqvNvF%y)gv&Rcy!{g8A1j)gDs17hciLfvbwRtY$+~(wEb8mCt^JRb zFBLU4R+l?%pTLW~I=3Rtn!WeD-L)0Pe0c1Ic-qV+N*=2nI!^Z zoW6Pbav}qpSFqr^6XQ9oWy*+NE~EmmMZSWRyYoU+)cU9s*) zjvaKm^pA(czXz~k#26dFlGG#r+X85ModztX-F99S$N#5QI2LFXuj18f%qPGDaY}Ur zg>YYIMa$^_wu*z1KSJ9O`tJb*v{41@)GW?_IQ_R(IxNsCw#$3=zb#+_yddx&bJJud zH?UzBib(m)afl>D5^J14Z1AcLyPmYZma47gVN8YozEk)Dy%LoV94tRAo*unK!wBEa;M%z?MyvK%T&#v_e&;#yljd zVVbz5^u#fQP@4SYL8dw*m?5>dykPzc^0vXox|*%x`o2F%3*UHM6{u6W`aRSpYiTLf z*9BPj)f8sJwX(pdzr@SFgL3E+e1V?aK`X(<;mI*^4NUbAX?khq6BX6*6*16bY;vF< zLk@hep|W?;Y$!ccw#ui?>8e#SC92Xi;;Ml?zn5<`fg<23y>%vQ!DcmrVgw)zC{W~% zpM9A?m|)xNbq87%0D0no3A;7Pk;;S2zE;m)fwPYd-^wltc#tmB9tiVjRo4ir^%gRL zFmZxGe~pZ}KUf~qMf^_#?I-yr^;yBqE4Th!j(yFyF zs>%;~RazmV2BF}AEYr`>VISnPF5-FBy_o)|DPwz!V|wWKfaC6pTS zlK#Bv9{M|N26{ z>PnDEf^EDk!r5FgzV7u_B9GhlUi@W13H~jbNfVXwq59tYBJk3lO@W<8SCwZg)}^-;fj+YqB7dL;_DF4ny9%k9&6Cjt2f{I$S_ zA)qmG_>>VOnbM!oLq6_qu0-t?;segw1lY4_DnW8F+8>?T!l^5cq=mQ_iahmrFz5~? z#nvf0<eV#9!KXd-<1BRq-rzaD&~LOulVYh&ceX|D)VL*GInQ5nha z)e^oR1ZGvB1mr=e`$ivv z!?{18A6s6_kxMj_PFb>qNG0(Jqt*nQ3|7zuSceZEj5tfFvc3&s>W=m*Q)|vd<|6Bi z`Pwl3xPkHV*eO<=x-!Ucq%9(vZzYbWB(d#;*?{v=<2t@OTq@3*p7GL4&Vy%vp6ksH#@i|3_67lI(?y_kP*c?R zD7d=Uk2i_R1aD=5(T(bVl7Ab60Kk-@sqv60gQDlAQ&=p0FA<7}KCC}P% zQ8th|Blhx#;M0_tIEVdGhYhIbwt>di(KxcL$EY^r`Yz1dQMZ9KiTz2Fl~$k%MDm_! z;sMAJBmuwouVsn?IesnN1`bU?nU$-<&X2`u`1=BOEVd9{LigPN_mLNv>J63Ua-8W0?g zYuv|kh(v@o9{YkYat)$a8O6x>t$^=2)Il3`bo@(8HuSkF;5dv#+&~@%3C-7_G7*%B ztstSuUa*&51yXR1*>9_|H$0#f;@3tVu$b$V zwGubu)?a&B`EP)j)yJoO8B@#vmAoFh;>)x66ISST!K_Gvi%u*<80u@5wtNGCRow?F zduW$ySpW^Z%evYwi1>rRK?uq<9`zy&5M(&1khZ;bS*sM_m6S1|Y4&ES046kXSIBv= z?!P3JXYPG_!?Ylk+tS)^Hb z!7TL;2DW}40BcbK21Vjy3+$7LJ&xeBHnNMl>W2=>T-p*}n z1&!J>l+{zD-9RRdHQe_M4U?=mBzB^k*V8G|5n4s{B6%$S+KvIR9SNgm9U-71FHnt2 zS`qiLvZ33AC9=5s-OkBgQm1>#C|`s6=LSF(DNKd1sRi7LfyP@yA96)i){tJS`a#-B zADT%+bjIYN?rD8rsF>*W9q^u1ZQ39k#)7|s)t>~70zU8=l~y2G$^6HK}s None: + with patch('zerver.management.commands.convert_mattermost_data.do_convert_data') as m: + mm_fixtures = self.fixture_file_name("", "mattermost_fixtures") + output_dir = self.make_import_output_dir("mattermost") + call_command(self.COMMAND_NAME, mm_fixtures, "--output={}".format(output_dir)) + + m.assert_called_with( + masking_content=False, + mattermost_data_dir=os.path.realpath(mm_fixtures), + output_dir=os.path.realpath(output_dir), + ) diff --git a/zerver/tests/test_mattermost_importer.py b/zerver/tests/test_mattermost_importer.py new file mode 100644 index 0000000000..0950a25d20 --- /dev/null +++ b/zerver/tests/test_mattermost_importer.py @@ -0,0 +1,507 @@ +import os +import ujson +import filecmp +import logging + +from typing import Dict, Any, List, Set + +from zerver.lib.import_realm import ( + do_import_realm, +) +from zerver.lib.test_classes import ( + ZulipTestCase, +) + +from zerver.data_import.mattermost_user import UserHandler +from zerver.data_import.mattermost import mattermost_data_file_to_dict, process_user, convert_user_data, \ + create_username_to_user_mapping, label_mirror_dummy_users, reset_mirror_dummy_users, \ + convert_channel_data, write_emoticon_data, get_mentioned_user_ids, check_user_in_team, \ + build_reactions, get_name_to_codepoint_dict, do_convert_data +from zerver.data_import.sequencer import IdMapper +from zerver.data_import.import_util import SubscriberHandler +from zerver.models import Reaction, UserProfile, Message, get_realm + +class MatterMostImporter(ZulipTestCase): + logger = logging.getLogger() + # set logger to a higher level to suppress 'logger.INFO' outputs + logger.setLevel(logging.WARNING) + + def setUp(self) -> None: + fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures") + self.mattermost_data = mattermost_data_file_to_dict(fixture_file_name) + self.username_to_user = create_username_to_user_mapping(self.mattermost_data["user"]) + reset_mirror_dummy_users(self.username_to_user) + + def test_mattermost_data_file_to_dict(self) -> None: + self.assertEqual(len(self.mattermost_data), 6) + + self.assertEqual(self.mattermost_data["version"], [1]) + + self.assertEqual(len(self.mattermost_data["team"]), 2) + self.assertEqual(self.mattermost_data["team"][0]["name"], "gryffindor") + + self.assertEqual(len(self.mattermost_data["channel"]), 5) + self.assertEqual(self.mattermost_data["channel"][0]["name"], "gryffindor-common-room") + self.assertEqual(self.mattermost_data["channel"][0]["team"], "gryffindor") + + self.assertEqual(len(self.mattermost_data["user"]), 5) + self.assertEqual(self.mattermost_data["user"][1]["username"], "harry") + self.assertEqual(len(self.mattermost_data["user"][1]["teams"]), 1) + + self.assertEqual(len(self.mattermost_data["post"]), 20) + self.assertEqual(self.mattermost_data["post"][0]["team"], "gryffindor") + self.assertEqual(self.mattermost_data["post"][0]["channel"], "dumbledores-army") + self.assertEqual(self.mattermost_data["post"][0]["user"], "harry") + self.assertEqual(len(self.mattermost_data["post"][0]["replies"]), 1) + + self.assertEqual(len(self.mattermost_data["emoji"]), 2) + self.assertEqual(self.mattermost_data["emoji"][0]["name"], "peerdium") + + def test_process_user(self) -> None: + user_id_mapper = IdMapper() + + harry_dict = self.username_to_user["harry"] + harry_dict["is_mirror_dummy"] = False + + realm_id = 3 + + team_name = "gryffindor" + user = process_user(harry_dict, realm_id, team_name, user_id_mapper) + self.assertEqual(user["avatar_source"], 'G') + self.assertEqual(user["delivery_email"], "harry@zulip.com") + self.assertEqual(user["email"], "harry@zulip.com") + self.assertEqual(user["full_name"], "Harry Potter") + self.assertEqual(user["id"], 1) + self.assertEqual(user["is_active"], True) + self.assertEqual(user["is_realm_admin"], True) + self.assertEqual(user["is_guest"], False) + self.assertEqual(user["is_mirror_dummy"], False) + self.assertEqual(user["realm"], 3) + self.assertEqual(user["short_name"], "harry") + self.assertEqual(user["timezone"], "UTC") + + team_name = "slytherin" + snape_dict = self.username_to_user["snape"] + snape_dict["is_mirror_dummy"] = True + user = process_user(snape_dict, realm_id, team_name, user_id_mapper) + self.assertEqual(user["avatar_source"], 'G') + self.assertEqual(user["delivery_email"], "snape@zulip.com") + self.assertEqual(user["email"], "snape@zulip.com") + self.assertEqual(user["full_name"], "Severus Snape") + self.assertEqual(user["id"], 2) + self.assertEqual(user["is_active"], False) + self.assertEqual(user["is_realm_admin"], False) + self.assertEqual(user["is_guest"], False) + self.assertEqual(user["is_mirror_dummy"], True) + self.assertEqual(user["realm"], 3) + self.assertEqual(user["short_name"], "snape") + self.assertEqual(user["timezone"], "UTC") + + def test_convert_user_data(self) -> None: + user_id_mapper = IdMapper() + realm_id = 3 + + team_name = "gryffindor" + user_handler = UserHandler() + convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name) + self.assertTrue(user_id_mapper.has("harry")) + self.assertTrue(user_id_mapper.has("ron")) + self.assertEqual(user_handler.get_user(user_id_mapper.get("harry"))["full_name"], "Harry Potter") + self.assertEqual(user_handler.get_user(user_id_mapper.get("ron"))["full_name"], "Ron Weasley") + + team_name = "slytherin" + user_handler = UserHandler() + convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name) + self.assertEqual(len(user_handler.get_all_users()), 3) + self.assertTrue(user_id_mapper.has("malfoy")) + self.assertTrue(user_id_mapper.has("pansy")) + self.assertTrue(user_id_mapper.has("snape")) + + team_name = "gryffindor" + # Snape is a mirror dummy user in Harry's team. + label_mirror_dummy_users(team_name, self.mattermost_data, self.username_to_user) + user_handler = UserHandler() + convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name) + self.assertEqual(len(user_handler.get_all_users()), 3) + self.assertTrue(user_id_mapper.has("snape")) + + team_name = "slytherin" + user_handler = UserHandler() + convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name) + self.assertEqual(len(user_handler.get_all_users()), 3) + + def test_convert_channel_data(self) -> None: + user_handler = UserHandler() + subscriber_handler = SubscriberHandler() + stream_id_mapper = IdMapper() + user_id_mapper = IdMapper() + team_name = "gryffindor" + + convert_user_data( + user_handler=user_handler, + user_id_mapper=user_id_mapper, + user_data_map=self.username_to_user, + realm_id=3, + team_name=team_name, + ) + + zerver_stream = convert_channel_data( + channel_data=self.mattermost_data["channel"], + user_data_map=self.username_to_user, + subscriber_handler=subscriber_handler, + stream_id_mapper=stream_id_mapper, + user_id_mapper=user_id_mapper, + realm_id=3, + team_name=team_name, + ) + + self.assertEqual(len(zerver_stream), 3) + + self.assertEqual(zerver_stream[0]["name"], "Gryffindor common room") + self.assertEqual(zerver_stream[0]["invite_only"], False) + self.assertEqual(zerver_stream[0]["description"], "A place for talking about Gryffindor common room") + self.assertEqual(zerver_stream[0]["rendered_description"], "") + self.assertEqual(zerver_stream[0]["realm"], 3) + + self.assertEqual(zerver_stream[1]["name"], "Gryffindor quidditch team") + self.assertEqual(zerver_stream[1]["invite_only"], False) + self.assertEqual(zerver_stream[1]["description"], "A place for talking about Gryffindor quidditch team") + self.assertEqual(zerver_stream[1]["rendered_description"], "") + self.assertEqual(zerver_stream[1]["realm"], 3) + + self.assertEqual(zerver_stream[2]["name"], "Dumbledores army") + self.assertEqual(zerver_stream[2]["invite_only"], True) + self.assertEqual(zerver_stream[2]["description"], "A place for talking about Dumbledores army") + self.assertEqual(zerver_stream[2]["rendered_description"], "") + self.assertEqual(zerver_stream[2]["realm"], 3) + + self.assertTrue(stream_id_mapper.has("gryffindor-common-room")) + self.assertTrue(stream_id_mapper.has("gryffindor-quidditch-team")) + self.assertTrue(stream_id_mapper.has("dumbledores-army")) + + # TODO: Add ginny + self.assertEqual(subscriber_handler.get_users(stream_id_mapper.get("gryffindor-common-room")), {1, 2}) + self.assertEqual(subscriber_handler.get_users(stream_id_mapper.get("gryffindor-quidditch-team")), {1, 2}) + self.assertEqual(subscriber_handler.get_users(stream_id_mapper.get("dumbledores-army")), {1, 2}) + + team_name = "slytherin" + zerver_stream = convert_channel_data( + channel_data=self.mattermost_data["channel"], + user_data_map=self.username_to_user, + subscriber_handler=subscriber_handler, + stream_id_mapper=stream_id_mapper, + user_id_mapper=user_id_mapper, + realm_id=4, + team_name=team_name, + ) + + self.assertEqual(subscriber_handler.get_users(stream_id_mapper.get("slytherin-common-room")), {3, 4, 5}) + self.assertEqual(subscriber_handler.get_users(stream_id_mapper.get("slytherin-quidditch-team")), {3, 4}) + + def test_write_emoticon_data(self) -> None: + zerver_realm_emoji = write_emoticon_data( + realm_id=3, + custom_emoji_data=self.mattermost_data["emoji"], + data_dir=self.fixture_file_name("", "mattermost_fixtures"), + output_dir=self.make_import_output_dir("mattermost") + ) + self.assertEqual(len(zerver_realm_emoji), 2) + self.assertEqual(zerver_realm_emoji[0]["file_name"], "peerdium") + self.assertEqual(zerver_realm_emoji[0]["realm"], 3) + self.assertEqual(zerver_realm_emoji[0]["deactivated"], False) + + self.assertEqual(zerver_realm_emoji[1]["file_name"], "tick") + self.assertEqual(zerver_realm_emoji[1]["realm"], 3) + self.assertEqual(zerver_realm_emoji[1]["deactivated"], False) + + records_file = os.path.join('var', 'test-mattermost-import', "emoji", "records.json") + with open(records_file, "r") as f: + records_json = ujson.load(f) + + self.assertEqual(records_json[0]["file_name"], "peerdium") + self.assertEqual(records_json[0]["realm_id"], 3) + exported_emoji_path = self.fixture_file_name(self.mattermost_data["emoji"][0]["image"], "mattermost_fixtures") + self.assertTrue(filecmp.cmp(records_json[0]["path"], exported_emoji_path)) + + self.assertEqual(records_json[1]["file_name"], "tick") + self.assertEqual(records_json[1]["realm_id"], 3) + exported_emoji_path = self.fixture_file_name(self.mattermost_data["emoji"][1]["image"], "mattermost_fixtures") + self.assertTrue(filecmp.cmp(records_json[1]["path"], exported_emoji_path)) + + def test_get_mentioned_user_ids(self) -> None: + user_id_mapper = IdMapper() + harry_id = user_id_mapper.get("harry") + + raw_message = { + "content": "Hello @harry" + } + ids = get_mentioned_user_ids(raw_message, user_id_mapper) + self.assertEqual(list(ids), [harry_id]) + + raw_message = { + "content": "Hello" + } + ids = get_mentioned_user_ids(raw_message, user_id_mapper) + self.assertEqual(list(ids), []) + + raw_message = { + "content": "@harry How are you?" + } + ids = get_mentioned_user_ids(raw_message, user_id_mapper) + self.assertEqual(list(ids), [harry_id]) + + raw_message = { + "content": "@harry @ron Where are you folks?" + } + ron_id = user_id_mapper.get("ron") + ids = get_mentioned_user_ids(raw_message, user_id_mapper) + self.assertEqual(list(ids), [harry_id, ron_id]) + + raw_message = { + "content": "@harry.com How are you?" + } + ids = get_mentioned_user_ids(raw_message, user_id_mapper) + self.assertEqual(list(ids), []) + + raw_message = { + "content": "hello@harry.com How are you?" + } + ids = get_mentioned_user_ids(raw_message, user_id_mapper) + self.assertEqual(list(ids), []) + + harry_id = user_id_mapper.get("harry_") + raw_message = { + "content": "Hello @harry_" + } + ids = get_mentioned_user_ids(raw_message, user_id_mapper) + self.assertEqual(list(ids), [harry_id]) + + harry_id = user_id_mapper.get("harry.") + raw_message = { + "content": "Hello @harry." + } + ids = get_mentioned_user_ids(raw_message, user_id_mapper) + self.assertEqual(list(ids), [harry_id]) + + harry_id = user_id_mapper.get("ha_rry.") + raw_message = { + "content": "Hello @ha_rry." + } + ids = get_mentioned_user_ids(raw_message, user_id_mapper) + self.assertEqual(list(ids), [harry_id]) + + ron_id = user_id_mapper.get("ron") + raw_message = { + "content": "Hello @ron." + } + ids = get_mentioned_user_ids(raw_message, user_id_mapper) + self.assertEqual(list(ids), []) + + raw_message = { + "content": "Hello @ron_" + } + ids = get_mentioned_user_ids(raw_message, user_id_mapper) + self.assertEqual(list(ids), []) + + def test_check_user_in_team(self) -> None: + harry = self.username_to_user["harry"] + self.assertTrue(check_user_in_team(harry, "gryffindor")) + self.assertFalse(check_user_in_team(harry, "slytherin")) + + snape = self.username_to_user["snape"] + self.assertFalse(check_user_in_team(snape, "gryffindor")) + self.assertTrue(check_user_in_team(snape, "slytherin")) + + def test_label_mirror_dummy_users(self) -> None: + label_mirror_dummy_users( + team_name="gryffindor", + mattermost_data=self.mattermost_data, + username_to_user=self.username_to_user, + ) + self.assertFalse(self.username_to_user["harry"]["is_mirror_dummy"]) + self.assertFalse(self.username_to_user["ron"]["is_mirror_dummy"]) + self.assertFalse(self.username_to_user["malfoy"]["is_mirror_dummy"]) + + # snape is mirror dummy since the user sent a message in gryffindor and + # left the team + self.assertTrue(self.username_to_user["snape"]["is_mirror_dummy"]) + + def test_build_reactions(self) -> None: + total_reactions = [] # type: List[Dict[str, Any]] + + reactions = [ + {"user": "harry", "create_at": 1553165521410, "emoji_name": "tick"}, + {"user": "ron", "create_at": 1553166530805, "emoji_name": "smile"}, + {"user": "ron", "create_at": 1553166540953, "emoji_name": "world_map"}, + {"user": "harry", "create_at": 1553166540957, "emoji_name": "world_map"} + ] + + zerver_realmemoji = write_emoticon_data( + realm_id=3, + custom_emoji_data=self.mattermost_data["emoji"], + data_dir=self.fixture_file_name("", "mattermost_fixtures"), + output_dir=self.make_import_output_dir("mattermost") + ) + + # Make sure tick is present in fixture data + self.assertEqual(zerver_realmemoji[1]["name"], "tick") + tick_emoji_code = zerver_realmemoji[1]["id"] + + name_to_codepoint = get_name_to_codepoint_dict() + user_id_mapper = IdMapper() + harry_id = user_id_mapper.get("harry") + ron_id = user_id_mapper.get("ron") + + build_reactions( + realm_id=3, + total_reactions=total_reactions, + reactions=reactions, + message_id=5, + name_to_codepoint=name_to_codepoint, + user_id_mapper=user_id_mapper, + zerver_realmemoji=zerver_realmemoji + ) + + smile_emoji_code = name_to_codepoint["smile"] + world_map_emoji_code = name_to_codepoint["world_map"] + + expected_total_reactions = [ + { + 'user_profile': harry_id, 'message': 5, 'id': 1, 'reaction_type': Reaction.REALM_EMOJI, + 'emoji_code': tick_emoji_code, 'emoji_name': 'tick' + }, + { + 'user_profile': ron_id, 'message': 5, 'id': 2, 'reaction_type': Reaction.UNICODE_EMOJI, + 'emoji_code': smile_emoji_code, 'emoji_name': 'smile' + }, + { + 'user_profile': ron_id, 'message': 5, 'id': 3, 'reaction_type': Reaction.UNICODE_EMOJI, + 'emoji_code': world_map_emoji_code, 'emoji_name': 'world_map' + }, + { + 'user_profile': harry_id, 'message': 5, 'id': 4, 'reaction_type': Reaction.UNICODE_EMOJI, + 'emoji_code': world_map_emoji_code, 'emoji_name': 'world_map' + } + ] + + self.assertEqual(total_reactions, expected_total_reactions) + + def team_output_dir(self, output_dir: str, team_name: str) -> str: + return os.path.join(output_dir, team_name) + + def read_file(self, team_output_dir: str, output_file: str) -> Any: + full_path = os.path.join(team_output_dir, output_file) + with open(full_path) as f: + return ujson.load(f) + + def get_set(self, data: List[Dict[str, Any]], field: str) -> Set[str]: + values = set(r[field] for r in data) + return values + + def test_do_convert_data(self) -> None: + mattermost_data_dir = self.fixture_file_name("", "mattermost_fixtures") + output_dir = self.make_import_output_dir("mattermost") + + do_convert_data( + mattermost_data_dir=mattermost_data_dir, + output_dir=output_dir, + masking_content=False + ) + + harry_team_output_dir = self.team_output_dir(output_dir, "gryffindor") + self.assertEqual(os.path.exists(os.path.join(harry_team_output_dir, 'avatars')), True) + self.assertEqual(os.path.exists(os.path.join(harry_team_output_dir, 'emoji')), True) + self.assertEqual(os.path.exists(os.path.join(harry_team_output_dir, 'attachment.json')), True) + + realm = self.read_file(harry_team_output_dir, 'realm.json') + + self.assertEqual('Organization imported from Mattermost!', + realm['zerver_realm'][0]['description']) + + exported_user_ids = self.get_set(realm['zerver_userprofile'], 'id') + exported_user_full_names = self.get_set(realm['zerver_userprofile'], 'full_name') + self.assertEqual(set(['Harry Potter', 'Ron Weasley', 'Severus Snape']), exported_user_full_names) + + exported_user_emails = self.get_set(realm['zerver_userprofile'], 'email') + self.assertEqual(set(['harry@zulip.com', 'ron@zulip.com', 'snape@zulip.com']), exported_user_emails) + + self.assertEqual(len(realm['zerver_stream']), 3) + exported_stream_names = self.get_set(realm['zerver_stream'], 'name') + self.assertEqual(exported_stream_names, set(['Gryffindor common room', 'Gryffindor quidditch team', 'Dumbledores army'])) + self.assertEqual(self.get_set(realm['zerver_stream'], 'realm'), set([realm['zerver_realm'][0]['id']])) + self.assertEqual(self.get_set(realm['zerver_stream'], 'deactivated'), set([False])) + + self.assertEqual(len(realm['zerver_defaultstream']), 0) + + exported_recipient_ids = self.get_set(realm['zerver_recipient'], 'id') + self.assertEqual(exported_recipient_ids, set([1, 2, 3, 4, 5, 6])) + exported_recipient_types = self.get_set(realm['zerver_recipient'], 'type') + self.assertEqual(exported_recipient_types, set([1, 2])) + exported_recipient_type_ids = self.get_set(realm['zerver_recipient'], 'type_id') + self.assertEqual(exported_recipient_type_ids, set([1, 2, 3])) + + exported_subscription_userprofile = self.get_set(realm['zerver_subscription'], 'user_profile') + self.assertEqual(exported_subscription_userprofile, set([1, 2, 3])) + exported_subscription_recipients = self.get_set(realm['zerver_subscription'], 'recipient') + self.assertEqual(exported_subscription_recipients, set([1, 2, 3, 4, 5, 6])) + + messages = self.read_file(harry_team_output_dir, 'messages-000001.json') + + exported_messages_id = self.get_set(messages['zerver_message'], 'id') + self.assertIn(messages['zerver_message'][0]['sender'], exported_user_ids) + self.assertIn(messages['zerver_message'][0]['recipient'], exported_recipient_ids) + self.assertIn(messages['zerver_message'][0]['content'], 'harry joined the channel.\n\n') + + exported_usermessage_userprofiles = self.get_set(messages['zerver_usermessage'], 'user_profile') + self.assertEqual(len(exported_usermessage_userprofiles), 2) + exported_usermessage_messages = self.get_set(messages['zerver_usermessage'], 'message') + self.assertEqual(exported_usermessage_messages, exported_messages_id) + + do_import_realm( + import_dir=harry_team_output_dir, + subdomain='gryffindor' + ) + realm = get_realm('gryffindor') + + realm_users = UserProfile.objects.filter(realm=realm) + messages = Message.objects.filter(sender__in=realm_users) + for message in messages: + self.assertIsNotNone(message.rendered_content) + + def test_do_convert_data_with_masking(self) -> None: + mattermost_data_dir = self.fixture_file_name("", "mattermost_fixtures") + output_dir = self.make_import_output_dir("mattermost") + + do_convert_data( + mattermost_data_dir=mattermost_data_dir, + output_dir=output_dir, + masking_content=True + ) + + harry_team_output_dir = self.team_output_dir(output_dir, "gryffindor") + messages = self.read_file(harry_team_output_dir, 'messages-000001.json') + + self.assertIn(messages['zerver_message'][0]['content'], 'xxxxx xxxxxx xxx xxxxxxx.\n\n') + + def test_import_data_to_existing_database(self) -> None: + mattermost_data_dir = self.fixture_file_name("", "mattermost_fixtures") + output_dir = self.make_import_output_dir("mattermost") + + do_convert_data( + mattermost_data_dir=mattermost_data_dir, + output_dir=output_dir, + masking_content=True + ) + + harry_team_output_dir = self.team_output_dir(output_dir, "gryffindor") + + do_import_realm( + import_dir=harry_team_output_dir, + subdomain='gryffindor' + ) + realm = get_realm('gryffindor') + + realm_users = UserProfile.objects.filter(realm=realm) + messages = Message.objects.filter(sender__in=realm_users) + for message in messages: + self.assertIsNotNone(message.rendered_content)