import: Add tool for importing teams from mattermost.

templates/zerver/create_realm.html

@@ -36,8 +36,8 @@ page can be easily identified in it's respective JavaScript file -->
             {% endif %}
             <div class="bottom-text">
                 Or import
-                from <a href="/help/import-from-slack">Slack</a>, <a href="/help/import-from-hipchat">HipChat</a> or
+                from <a href="/help/import-from-slack">Slack</a>, <a href="/help/import-from-mattermost">Mattermost</a>,
-                <a href="/help/import-from-gitter">Gitter</a>.
+                <a href="/help/import-from-hipchat">HipChat</a> or <a href="/help/import-from-gitter">Gitter</a>.
             </div>
         </div>
     </div>

templates/zerver/features.html

@@ -242,10 +242,10 @@
             </p>
         </div>
         <div class="feature-block">
-            <h3>SLACK/HIPCHAT/GITTER IMPORT</h3>
+            <h3>DATA IMPORT</h3>
             <p>
-                Import an existing Slack, HipChat, Stride, or Gitter workspace into
+                Import an existing Slack, Mattermost, HipChat, Stride,
-                Zulip.
+                or Gitter workspace into Zulip.
             </p>
         </div>
         <div class="feature-block">

templates/zerver/help/import-from-mattermost.md

@@ -0,0 +1,104 @@
+# Import from Mattermost
+
+Starting with Zulip 2.1, Zulip supports importing data from Mattermost,
+including users, channels, messages, and custom emoji.
+
+
+**Note:** You can only import a Mattermost team as a new Zulip
+organization. In particular, you cannot use this tool to import data
+into an existing Zulip organization.
+
+## Import from Mattermost
+
+First, export your data.  The following instructions assume you're
+running Mattermost inside a Docker container:
+
+1. SSH into your Mattermost app server
+
+2. Run the following command to export the data.
+  `docker exec -it mattermost-docker_app_1 mattermost export bulk export.json --all-teams`
+
+3. This will generate `export.json` and possibly an `exported_emoji`
+   directory inside the **mattermost-docker_app_1** container.  The
+   `exported_emoji` folder will only be created if your users had
+   uploaded custom emoji to the Mattermost server.
+
+4. SSH into to **mattermost-docker_app_1** container by running the following command.
+  `docker exec -it mattermost-docker_app_1 sh`
+
+4. Tar the exported files by running the following command.
+  `tar --transform 's|^|mattermost/|' -czf export.tar.gz exported_emoji/ export.json`
+
+5. Now download the `export.tar.gz` file from the Docker container to your local computer.
+
+### Import into zulipchat.com
+
+Email support@zulipchat.com with your exported archive and your desired Zulip
+subdomain. Your imported organization will be hosted at
+`<subdomain>.zulipchat.com`.
+
+If you've already created a test organization at
+`<subdomain>.zulipchat.com`, let us know, and we can rename the old
+organization first.
+
+### Import into a self-hosted Zulip server
+
+First
+[install a new Zulip server](https://zulip.readthedocs.io/en/stable/production/install.html),
+skipping "Step 3: Create a Zulip organization, and log in" (you'll
+create your Zulip organization via the data import tool instead).
+
+Use [upgrade-zulip-from-git][upgrade-zulip-from-git] to
+upgrade your Zulip server to the latest `master` branch.
+
+Log in to a shell on your Zulip server as the `zulip` user.
+
+Extract the `export.tar.gz` to `/home/zulip/mattermost` as follows.
+
+```bash
+cd /home/zulip
+tar -xzvf export.tar.gz
+```
+
+To import with the most common configuration, run the following commands
+replacing `<team-name>` with the name of the team you want to import from
+Mattermost export.
+
+```
+cd /home/zulip/deployments/current
+./manage.py convert_mattermost_data /home/zulip/mattermost --output /home/zulip/converted_mattermost_data
+./manage.py import "" /home/zulip/converted_mattermost_data/<team-name>
+```
+
+This could take several minutes to run, depending on how much data you're
+importing.
+
+**Import options**
+
+The commands above create an imported organization on the root domain
+(`EXTERNAL_HOST`) of the Zulip installation. You can also import into a
+custom subdomain, e.g. if you already have an existing organization on the
+root domain. Replace the last line above with the following, after replacing
+`<subdomain>` with the desired subdomain.
+
+```
+./manage.py import <subdomain> /home/zulip/converted_mattermost_data/<team-name>
+```
+
+{!import-login.md!}
+
+[upgrade-zulip-from-git]: https://zulip.readthedocs.io/en/latest/production/maintain-secure-upgrade.html#upgrading-from-a-git-repository
+
+## Limitations
+
+Mattermost's export tool is incomplete and does not support exporting
+the following data:
+
+* private messages and group private messages between users
+* user avatars
+* uploaded files and message attachments.
+
+We expect to add support for importing these data from Mattermost once
+Mattermost's export tool includes them.
+
+[upgrade-zulip-from-git]: https://zulip.readthedocs.io/en/latest/production/maintain-secure-upgrade.html#upgrading-from-a-git-repository

templates/zerver/help/include/sidebar_index.md

@@ -105,6 +105,7 @@
 * [Create your organization profile](/help/create-your-organization-profile)
 * [Link to your Zulip from the web](/help/join-zulip-chat-badge)
 * [Import from HipChat/Stride](/help/import-from-hipchat)
+* [Import from Mattermost](/help/import-from-mattermost)
 * [Import from Slack](/help/import-from-slack)
 * [Import from Gitter](/help/import-from-gitter)
 * [Roles and permissions](/help/roles-and-permissions)

zerver/data_import/mattermost.py

@@ -0,0 +1,705 @@
+"""
+spec:
+https://docs.mattermost.com/administration/bulk-export.html
+"""
+import os
+import logging
+import subprocess
+import ujson
+import re
+import shutil
+
+from typing import Any, Callable, Dict, List, Set
+
+from django.conf import settings
+from django.utils.timezone import now as timezone_now
+from django.forms.models import model_to_dict
+
+from zerver.models import Recipient, RealmEmoji, Reaction
+from zerver.lib.utils import (
+    process_list_in_batches,
+)
+from zerver.lib.emoji import NAME_TO_CODEPOINT_PATH
+from zerver.data_import.import_util import ZerverFieldsT, build_zerver_realm, \
+    build_stream, build_realm, build_message, create_converted_data_files, \
+    make_subscriber_map, build_recipients, build_user_profile, \
+    build_stream_subscriptions, build_personal_subscriptions, SubscriberHandler, \
+    build_realm_emoji, make_user_messages
+
+from zerver.data_import.mattermost_user import UserHandler
+from zerver.data_import.sequencer import NEXT_ID, IdMapper
+
+def make_realm(realm_id: int, team: Dict[str, Any]) -> ZerverFieldsT:
+    # set correct realm details
+    NOW = float(timezone_now().timestamp())
+    domain_name = settings.EXTERNAL_HOST
+    realm_subdomain = team["name"]
+
+    zerver_realm = build_zerver_realm(realm_id, realm_subdomain, NOW, 'Mattermost')
+    realm = build_realm(zerver_realm, realm_id, domain_name)
+
+    # We may override these later.
+    realm['zerver_defaultstream'] = []
+
+    return realm
+
+def process_user(user_dict: Dict[str, Any], realm_id: int, team_name: str,
+                 user_id_mapper: IdMapper) -> ZerverFieldsT:
+    def is_team_admin(user_dict: Dict[str, Any]) -> bool:
+        for team in user_dict["teams"]:
+            if team["name"] == team_name and "team_admin" in team["roles"]:
+                return True
+        return False
+
+    def get_full_name(user_dict: Dict[str, Any]) -> str:
+        full_name = "{} {}".format(user_dict["first_name"], user_dict["last_name"])
+        if full_name.strip():
+            return full_name
+        return user_dict['username']
+
+    avatar_source = 'G'
+    full_name = get_full_name(user_dict)
+    id = user_id_mapper.get(user_dict['username'])
+    delivery_email = user_dict['email']
+    email = user_dict['email']
+    is_realm_admin = is_team_admin(user_dict)
+    is_guest = False
+    short_name = user_dict['username']
+    date_joined = int(timezone_now().timestamp())
+    timezone = 'UTC'
+
+    if user_dict["is_mirror_dummy"]:
+        is_active = False
+        is_mirror_dummy = True
+    else:
+        is_active = True
+        is_mirror_dummy = False
+
+    return build_user_profile(
+        avatar_source=avatar_source,
+        date_joined=date_joined,
+        delivery_email=delivery_email,
+        email=email,
+        full_name=full_name,
+        id=id,
+        is_active=is_active,
+        is_realm_admin=is_realm_admin,
+        is_guest=is_guest,
+        is_mirror_dummy=is_mirror_dummy,
+        realm_id=realm_id,
+        short_name=short_name,
+        timezone=timezone,
+    )
+
+def convert_user_data(user_handler: UserHandler,
+                      user_id_mapper: IdMapper,
+                      user_data_map: Dict[str, Dict[str, Any]],
+                      realm_id: int,
+                      team_name: str) -> None:
+
+    user_data_list = []
+    for username in user_data_map:
+        user = user_data_map[username]
+        if check_user_in_team(user, team_name) or user["is_mirror_dummy"]:
+            user_data_list.append(user)
+
+    for raw_item in user_data_list:
+        user = process_user(raw_item, realm_id, team_name, user_id_mapper)
+        user_handler.add_user(user)
+
+def convert_channel_data(channel_data: List[ZerverFieldsT],
+                         user_data_map: Dict[str, Dict[str, Any]],
+                         subscriber_handler: SubscriberHandler,
+                         stream_id_mapper: IdMapper,
+                         user_id_mapper: IdMapper,
+                         realm_id: int,
+                         team_name: str) -> List[ZerverFieldsT]:
+    channel_data_list = [
+        d
+        for d in channel_data
+        if d['team'] == team_name
+    ]
+
+    channel_members_map = {}  # type: Dict[str, List[str]]
+    channel_admins_map = {}  # type: Dict[str, List[str]]
+
+    def initialize_stream_membership_dicts() -> None:
+        for channel in channel_data:
+            channel_name = channel["name"]
+            channel_members_map[channel_name] = []
+            channel_admins_map[channel_name] = []
+
+        for username in user_data_map:
+            user_dict = user_data_map[username]
+            teams = user_dict["teams"]
+            for team in teams:
+                if team["name"] != team_name:
+                    continue
+                for channel in team["channels"]:
+                    channel_roles = channel["roles"]
+                    channel_name = channel["name"]
+                    if "channel_admin" in channel_roles:
+                        channel_admins_map[channel_name].append(username)
+                    elif "channel_user" in channel_roles:
+                        channel_members_map[channel_name].append(username)
+
+    def get_invite_only_value_from_channel_type(channel_type: str) -> bool:
+        # Channel can have two types in Mattermost
+        # "O" for a public channel.
+        # "P" for a private channel.
+        if channel_type == 'O':
+            return False
+        elif channel_type == 'P':
+            return True
+        else:  # nocoverage
+            raise Exception('unexpected value')
+
+    streams = []
+    initialize_stream_membership_dicts()
+
+    for channel_dict in channel_data_list:
+        now = int(timezone_now().timestamp())
+        stream_id = stream_id_mapper.get(channel_dict['name'])
+        stream_name = channel_dict["name"]
+        invite_only = get_invite_only_value_from_channel_type(channel_dict['type'])
+
+        stream = build_stream(
+            date_created=now,
+            realm_id=realm_id,
+            name=channel_dict['display_name'],
+            # Purpose describes how the channel should be used. It is similar to
+            # stream description and is shown in channel list to help others decide
+            # whether to join.
+            # Header text always appears right next to channel name in channel header.
+            # Can be used for advertising the purpose of stream, making announcements as
+            # well as including frequently used links. So probably not a bad idea to use
+            # this as description if the channel purpose is empty.
+            description=channel_dict["purpose"] or channel_dict['header'],
+            stream_id=stream_id,
+            # Mattermost export don't include data of archived(~ deactivated) channels.
+            deactivated=False,
+            invite_only=invite_only,
+        )
+
+        channel_users = set()
+        for username in channel_admins_map[stream_name]:
+            channel_users.add(user_id_mapper.get(username))
+
+        for username in channel_members_map[stream_name]:
+            channel_users.add(user_id_mapper.get(username))
+
+        if channel_users:
+            subscriber_handler.set_info(
+                stream_id=stream_id,
+                users=channel_users,
+            )
+        streams.append(stream)
+
+    return streams
+
+def get_name_to_codepoint_dict() -> Dict[str, str]:
+    with open(NAME_TO_CODEPOINT_PATH) as fp:
+        return ujson.load(fp)
+
+def build_reactions(realm_id: int, total_reactions: List[ZerverFieldsT], reactions: List[ZerverFieldsT],
+                    message_id: int, name_to_codepoint: ZerverFieldsT,
+                    user_id_mapper: IdMapper, zerver_realmemoji: List[ZerverFieldsT]) -> None:
+    realmemoji = {}
+    for realm_emoji in zerver_realmemoji:
+        realmemoji[realm_emoji['name']] = realm_emoji['id']
+
+    # For the unicode emoji codes, we use equivalent of
+    # function 'emoji_name_to_emoji_code' in 'zerver/lib/emoji' here
+    for mattermost_reaction in reactions:
+        emoji_name = mattermost_reaction['emoji_name']
+        username = mattermost_reaction["user"]
+        # Check in unicode emoji
+        if emoji_name in name_to_codepoint:
+            emoji_code = name_to_codepoint[emoji_name]
+            reaction_type = Reaction.UNICODE_EMOJI
+        # Check in realm emoji
+        elif emoji_name in realmemoji:
+            emoji_code = realmemoji[emoji_name]
+            reaction_type = Reaction.REALM_EMOJI
+        else:  # nocoverage
+            continue
+
+        if not user_id_mapper.has(username):
+            continue
+
+        reaction_id = NEXT_ID('reaction')
+        reaction = Reaction(
+            id=reaction_id,
+            emoji_code=emoji_code,
+            emoji_name=emoji_name,
+            reaction_type=reaction_type)
+
+        reaction_dict = model_to_dict(reaction, exclude=['message', 'user_profile'])
+        reaction_dict['message'] = message_id
+        reaction_dict['user_profile'] = user_id_mapper.get(username)
+        total_reactions.append(reaction_dict)
+
+def get_mentioned_user_ids(raw_message: Dict[str, Any], user_id_mapper: IdMapper) -> Set[int]:
+    user_ids = set()
+    content = raw_message["content"]
+
+    # usernames can be of the form user.name, user_name, username., username_, user.name_ etc
+    matches = re.findall("(?<=^|(?<=[^a-zA-Z0-9-_.]))@(([A-Za-z0-9]+[_.]?)+)", content)
+
+    for match in matches:
+        possible_username = match[0]
+        if user_id_mapper.has(possible_username):
+            user_ids.add(user_id_mapper.get(possible_username))
+    return user_ids
+
+def process_raw_message_batch(realm_id: int,
+                              raw_messages: List[Dict[str, Any]],
+                              subscriber_map: Dict[int, Set[int]],
+                              user_id_mapper: IdMapper,
+                              user_handler: UserHandler,
+                              get_recipient_id: Callable[[ZerverFieldsT], int],
+                              is_pm_data: bool,
+                              output_dir: str,
+                              zerver_realmemoji: List[Dict[str, Any]],
+                              total_reactions: List[Dict[str, Any]],
+                              ) -> None:
+
+    def fix_mentions(content: str, mention_user_ids: Set[int]) -> str:
+        for user_id in mention_user_ids:
+            user = user_handler.get_user(user_id=user_id)
+            mattermost_mention = '@{short_name}'.format(**user)
+            zulip_mention = '@**{full_name}**'.format(**user)
+            content = content.replace(mattermost_mention, zulip_mention)
+
+        content = content.replace('@channel', '@**all**')
+        content = content.replace('@all', '@**all**')
+        # We don't have an equivalent for Mattermost's @here mention which mentions all users
+        # online in the channel.
+        content = content.replace('@here', '@**all**')
+        return content
+
+    mention_map = dict()  # type: Dict[int, Set[int]]
+    zerver_message = []
+
+    import html2text
+    h = html2text.HTML2Text()
+
+    name_to_codepoint = get_name_to_codepoint_dict()
+
+    for raw_message in raw_messages:
+        message_id = NEXT_ID('message')
+        mention_user_ids = get_mentioned_user_ids(raw_message, user_id_mapper)
+        mention_map[message_id] = mention_user_ids
+
+        content = fix_mentions(
+            content=raw_message['content'],
+            mention_user_ids=mention_user_ids,
+        )
+        content = h.handle(content)
+
+        if len(content) > 10000:  # nocoverage
+            logging.info('skipping too-long message of length %s' % (len(content),))
+            continue
+
+        pub_date = raw_message['pub_date']
+        try:
+            recipient_id = get_recipient_id(raw_message)
+        except KeyError:
+            logging.debug("Could not find recipient_id for a message, skipping.")
+            continue
+
+        rendered_content = None
+
+        topic_name = 'imported from mattermost'
+        user_id = raw_message['sender_id']
+
+        message = build_message(
+            content=content,
+            message_id=message_id,
+            pub_date=pub_date,
+            recipient_id=recipient_id,
+            rendered_content=rendered_content,
+            topic_name=topic_name,
+            user_id=user_id,
+            has_attachment=False,
+        )
+        zerver_message.append(message)
+        build_reactions(realm_id, total_reactions, raw_message["reactions"], message_id,
+                        name_to_codepoint, user_id_mapper, zerver_realmemoji)
+
+    zerver_usermessage = make_user_messages(
+        zerver_message=zerver_message,
+        subscriber_map=subscriber_map,
+        is_pm_data=is_pm_data,
+        mention_map=mention_map,
+    )
+
+    message_json = dict(
+        zerver_message=zerver_message,
+        zerver_usermessage=zerver_usermessage,
+    )
+
+    dump_file_id = NEXT_ID('dump_file_id' + str(realm_id))
+    message_file = "/messages-%06d.json" % (dump_file_id,)
+    create_converted_data_files(message_json, output_dir, message_file)
+
+def process_posts(team_name: str,
+                  realm_id: int,
+                  post_data: List[Dict[str, Any]],
+                  get_recipient_id: Callable[[ZerverFieldsT], int],
+                  subscriber_map: Dict[int, Set[int]],
+                  output_dir: str,
+                  is_pm_data: bool,
+                  masking_content: bool,
+                  user_id_mapper: IdMapper,
+                  user_handler: UserHandler,
+                  username_to_user: Dict[str, Dict[str, Any]],
+                  zerver_realmemoji: List[Dict[str, Any]],
+                  total_reactions: List[Dict[str, Any]]) -> None:
+
+    post_data_list = [
+        d
+        for d in post_data
+        if d["team"] == team_name
+    ]
+
+    def message_to_dict(post_dict: Dict[str, Any]) -> Dict[str, Any]:
+        sender_id = user_id_mapper.get(post_dict["user"])
+        content = post_dict['message']
+
+        if masking_content:
+            content = re.sub('[a-z]', 'x', content)
+            content = re.sub('[A-Z]', 'X', content)
+
+        if "reactions" in post_dict:
+            reactions = post_dict["reactions"] or []
+        else:
+            reactions = []
+
+        return dict(
+            sender_id=sender_id,
+            receiver_id=post_dict["channel"],
+            content=content,
+            pub_date=int(post_dict['create_at'] / 1000),
+            reactions=reactions
+        )
+
+    raw_messages = []
+    for post_dict in post_data_list:
+        raw_messages.append(message_to_dict(post_dict))
+        message_replies = post_dict["replies"]
+        # Replies to a message in Mattermost are stored in the main message object.
+        # For now, we just append the replies immediately after the original message.
+        if message_replies is not None:
+            for reply in message_replies:
+                reply["channel"] = post_dict["channel"]
+                raw_messages.append(message_to_dict(reply))
+
+    def process_batch(lst: List[Dict[str, Any]]) -> None:
+        process_raw_message_batch(
+            realm_id=realm_id,
+            raw_messages=lst,
+            subscriber_map=subscriber_map,
+            user_id_mapper=user_id_mapper,
+            user_handler=user_handler,
+            get_recipient_id=get_recipient_id,
+            is_pm_data=is_pm_data,
+            output_dir=output_dir,
+            zerver_realmemoji=zerver_realmemoji,
+            total_reactions=total_reactions,
+        )
+
+    chunk_size = 1000
+
+    process_list_in_batches(
+        lst=raw_messages,
+        chunk_size=chunk_size,
+        process_batch=process_batch,
+    )
+
+def write_message_data(team_name: str,
+                       realm_id: int,
+                       post_data: List[Dict[str, Any]],
+                       zerver_recipient: List[ZerverFieldsT],
+                       subscriber_map: Dict[int, Set[int]],
+                       output_dir: str,
+                       masking_content: bool,
+                       stream_id_mapper: IdMapper,
+                       user_id_mapper: IdMapper,
+                       user_handler: UserHandler,
+                       username_to_user: Dict[str, Dict[str, Any]],
+                       zerver_realmemoji: List[Dict[str, Any]],
+                       total_reactions: List[Dict[str, Any]]) -> None:
+
+    stream_id_to_recipient_id = {
+        d['type_id']: d['id']
+        for d in zerver_recipient
+        if d['type'] == Recipient.STREAM
+    }
+
+    def get_stream_recipient_id(raw_message: ZerverFieldsT) -> int:
+        receiver_id = raw_message['receiver_id']
+        stream_id = stream_id_mapper.get(receiver_id)
+        recipient_id = stream_id_to_recipient_id[stream_id]
+        return recipient_id
+
+    process_posts(
+        team_name=team_name,
+        realm_id=realm_id,
+        post_data=post_data,
+        get_recipient_id=get_stream_recipient_id,
+        subscriber_map=subscriber_map,
+        output_dir=output_dir,
+        is_pm_data=False,
+        masking_content=masking_content,
+        user_id_mapper=user_id_mapper,
+        user_handler=user_handler,
+        username_to_user=username_to_user,
+        zerver_realmemoji=zerver_realmemoji,
+        total_reactions=total_reactions,
+    )
+
+def write_emoticon_data(realm_id: int,
+                        custom_emoji_data: List[Dict[str, Any]],
+                        data_dir: str,
+                        output_dir: str) -> List[ZerverFieldsT]:
+    '''
+    This function does most of the work for processing emoticons, the bulk
+    of which is copying files.  We also write a json file with metadata.
+    Finally, we return a list of RealmEmoji dicts to our caller.
+
+    In our data_dir we have a pretty simple setup:
+
+        The exported JSON file will have emoji rows if it contains any custom emoji
+            {
+                "type": "emoji",
+                "emoji": {"name": "peerdium", "image": "exported_emoji/h15ni7kf1bnj7jeua4qhmctsdo/image"}
+            }
+            {
+                "type": "emoji",
+                "emoji": {"name": "tick", "image": "exported_emoji/7u7x8ytgp78q8jir81o9ejwwnr/image"}
+            }
+
+        exported_emoji/ - contains a bunch of image files:
+            exported_emoji/7u7x8ytgp78q8jir81o9ejwwnr/image
+            exported_emoji/h15ni7kf1bnj7jeua4qhmctsdo/image
+
+    We move all the relevant files to Zulip's more nested
+    directory structure.
+    '''
+
+    logging.info('Starting to process emoticons')
+
+    flat_data = [
+        dict(
+            path=d['image'],
+            name=d['name'],
+        )
+        for d in custom_emoji_data
+    ]
+
+    emoji_folder = os.path.join(output_dir, 'emoji')
+    os.makedirs(emoji_folder, exist_ok=True)
+
+    def process(data: ZerverFieldsT) -> ZerverFieldsT:
+        source_sub_path = data['path']
+        source_path = os.path.join(data_dir, source_sub_path)
+
+        target_fn = data["name"]
+        target_sub_path = RealmEmoji.PATH_ID_TEMPLATE.format(
+            realm_id=realm_id,
+            emoji_file_name=target_fn,
+        )
+        target_path = os.path.join(emoji_folder, target_sub_path)
+
+        os.makedirs(os.path.dirname(target_path), exist_ok=True)
+
+        source_path = os.path.abspath(source_path)
+        target_path = os.path.abspath(target_path)
+
+        shutil.copyfile(source_path, target_path)
+
+        return dict(
+            path=target_path,
+            s3_path=target_path,
+            file_name=target_fn,
+            realm_id=realm_id,
+            name=data['name'],
+        )
+
+    emoji_records = list(map(process, flat_data))
+    create_converted_data_files(emoji_records, output_dir, '/emoji/records.json')
+
+    realmemoji = [
+        build_realm_emoji(
+            realm_id=realm_id,
+            name=rec['name'],
+            id=NEXT_ID('realmemoji'),
+            file_name=rec['file_name'],
+        )
+        for rec in emoji_records
+    ]
+    logging.info('Done processing emoticons')
+
+    return realmemoji
+
+def create_username_to_user_mapping(user_data_list: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]:
+    username_to_user = {}
+    for user in user_data_list:
+        username_to_user[user["username"]] = user
+    return username_to_user
+
+def check_user_in_team(user: Dict[str, Any], team_name: str) -> bool:
+    for team in user["teams"]:
+        if team["name"] == team_name:
+            return True
+    return False
+
+def label_mirror_dummy_users(team_name: str, mattermost_data: Dict[str, List[Dict[str, Any]]],
+                             username_to_user: Dict[str, Dict[str, Any]]) -> None:
+    # This function might looks like a great place to label admin users. But
+    # that won't be fully correct since we are iterating only though posts and
+    # it covers only users that has sent atleast one message.
+    for post in mattermost_data["post"]:
+        if post["team"] == team_name:
+            user = username_to_user[post["user"]]
+            if not check_user_in_team(user, team_name):
+                user["is_mirror_dummy"] = True
+
+def reset_mirror_dummy_users(username_to_user: Dict[str, Dict[str, Any]]) -> None:
+    for username in username_to_user:
+        user = username_to_user[username]
+        user["is_mirror_dummy"] = False
+
+def mattermost_data_file_to_dict(mattermost_data_file: str) -> Dict[str, List[Dict[str, Any]]]:
+    mattermost_data = {}  # type: Dict[str, List[Dict[str, Any]]]
+    mattermost_data["version"] = []
+    mattermost_data["team"] = []
+    mattermost_data["channel"] = []
+    mattermost_data["user"] = []
+    mattermost_data["post"] = []
+    mattermost_data["emoji"] = []
+
+    with open(mattermost_data_file, "r") as fp:
+        for line in fp:
+            row = ujson.loads(line.rstrip("\n"))
+            data_type = row["type"]
+            mattermost_data[data_type].append(row[data_type])
+    return mattermost_data
+
+def do_convert_data(mattermost_data_dir: str, output_dir: str, masking_content: bool) -> None:
+    username_to_user = {}  # type: Dict[str, Dict[str, Any]]
+
+    os.makedirs(output_dir, exist_ok=True)
+    if os.listdir(output_dir):  # nocoverage
+        raise Exception("Output directory should be empty!")
+
+    mattermost_data_file = os.path.join(mattermost_data_dir, "export.json")
+    mattermost_data = mattermost_data_file_to_dict(mattermost_data_file)
+
+    username_to_user = create_username_to_user_mapping(mattermost_data["user"])
+
+    for team in mattermost_data["team"]:
+        realm_id = NEXT_ID("realm_id")
+        team_name = team["name"]
+
+        user_handler = UserHandler()
+        subscriber_handler = SubscriberHandler()
+        user_id_mapper = IdMapper()
+        stream_id_mapper = IdMapper()
+
+        print("Generating data for", team_name)
+        realm = make_realm(realm_id, team)
+        realm_output_dir = os.path.join(output_dir, team_name)
+
+        reset_mirror_dummy_users(username_to_user)
+        label_mirror_dummy_users(team_name, mattermost_data, username_to_user)
+
+        convert_user_data(
+            user_handler=user_handler,
+            user_id_mapper=user_id_mapper,
+            user_data_map=username_to_user,
+            realm_id=realm_id,
+            team_name=team_name,
+        )
+
+        zerver_stream = convert_channel_data(
+            channel_data=mattermost_data["channel"],
+            user_data_map=username_to_user,
+            subscriber_handler=subscriber_handler,
+            stream_id_mapper=stream_id_mapper,
+            user_id_mapper=user_id_mapper,
+            realm_id=realm_id,
+            team_name=team_name,
+        )
+
+        realm['zerver_stream'] = zerver_stream
+
+        all_users = user_handler.get_all_users()
+
+        zerver_recipient = build_recipients(
+            zerver_userprofile=all_users,
+            zerver_stream=zerver_stream,
+        )
+        realm['zerver_recipient'] = zerver_recipient
+
+        stream_subscriptions = build_stream_subscriptions(
+            get_users=subscriber_handler.get_users,
+            zerver_recipient=zerver_recipient,
+            zerver_stream=zerver_stream,
+        )
+
+        personal_subscriptions = build_personal_subscriptions(
+            zerver_recipient=zerver_recipient,
+        )
+
+        # Mattermost currently supports only exporting messages from channels.
+        # Personal messages and huddles are not exported.
+        zerver_subscription = personal_subscriptions + stream_subscriptions
+        realm['zerver_subscription'] = zerver_subscription
+
+        zerver_realmemoji = write_emoticon_data(
+            realm_id=realm_id,
+            custom_emoji_data=mattermost_data["emoji"],
+            data_dir=mattermost_data_dir,
+            output_dir=realm_output_dir,
+        )
+        realm['zerver_realmemoji'] = zerver_realmemoji
+
+        subscriber_map = make_subscriber_map(
+            zerver_subscription=zerver_subscription,
+        )
+
+        total_reactions = []  # type: List[Dict[str, Any]]
+        write_message_data(
+            team_name=team_name,
+            realm_id=realm_id,
+            post_data=mattermost_data["post"],
+            zerver_recipient=zerver_recipient,
+            subscriber_map=subscriber_map,
+            output_dir=realm_output_dir,
+            masking_content=masking_content,
+            stream_id_mapper=stream_id_mapper,
+            user_id_mapper=user_id_mapper,
+            user_handler=user_handler,
+            username_to_user=username_to_user,
+            zerver_realmemoji=zerver_realmemoji,
+            total_reactions=total_reactions,
+        )
+        realm['zerver_reaction'] = total_reactions
+        realm['zerver_userprofile'] = user_handler.get_all_users()
+        realm['sort_by_date'] = True
+
+        create_converted_data_files(realm, realm_output_dir, '/realm.json')
+        # Mattermost currently doesn't support exporting avatars
+        create_converted_data_files([], realm_output_dir, '/avatars/records.json')
+        # Mattermost currently doesn't support exporting uploads
+        create_converted_data_files([], realm_output_dir, '/uploads/records.json')
+
+        # Mattermost currently doesn't support exporting attachments
+        attachment = {"zerver_attachment": []}  # type: Dict[str, List[Any]]
+        create_converted_data_files(attachment, realm_output_dir, '/attachment.json')
+
+        logging.info('Start making tarball')
+        subprocess.check_call(["tar", "-czf", realm_output_dir + '.tar.gz', realm_output_dir, '-P'])
+        logging.info('Done making tarball')

zerver/data_import/mattermost_user.py

@@ -0,0 +1,25 @@
+from typing import Any, Dict, List
+
+class UserHandler:
+    '''
+    Our UserHandler class is a glorified wrapper
+    around the data that eventually goes into
+    zerver_userprofile.
+
+    The class helps us do things like map ids
+    to names for mentions.
+    '''
+    def __init__(self) -> None:
+        self.id_to_user_map = dict()  # type: Dict[int, Dict[str, Any]]
+
+    def add_user(self, user: Dict[str, Any]) -> None:
+        user_id = user['id']
+        self.id_to_user_map[user_id] = user
+
+    def get_user(self, user_id: int) -> Dict[str, Any]:
+        user = self.id_to_user_map[user_id]
+        return user
+
+    def get_all_users(self) -> List[Dict[str, Any]]:
+        users = list(self.id_to_user_map.values())
+        return users

zerver/management/commands/convert_mattermost_data.py

@@ -0,0 +1,68 @@
+import argparse
+import os
+from typing import Any
+
+'''
+Example usage for testing purposes. For testing data see the mattermost_fixtures
+in zerver/tests/.
+
+    ./manage.py convert_mattermost_data mattermost_fixtures --output mm_export
+    ./manage.py import --destroy-rebuild-database mattermost mm_export/gryffindor
+
+Test out the realm:
+    ./tools/run-dev.py
+    go to browser and use your dev url
+'''
+
+from django.core.management.base import BaseCommand, CommandParser
+
+from zerver.data_import.mattermost import do_convert_data
+
+class Command(BaseCommand):
+    help = """Convert the mattermost data into Zulip data format."""
+
+    def add_arguments(self, parser: CommandParser) -> None:
+        dir_help = "Directory containing exported JSON file and exported_emoji (optional) directory."
+        parser.add_argument('mattermost_data_dir',
+                            metavar='<mattermost data directory>',
+                            help=dir_help)
+
+        parser.add_argument('--output', dest='output_dir',
+                            action="store",
+                            help='Directory to write converted data to.')
+
+        parser.add_argument('--mask', dest='masking_content',
+                            action="store_true",
+                            help='Mask the content for privacy during QA.')
+
+        parser.formatter_class = argparse.RawTextHelpFormatter
+
+    def handle(self, *args: Any, **options: Any) -> None:
+        output_dir = options["output_dir"]
+        if output_dir is None:
+            print("You need to specify --output <output directory>")
+            exit(1)
+
+        if os.path.exists(output_dir) and not os.path.isdir(output_dir):
+            print(output_dir + " is not a directory")
+            exit(1)
+
+        os.makedirs(output_dir, exist_ok=True)
+
+        if os.listdir(output_dir):
+            print('Output directory should be empty!')
+            exit(1)
+        output_dir = os.path.realpath(output_dir)
+
+        data_dir = options['mattermost_data_dir']
+        if not os.path.exists(data_dir):
+            print("Directory not found: '%s'" % (data_dir,))
+            exit(1)
+        data_dir = os.path.realpath(data_dir)
+
+        print("Converting Data ...")
+        do_convert_data(
+            mattermost_data_dir=data_dir,
+            output_dir=output_dir,
+            masking_content=options.get('masking_content', False),
+        )

zerver/tests/fixtures/mattermost_fixtures/export.json

@@ -0,0 +1,35 @@
+{"type":"version","version":1}
+{"type":"team","team":{"name":"gryffindor","display_name":"Iago Realm","type":"O","description":"","allow_open_invite":true}}
+{"type":"team","team":{"name":"slytherin","display_name":"Othello Team","type":"O","description":"","allow_open_invite":true}}
+{"type":"channel","channel":{"team":"gryffindor","name":"gryffindor-common-room","display_name":"Gryffindor common room","type":"O","header":"","purpose":"A place for talking about Gryffindor common room"}}
+{"type":"channel","channel":{"team":"gryffindor","name":"gryffindor-quidditch-team","display_name":"Gryffindor quidditch team","type":"O","header":"","purpose":"A place for talking about Gryffindor quidditch team"}}
+{"type":"channel","channel":{"team":"slytherin","name":"slytherin-common-room","display_name":"Slytherin common room","type":"O","header":"","purpose":""}}
+{"type":"channel","channel":{"team":"gryffindor","name":"dumbledores-army","display_name":"Dumbledores army","type":"P","header":"https//:github.com/zulip/zulip","purpose":"A place for talking about Dumbledores army"}}
+{"type":"channel","channel":{"team":"slytherin","name":"slytherin-quidditch-team","display_name":"Slytherin quidditch team","type":"O","header":"","purpose":""}}
+{"type":"user","user":{"username":"ron","email":"ron@zulip.com","auth_service":"","nickname":"","first_name":"Ron","last_name":"Weasley","position":"","roles":"system_user","locale":"en","teams":[{"name":"gryffindor","roles":"team_user","channels":[{"name":"gryffindor-quidditch-team","roles":"channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"gryffindor-common-room","roles":"channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"dumbledores-army","roles":"channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false}]}],"notify_props":{"desktop":"mention","desktop_sound":"true","email":"true","mobile":"mention","mobile_push_status":"away","channel":"true","comments":"never","mention_keys":"ron,@ron"}}}
+{"type":"user","user":{"username":"harry","email":"harry@zulip.com","auth_service":"","nickname":"","first_name":"Harry","last_name":"Potter","position":"","roles":"system_admin system_user","locale":"en","teams":[{"name":"gryffindor","roles":"team_admin team_user","channels":[{"name":"dumbledores-army","roles":"channel_admin channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"gryffindor-common-room","roles":"channel_admin channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"gryffindor-quidditch-team","roles":"channel_admin channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false}]}],"notify_props":{"desktop":"mention","desktop_sound":"true","email":"true","mobile":"mention","mobile_push_status":"away","channel":"true","comments":"never","mention_keys":"harry,@harry"}}}
+{"type":"user","user":{"username":"malfoy","email":"malfoy@zulip.com","auth_service":"","nickname":"","first_name":"","last_name":"","position":"","roles":"system_user","locale":"en","teams":[{"name":"slytherin","roles":"team_admin team_user","channels":[{"name":"slytherin-common-room","roles":"channel_admin channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"slytherin-quidditch-team","roles":"channel_admin channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false}]}],"notify_props":{"desktop":"mention","desktop_sound":"true","email":"true","mobile":"mention","mobile_push_status":"away","channel":"true","comments":"never","mention_keys":"malfoy,@malfoy"}}}
+{"type":"user","user":{"username":"pansy","email":"pansy@zulip.com","auth_service":"","nickname":"","first_name":"","last_name":"","position":"","roles":"system_user","locale":"en","teams":[{"name":"slytherin","roles":"team_admin team_user","channels":[{"name":"slytherin-common-room","roles":"channel_admin channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false},{"name":"slytherin-quidditch-team","roles":"channel_admin channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false}]}],"notify_props":{"desktop":"mention","desktop_sound":"true","email":"true","mobile":"mention","mobile_push_status":"away","channel":"true","comments":"never","mention_keys":"malfoy,@malfoy"}}}
+{"type":"user","user":{"username":"snape","email":"snape@zulip.com","auth_service":"","nickname":"","first_name":"Severus","last_name":"Snape","position":"","roles":"system_user","locale":"en","teams":[{"name":"slytherin","roles":"team_user","channels":[{"name":"slytherin-common-room","roles":"channel_user","notify_props":{"desktop":"default","mobile":"default","mark_unread":"all"},"favorite":false}]}],"notify_props":{"desktop":"mention","desktop_sound":"true","email":"true","mobile":"mention","mobile_push_status":"away","channel":"true","comments":"never","mention_keys":"snape,@snape"}}}
+{"type":"post","post":{"team":"gryffindor","channel":"dumbledores-army","user":"harry","message":"harry joined the channel.","create_at":1553166657086,"reactions":null,"replies":[{"user":"ron","message":"The weather is so hot!","create_at":1553166584976}]}}
+{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-common-room","user":"ron","message":"ron joined the channel.","create_at":1553166512493,"reactions":null,"replies":null}}
+{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-quidditch-team","user":"harry","message":"harry joined the team.","create_at":1553165141670,"reactions":null,"replies":null}}
+{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-quidditch-team","user":"harry","message":"Awesome!","create_at":1553166557928,"reactions":[{"user":"malfoy","create_at":1553166812156,"emoji_name":"tick"}],"replies":null}}
+{"type":"post","post":{"team":"slytherin","channel":"slytherin-quidditch-team","user":"malfoy","message":"malfoy joined the team.","create_at":1553166852598,"reactions":null,"replies":null}}
+{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-quidditch-team","user":"ron","message":"ron joined the team.","create_at":1553166512482,"reactions":null,"replies":null}}
+{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-quidditch-team","user":"ron","message":"Hey folks","create_at":1553166519720,"reactions":null,"replies":null}}
+{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-quidditch-team","user":"harry","message":"@ron Welcome mate!","create_at":1553166519726,"reactions":null,"replies":null}}
+{"type":"post","post":{"team":"gryffindor","channel":"dumbledores-army","user":"harry","message":"ron added to the channel by harry.","create_at":1553166681045,"reactions":null,"replies":null}}
+{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-quidditch-team","user":"harry","message":"Hello world","create_at":1553165193242,"reactions":[{"user":"harry","create_at":1553165521410,"emoji_name":"tick"},{"user":"ron","create_at":1553166530805,"emoji_name":"smile"},{"user":"ron","create_at":1553166540953,"emoji_name":"world_map"}],"replies":null}}
+{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-common-room","user":"harry","message":"Looks like this channel is empty","create_at":1553166567370,"reactions":[{"user":"ron","create_at":1553166584976,"emoji_name":"rocket"}],"replies":null}}
+{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-quidditch-team","user":"ron","message":"How is everything going","create_at":1553166525124,"reactions":[{"user":"harry","create_at":1553166552827,"emoji_name":"apple"}],"replies":null}}
+{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-common-room","user":"ron","message":"Not really","create_at":1553166593455,"reactions":null,"replies":null}}
+{"type":"post","post":{"team":"gryffindor","channel":"dumbledores-army","user":"ron","message":"hello","create_at":1553166686344,"reactions":null,"replies":null}}
+{"type":"post","post":{"team":"gryffindor","channel":"dumbledores-army","user":"harry","message":"hey everyone","create_at":1553166668668,"reactions":[{"user":"ron","create_at":1553166695260,"emoji_name":"grin"}],"replies":null}}
+{"type":"post","post":{"team":"slytherin","channel":"slytherin-common-room","user":"malfoy","message":"malfoy joined the channel.","create_at":1553166852612,"reactions":null,"replies":null}}
+{"type":"post","post":{"team":"slytherin","channel":"slytherin-quidditch-team","user":"malfoy","message":":rofl: 4","create_at":1553166916448,"reactions":[{"user":"harry","create_at":1553167016056,"emoji_name":"peerdium"}],"replies":null}}
+{"type":"post","post":{"team":"slytherin","channel":"slytherin-quidditch-team","user":"malfoy","message":"Hello folks","create_at":1553166858280,"reactions":[{"user":"harry","create_at":1553166903980,"emoji_name":"joy"}],"replies":null}}
+{"type":"post","post":{"team":"gryffindor","channel":"gryffindor-common-room","user":"harry","message":"harry joined the channel.","create_at":1553165141689,"reactions":null,"replies":null}}
+{"type":"post","post":{"team":"gryffindor","channel":"slytherin-quidditch-team","user":"snape","message":"Hey folks! I was always in your team. Time to go now.","create_at":1553166740759,"reactions":null,"replies":null}}
+{"type":"emoji","emoji":{"name":"peerdium","image":"exported_emoji/h15ni7kf1bnj7jeua4qhmctsdo/image.png"}}
+{"type":"emoji","emoji":{"name":"tick","image":"exported_emoji/7u7x8ytgp78q8jir81o9ejwwnr/image.png"}}

zerver/tests/test_management_commands.py

@@ -374,3 +374,18 @@ class TestSendToEmailMirror(ZulipTestCase):
         stream_id = get_stream("Denmark2", message.sender.realm).id
         self.assertEqual(message.recipient.type, Recipient.STREAM)
         self.assertEqual(message.recipient.type_id, stream_id)
+
+class TestConvertMattermostData(ZulipTestCase):
+    COMMAND_NAME = 'convert_mattermost_data'
+
+    def test_check_if_command_calls_do_convert_data(self) -> None:
+        with patch('zerver.management.commands.convert_mattermost_data.do_convert_data') as m:
+            mm_fixtures = self.fixture_file_name("", "mattermost_fixtures")
+            output_dir = self.make_import_output_dir("mattermost")
+            call_command(self.COMMAND_NAME, mm_fixtures, "--output={}".format(output_dir))
+
+        m.assert_called_with(
+            masking_content=False,
+            mattermost_data_dir=os.path.realpath(mm_fixtures),
+            output_dir=os.path.realpath(output_dir),
+        )

zerver/tests/test_mattermost_importer.py

@@ -0,0 +1,507 @@
+import os
+import ujson
+import filecmp
+import logging
+
+from typing import Dict, Any, List, Set
+
+from zerver.lib.import_realm import (
+    do_import_realm,
+)
+from zerver.lib.test_classes import (
+    ZulipTestCase,
+)
+
+from zerver.data_import.mattermost_user import UserHandler
+from zerver.data_import.mattermost import mattermost_data_file_to_dict, process_user, convert_user_data, \
+    create_username_to_user_mapping, label_mirror_dummy_users, reset_mirror_dummy_users, \
+    convert_channel_data, write_emoticon_data, get_mentioned_user_ids, check_user_in_team, \
+    build_reactions, get_name_to_codepoint_dict, do_convert_data
+from zerver.data_import.sequencer import IdMapper
+from zerver.data_import.import_util import SubscriberHandler
+from zerver.models import Reaction, UserProfile, Message, get_realm
+
+class MatterMostImporter(ZulipTestCase):
+    logger = logging.getLogger()
+    # set logger to a higher level to suppress 'logger.INFO' outputs
+    logger.setLevel(logging.WARNING)
+
+    def setUp(self) -> None:
+        fixture_file_name = self.fixture_file_name("export.json", "mattermost_fixtures")
+        self.mattermost_data = mattermost_data_file_to_dict(fixture_file_name)
+        self.username_to_user = create_username_to_user_mapping(self.mattermost_data["user"])
+        reset_mirror_dummy_users(self.username_to_user)
+
+    def test_mattermost_data_file_to_dict(self) -> None:
+        self.assertEqual(len(self.mattermost_data), 6)
+
+        self.assertEqual(self.mattermost_data["version"], [1])
+
+        self.assertEqual(len(self.mattermost_data["team"]), 2)
+        self.assertEqual(self.mattermost_data["team"][0]["name"], "gryffindor")
+
+        self.assertEqual(len(self.mattermost_data["channel"]), 5)
+        self.assertEqual(self.mattermost_data["channel"][0]["name"], "gryffindor-common-room")
+        self.assertEqual(self.mattermost_data["channel"][0]["team"], "gryffindor")
+
+        self.assertEqual(len(self.mattermost_data["user"]), 5)
+        self.assertEqual(self.mattermost_data["user"][1]["username"], "harry")
+        self.assertEqual(len(self.mattermost_data["user"][1]["teams"]), 1)
+
+        self.assertEqual(len(self.mattermost_data["post"]), 20)
+        self.assertEqual(self.mattermost_data["post"][0]["team"], "gryffindor")
+        self.assertEqual(self.mattermost_data["post"][0]["channel"], "dumbledores-army")
+        self.assertEqual(self.mattermost_data["post"][0]["user"], "harry")
+        self.assertEqual(len(self.mattermost_data["post"][0]["replies"]), 1)
+
+        self.assertEqual(len(self.mattermost_data["emoji"]), 2)
+        self.assertEqual(self.mattermost_data["emoji"][0]["name"], "peerdium")
+
+    def test_process_user(self) -> None:
+        user_id_mapper = IdMapper()
+
+        harry_dict = self.username_to_user["harry"]
+        harry_dict["is_mirror_dummy"] = False
+
+        realm_id = 3
+
+        team_name = "gryffindor"
+        user = process_user(harry_dict, realm_id, team_name, user_id_mapper)
+        self.assertEqual(user["avatar_source"], 'G')
+        self.assertEqual(user["delivery_email"], "harry@zulip.com")
+        self.assertEqual(user["email"], "harry@zulip.com")
+        self.assertEqual(user["full_name"], "Harry Potter")
+        self.assertEqual(user["id"], 1)
+        self.assertEqual(user["is_active"], True)
+        self.assertEqual(user["is_realm_admin"], True)
+        self.assertEqual(user["is_guest"], False)
+        self.assertEqual(user["is_mirror_dummy"], False)
+        self.assertEqual(user["realm"], 3)
+        self.assertEqual(user["short_name"], "harry")
+        self.assertEqual(user["timezone"], "UTC")
+
+        team_name = "slytherin"
+        snape_dict = self.username_to_user["snape"]
+        snape_dict["is_mirror_dummy"] = True
+        user = process_user(snape_dict, realm_id, team_name, user_id_mapper)
+        self.assertEqual(user["avatar_source"], 'G')
+        self.assertEqual(user["delivery_email"], "snape@zulip.com")
+        self.assertEqual(user["email"], "snape@zulip.com")
+        self.assertEqual(user["full_name"], "Severus Snape")
+        self.assertEqual(user["id"], 2)
+        self.assertEqual(user["is_active"], False)
+        self.assertEqual(user["is_realm_admin"], False)
+        self.assertEqual(user["is_guest"], False)
+        self.assertEqual(user["is_mirror_dummy"], True)
+        self.assertEqual(user["realm"], 3)
+        self.assertEqual(user["short_name"], "snape")
+        self.assertEqual(user["timezone"], "UTC")
+
+    def test_convert_user_data(self) -> None:
+        user_id_mapper = IdMapper()
+        realm_id = 3
+
+        team_name = "gryffindor"
+        user_handler = UserHandler()
+        convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name)
+        self.assertTrue(user_id_mapper.has("harry"))
+        self.assertTrue(user_id_mapper.has("ron"))
+        self.assertEqual(user_handler.get_user(user_id_mapper.get("harry"))["full_name"], "Harry Potter")
+        self.assertEqual(user_handler.get_user(user_id_mapper.get("ron"))["full_name"], "Ron Weasley")
+
+        team_name = "slytherin"
+        user_handler = UserHandler()
+        convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name)
+        self.assertEqual(len(user_handler.get_all_users()), 3)
+        self.assertTrue(user_id_mapper.has("malfoy"))
+        self.assertTrue(user_id_mapper.has("pansy"))
+        self.assertTrue(user_id_mapper.has("snape"))
+
+        team_name = "gryffindor"
+        # Snape is a mirror dummy user in Harry's team.
+        label_mirror_dummy_users(team_name, self.mattermost_data, self.username_to_user)
+        user_handler = UserHandler()
+        convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name)
+        self.assertEqual(len(user_handler.get_all_users()), 3)
+        self.assertTrue(user_id_mapper.has("snape"))
+
+        team_name = "slytherin"
+        user_handler = UserHandler()
+        convert_user_data(user_handler, user_id_mapper, self.username_to_user, realm_id, team_name)
+        self.assertEqual(len(user_handler.get_all_users()), 3)
+
+    def test_convert_channel_data(self) -> None:
+        user_handler = UserHandler()
+        subscriber_handler = SubscriberHandler()
+        stream_id_mapper = IdMapper()
+        user_id_mapper = IdMapper()
+        team_name = "gryffindor"
+
+        convert_user_data(
+            user_handler=user_handler,
+            user_id_mapper=user_id_mapper,
+            user_data_map=self.username_to_user,
+            realm_id=3,
+            team_name=team_name,
+        )
+
+        zerver_stream = convert_channel_data(
+            channel_data=self.mattermost_data["channel"],
+            user_data_map=self.username_to_user,
+            subscriber_handler=subscriber_handler,
+            stream_id_mapper=stream_id_mapper,
+            user_id_mapper=user_id_mapper,
+            realm_id=3,
+            team_name=team_name,
+        )
+
+        self.assertEqual(len(zerver_stream), 3)
+
+        self.assertEqual(zerver_stream[0]["name"], "Gryffindor common room")
+        self.assertEqual(zerver_stream[0]["invite_only"], False)
+        self.assertEqual(zerver_stream[0]["description"], "A place for talking about Gryffindor common room")
+        self.assertEqual(zerver_stream[0]["rendered_description"], "")
+        self.assertEqual(zerver_stream[0]["realm"], 3)
+
+        self.assertEqual(zerver_stream[1]["name"], "Gryffindor quidditch team")
+        self.assertEqual(zerver_stream[1]["invite_only"], False)
+        self.assertEqual(zerver_stream[1]["description"], "A place for talking about Gryffindor quidditch team")
+        self.assertEqual(zerver_stream[1]["rendered_description"], "")
+        self.assertEqual(zerver_stream[1]["realm"], 3)
+
+        self.assertEqual(zerver_stream[2]["name"], "Dumbledores army")
+        self.assertEqual(zerver_stream[2]["invite_only"], True)
+        self.assertEqual(zerver_stream[2]["description"], "A place for talking about Dumbledores army")
+        self.assertEqual(zerver_stream[2]["rendered_description"], "")
+        self.assertEqual(zerver_stream[2]["realm"], 3)
+
+        self.assertTrue(stream_id_mapper.has("gryffindor-common-room"))
+        self.assertTrue(stream_id_mapper.has("gryffindor-quidditch-team"))
+        self.assertTrue(stream_id_mapper.has("dumbledores-army"))
+
+        # TODO: Add ginny
+        self.assertEqual(subscriber_handler.get_users(stream_id_mapper.get("gryffindor-common-room")), {1, 2})
+        self.assertEqual(subscriber_handler.get_users(stream_id_mapper.get("gryffindor-quidditch-team")), {1, 2})
+        self.assertEqual(subscriber_handler.get_users(stream_id_mapper.get("dumbledores-army")), {1, 2})
+
+        team_name = "slytherin"
+        zerver_stream = convert_channel_data(
+            channel_data=self.mattermost_data["channel"],
+            user_data_map=self.username_to_user,
+            subscriber_handler=subscriber_handler,
+            stream_id_mapper=stream_id_mapper,
+            user_id_mapper=user_id_mapper,
+            realm_id=4,
+            team_name=team_name,
+        )
+
+        self.assertEqual(subscriber_handler.get_users(stream_id_mapper.get("slytherin-common-room")), {3, 4, 5})
+        self.assertEqual(subscriber_handler.get_users(stream_id_mapper.get("slytherin-quidditch-team")), {3, 4})
+
+    def test_write_emoticon_data(self) -> None:
+        zerver_realm_emoji = write_emoticon_data(
+            realm_id=3,
+            custom_emoji_data=self.mattermost_data["emoji"],
+            data_dir=self.fixture_file_name("", "mattermost_fixtures"),
+            output_dir=self.make_import_output_dir("mattermost")
+        )
+        self.assertEqual(len(zerver_realm_emoji), 2)
+        self.assertEqual(zerver_realm_emoji[0]["file_name"], "peerdium")
+        self.assertEqual(zerver_realm_emoji[0]["realm"], 3)
+        self.assertEqual(zerver_realm_emoji[0]["deactivated"], False)
+
+        self.assertEqual(zerver_realm_emoji[1]["file_name"], "tick")
+        self.assertEqual(zerver_realm_emoji[1]["realm"], 3)
+        self.assertEqual(zerver_realm_emoji[1]["deactivated"], False)
+
+        records_file = os.path.join('var', 'test-mattermost-import', "emoji", "records.json")
+        with open(records_file, "r") as f:
+            records_json = ujson.load(f)
+
+        self.assertEqual(records_json[0]["file_name"], "peerdium")
+        self.assertEqual(records_json[0]["realm_id"], 3)
+        exported_emoji_path = self.fixture_file_name(self.mattermost_data["emoji"][0]["image"], "mattermost_fixtures")
+        self.assertTrue(filecmp.cmp(records_json[0]["path"], exported_emoji_path))
+
+        self.assertEqual(records_json[1]["file_name"], "tick")
+        self.assertEqual(records_json[1]["realm_id"], 3)
+        exported_emoji_path = self.fixture_file_name(self.mattermost_data["emoji"][1]["image"], "mattermost_fixtures")
+        self.assertTrue(filecmp.cmp(records_json[1]["path"], exported_emoji_path))
+
+    def test_get_mentioned_user_ids(self) -> None:
+        user_id_mapper = IdMapper()
+        harry_id = user_id_mapper.get("harry")
+
+        raw_message = {
+            "content": "Hello @harry"
+        }
+        ids = get_mentioned_user_ids(raw_message, user_id_mapper)
+        self.assertEqual(list(ids), [harry_id])
+
+        raw_message = {
+            "content": "Hello"
+        }
+        ids = get_mentioned_user_ids(raw_message, user_id_mapper)
+        self.assertEqual(list(ids), [])
+
+        raw_message = {
+            "content": "@harry How are you?"
+        }
+        ids = get_mentioned_user_ids(raw_message, user_id_mapper)
+        self.assertEqual(list(ids), [harry_id])
+
+        raw_message = {
+            "content": "@harry @ron Where are you folks?"
+        }
+        ron_id = user_id_mapper.get("ron")
+        ids = get_mentioned_user_ids(raw_message, user_id_mapper)
+        self.assertEqual(list(ids), [harry_id, ron_id])
+
+        raw_message = {
+            "content": "@harry.com How are you?"
+        }
+        ids = get_mentioned_user_ids(raw_message, user_id_mapper)
+        self.assertEqual(list(ids), [])
+
+        raw_message = {
+            "content": "hello@harry.com How are you?"
+        }
+        ids = get_mentioned_user_ids(raw_message, user_id_mapper)
+        self.assertEqual(list(ids), [])
+
+        harry_id = user_id_mapper.get("harry_")
+        raw_message = {
+            "content": "Hello @harry_"
+        }
+        ids = get_mentioned_user_ids(raw_message, user_id_mapper)
+        self.assertEqual(list(ids), [harry_id])
+
+        harry_id = user_id_mapper.get("harry.")
+        raw_message = {
+            "content": "Hello @harry."
+        }
+        ids = get_mentioned_user_ids(raw_message, user_id_mapper)
+        self.assertEqual(list(ids), [harry_id])
+
+        harry_id = user_id_mapper.get("ha_rry.")
+        raw_message = {
+            "content": "Hello @ha_rry."
+        }
+        ids = get_mentioned_user_ids(raw_message, user_id_mapper)
+        self.assertEqual(list(ids), [harry_id])
+
+        ron_id = user_id_mapper.get("ron")
+        raw_message = {
+            "content": "Hello @ron."
+        }
+        ids = get_mentioned_user_ids(raw_message, user_id_mapper)
+        self.assertEqual(list(ids), [])
+
+        raw_message = {
+            "content": "Hello @ron_"
+        }
+        ids = get_mentioned_user_ids(raw_message, user_id_mapper)
+        self.assertEqual(list(ids), [])
+
+    def test_check_user_in_team(self) -> None:
+        harry = self.username_to_user["harry"]
+        self.assertTrue(check_user_in_team(harry, "gryffindor"))
+        self.assertFalse(check_user_in_team(harry, "slytherin"))
+
+        snape = self.username_to_user["snape"]
+        self.assertFalse(check_user_in_team(snape, "gryffindor"))
+        self.assertTrue(check_user_in_team(snape, "slytherin"))
+
+    def test_label_mirror_dummy_users(self) -> None:
+        label_mirror_dummy_users(
+            team_name="gryffindor",
+            mattermost_data=self.mattermost_data,
+            username_to_user=self.username_to_user,
+        )
+        self.assertFalse(self.username_to_user["harry"]["is_mirror_dummy"])
+        self.assertFalse(self.username_to_user["ron"]["is_mirror_dummy"])
+        self.assertFalse(self.username_to_user["malfoy"]["is_mirror_dummy"])
+
+        # snape is mirror dummy since the user sent a message in gryffindor and
+        # left the team
+        self.assertTrue(self.username_to_user["snape"]["is_mirror_dummy"])
+
+    def test_build_reactions(self) -> None:
+        total_reactions = []  # type: List[Dict[str, Any]]
+
+        reactions = [
+            {"user": "harry", "create_at": 1553165521410, "emoji_name": "tick"},
+            {"user": "ron", "create_at": 1553166530805, "emoji_name": "smile"},
+            {"user": "ron", "create_at": 1553166540953, "emoji_name": "world_map"},
+            {"user": "harry", "create_at": 1553166540957, "emoji_name": "world_map"}
+        ]
+
+        zerver_realmemoji = write_emoticon_data(
+            realm_id=3,
+            custom_emoji_data=self.mattermost_data["emoji"],
+            data_dir=self.fixture_file_name("", "mattermost_fixtures"),
+            output_dir=self.make_import_output_dir("mattermost")
+        )
+
+        # Make sure tick is present in fixture data
+        self.assertEqual(zerver_realmemoji[1]["name"], "tick")
+        tick_emoji_code = zerver_realmemoji[1]["id"]
+
+        name_to_codepoint = get_name_to_codepoint_dict()
+        user_id_mapper = IdMapper()
+        harry_id = user_id_mapper.get("harry")
+        ron_id = user_id_mapper.get("ron")
+
+        build_reactions(
+            realm_id=3,
+            total_reactions=total_reactions,
+            reactions=reactions,
+            message_id=5,
+            name_to_codepoint=name_to_codepoint,
+            user_id_mapper=user_id_mapper,
+            zerver_realmemoji=zerver_realmemoji
+        )
+
+        smile_emoji_code = name_to_codepoint["smile"]
+        world_map_emoji_code = name_to_codepoint["world_map"]
+
+        expected_total_reactions = [
+            {
+                'user_profile': harry_id, 'message': 5, 'id': 1, 'reaction_type': Reaction.REALM_EMOJI,
+                'emoji_code': tick_emoji_code, 'emoji_name': 'tick'
+            },
+            {
+                'user_profile': ron_id, 'message': 5, 'id': 2, 'reaction_type': Reaction.UNICODE_EMOJI,
+                'emoji_code': smile_emoji_code, 'emoji_name': 'smile'
+            },
+            {
+                'user_profile': ron_id, 'message': 5, 'id': 3, 'reaction_type': Reaction.UNICODE_EMOJI,
+                'emoji_code': world_map_emoji_code, 'emoji_name': 'world_map'
+            },
+            {
+                'user_profile': harry_id, 'message': 5, 'id': 4, 'reaction_type': Reaction.UNICODE_EMOJI,
+                'emoji_code': world_map_emoji_code, 'emoji_name': 'world_map'
+            }
+        ]
+
+        self.assertEqual(total_reactions, expected_total_reactions)
+
+    def team_output_dir(self, output_dir: str, team_name: str) -> str:
+        return os.path.join(output_dir, team_name)
+
+    def read_file(self, team_output_dir: str, output_file: str) -> Any:
+        full_path = os.path.join(team_output_dir, output_file)
+        with open(full_path) as f:
+            return ujson.load(f)
+
+    def get_set(self, data: List[Dict[str, Any]], field: str) -> Set[str]:
+        values = set(r[field] for r in data)
+        return values
+
+    def test_do_convert_data(self) -> None:
+        mattermost_data_dir = self.fixture_file_name("", "mattermost_fixtures")
+        output_dir = self.make_import_output_dir("mattermost")
+
+        do_convert_data(
+            mattermost_data_dir=mattermost_data_dir,
+            output_dir=output_dir,
+            masking_content=False
+        )
+
+        harry_team_output_dir = self.team_output_dir(output_dir, "gryffindor")
+        self.assertEqual(os.path.exists(os.path.join(harry_team_output_dir, 'avatars')), True)
+        self.assertEqual(os.path.exists(os.path.join(harry_team_output_dir, 'emoji')), True)
+        self.assertEqual(os.path.exists(os.path.join(harry_team_output_dir, 'attachment.json')), True)
+
+        realm = self.read_file(harry_team_output_dir, 'realm.json')
+
+        self.assertEqual('Organization imported from Mattermost!',
+                         realm['zerver_realm'][0]['description'])
+
+        exported_user_ids = self.get_set(realm['zerver_userprofile'], 'id')
+        exported_user_full_names = self.get_set(realm['zerver_userprofile'], 'full_name')
+        self.assertEqual(set(['Harry Potter', 'Ron Weasley', 'Severus Snape']), exported_user_full_names)
+
+        exported_user_emails = self.get_set(realm['zerver_userprofile'], 'email')
+        self.assertEqual(set(['harry@zulip.com', 'ron@zulip.com', 'snape@zulip.com']), exported_user_emails)
+
+        self.assertEqual(len(realm['zerver_stream']), 3)
+        exported_stream_names = self.get_set(realm['zerver_stream'], 'name')
+        self.assertEqual(exported_stream_names, set(['Gryffindor common room', 'Gryffindor quidditch team', 'Dumbledores army']))
+        self.assertEqual(self.get_set(realm['zerver_stream'], 'realm'), set([realm['zerver_realm'][0]['id']]))
+        self.assertEqual(self.get_set(realm['zerver_stream'], 'deactivated'), set([False]))
+
+        self.assertEqual(len(realm['zerver_defaultstream']), 0)
+
+        exported_recipient_ids = self.get_set(realm['zerver_recipient'], 'id')
+        self.assertEqual(exported_recipient_ids, set([1, 2, 3, 4, 5, 6]))
+        exported_recipient_types = self.get_set(realm['zerver_recipient'], 'type')
+        self.assertEqual(exported_recipient_types, set([1, 2]))
+        exported_recipient_type_ids = self.get_set(realm['zerver_recipient'], 'type_id')
+        self.assertEqual(exported_recipient_type_ids, set([1, 2, 3]))
+
+        exported_subscription_userprofile = self.get_set(realm['zerver_subscription'], 'user_profile')
+        self.assertEqual(exported_subscription_userprofile, set([1, 2, 3]))
+        exported_subscription_recipients = self.get_set(realm['zerver_subscription'], 'recipient')
+        self.assertEqual(exported_subscription_recipients, set([1, 2, 3, 4, 5, 6]))
+
+        messages = self.read_file(harry_team_output_dir, 'messages-000001.json')
+
+        exported_messages_id = self.get_set(messages['zerver_message'], 'id')
+        self.assertIn(messages['zerver_message'][0]['sender'], exported_user_ids)
+        self.assertIn(messages['zerver_message'][0]['recipient'], exported_recipient_ids)
+        self.assertIn(messages['zerver_message'][0]['content'], 'harry joined the channel.\n\n')
+
+        exported_usermessage_userprofiles = self.get_set(messages['zerver_usermessage'], 'user_profile')
+        self.assertEqual(len(exported_usermessage_userprofiles), 2)
+        exported_usermessage_messages = self.get_set(messages['zerver_usermessage'], 'message')
+        self.assertEqual(exported_usermessage_messages, exported_messages_id)
+
+        do_import_realm(
+            import_dir=harry_team_output_dir,
+            subdomain='gryffindor'
+        )
+        realm = get_realm('gryffindor')
+
+        realm_users = UserProfile.objects.filter(realm=realm)
+        messages = Message.objects.filter(sender__in=realm_users)
+        for message in messages:
+            self.assertIsNotNone(message.rendered_content)
+
+    def test_do_convert_data_with_masking(self) -> None:
+        mattermost_data_dir = self.fixture_file_name("", "mattermost_fixtures")
+        output_dir = self.make_import_output_dir("mattermost")
+
+        do_convert_data(
+            mattermost_data_dir=mattermost_data_dir,
+            output_dir=output_dir,
+            masking_content=True
+        )
+
+        harry_team_output_dir = self.team_output_dir(output_dir, "gryffindor")
+        messages = self.read_file(harry_team_output_dir, 'messages-000001.json')
+
+        self.assertIn(messages['zerver_message'][0]['content'], 'xxxxx xxxxxx xxx xxxxxxx.\n\n')
+
+    def test_import_data_to_existing_database(self) -> None:
+        mattermost_data_dir = self.fixture_file_name("", "mattermost_fixtures")
+        output_dir = self.make_import_output_dir("mattermost")
+
+        do_convert_data(
+            mattermost_data_dir=mattermost_data_dir,
+            output_dir=output_dir,
+            masking_content=True
+        )
+
+        harry_team_output_dir = self.team_output_dir(output_dir, "gryffindor")
+
+        do_import_realm(
+            import_dir=harry_team_output_dir,
+            subdomain='gryffindor'
+        )
+        realm = get_realm('gryffindor')
+
+        realm_users = UserProfile.objects.filter(realm=realm)
+        messages = Message.objects.filter(sender__in=realm_users)
+        for message in messages:
+            self.assertIsNotNone(message.rendered_content)