gitter import: Process user mentions.

The gitter mentions are in the format '@usermention'
and the mentions are included in the export data as:

"mentions": [
   {
    "screenName": "usermention",
    "userId": "54d7876c15522ed4b3dbbefb",
    "userIds": []
   }]

We extract this data and map this mention to @**usermention**
for Zulip.

zerver/lib/gitter_import.py

@@ -223,6 +223,7 @@ def build_subscription(recipient_id: int, user_id: int,
 def convert_gitter_workspace_messages(gitter_data: GitterDataT, output_dir: str,
                                       zerver_subscription: List[ZerverFieldsT],
                                       user_map: Dict[str, int],
+                                      user_short_name_to_full_name: Dict[str, str],
                                       chunk_size: int=MESSAGE_BATCH_CHUNK_SIZE) -> None:
     """
     Messages are stored in batches
@@ -244,6 +245,8 @@ def convert_gitter_workspace_messages(gitter_data: GitterDataT, output_dir: str,
             break
         for message in message_data:
             message_time = dateutil.parser.parse(message['sent']).timestamp()
+            mentioned_user_ids = get_usermentions(message, user_map,
+                                                  user_short_name_to_full_name)
             rendered_content = None
 
             zulip_message = dict(
@@ -266,6 +269,9 @@ def convert_gitter_workspace_messages(gitter_data: GitterDataT, output_dir: str,
             for subscription in zerver_subscription:
                 if subscription['recipient'] == recipient_id:
                     flags_mask = 1  # For read
+                    if subscription['user_profile'] in mentioned_user_ids:
+                        flags_mask = 9  # For read and mentioned
+
                     usermessage = dict(
                         user_profile=subscription['user_profile'],
                         id=usermessage_id,
@@ -287,6 +293,20 @@ def convert_gitter_workspace_messages(gitter_data: GitterDataT, output_dir: str,
 
     logging.info('######### IMPORTING MESSAGES FINISHED #########\n')
 
+def get_usermentions(message: Dict[str, Any], user_map: Dict[str, int],
+                     user_short_name_to_full_name: Dict[str, str]) -> List[int]:
+    mentioned_user_ids = []
+    if 'mentions' in message:
+        for mention in message['mentions']:
+            if mention.get('userId') in user_map:
+                gitter_mention = '@%s' % (mention['screenName'])
+                zulip_mention = ('@**%s**' %
+                                 (user_short_name_to_full_name[mention['screenName']]))
+                message['text'] = message['text'].replace(gitter_mention, zulip_mention)
+
+                mentioned_user_ids.append(user_map[mention['userId']])
+    return mentioned_user_ids
+
 def do_convert_data(gitter_data_file: str, output_dir: str, threads: int=6) -> None:
     #  Subdomain is set by the user while running the import commands
     realm_subdomain = ""
@@ -302,8 +322,15 @@ def do_convert_data(gitter_data_file: str, output_dir: str, threads: int=6) -> N
 
     realm, avatar_list, user_map = gitter_workspace_to_realm(
         domain_name, gitter_data, realm_subdomain)
+
+    # For user mentions
+    user_short_name_to_full_name = {}
+    for userprofile in realm['zerver_userprofile']:
+        user_short_name_to_full_name[userprofile['short_name']] = userprofile['full_name']
+
     convert_gitter_workspace_messages(
-        gitter_data, output_dir, realm['zerver_subscription'], user_map)
+        gitter_data, output_dir, realm['zerver_subscription'], user_map,
+        user_short_name_to_full_name)
 
     avatar_folder = os.path.join(output_dir, 'avatars')
     avatar_realm_folder = os.path.join(avatar_folder, str(realm_id))

zerver/tests/test_gitter_importer.py

@@ -20,6 +20,7 @@ from zerver.models import (
 )
 from zerver.lib.gitter_import import (
     do_convert_data,
+    get_usermentions,
 )
 
 import ujson
@@ -106,3 +107,26 @@ class GitterImporter(ZulipTestCase):
         self.assertEqual(exported_user_ids, exported_usermessage_userprofile)
         exported_usermessage_message = get_set(messages['zerver_usermessage'], 'message')
         self.assertEqual(exported_usermessage_message, exported_messages_id)
+
+    def test_get_usermentions(self) -> None:
+        user_map = {'57124a4': 3, '57124b4': 5, '57124c4': 8}
+        user_short_name_to_full_name = {'user': 'user name', 'user2': 'user2',
+                                        'user3': 'user name 3', 'user4': 'user 4'}
+        messages = [{'text': 'hi @user',
+                     'mentions': [{'screenName': 'user', 'userId': '57124a4'}]},
+                    {'text': 'hi @user2 @user3',
+                     'mentions': [{'screenName': 'user2', 'userId': '57124b4'},
+                                  {'screenName': 'user3', 'userId': '57124c4'}]},
+                    {'text': 'hi @user4',
+                     'mentions': [{'screenName': 'user4'}]},
+                    {'text': 'hi @user5',
+                     'mentions': [{'screenName': 'user', 'userId': '5712ds4'}]}]
+
+        self.assertEqual(get_usermentions(messages[0], user_map, user_short_name_to_full_name), [3])
+        self.assertEqual(messages[0]['text'], 'hi @**user name**')
+        self.assertEqual(get_usermentions(messages[1], user_map, user_short_name_to_full_name), [5, 8])
+        self.assertEqual(messages[1]['text'], 'hi @**user2** @**user name 3**')
+        self.assertEqual(get_usermentions(messages[2], user_map, user_short_name_to_full_name), [])
+        self.assertEqual(messages[2]['text'], 'hi @user4')
+        self.assertEqual(get_usermentions(messages[3], user_map, user_short_name_to_full_name), [])
+        self.assertEqual(messages[3]['text'], 'hi @user5')