slack importer: Get user data from a get request to slack users api.

The fresh imported data shows that the users emails are not included
in the data. However, the data received from the older method of slack
(which is using legacy tokens) contains the email data of the users.

zerver/lib/slack_data_to_zulip_data.py

@@ -7,6 +7,7 @@ import shutil
 import subprocess
 import re
 import logging
+import requests
 
 from django.utils.timezone import now as timezone_now
 from typing import Any, Dict, List, Tuple
@@ -45,7 +46,8 @@ def get_model_id(model: Any, table_name: str, sequence_increase_factor: int) ->
     os.system('echo %s | ./manage.py dbshell' % (increment_id_command))
     return start_id_sequence
 
-def slack_workspace_to_realm(REALM_ID: int, realm_subdomain: str, fixtures_path: str,
+def slack_workspace_to_realm(REALM_ID: int, user_list: List[ZerverFieldsT],
+                             realm_subdomain: str, fixtures_path: str,
                              slack_data_dir: str) -> Tuple[ZerverFieldsT, AddedUsersT,
                                                            AddedRecipientsT, AddedChannelsT]:
     """
@@ -79,6 +81,7 @@ def slack_workspace_to_realm(REALM_ID: int, realm_subdomain: str, fixtures_path:
                  zerver_realmemoji=[])
 
     zerver_userprofile, added_users = users_to_zerver_userprofile(slack_data_dir,
+                                                                  user_list,
                                                                   REALM_ID,
                                                                   int(NOW),
                                                                   DOMAIN_NAME)
@@ -111,8 +114,9 @@ def build_zerver_realm(fixtures_path: str, REALM_ID: int, realm_subdomain: str,
 
     return zerver_realm_skeleton
 
-def users_to_zerver_userprofile(slack_data_dir: str, realm_id: int, timestamp: Any,
+def users_to_zerver_userprofile(slack_data_dir: str, users: List[ZerverFieldsT], realm_id: int,
-                                domain_name: str) -> Tuple[List[ZerverFieldsT], AddedUsersT]:
+                                timestamp: Any, domain_name: str) -> Tuple[List[ZerverFieldsT],
+                                                                           AddedUsersT]:
     """
     Returns:
     1. zerver_userprofile, which is a list of user profile
@@ -120,7 +124,6 @@ def users_to_zerver_userprofile(slack_data_dir: str, realm_id: int, timestamp: A
        user id
     """
     logging.info('######### IMPORTING USERS STARTED #########\n')
-    users = get_data_file(slack_data_dir + '/users.json')
     total_users = len(users)
     zerver_userprofile = []
     added_users = {}
@@ -435,8 +438,9 @@ def build_subscription(channel_members: List[str], zerver_subscription: List[Zer
         subscription_id += 1
     return zerver_subscription, subscription_id
 
-def convert_slack_workspace_messages(slack_data_dir: str, REALM_ID: int, added_users: AddedUsersT,
+def convert_slack_workspace_messages(slack_data_dir: str, users: List[ZerverFieldsT], REALM_ID: int,
-                                     added_recipient: AddedRecipientsT, added_channels: AddedChannelsT,
+                                     added_users: AddedUsersT, added_recipient: AddedRecipientsT,
+                                     added_channels: AddedChannelsT,
                                      realm: ZerverFieldsT) -> ZerverFieldsT:
     """
     Returns:
@@ -465,7 +469,7 @@ def convert_slack_workspace_messages(slack_data_dir: str, REALM_ID: int, added_u
         message_id = len(zerver_message) + message_id_count  # For the id of the messages
         usermessage_id = len(zerver_usermessage) + usermessage_id_count
         id_list = [message_id, usermessage_id]
-        zm, zum = channel_message_to_zerver_message(constants, channel,
+        zm, zum = channel_message_to_zerver_message(constants, channel, users,
                                                     added_users, added_recipient,
                                                     realm['zerver_subscription'],
                                                     id_list)
@@ -506,7 +510,8 @@ def get_total_messages_and_usermessages(slack_data_dir: str, channel_name: str,
     return total_messages, total_usermessages
 
 def channel_message_to_zerver_message(constants: List[Any], channel: str,
-                                      added_users: AddedUsersT, added_recipient: AddedRecipientsT,
+                                      users: List[ZerverFieldsT], added_users: AddedUsersT,
+                                      added_recipient: AddedRecipientsT,
                                       zerver_subscription: List[ZerverFieldsT],
                                       ids: List[int]) -> Tuple[List[ZerverFieldsT],
                                                                List[ZerverFieldsT]]:
@@ -518,7 +523,6 @@ def channel_message_to_zerver_message(constants: List[Any], channel: str,
     slack_data_dir, REALM_ID = constants
     message_id, usermessage_id = ids
     json_names = os.listdir(slack_data_dir + '/' + channel)
-    users = get_data_file(slack_data_dir + '/users.json')
     zerver_message = []
     zerver_usermessage = []  # type: List[ZerverFieldsT]
 
@@ -591,7 +595,7 @@ def build_zerver_usermessage(zerver_usermessage: List[ZerverFieldsT], usermessag
             zerver_usermessage.append(usermessage)
     return zerver_usermessage, usermessage_id
 
-def do_convert_data(slack_zip_file: str, realm_subdomain: str, output_dir: str) -> None:
+def do_convert_data(slack_zip_file: str, realm_subdomain: str, output_dir: str, token: str) -> None:
     check_subdomain_available(realm_subdomain)
     slack_data_dir = slack_zip_file.replace('.zip', '')
     if not os.path.exists(slack_data_dir):
@@ -604,12 +608,16 @@ def do_convert_data(slack_zip_file: str, realm_subdomain: str, output_dir: str)
     fixtures_path = script_path + '../fixtures/'
 
     REALM_ID = get_model_id(Realm, 'zerver_realm', 1)
+
+    user_list = get_user_data(token)
     realm, added_users, added_recipient, added_channels = slack_workspace_to_realm(REALM_ID,
+                                                                                   user_list,
                                                                                    realm_subdomain,
                                                                                    fixtures_path,
                                                                                    slack_data_dir)
-    message_json = convert_slack_workspace_messages(slack_data_dir, REALM_ID, added_users,
+    message_json = convert_slack_workspace_messages(slack_data_dir, user_list, REALM_ID,
-                                                    added_recipient, added_channels, realm)
+                                                    added_users, added_recipient, added_channels,
+                                                    realm)
 
     zerver_attachment = []  # type: List[ZerverFieldsT]
     attachment = {"zerver_attachment": zerver_attachment}
@@ -637,6 +645,15 @@ def get_data_file(path: str) -> Any:
     data = json.load(open(path))
     return data
 
+def get_user_data(token: str) -> List[ZerverFieldsT]:
+    slack_user_list_url = "https://slack.com/api/users.list"
+    user_list = requests.get('%s?token=%s' % (slack_user_list_url, token))
+    if user_list.status_code == requests.codes.ok:
+        user_list_json = user_list.json()['members']
+        return user_list_json
+    else:
+        raise Exception('Enter a valid token!')
+
 def create_converted_data_files(data: Any, output_dir: str, file_path: str,
                                 make_new_dir: bool) -> None:
     output_file = output_dir + file_path

zerver/management/commands/convert_slack_data.py

@@ -21,6 +21,9 @@ class Command(BaseCommand):
         parser.add_argument('realm_name', metavar='<realm_name>',
                             type=str, help="Realm Name")
 
+        parser.add_argument('--token', metavar='<slack_token>',
+                            type=str, help='Slack legacy token of the organsation')
+
         parser.add_argument('--output', dest='output_dir',
                             action="store", default=None,
                             help='Directory to write exported data to.')
@@ -37,14 +40,19 @@ class Command(BaseCommand):
         os.makedirs(output_dir)
 
         realm_name = options['realm_name']
+        token = options['token']
         if realm_name is None:
             print("Enter realm name!")
             exit(1)
 
+        if token is None:
+            print("Enter slack legacy token!")
+            exit(1)
+
         for path in options['slack_data_zip']:
             if not os.path.exists(path):
                 print("Slack data directory not found: '%s'" % (path,))
                 exit(1)
 
             print("Converting Data ...")
-            do_convert_data(path, realm_name, output_dir)
+            do_convert_data(path, realm_name, output_dir, token)

zerver/tests/test_slack_importer.py

@@ -4,6 +4,7 @@ from django.utils.timezone import now as timezone_now
 
 from zerver.lib.slack_data_to_zulip_data import (
     get_model_id,
+    get_user_data,
     build_zerver_realm,
     get_user_email,
     get_admin,
@@ -33,10 +34,26 @@ import ujson
 import json
 
 import logging
+import requests
 import os
 import mock
 from typing import Any, AnyStr, Dict, List, Optional, Set, Tuple, Text
 
+# This method will be used by the mock to replace requests.get
+def mocked_requests_get(*args: List[str], **kwargs: List[str]) -> mock.Mock:
+    class MockResponse:
+        def __init__(self, json_data: Dict[str, Any], status_code: int) -> None:
+            self.json_data = json_data
+            self.status_code = status_code
+
+        def json(self) -> Dict[str, Any]:
+            return self.json_data
+
+    if args[0] == 'https://slack.com/api/users.list?token=valid-token':
+        return MockResponse({"members": "user_data"}, 200)
+    else:
+        return MockResponse(None, 404)
+
 class SlackImporter(ZulipTestCase):
     logger = logging.getLogger()
     # set logger to a higher level to suppress 'logger.INFO' outputs
@@ -49,6 +66,15 @@ class SlackImporter(ZulipTestCase):
 
         self.assertEqual(start_id_sequence, test_id_sequence)
 
+    @mock.patch('requests.get', side_effect=mocked_requests_get)
+    def test_get_user_data(self, mock_get: mock.Mock) -> None:
+        token = 'valid-token'
+        self.assertEqual(get_user_data(token), "user_data")
+        token = 'invalid-token'
+        with self.assertRaises(Exception) as invalid:
+            get_user_data(token)
+        self.assertEqual(invalid.exception.args, ('Enter a valid token!',),)
+
     def test_build_zerver_realm(self) -> None:
         fixtures_path = os.path.dirname(os.path.abspath(__file__)) + '/../fixtures/'
         realm_id = 2
@@ -87,10 +113,8 @@ class SlackImporter(ZulipTestCase):
         self.assertEqual(get_user_timezone(user_timezone_none), "America/New_York")
         self.assertEqual(get_user_timezone(user_no_timezone), "America/New_York")
 
-    @mock.patch("zerver.lib.slack_data_to_zulip_data.get_data_file")
     @mock.patch("zerver.lib.slack_data_to_zulip_data.get_model_id", return_value=1)
-    def test_users_to_zerver_userprofile(self, mock_get_model_id: mock.Mock,
+    def test_users_to_zerver_userprofile(self, mock_get_model_id: mock.Mock) -> None:
-                                         mock_get_data_file: mock.Mock) -> None:
         user_data = [{"id": "U08RGD1RD",
                       "name": "john",
                       "deleted": False,
@@ -113,7 +137,6 @@ class SlackImporter(ZulipTestCase):
                       "deleted": False,
                       "profile": {"image_32": "https:\/\/secure.gravatar.com\/avatar\/random1.png",
                                   "email": "bot1@zulipchat.com"}}]
-        mock_get_data_file.return_value = user_data
 
         # As user with slack_id 'U0CBK5KAT' is the primary owner, that user should be imported first
         # and hence has zulip_id = 1
@@ -122,7 +145,8 @@ class SlackImporter(ZulipTestCase):
                             'U09TYF5Sk': 3}
         slack_data_dir = './random_path'
         timestamp = int(timezone_now().timestamp())
-        zerver_userprofile, added_users = users_to_zerver_userprofile(slack_data_dir, 1, timestamp, 'test_domain')
+        zerver_userprofile, added_users = users_to_zerver_userprofile(slack_data_dir, user_data,
+                                                                      1, timestamp, 'test_domain')
 
         # test that the primary owner should always be imported first
         self.assertDictEqual(added_users, test_added_users)
@@ -264,7 +288,9 @@ class SlackImporter(ZulipTestCase):
                                       mock_build_zerver_realm: mock.Mock) -> None:
 
         realm_id = 1
+        user_list = []  # type: List[Dict[str, Any]]
         realm, added_users, added_recipient, added_channels = slack_workspace_to_realm(realm_id,
+                                                                                       user_list,
                                                                                        'test-realm',
                                                                                        './fixture',
                                                                                        './random_path')
@@ -371,7 +397,7 @@ class SlackImporter(ZulipTestCase):
                  {"text": "random test", "user": "U061A1R2R",
                   "ts": "1433868669.000012"}]
 
-        mock_get_data_file.side_effect = [user_data, date1, date2]
+        mock_get_data_file.side_effect = [date1, date2]
         added_recipient = {'random': 2}
         constants = ['./random_path', 2]
         ids = [3, 7]
@@ -380,7 +406,8 @@ class SlackImporter(ZulipTestCase):
         zerver_usermessage = []  # type: List[Dict[str, Any]]
         zerver_subscription = []  # type: List[Dict[str, Any]]
         zerver_message, zerver_usermessage = channel_message_to_zerver_message(constants, channel_name,
-                                                                               added_users, added_recipient,
+                                                                               user_data, added_users,
+                                                                               added_recipient,
                                                                                zerver_subscription, ids)
         # functioning already tested in helper function
         self.assertEqual(zerver_usermessage, [])
@@ -420,13 +447,14 @@ class SlackImporter(ZulipTestCase):
         zerver_message2 = [{'id': 5}]
 
         realm = {'zerver_subscription': []}  # type: Dict[str, Any]
+        user_list = []  # type: List[Dict[str, Any]]
 
         zerver_usermessage1 = [{'id': 3}, {'id': 5}]
         zerver_usermessage2 = [{'id': 6}, {'id': 9}]
 
         mock_message.side_effect = [[zerver_message1, zerver_usermessage1],
                                     [zerver_message2, zerver_usermessage2]]
-        message_json = convert_slack_workspace_messages('./random_path', 2, {},
+        message_json = convert_slack_workspace_messages('./random_path', user_list, 2, {},
                                                         {}, added_channels,
                                                         realm)
         self.assertEqual(message_json['zerver_message'], zerver_message1 + zerver_message2)