diff --git a/zerver/lib/slack_data_to_zulip_data.py b/zerver/lib/slack_data_to_zulip_data.py index 8aa62ac0e8..a79e6b3634 100755 --- a/zerver/lib/slack_data_to_zulip_data.py +++ b/zerver/lib/slack_data_to_zulip_data.py @@ -7,6 +7,7 @@ import shutil import subprocess import re import logging +import requests from django.utils.timezone import now as timezone_now from typing import Any, Dict, List, Tuple @@ -45,7 +46,8 @@ def get_model_id(model: Any, table_name: str, sequence_increase_factor: int) -> os.system('echo %s | ./manage.py dbshell' % (increment_id_command)) return start_id_sequence -def slack_workspace_to_realm(REALM_ID: int, realm_subdomain: str, fixtures_path: str, +def slack_workspace_to_realm(REALM_ID: int, user_list: List[ZerverFieldsT], + realm_subdomain: str, fixtures_path: str, slack_data_dir: str) -> Tuple[ZerverFieldsT, AddedUsersT, AddedRecipientsT, AddedChannelsT]: """ @@ -79,6 +81,7 @@ def slack_workspace_to_realm(REALM_ID: int, realm_subdomain: str, fixtures_path: zerver_realmemoji=[]) zerver_userprofile, added_users = users_to_zerver_userprofile(slack_data_dir, + user_list, REALM_ID, int(NOW), DOMAIN_NAME) @@ -111,8 +114,9 @@ def build_zerver_realm(fixtures_path: str, REALM_ID: int, realm_subdomain: str, return zerver_realm_skeleton -def users_to_zerver_userprofile(slack_data_dir: str, realm_id: int, timestamp: Any, - domain_name: str) -> Tuple[List[ZerverFieldsT], AddedUsersT]: +def users_to_zerver_userprofile(slack_data_dir: str, users: List[ZerverFieldsT], realm_id: int, + timestamp: Any, domain_name: str) -> Tuple[List[ZerverFieldsT], + AddedUsersT]: """ Returns: 1. zerver_userprofile, which is a list of user profile @@ -120,7 +124,6 @@ def users_to_zerver_userprofile(slack_data_dir: str, realm_id: int, timestamp: A user id """ logging.info('######### IMPORTING USERS STARTED #########\n') - users = get_data_file(slack_data_dir + '/users.json') total_users = len(users) zerver_userprofile = [] added_users = {} @@ -435,8 +438,9 @@ def build_subscription(channel_members: List[str], zerver_subscription: List[Zer subscription_id += 1 return zerver_subscription, subscription_id -def convert_slack_workspace_messages(slack_data_dir: str, REALM_ID: int, added_users: AddedUsersT, - added_recipient: AddedRecipientsT, added_channels: AddedChannelsT, +def convert_slack_workspace_messages(slack_data_dir: str, users: List[ZerverFieldsT], REALM_ID: int, + added_users: AddedUsersT, added_recipient: AddedRecipientsT, + added_channels: AddedChannelsT, realm: ZerverFieldsT) -> ZerverFieldsT: """ Returns: @@ -465,7 +469,7 @@ def convert_slack_workspace_messages(slack_data_dir: str, REALM_ID: int, added_u message_id = len(zerver_message) + message_id_count # For the id of the messages usermessage_id = len(zerver_usermessage) + usermessage_id_count id_list = [message_id, usermessage_id] - zm, zum = channel_message_to_zerver_message(constants, channel, + zm, zum = channel_message_to_zerver_message(constants, channel, users, added_users, added_recipient, realm['zerver_subscription'], id_list) @@ -506,7 +510,8 @@ def get_total_messages_and_usermessages(slack_data_dir: str, channel_name: str, return total_messages, total_usermessages def channel_message_to_zerver_message(constants: List[Any], channel: str, - added_users: AddedUsersT, added_recipient: AddedRecipientsT, + users: List[ZerverFieldsT], added_users: AddedUsersT, + added_recipient: AddedRecipientsT, zerver_subscription: List[ZerverFieldsT], ids: List[int]) -> Tuple[List[ZerverFieldsT], List[ZerverFieldsT]]: @@ -518,7 +523,6 @@ def channel_message_to_zerver_message(constants: List[Any], channel: str, slack_data_dir, REALM_ID = constants message_id, usermessage_id = ids json_names = os.listdir(slack_data_dir + '/' + channel) - users = get_data_file(slack_data_dir + '/users.json') zerver_message = [] zerver_usermessage = [] # type: List[ZerverFieldsT] @@ -591,7 +595,7 @@ def build_zerver_usermessage(zerver_usermessage: List[ZerverFieldsT], usermessag zerver_usermessage.append(usermessage) return zerver_usermessage, usermessage_id -def do_convert_data(slack_zip_file: str, realm_subdomain: str, output_dir: str) -> None: +def do_convert_data(slack_zip_file: str, realm_subdomain: str, output_dir: str, token: str) -> None: check_subdomain_available(realm_subdomain) slack_data_dir = slack_zip_file.replace('.zip', '') if not os.path.exists(slack_data_dir): @@ -604,12 +608,16 @@ def do_convert_data(slack_zip_file: str, realm_subdomain: str, output_dir: str) fixtures_path = script_path + '../fixtures/' REALM_ID = get_model_id(Realm, 'zerver_realm', 1) + + user_list = get_user_data(token) realm, added_users, added_recipient, added_channels = slack_workspace_to_realm(REALM_ID, + user_list, realm_subdomain, fixtures_path, slack_data_dir) - message_json = convert_slack_workspace_messages(slack_data_dir, REALM_ID, added_users, - added_recipient, added_channels, realm) + message_json = convert_slack_workspace_messages(slack_data_dir, user_list, REALM_ID, + added_users, added_recipient, added_channels, + realm) zerver_attachment = [] # type: List[ZerverFieldsT] attachment = {"zerver_attachment": zerver_attachment} @@ -637,6 +645,15 @@ def get_data_file(path: str) -> Any: data = json.load(open(path)) return data +def get_user_data(token: str) -> List[ZerverFieldsT]: + slack_user_list_url = "https://slack.com/api/users.list" + user_list = requests.get('%s?token=%s' % (slack_user_list_url, token)) + if user_list.status_code == requests.codes.ok: + user_list_json = user_list.json()['members'] + return user_list_json + else: + raise Exception('Enter a valid token!') + def create_converted_data_files(data: Any, output_dir: str, file_path: str, make_new_dir: bool) -> None: output_file = output_dir + file_path diff --git a/zerver/management/commands/convert_slack_data.py b/zerver/management/commands/convert_slack_data.py index 821b910027..4db91025dc 100644 --- a/zerver/management/commands/convert_slack_data.py +++ b/zerver/management/commands/convert_slack_data.py @@ -21,6 +21,9 @@ class Command(BaseCommand): parser.add_argument('realm_name', metavar='', type=str, help="Realm Name") + parser.add_argument('--token', metavar='', + type=str, help='Slack legacy token of the organsation') + parser.add_argument('--output', dest='output_dir', action="store", default=None, help='Directory to write exported data to.') @@ -37,14 +40,19 @@ class Command(BaseCommand): os.makedirs(output_dir) realm_name = options['realm_name'] + token = options['token'] if realm_name is None: print("Enter realm name!") exit(1) + if token is None: + print("Enter slack legacy token!") + exit(1) + for path in options['slack_data_zip']: if not os.path.exists(path): print("Slack data directory not found: '%s'" % (path,)) exit(1) print("Converting Data ...") - do_convert_data(path, realm_name, output_dir) + do_convert_data(path, realm_name, output_dir, token) diff --git a/zerver/tests/test_slack_importer.py b/zerver/tests/test_slack_importer.py index 2f87831e73..e7cb7da23b 100644 --- a/zerver/tests/test_slack_importer.py +++ b/zerver/tests/test_slack_importer.py @@ -4,6 +4,7 @@ from django.utils.timezone import now as timezone_now from zerver.lib.slack_data_to_zulip_data import ( get_model_id, + get_user_data, build_zerver_realm, get_user_email, get_admin, @@ -33,10 +34,26 @@ import ujson import json import logging +import requests import os import mock from typing import Any, AnyStr, Dict, List, Optional, Set, Tuple, Text +# This method will be used by the mock to replace requests.get +def mocked_requests_get(*args: List[str], **kwargs: List[str]) -> mock.Mock: + class MockResponse: + def __init__(self, json_data: Dict[str, Any], status_code: int) -> None: + self.json_data = json_data + self.status_code = status_code + + def json(self) -> Dict[str, Any]: + return self.json_data + + if args[0] == 'https://slack.com/api/users.list?token=valid-token': + return MockResponse({"members": "user_data"}, 200) + else: + return MockResponse(None, 404) + class SlackImporter(ZulipTestCase): logger = logging.getLogger() # set logger to a higher level to suppress 'logger.INFO' outputs @@ -49,6 +66,15 @@ class SlackImporter(ZulipTestCase): self.assertEqual(start_id_sequence, test_id_sequence) + @mock.patch('requests.get', side_effect=mocked_requests_get) + def test_get_user_data(self, mock_get: mock.Mock) -> None: + token = 'valid-token' + self.assertEqual(get_user_data(token), "user_data") + token = 'invalid-token' + with self.assertRaises(Exception) as invalid: + get_user_data(token) + self.assertEqual(invalid.exception.args, ('Enter a valid token!',),) + def test_build_zerver_realm(self) -> None: fixtures_path = os.path.dirname(os.path.abspath(__file__)) + '/../fixtures/' realm_id = 2 @@ -87,10 +113,8 @@ class SlackImporter(ZulipTestCase): self.assertEqual(get_user_timezone(user_timezone_none), "America/New_York") self.assertEqual(get_user_timezone(user_no_timezone), "America/New_York") - @mock.patch("zerver.lib.slack_data_to_zulip_data.get_data_file") @mock.patch("zerver.lib.slack_data_to_zulip_data.get_model_id", return_value=1) - def test_users_to_zerver_userprofile(self, mock_get_model_id: mock.Mock, - mock_get_data_file: mock.Mock) -> None: + def test_users_to_zerver_userprofile(self, mock_get_model_id: mock.Mock) -> None: user_data = [{"id": "U08RGD1RD", "name": "john", "deleted": False, @@ -113,7 +137,6 @@ class SlackImporter(ZulipTestCase): "deleted": False, "profile": {"image_32": "https:\/\/secure.gravatar.com\/avatar\/random1.png", "email": "bot1@zulipchat.com"}}] - mock_get_data_file.return_value = user_data # As user with slack_id 'U0CBK5KAT' is the primary owner, that user should be imported first # and hence has zulip_id = 1 @@ -122,7 +145,8 @@ class SlackImporter(ZulipTestCase): 'U09TYF5Sk': 3} slack_data_dir = './random_path' timestamp = int(timezone_now().timestamp()) - zerver_userprofile, added_users = users_to_zerver_userprofile(slack_data_dir, 1, timestamp, 'test_domain') + zerver_userprofile, added_users = users_to_zerver_userprofile(slack_data_dir, user_data, + 1, timestamp, 'test_domain') # test that the primary owner should always be imported first self.assertDictEqual(added_users, test_added_users) @@ -264,7 +288,9 @@ class SlackImporter(ZulipTestCase): mock_build_zerver_realm: mock.Mock) -> None: realm_id = 1 + user_list = [] # type: List[Dict[str, Any]] realm, added_users, added_recipient, added_channels = slack_workspace_to_realm(realm_id, + user_list, 'test-realm', './fixture', './random_path') @@ -371,7 +397,7 @@ class SlackImporter(ZulipTestCase): {"text": "random test", "user": "U061A1R2R", "ts": "1433868669.000012"}] - mock_get_data_file.side_effect = [user_data, date1, date2] + mock_get_data_file.side_effect = [date1, date2] added_recipient = {'random': 2} constants = ['./random_path', 2] ids = [3, 7] @@ -380,7 +406,8 @@ class SlackImporter(ZulipTestCase): zerver_usermessage = [] # type: List[Dict[str, Any]] zerver_subscription = [] # type: List[Dict[str, Any]] zerver_message, zerver_usermessage = channel_message_to_zerver_message(constants, channel_name, - added_users, added_recipient, + user_data, added_users, + added_recipient, zerver_subscription, ids) # functioning already tested in helper function self.assertEqual(zerver_usermessage, []) @@ -420,13 +447,14 @@ class SlackImporter(ZulipTestCase): zerver_message2 = [{'id': 5}] realm = {'zerver_subscription': []} # type: Dict[str, Any] + user_list = [] # type: List[Dict[str, Any]] zerver_usermessage1 = [{'id': 3}, {'id': 5}] zerver_usermessage2 = [{'id': 6}, {'id': 9}] mock_message.side_effect = [[zerver_message1, zerver_usermessage1], [zerver_message2, zerver_usermessage2]] - message_json = convert_slack_workspace_messages('./random_path', 2, {}, + message_json = convert_slack_workspace_messages('./random_path', user_list, 2, {}, {}, added_channels, realm) self.assertEqual(message_json['zerver_message'], zerver_message1 + zerver_message2)