paulmataruso/zulip
1
0
Fork 0
mirror of https://github.com/zulip/zulip.git synced 2025-11-04 14:03:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

slack importer: Get user data from a get request to slack users api.

Browse Source 
The fresh imported data shows that the users emails are not included
in the data. However, the data received from the older method of slack
(which is using legacy tokens) contains the email data of the users.
This commit is contained in:
Rhea Parekh
2018-02-01 05:26:57 +05:30
committed by Tim Abbott
parent 6e30da9f92
commit c0e30079f6
3 changed files with 74 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
zerver/lib/slack_data_to_zulip_data.py
View File

@@ -7,6 +7,7 @@ import shutil
import subprocess
import re
import logging
import requests
from django.utils.timezone import now as timezone_now
from typing import Any, Dict, List, Tuple
@@ -45,7 +46,8 @@ def get_model_id(model: Any, table_name: str, sequence_increase_factor: int) ->
os.system('echo %s | ./manage.py dbshell' % (increment_id_command))
return start_id_sequence
def slack_workspace_to_realm(REALM_ID: int, realm_subdomain: str, fixtures_path: str,
def slack_workspace_to_realm(REALM_ID: int, user_list: List[ZerverFieldsT],
realm_subdomain: str, fixtures_path: str,
slack_data_dir: str) -> Tuple[ZerverFieldsT, AddedUsersT,
AddedRecipientsT, AddedChannelsT]:
"""
@@ -79,6 +81,7 @@ def slack_workspace_to_realm(REALM_ID: int, realm_subdomain: str, fixtures_path:
zerver_realmemoji=[])
zerver_userprofile, added_users = users_to_zerver_userprofile(slack_data_dir,
user_list,
REALM_ID,
int(NOW),
DOMAIN_NAME)
@@ -111,8 +114,9 @@ def build_zerver_realm(fixtures_path: str, REALM_ID: int, realm_subdomain: str,
return zerver_realm_skeleton
def users_to_zerver_userprofile(slack_data_dir: str, realm_id: int, timestamp: Any,
domain_name: str) -> Tuple[List[ZerverFieldsT], AddedUsersT]:
def users_to_zerver_userprofile(slack_data_dir: str, users: List[ZerverFieldsT], realm_id: int,
timestamp: Any, domain_name: str) -> Tuple[List[ZerverFieldsT],
AddedUsersT]:
"""
Returns:
1. zerver_userprofile, which is a list of user profile
@@ -120,7 +124,6 @@ def users_to_zerver_userprofile(slack_data_dir: str, realm_id: int, timestamp: A
user id
"""
logging.info('######### IMPORTING USERS STARTED #########\n')
users = get_data_file(slack_data_dir + '/users.json')
total_users = len(users)
zerver_userprofile = []
added_users = {}
@@ -435,8 +438,9 @@ def build_subscription(channel_members: List[str], zerver_subscription: List[Zer
subscription_id += 1
return zerver_subscription, subscription_id
def convert_slack_workspace_messages(slack_data_dir: str, REALM_ID: int, added_users: AddedUsersT,
added_recipient: AddedRecipientsT, added_channels: AddedChannelsT,
def convert_slack_workspace_messages(slack_data_dir: str, users: List[ZerverFieldsT], REALM_ID: int,
added_users: AddedUsersT, added_recipient: AddedRecipientsT,
added_channels: AddedChannelsT,
realm: ZerverFieldsT) -> ZerverFieldsT:
"""
Returns:
@@ -465,7 +469,7 @@ def convert_slack_workspace_messages(slack_data_dir: str, REALM_ID: int, added_u
message_id = len(zerver_message) + message_id_count # For the id of the messages
usermessage_id = len(zerver_usermessage) + usermessage_id_count
id_list = [message_id, usermessage_id]
zm, zum = channel_message_to_zerver_message(constants, channel,
zm, zum = channel_message_to_zerver_message(constants, channel, users,
added_users, added_recipient,
realm['zerver_subscription'],
id_list)
@@ -506,7 +510,8 @@ def get_total_messages_and_usermessages(slack_data_dir: str, channel_name: str,
return total_messages, total_usermessages
def channel_message_to_zerver_message(constants: List[Any], channel: str,
added_users: AddedUsersT, added_recipient: AddedRecipientsT,
users: List[ZerverFieldsT], added_users: AddedUsersT,
added_recipient: AddedRecipientsT,
zerver_subscription: List[ZerverFieldsT],
ids: List[int]) -> Tuple[List[ZerverFieldsT],
List[ZerverFieldsT]]:
@@ -518,7 +523,6 @@ def channel_message_to_zerver_message(constants: List[Any], channel: str,
slack_data_dir, REALM_ID = constants
message_id, usermessage_id = ids
json_names = os.listdir(slack_data_dir + '/' + channel)
users = get_data_file(slack_data_dir + '/users.json')
zerver_message = []
zerver_usermessage = [] # type: List[ZerverFieldsT]
@@ -591,7 +595,7 @@ def build_zerver_usermessage(zerver_usermessage: List[ZerverFieldsT], usermessag
zerver_usermessage.append(usermessage)
return zerver_usermessage, usermessage_id
def do_convert_data(slack_zip_file: str, realm_subdomain: str, output_dir: str) -> None:
def do_convert_data(slack_zip_file: str, realm_subdomain: str, output_dir: str, token: str) -> None:
check_subdomain_available(realm_subdomain)
slack_data_dir = slack_zip_file.replace('.zip', '')
if not os.path.exists(slack_data_dir):
@@ -604,12 +608,16 @@ def do_convert_data(slack_zip_file: str, realm_subdomain: str, output_dir: str)
fixtures_path = script_path + '../fixtures/'
REALM_ID = get_model_id(Realm, 'zerver_realm', 1)
user_list = get_user_data(token)
realm, added_users, added_recipient, added_channels = slack_workspace_to_realm(REALM_ID,
user_list,
realm_subdomain,
fixtures_path,
slack_data_dir)
message_json = convert_slack_workspace_messages(slack_data_dir, REALM_ID, added_users,
added_recipient, added_channels, realm)
message_json = convert_slack_workspace_messages(slack_data_dir, user_list, REALM_ID,
added_users, added_recipient, added_channels,
realm)
zerver_attachment = [] # type: List[ZerverFieldsT]
attachment = {"zerver_attachment": zerver_attachment}
@@ -637,6 +645,15 @@ def get_data_file(path: str) -> Any:
data = json.load(open(path))
return data
def get_user_data(token: str) -> List[ZerverFieldsT]:
slack_user_list_url = "https://slack.com/api/users.list"
user_list = requests.get('%s?token=%s' % (slack_user_list_url, token))
if user_list.status_code == requests.codes.ok:
user_list_json = user_list.json()['members']
return user_list_json
else:
raise Exception('Enter a valid token!')
def create_converted_data_files(data: Any, output_dir: str, file_path: str,
make_new_dir: bool) -> None:
output_file = output_dir + file_path

10
zerver/management/commands/convert_slack_data.py
View File

@@ -21,6 +21,9 @@ class Command(BaseCommand):
parser.add_argument('realm_name', metavar='<realm_name>',
type=str, help="Realm Name")
parser.add_argument('--token', metavar='<slack_token>',
type=str, help='Slack legacy token of the organsation')
parser.add_argument('--output', dest='output_dir',
action="store", default=None,
help='Directory to write exported data to.')
@@ -37,14 +40,19 @@ class Command(BaseCommand):
os.makedirs(output_dir)
realm_name = options['realm_name']
token = options['token']
if realm_name is None:
print("Enter realm name!")
exit(1)
if token is None:
print("Enter slack legacy token!")
exit(1)
for path in options['slack_data_zip']:
if not os.path.exists(path):
print("Slack data directory not found: '%s'" % (path,))
exit(1)
print("Converting Data ...")
do_convert_data(path, realm_name, output_dir)
do_convert_data(path, realm_name, output_dir, token)

44
zerver/tests/test_slack_importer.py
View File

@@ -4,6 +4,7 @@ from django.utils.timezone import now as timezone_now
from zerver.lib.slack_data_to_zulip_data import (
get_model_id,
get_user_data,
build_zerver_realm,
get_user_email,
get_admin,
@@ -33,10 +34,26 @@ import ujson
import json
import logging
import requests
import os
import mock
from typing import Any, AnyStr, Dict, List, Optional, Set, Tuple, Text
# This method will be used by the mock to replace requests.get
def mocked_requests_get(*args: List[str], **kwargs: List[str]) -> mock.Mock:
class MockResponse:
def __init__(self, json_data: Dict[str, Any], status_code: int) -> None:
self.json_data = json_data
self.status_code = status_code
def json(self) -> Dict[str, Any]:
return self.json_data
if args[0] == 'https://slack.com/api/users.list?token=valid-token':
return MockResponse({"members": "user_data"}, 200)
else:
return MockResponse(None, 404)
class SlackImporter(ZulipTestCase):
logger = logging.getLogger()
# set logger to a higher level to suppress 'logger.INFO' outputs
@@ -49,6 +66,15 @@ class SlackImporter(ZulipTestCase):
self.assertEqual(start_id_sequence, test_id_sequence)
@mock.patch('requests.get', side_effect=mocked_requests_get)
def test_get_user_data(self, mock_get: mock.Mock) -> None:
token = 'valid-token'
self.assertEqual(get_user_data(token), "user_data")
token = 'invalid-token'
with self.assertRaises(Exception) as invalid:
get_user_data(token)
self.assertEqual(invalid.exception.args, ('Enter a valid token!',),)
def test_build_zerver_realm(self) -> None:
fixtures_path = os.path.dirname(os.path.abspath(__file__)) + '/../fixtures/'
realm_id = 2
@@ -87,10 +113,8 @@ class SlackImporter(ZulipTestCase):
self.assertEqual(get_user_timezone(user_timezone_none), "America/New_York")
self.assertEqual(get_user_timezone(user_no_timezone), "America/New_York")
@mock.patch("zerver.lib.slack_data_to_zulip_data.get_data_file")
@mock.patch("zerver.lib.slack_data_to_zulip_data.get_model_id", return_value=1)
def test_users_to_zerver_userprofile(self, mock_get_model_id: mock.Mock,
mock_get_data_file: mock.Mock) -> None:
def test_users_to_zerver_userprofile(self, mock_get_model_id: mock.Mock) -> None:
user_data = [{"id": "U08RGD1RD",
"name": "john",
"deleted": False,
@@ -113,7 +137,6 @@ class SlackImporter(ZulipTestCase):
"deleted": False,
"profile": {"image_32": "https:\/\/secure.gravatar.com\/avatar\/random1.png",
"email": "bot1@zulipchat.com"}}]
mock_get_data_file.return_value = user_data
# As user with slack_id 'U0CBK5KAT' is the primary owner, that user should be imported first
# and hence has zulip_id = 1
@@ -122,7 +145,8 @@ class SlackImporter(ZulipTestCase):
'U09TYF5Sk': 3}
slack_data_dir = './random_path'
timestamp = int(timezone_now().timestamp())
zerver_userprofile, added_users = users_to_zerver_userprofile(slack_data_dir, 1, timestamp, 'test_domain')
zerver_userprofile, added_users = users_to_zerver_userprofile(slack_data_dir, user_data,
1, timestamp, 'test_domain')
# test that the primary owner should always be imported first
self.assertDictEqual(added_users, test_added_users)
@@ -264,7 +288,9 @@ class SlackImporter(ZulipTestCase):
mock_build_zerver_realm: mock.Mock) -> None:
realm_id = 1
user_list = [] # type: List[Dict[str, Any]]
realm, added_users, added_recipient, added_channels = slack_workspace_to_realm(realm_id,
user_list,
'test-realm',
'./fixture',
'./random_path')
@@ -371,7 +397,7 @@ class SlackImporter(ZulipTestCase):
{"text": "random test", "user": "U061A1R2R",
"ts": "1433868669.000012"}]
mock_get_data_file.side_effect = [user_data, date1, date2]
mock_get_data_file.side_effect = [date1, date2]
added_recipient = {'random': 2}
constants = ['./random_path', 2]
ids = [3, 7]
@@ -380,7 +406,8 @@ class SlackImporter(ZulipTestCase):
zerver_usermessage = [] # type: List[Dict[str, Any]]
zerver_subscription = [] # type: List[Dict[str, Any]]
zerver_message, zerver_usermessage = channel_message_to_zerver_message(constants, channel_name,
added_users, added_recipient,
user_data, added_users,
added_recipient,
zerver_subscription, ids)
# functioning already tested in helper function
self.assertEqual(zerver_usermessage, [])
@@ -420,13 +447,14 @@ class SlackImporter(ZulipTestCase):
zerver_message2 = [{'id': 5}]
realm = {'zerver_subscription': []} # type: Dict[str, Any]
user_list = [] # type: List[Dict[str, Any]]
zerver_usermessage1 = [{'id': 3}, {'id': 5}]
zerver_usermessage2 = [{'id': 6}, {'id': 9}]
mock_message.side_effect = [[zerver_message1, zerver_usermessage1],
[zerver_message2, zerver_usermessage2]]
message_json = convert_slack_workspace_messages('./random_path', 2, {},
message_json = convert_slack_workspace_messages('./random_path', user_list, 2, {},
{}, added_channels,
realm)
self.assertEqual(message_json['zerver_message'], zerver_message1 + zerver_message2)