paulmataruso/zulip
1
0
Fork 0
mirror of https://github.com/zulip/zulip.git synced 2025-11-04 22:13:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

slack importer: Get user data from a get request to slack users api.

Browse Source 
The fresh imported data shows that the users emails are not included
in the data. However, the data received from the older method of slack
(which is using legacy tokens) contains the email data of the users.
This commit is contained in:
Rhea Parekh
2018-02-01 05:26:57 +05:30
committed by Tim Abbott
parent 6e30da9f92
commit c0e30079f6
3 changed files with 74 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
zerver/lib/slack_data_to_zulip_data.py
View File

@@ -7,6 +7,7 @@ import shutil
import subprocess
import re
import logging
import requests
from django.utils.timezone import now as timezone_now
from typing import Any, Dict, List, Tuple
@@ -45,7 +46,8 @@ def get_model_id(model: Any, table_name: str, sequence_increase_factor: int) ->
os.system('echo %s | ./manage.py dbshell' % (increment_id_command))
return start_id_sequence
def slack_workspace_to_realm(REALM_ID: int, realm_subdomain: str, fixtures_path: str,
def slack_workspace_to_realm(REALM_ID: int, user_list: List[ZerverFieldsT],
realm_subdomain: str, fixtures_path: str,
slack_data_dir: str) -> Tuple[ZerverFieldsT, AddedUsersT,
AddedRecipientsT, AddedChannelsT]:
"""
@@ -79,6 +81,7 @@ def slack_workspace_to_realm(REALM_ID: int, realm_subdomain: str, fixtures_path:
zerver_realmemoji=[])
zerver_userprofile, added_users = users_to_zerver_userprofile(slack_data_dir,
user_list,
REALM_ID,
int(NOW),
DOMAIN_NAME)
@@ -111,8 +114,9 @@ def build_zerver_realm(fixtures_path: str, REALM_ID: int, realm_subdomain: str,
return zerver_realm_skeleton
def users_to_zerver_userprofile(slack_data_dir: str, realm_id: int, timestamp: Any,
domain_name: str) -> Tuple[List[ZerverFieldsT], AddedUsersT]:
def users_to_zerver_userprofile(slack_data_dir: str, users: List[ZerverFieldsT], realm_id: int,
timestamp: Any, domain_name: str) -> Tuple[List[ZerverFieldsT],
AddedUsersT]:
"""
Returns:
1. zerver_userprofile, which is a list of user profile
@@ -120,7 +124,6 @@ def users_to_zerver_userprofile(slack_data_dir: str, realm_id: int, timestamp: A
user id
"""
logging.info('######### IMPORTING USERS STARTED #########\n')
users = get_data_file(slack_data_dir + '/users.json')
total_users = len(users)
zerver_userprofile = []
added_users = {}
@@ -435,8 +438,9 @@ def build_subscription(channel_members: List[str], zerver_subscription: List[Zer
subscription_id += 1
return zerver_subscription, subscription_id
def convert_slack_workspace_messages(slack_data_dir: str, REALM_ID: int, added_users: AddedUsersT,
added_recipient: AddedRecipientsT, added_channels: AddedChannelsT,
def convert_slack_workspace_messages(slack_data_dir: str, users: List[ZerverFieldsT], REALM_ID: int,
added_users: AddedUsersT, added_recipient: AddedRecipientsT,
added_channels: AddedChannelsT,
realm: ZerverFieldsT) -> ZerverFieldsT:
"""
Returns:
@@ -465,7 +469,7 @@ def convert_slack_workspace_messages(slack_data_dir: str, REALM_ID: int, added_u
message_id = len(zerver_message) + message_id_count # For the id of the messages
usermessage_id = len(zerver_usermessage) + usermessage_id_count
id_list = [message_id, usermessage_id]
zm, zum = channel_message_to_zerver_message(constants, channel,
zm, zum = channel_message_to_zerver_message(constants, channel, users,
added_users, added_recipient,
realm['zerver_subscription'],
id_list)
@@ -506,7 +510,8 @@ def get_total_messages_and_usermessages(slack_data_dir: str, channel_name: str,
return total_messages, total_usermessages
def channel_message_to_zerver_message(constants: List[Any], channel: str,
added_users: AddedUsersT, added_recipient: AddedRecipientsT,
users: List[ZerverFieldsT], added_users: AddedUsersT,
added_recipient: AddedRecipientsT,
zerver_subscription: List[ZerverFieldsT],
ids: List[int]) -> Tuple[List[ZerverFieldsT],
List[ZerverFieldsT]]:
@@ -518,7 +523,6 @@ def channel_message_to_zerver_message(constants: List[Any], channel: str,
slack_data_dir, REALM_ID = constants
message_id, usermessage_id = ids
json_names = os.listdir(slack_data_dir + '/' + channel)
users = get_data_file(slack_data_dir + '/users.json')
zerver_message = []
zerver_usermessage = [] # type: List[ZerverFieldsT]
@@ -591,7 +595,7 @@ def build_zerver_usermessage(zerver_usermessage: List[ZerverFieldsT], usermessag
zerver_usermessage.append(usermessage)
return zerver_usermessage, usermessage_id
def do_convert_data(slack_zip_file: str, realm_subdomain: str, output_dir: str) -> None:
def do_convert_data(slack_zip_file: str, realm_subdomain: str, output_dir: str, token: str) -> None:
check_subdomain_available(realm_subdomain)
slack_data_dir = slack_zip_file.replace('.zip', '')
if not os.path.exists(slack_data_dir):
@@ -604,12 +608,16 @@ def do_convert_data(slack_zip_file: str, realm_subdomain: str, output_dir: str)
fixtures_path = script_path + '../fixtures/'
REALM_ID = get_model_id(Realm, 'zerver_realm', 1)
user_list = get_user_data(token)
realm, added_users, added_recipient, added_channels = slack_workspace_to_realm(REALM_ID,
user_list,
realm_subdomain,
fixtures_path,
slack_data_dir)
message_json = convert_slack_workspace_messages(slack_data_dir, REALM_ID, added_users,
added_recipient, added_channels, realm)
message_json = convert_slack_workspace_messages(slack_data_dir, user_list, REALM_ID,
added_users, added_recipient, added_channels,
realm)
zerver_attachment = [] # type: List[ZerverFieldsT]
attachment = {"zerver_attachment": zerver_attachment}
@@ -637,6 +645,15 @@ def get_data_file(path: str) -> Any:
data = json.load(open(path))
return data
def get_user_data(token: str) -> List[ZerverFieldsT]:
slack_user_list_url = "https://slack.com/api/users.list"
user_list = requests.get('%s?token=%s' % (slack_user_list_url, token))
if user_list.status_code == requests.codes.ok:
user_list_json = user_list.json()['members']
return user_list_json
else:
raise Exception('Enter a valid token!')
def create_converted_data_files(data: Any, output_dir: str, file_path: str,
make_new_dir: bool) -> None:
output_file = output_dir + file_path