paulmataruso/zulip
1
0
Fork 0
mirror of https://github.com/zulip/zulip.git synced 2025-11-05 14:35:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

slack: Rename added_users to slack_user_id_to_zulip_user_id.

Browse Source
This commit is contained in:
Vishnu KS
2019-08-12 17:14:07 +05:30
committed by Tim Abbott
parent 3650f19692
commit 9d51a1b527
3 changed files with 94 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
zerver/data_import/slack.py
View File

@@ -28,7 +28,7 @@ from zerver.lib.emoji import NAME_TO_CODEPOINT_PATH
from urllib.parse import urlencode from urllib.parse import urlencode
# stubs # stubs
AddedUsersT = Dict[str, int] SlackToZulipUserIDT = Dict[str, int]
AddedChannelsT = Dict[str, Tuple[str, int]] AddedChannelsT = Dict[str, Tuple[str, int]]
AddedMPIMsT = Dict[str, Tuple[str, int]] AddedMPIMsT = Dict[str, Tuple[str, int]]
DMMembersT = Dict[str, Tuple[str, str]] DMMembersT = Dict[str, Tuple[str, str]]
@@ -40,7 +40,8 @@ def rm_tree(path: str) -> None:
def slack_workspace_to_realm(domain_name: str, realm_id: int, user_list: List[ZerverFieldsT], def slack_workspace_to_realm(domain_name: str, realm_id: int, user_list: List[ZerverFieldsT],
realm_subdomain: str, slack_data_dir: str, realm_subdomain: str, slack_data_dir: str,
custom_emoji_list: ZerverFieldsT) -> Tuple[ZerverFieldsT, AddedUsersT, custom_emoji_list: ZerverFieldsT) -> Tuple[ZerverFieldsT,
SlackToZulipUserIDT,
AddedRecipientsT, AddedRecipientsT,
AddedChannelsT, AddedChannelsT,
AddedMPIMsT, AddedMPIMsT,
@@ -50,7 +51,7 @@ def slack_workspace_to_realm(domain_name: str, realm_id: int, user_list: List[Ze
""" """
Returns: Returns:
1. realm, Converted Realm data 1. realm, Converted Realm data
2. added_users, which is a dictionary to map from slack user id to zulip user id 2. slack_user_id_to_zulip_user_id, which is a dictionary to map from slack user id to zulip user id
3. added_recipient, which is a dictionary to map from channel name to zulip recipient_id 3. added_recipient, which is a dictionary to map from channel name to zulip recipient_id
4. added_channels, which is a dictionary to map from channel name to channel id, zulip stream_id 4. added_channels, which is a dictionary to map from channel name to channel id, zulip stream_id
5. added_mpims, which is a dictionary to map from MPIM name to MPIM id, zulip huddle_id 5. added_mpims, which is a dictionary to map from MPIM name to MPIM id, zulip huddle_id
@@ -63,12 +64,12 @@ def slack_workspace_to_realm(domain_name: str, realm_id: int, user_list: List[Ze
zerver_realm = build_zerver_realm(realm_id, realm_subdomain, NOW, 'Slack') # type: List[ZerverFieldsT] zerver_realm = build_zerver_realm(realm_id, realm_subdomain, NOW, 'Slack') # type: List[ZerverFieldsT]
realm = build_realm(zerver_realm, realm_id, domain_name) realm = build_realm(zerver_realm, realm_id, domain_name)
zerver_userprofile, avatars, added_users, zerver_customprofilefield, \ zerver_userprofile, avatars, slack_user_id_to_zulip_user_id, zerver_customprofilefield, \
zerver_customprofilefield_value = users_to_zerver_userprofile(slack_data_dir, user_list, zerver_customprofilefield_value = users_to_zerver_userprofile(slack_data_dir, user_list,
realm_id, int(NOW), domain_name) realm_id, int(NOW), domain_name)
realm, added_channels, added_mpims, dm_members, \ realm, added_channels, added_mpims, dm_members, \
added_recipient = channels_to_zerver_stream(slack_data_dir, realm_id, realm, added_recipient = channels_to_zerver_stream(slack_data_dir, realm_id, realm,
added_users, zerver_userprofile) slack_user_id_to_zulip_user_id, zerver_userprofile)
zerver_realmemoji, emoji_url_map = build_realmemoji(custom_emoji_list, realm_id) zerver_realmemoji, emoji_url_map = build_realmemoji(custom_emoji_list, realm_id)
realm['zerver_realmemoji'] = zerver_realmemoji realm['zerver_realmemoji'] = zerver_realmemoji
@@ -81,7 +82,7 @@ def slack_workspace_to_realm(domain_name: str, realm_id: int, user_list: List[Ze
realm['zerver_customprofilefield'] = zerver_customprofilefield realm['zerver_customprofilefield'] = zerver_customprofilefield
realm['zerver_customprofilefieldvalue'] = zerver_customprofilefield_value realm['zerver_customprofilefieldvalue'] = zerver_customprofilefield_value
return realm, added_users, added_recipient, added_channels, added_mpims, \ return realm, slack_user_id_to_zulip_user_id, added_recipient, added_channels, added_mpims, \
dm_members, avatars, emoji_url_map dm_members, avatars, emoji_url_map
def build_realmemoji(custom_emoji_list: ZerverFieldsT, def build_realmemoji(custom_emoji_list: ZerverFieldsT,
@@ -112,14 +113,14 @@ def build_realmemoji(custom_emoji_list: ZerverFieldsT,
def users_to_zerver_userprofile(slack_data_dir: str, users: List[ZerverFieldsT], realm_id: int, def users_to_zerver_userprofile(slack_data_dir: str, users: List[ZerverFieldsT], realm_id: int,
timestamp: Any, domain_name: str) -> Tuple[List[ZerverFieldsT], timestamp: Any, domain_name: str) -> Tuple[List[ZerverFieldsT],
List[ZerverFieldsT], List[ZerverFieldsT],
AddedUsersT, SlackToZulipUserIDT,
List[ZerverFieldsT], List[ZerverFieldsT],
List[ZerverFieldsT]]: List[ZerverFieldsT]]:
""" """
Returns: Returns:
1. zerver_userprofile, which is a list of user profile 1. zerver_userprofile, which is a list of user profile
2. avatar_list, which is list to map avatars to zulip avatard records.json 2. avatar_list, which is list to map avatars to zulip avatard records.json
3. added_users, which is a dictionary to map from slack user id to zulip 3. slack_user_id_to_zulip_user_id, which is a dictionary to map from slack user id to zulip
user id user id
4. zerver_customprofilefield, which is a list of all custom profile fields 4. zerver_customprofilefield, which is a list of all custom profile fields
5. zerver_customprofilefield_values, which is a list of user profile fields 5. zerver_customprofilefield_values, which is a list of user profile fields
@@ -129,7 +130,7 @@ def users_to_zerver_userprofile(slack_data_dir: str, users: List[ZerverFieldsT],
zerver_customprofilefield = [] # type: List[ZerverFieldsT] zerver_customprofilefield = [] # type: List[ZerverFieldsT]
zerver_customprofilefield_values = [] # type: List[ZerverFieldsT] zerver_customprofilefield_values = [] # type: List[ZerverFieldsT]
avatar_list = [] # type: List[ZerverFieldsT] avatar_list = [] # type: List[ZerverFieldsT]
added_users = {} slack_user_id_to_zulip_user_id = {}
# The user data we get from the slack api does not contain custom profile data # The user data we get from the slack api does not contain custom profile data
# Hence we get it from the slack zip file # Hence we get it from the slack zip file
@@ -205,7 +206,7 @@ def users_to_zerver_userprofile(slack_data_dir: str, users: List[ZerverFieldsT],
userprofile_dict['realm'] = realm_id userprofile_dict['realm'] = realm_id
zerver_userprofile.append(userprofile_dict) zerver_userprofile.append(userprofile_dict)
added_users[slack_user_id] = user_id slack_user_id_to_zulip_user_id[slack_user_id] = user_id
if not user.get('is_primary_owner', False): if not user.get('is_primary_owner', False):
user_id_count += 1 user_id_count += 1
@@ -213,7 +214,7 @@ def users_to_zerver_userprofile(slack_data_dir: str, users: List[ZerverFieldsT],
process_customprofilefields(zerver_customprofilefield, zerver_customprofilefield_values) process_customprofilefields(zerver_customprofilefield, zerver_customprofilefield_values)
logging.info('######### IMPORTING USERS FINISHED #########\n') logging.info('######### IMPORTING USERS FINISHED #########\n')
return zerver_userprofile, avatar_list, added_users, zerver_customprofilefield, \ return zerver_userprofile, avatar_list, slack_user_id_to_zulip_user_id, zerver_customprofilefield, \
zerver_customprofilefield_values zerver_customprofilefield_values
def build_customprofile_field(customprofile_field: List[ZerverFieldsT], fields: ZerverFieldsT, def build_customprofile_field(customprofile_field: List[ZerverFieldsT], fields: ZerverFieldsT,
@@ -324,7 +325,8 @@ def get_user_timezone(user: ZerverFieldsT) -> str:
return timezone return timezone
def channels_to_zerver_stream(slack_data_dir: str, realm_id: int, def channels_to_zerver_stream(slack_data_dir: str, realm_id: int,
realm: Dict[str, Any], added_users: AddedUsersT, realm: Dict[str, Any],
slack_user_id_to_zulip_user_id: SlackToZulipUserIDT,
zerver_userprofile: List[ZerverFieldsT]) \ zerver_userprofile: List[ZerverFieldsT]) \
-> Tuple[Dict[str, List[ZerverFieldsT]], AddedChannelsT, AddedMPIMsT, DMMembersT, AddedRecipientsT]: -> Tuple[Dict[str, List[ZerverFieldsT]], AddedChannelsT, AddedMPIMsT, DMMembersT, AddedRecipientsT]:
""" """
@@ -392,12 +394,13 @@ def channels_to_zerver_stream(slack_data_dir: str, realm_id: int,
# construct the subscription object and append it to realm["zerver_subscription"] # construct the subscription object and append it to realm["zerver_subscription"]
subscription_id_count = get_subscription(channel['members'], realm["zerver_subscription"], subscription_id_count = get_subscription(channel['members'], realm["zerver_subscription"],
recipient_id, added_users, recipient_id, slack_user_id_to_zulip_user_id,
subscription_id_count) subscription_id_count)
# TODO add realm["zerver_subscription"] which correspond to # TODO add realm["zerver_subscription"] which correspond to
# huddles type recipient # huddles type recipient
# For huddles: # For huddles:
# sub['recipient']=recipient['id'] where recipient['type_id']=added_users[member] # sub['recipient']=recipient['id'] where
# recipient['type_id']=slack_user_id_to_zulip_user_id[member]
stream_id_count += 1 stream_id_count += 1
recipient_id_count += 1 recipient_id_count += 1
@@ -442,7 +445,7 @@ def channels_to_zerver_stream(slack_data_dir: str, realm_id: int,
added_recipient[mpim['name']] = recipient_id_count added_recipient[mpim['name']] = recipient_id_count
subscription_id_count = get_subscription(mpim['members'], realm["zerver_subscription"], subscription_id_count = get_subscription(mpim['members'], realm["zerver_subscription"],
recipient_id_count, added_users, recipient_id_count, slack_user_id_to_zulip_user_id,
subscription_id_count) subscription_id_count)
huddle_id_count += 1 huddle_id_count += 1
@@ -455,7 +458,7 @@ def channels_to_zerver_stream(slack_data_dir: str, realm_id: int,
mpims = [] mpims = []
process_mpims(mpims) process_mpims(mpims)
for slack_user_id, zulip_user_id in added_users.items(): for slack_user_id, zulip_user_id in slack_user_id_to_zulip_user_id.items():
recipient = build_recipient(zulip_user_id, recipient_id_count, Recipient.PERSONAL) recipient = build_recipient(zulip_user_id, recipient_id_count, Recipient.PERSONAL)
added_recipient[slack_user_id] = recipient_id_count added_recipient[slack_user_id] = recipient_id_count
sub = build_subscription(recipient_id_count, zulip_user_id, subscription_id_count) sub = build_subscription(recipient_id_count, zulip_user_id, subscription_id_count)
@@ -480,17 +483,19 @@ def channels_to_zerver_stream(slack_data_dir: str, realm_id: int,
return realm, added_channels, added_mpims, dm_members, added_recipient return realm, added_channels, added_mpims, dm_members, added_recipient
def get_subscription(channel_members: List[str], zerver_subscription: List[ZerverFieldsT], def get_subscription(channel_members: List[str], zerver_subscription: List[ZerverFieldsT],
recipient_id: int, added_users: AddedUsersT, recipient_id: int, slack_user_id_to_zulip_user_id: SlackToZulipUserIDT,
subscription_id: int) -> int: subscription_id: int) -> int:
for member in channel_members: for slack_user_id in channel_members:
sub = build_subscription(recipient_id, added_users[member], subscription_id) sub = build_subscription(recipient_id, slack_user_id_to_zulip_user_id[slack_user_id],
subscription_id)
# The recipient corresponds to a stream for stream-readable message. # The recipient corresponds to a stream for stream-readable message.
zerver_subscription.append(sub) zerver_subscription.append(sub)
subscription_id += 1 subscription_id += 1
return subscription_id return subscription_id
def process_long_term_idle_users(slack_data_dir: str, users: List[ZerverFieldsT], def process_long_term_idle_users(slack_data_dir: str, users: List[ZerverFieldsT],
added_users: AddedUsersT, added_channels: AddedChannelsT, slack_user_id_to_zulip_user_id: SlackToZulipUserIDT,
added_channels: AddedChannelsT,
added_mpims: AddedMPIMsT, dm_members: DMMembersT, added_mpims: AddedMPIMsT, dm_members: DMMembersT,
zerver_userprofile: List[ZerverFieldsT]) -> Set[int]: zerver_userprofile: List[ZerverFieldsT]) -> Set[int]:
"""Algorithmically, we treat users who have sent at least 10 messages """Algorithmically, we treat users who have sent at least 10 messages
@@ -527,7 +532,7 @@ def process_long_term_idle_users(slack_data_dir: str, users: List[ZerverFieldsT]
for slack_user in users: for slack_user in users:
if slack_user["id"] in recent_senders: if slack_user["id"] in recent_senders:
continue continue
zulip_user_id = added_users[slack_user['id']] zulip_user_id = slack_user_id_to_zulip_user_id[slack_user['id']]
long_term_idle.add(zulip_user_id) long_term_idle.add(zulip_user_id)
# Record long-term idle status in zerver_userprofile # Record long-term idle status in zerver_userprofile
@@ -542,7 +547,8 @@ def process_long_term_idle_users(slack_data_dir: str, users: List[ZerverFieldsT]
return long_term_idle return long_term_idle
def convert_slack_workspace_messages(slack_data_dir: str, users: List[ZerverFieldsT], realm_id: int, def convert_slack_workspace_messages(slack_data_dir: str, users: List[ZerverFieldsT], realm_id: int,
added_users: AddedUsersT, added_recipient: AddedRecipientsT, slack_user_id_to_zulip_user_id: SlackToZulipUserIDT,
added_recipient: AddedRecipientsT,
added_channels: AddedChannelsT, added_channels: AddedChannelsT,
added_mpims: AddedMPIMsT, added_mpims: AddedMPIMsT,
dm_members: DMMembersT, dm_members: DMMembersT,
@@ -560,8 +566,9 @@ def convert_slack_workspace_messages(slack_data_dir: str, users: List[ZerverFiel
3. attachment, which is a list of the attachments 3. attachment, which is a list of the attachments
""" """
long_term_idle = process_long_term_idle_users(slack_data_dir, users, added_users, added_channels, long_term_idle = process_long_term_idle_users(slack_data_dir, users, slack_user_id_to_zulip_user_id,
added_mpims, dm_members, zerver_userprofile) added_channels, added_mpims, dm_members,
zerver_userprofile)
# Now, we actually import the messages. # Now, we actually import the messages.
all_messages = get_messages_iterator(slack_data_dir, added_channels, added_mpims, dm_members) all_messages = get_messages_iterator(slack_data_dir, added_channels, added_mpims, dm_members)
@@ -591,7 +598,7 @@ def convert_slack_workspace_messages(slack_data_dir: str, users: List[ZerverFiel
zerver_message, zerver_usermessage, attachment, uploads, reactions = \ zerver_message, zerver_usermessage, attachment, uploads, reactions = \
channel_message_to_zerver_message( channel_message_to_zerver_message(
realm_id, users, added_users, added_recipient, message_data, realm_id, users, slack_user_id_to_zulip_user_id, added_recipient, message_data,
zerver_realmemoji, subscriber_map, added_channels, dm_members, zerver_realmemoji, subscriber_map, added_channels, dm_members,
domain_name, long_term_idle) domain_name, long_term_idle)
@@ -651,7 +658,7 @@ def get_messages_iterator(slack_data_dir: str, added_channels: Dict[str, Any],
def channel_message_to_zerver_message(realm_id: int, def channel_message_to_zerver_message(realm_id: int,
users: List[ZerverFieldsT], users: List[ZerverFieldsT],
added_users: AddedUsersT, slack_user_id_to_zulip_user_id: SlackToZulipUserIDT,
added_recipient: AddedRecipientsT, added_recipient: AddedRecipientsT,
all_messages: List[ZerverFieldsT], all_messages: List[ZerverFieldsT],
zerver_realmemoji: List[ZerverFieldsT], zerver_realmemoji: List[ZerverFieldsT],
@@ -685,9 +692,9 @@ def channel_message_to_zerver_message(realm_id: int,
total_user_messages = 0 total_user_messages = 0
total_skipped_user_messages = 0 total_skipped_user_messages = 0
for message in all_messages: for message in all_messages:
user = get_message_sending_user(message) slack_user_id = get_message_sending_user(message)
if not user: if not slack_user_id:
# Ignore messages without user names # Ignore messages without slack_user_id
# These are Sometimes produced by slack # These are Sometimes produced by slack
continue continue
@@ -705,7 +712,7 @@ def channel_message_to_zerver_message(realm_id: int,
try: try:
content, mentioned_user_ids, has_link = convert_to_zulip_markdown( content, mentioned_user_ids, has_link = convert_to_zulip_markdown(
message['text'], users, added_channels, added_users) message['text'], users, added_channels, slack_user_id_to_zulip_user_id)
except Exception: except Exception:
print("Slack message unexpectedly missing text representation:") print("Slack message unexpectedly missing text representation:")
print(ujson.dumps(message, indent=4)) print(ujson.dumps(message, indent=4))
@@ -733,9 +740,8 @@ def channel_message_to_zerver_message(realm_id: int,
# Process message reactions # Process message reactions
if 'reactions' in message.keys(): if 'reactions' in message.keys():
build_reactions(reaction_list, message['reactions'], added_users, build_reactions(reaction_list, message['reactions'], slack_user_id_to_zulip_user_id,
message_id, name_to_codepoint, message_id, name_to_codepoint, zerver_realmemoji)
zerver_realmemoji)
# Process different subtypes of slack messages # Process different subtypes of slack messages
@@ -755,9 +761,9 @@ def channel_message_to_zerver_message(realm_id: int,
domain_name=domain_name, domain_name=domain_name,
realm_id=realm_id, realm_id=realm_id,
message_id=message_id, message_id=message_id,
user=user, slack_user_id=slack_user_id,
users=users, users=users,
added_users=added_users, slack_user_id_to_zulip_user_id=slack_user_id_to_zulip_user_id,
zerver_attachment=zerver_attachment, zerver_attachment=zerver_attachment,
uploads_list=uploads_list, uploads_list=uploads_list,
) )
@@ -772,8 +778,8 @@ def channel_message_to_zerver_message(realm_id: int,
topic_name = 'imported from slack' topic_name = 'imported from slack'
zulip_message = build_message(topic_name, float(message['ts']), message_id, content, zulip_message = build_message(topic_name, float(message['ts']), message_id, content,
rendered_content, added_users[user], recipient_id, rendered_content, slack_user_id_to_zulip_user_id[slack_user_id],
has_image, has_link, has_attachment) recipient_id, has_image, has_link, has_attachment)
zerver_message.append(zulip_message) zerver_message.append(zulip_message)
# construct usermessages # construct usermessages
@@ -811,9 +817,9 @@ def process_message_files(message: ZerverFieldsT,
domain_name: str, domain_name: str,
realm_id: int, realm_id: int,
message_id: int, message_id: int,
user: str, slack_user_id: str,
users: List[ZerverFieldsT], users: List[ZerverFieldsT],
added_users: AddedUsersT, slack_user_id_to_zulip_user_id: SlackToZulipUserIDT,
zerver_attachment: List[ZerverFieldsT], zerver_attachment: List[ZerverFieldsT],
uploads_list: List[ZerverFieldsT]) -> Dict[str, Any]: uploads_list: List[ZerverFieldsT]) -> Dict[str, Any]:
has_attachment = False has_attachment = False
@@ -854,10 +860,10 @@ def process_message_files(message: ZerverFieldsT,
markdown_links.append(content_for_link) markdown_links.append(content_for_link)
# construct attachments # construct attachments
build_uploads(added_users[user], realm_id, file_user_email, fileinfo, s3_path, build_uploads(slack_user_id_to_zulip_user_id[slack_user_id], realm_id, file_user_email,
uploads_list) fileinfo, s3_path, uploads_list)
build_attachment(realm_id, {message_id}, added_users[user], build_attachment(realm_id, {message_id}, slack_user_id_to_zulip_user_id[slack_user_id],
fileinfo, s3_path, zerver_attachment) fileinfo, s3_path, zerver_attachment)
else: else:
# For attachments with link not from slack # For attachments with link not from slack
@@ -896,7 +902,7 @@ def get_attachment_path_and_content(fileinfo: ZerverFieldsT, realm_id: int) -> T
return s3_path, content return s3_path, content
def build_reactions(reaction_list: List[ZerverFieldsT], reactions: List[ZerverFieldsT], def build_reactions(reaction_list: List[ZerverFieldsT], reactions: List[ZerverFieldsT],
added_users: AddedUsersT, message_id: int, slack_user_id_to_zulip_user_id: SlackToZulipUserIDT, message_id: int,
name_to_codepoint: ZerverFieldsT, name_to_codepoint: ZerverFieldsT,
zerver_realmemoji: List[ZerverFieldsT]) -> None: zerver_realmemoji: List[ZerverFieldsT]) -> None:
realmemoji = {} realmemoji = {}
@@ -918,7 +924,7 @@ def build_reactions(reaction_list: List[ZerverFieldsT], reactions: List[ZerverFi
else: else:
continue continue
for user in slack_reaction['users']: for slack_user_id in slack_reaction['users']:
reaction_id = NEXT_ID('reaction') reaction_id = NEXT_ID('reaction')
reaction = Reaction( reaction = Reaction(
id=reaction_id, id=reaction_id,
@@ -929,7 +935,7 @@ def build_reactions(reaction_list: List[ZerverFieldsT], reactions: List[ZerverFi
reaction_dict = model_to_dict(reaction, reaction_dict = model_to_dict(reaction,
exclude=['message', 'user_profile']) exclude=['message', 'user_profile'])
reaction_dict['message'] = message_id reaction_dict['message'] = message_id
reaction_dict['user_profile'] = added_users[user] reaction_dict['user_profile'] = slack_user_id_to_zulip_user_id[slack_user_id]
reaction_list.append(reaction_dict) reaction_list.append(reaction_dict)
@@ -1019,14 +1025,15 @@ def do_convert_data(slack_zip_file: str, output_dir: str, token: str, threads: i
# Get custom emoji from slack api # Get custom emoji from slack api
custom_emoji_list = get_slack_api_data("https://slack.com/api/emoji.list", "emoji", token=token) custom_emoji_list = get_slack_api_data("https://slack.com/api/emoji.list", "emoji", token=token)
realm, added_users, added_recipient, added_channels, added_mpims, dm_members, avatar_list, \ realm, slack_user_id_to_zulip_user_id, added_recipient, added_channels, added_mpims, \
emoji_url_map = slack_workspace_to_realm(domain_name, realm_id, user_list, dm_members, avatar_list, emoji_url_map = slack_workspace_to_realm(domain_name, realm_id, user_list,
realm_subdomain, realm_subdomain, slack_data_dir,
slack_data_dir, custom_emoji_list) custom_emoji_list)
reactions, uploads_list, zerver_attachment = convert_slack_workspace_messages( reactions, uploads_list, zerver_attachment = convert_slack_workspace_messages(
slack_data_dir, user_list, realm_id, added_users, added_recipient, added_channels, added_mpims, slack_data_dir, user_list, realm_id, slack_user_id_to_zulip_user_id,
dm_members, realm, realm['zerver_userprofile'], realm['zerver_realmemoji'], domain_name, output_dir) added_recipient, added_channels, added_mpims, dm_members, realm,
realm['zerver_userprofile'], realm['zerver_realmemoji'], domain_name, output_dir)
# Move zerver_reactions to realm.json file # Move zerver_reactions to realm.json file
realm['zerver_reaction'] = reactions realm['zerver_reaction'] = reactions

13
zerver/data_import/slack_message_conversion.py
View File

@@ -3,7 +3,7 @@ from typing import Any, Dict, Tuple, List, Optional
# stubs # stubs
ZerverFieldsT = Dict[str, Any] ZerverFieldsT = Dict[str, Any]
AddedUsersT = Dict[str, int] SlackToZulipUserIDT = Dict[str, int]
AddedChannelsT = Dict[str, Tuple[str, int]] AddedChannelsT = Dict[str, Tuple[str, int]]
# Slack link can be in the format <http://www.foo.com|www.foo.com> and <http://foo.com/> # Slack link can be in the format <http://www.foo.com|www.foo.com> and <http://foo.com/>
@@ -67,7 +67,8 @@ def get_user_full_name(user: ZerverFieldsT) -> str:
# Markdown mapping # Markdown mapping
def convert_to_zulip_markdown(text: str, users: List[ZerverFieldsT], def convert_to_zulip_markdown(text: str, users: List[ZerverFieldsT],
added_channels: AddedChannelsT, added_channels: AddedChannelsT,
added_users: AddedUsersT) -> Tuple[str, List[int], bool]: slack_user_id_to_zulip_user_id: SlackToZulipUserIDT) -> \
Tuple[str, List[int], bool]:
mentioned_users_id = [] mentioned_users_id = []
text = convert_markdown_syntax(text, SLACK_BOLD_REGEX, "**") text = convert_markdown_syntax(text, SLACK_BOLD_REGEX, "**")
text = convert_markdown_syntax(text, SLACK_STRIKETHROUGH_REGEX, "~~") text = convert_markdown_syntax(text, SLACK_STRIKETHROUGH_REGEX, "~~")
@@ -92,8 +93,8 @@ def convert_to_zulip_markdown(text: str, users: List[ZerverFieldsT],
# Check user mentions and change mention format from # Check user mentions and change mention format from
# '<@slack_id|short_name>' to '@**full_name**' # '<@slack_id|short_name>' to '@**full_name**'
if (re.findall(SLACK_USERMENTION_REGEX, tokens[iterator], re.VERBOSE)): if (re.findall(SLACK_USERMENTION_REGEX, tokens[iterator], re.VERBOSE)):
tokens[iterator], user_id = get_user_mentions(tokens[iterator], tokens[iterator], user_id = get_user_mentions(tokens[iterator], users,
users, added_users) slack_user_id_to_zulip_user_id)
if user_id is not None: if user_id is not None:
mentioned_users_id.append(user_id) mentioned_users_id.append(user_id)
@@ -112,7 +113,7 @@ def convert_to_zulip_markdown(text: str, users: List[ZerverFieldsT],
return text, mentioned_users_id, message_has_link return text, mentioned_users_id, message_has_link
def get_user_mentions(token: str, users: List[ZerverFieldsT], def get_user_mentions(token: str, users: List[ZerverFieldsT],
added_users: AddedUsersT) -> Tuple[str, Optional[int]]: slack_user_id_to_zulip_user_id: SlackToZulipUserIDT) -> Tuple[str, Optional[int]]:
slack_usermention_match = re.search(SLACK_USERMENTION_REGEX, token, re.VERBOSE) slack_usermention_match = re.search(SLACK_USERMENTION_REGEX, token, re.VERBOSE)
assert slack_usermention_match is not None assert slack_usermention_match is not None
short_name = slack_usermention_match.group(4) short_name = slack_usermention_match.group(4)
@@ -121,7 +122,7 @@ def get_user_mentions(token: str, users: List[ZerverFieldsT],
if (user['id'] == slack_id and user['name'] == short_name and short_name) or \ if (user['id'] == slack_id and user['name'] == short_name and short_name) or \
(user['id'] == slack_id and short_name is None): (user['id'] == slack_id and short_name is None):
full_name = get_user_full_name(user) full_name = get_user_full_name(user)
user_id = added_users[slack_id] user_id = slack_user_id_to_zulip_user_id[slack_id]
mention = "@**" + full_name + "**" mention = "@**" + full_name + "**"
token = re.sub(SLACK_USERMENTION_REGEX, mention, token, flags=re.VERBOSE) token = re.sub(SLACK_USERMENTION_REGEX, mention, token, flags=re.VERBOSE)
return token, user_id return token, user_id

52
zerver/tests/test_slack_importer.py
View File

@@ -237,11 +237,11 @@ class SlackImporter(ZulipTestCase):
mock_get_data_file.return_value = user_data mock_get_data_file.return_value = user_data
# As user with slack_id 'U0CBK5KAT' is the primary owner, that user should be imported first # As user with slack_id 'U0CBK5KAT' is the primary owner, that user should be imported first
# and hence has zulip_id = 1 # and hence has zulip_id = 1
test_added_users = {'U08RGD1RD': 1, 'U0CBK5KAT': 0, 'U09TYF5Sk': 2, 'UHSG7OPQN': 3} test_slack_user_id_to_zulip_user_id = {'U08RGD1RD': 1, 'U0CBK5KAT': 0, 'U09TYF5Sk': 2, 'UHSG7OPQN': 3}
slack_data_dir = './random_path' slack_data_dir = './random_path'
timestamp = int(timezone_now().timestamp()) timestamp = int(timezone_now().timestamp())
mock_get_data_file.return_value = user_data mock_get_data_file.return_value = user_data
zerver_userprofile, avatar_list, added_users, customprofilefield, \ zerver_userprofile, avatar_list, slack_user_id_to_zulip_user_id, customprofilefield, \
customprofilefield_value = users_to_zerver_userprofile(slack_data_dir, user_data, 1, customprofilefield_value = users_to_zerver_userprofile(slack_data_dir, user_data, 1,
timestamp, 'test_domain') timestamp, 'test_domain')
@@ -263,7 +263,7 @@ class SlackImporter(ZulipTestCase):
self.assertEqual(customprofilefield_value[5]['value'], 'test_skype_name') self.assertEqual(customprofilefield_value[5]['value'], 'test_skype_name')
# test that the primary owner should always be imported first # test that the primary owner should always be imported first
self.assertDictEqual(added_users, test_added_users) self.assertDictEqual(slack_user_id_to_zulip_user_id, test_slack_user_id_to_zulip_user_id)
self.assertEqual(len(avatar_list), 4) self.assertEqual(len(avatar_list), 4)
self.assertEqual(len(zerver_userprofile), 4) self.assertEqual(len(zerver_userprofile), 4)
@@ -277,13 +277,13 @@ class SlackImporter(ZulipTestCase):
self.assertEqual(zerver_userprofile[0]['email'], 'jon@gmail.com') self.assertEqual(zerver_userprofile[0]['email'], 'jon@gmail.com')
self.assertEqual(zerver_userprofile[0]['full_name'], 'John Doe') self.assertEqual(zerver_userprofile[0]['full_name'], 'John Doe')
self.assertEqual(zerver_userprofile[1]['id'], test_added_users['U0CBK5KAT']) self.assertEqual(zerver_userprofile[1]['id'], test_slack_user_id_to_zulip_user_id['U0CBK5KAT'])
self.assertEqual(zerver_userprofile[1]['is_realm_admin'], True) self.assertEqual(zerver_userprofile[1]['is_realm_admin'], True)
self.assertEqual(zerver_userprofile[1]['is_staff'], False) self.assertEqual(zerver_userprofile[1]['is_staff'], False)
self.assertEqual(zerver_userprofile[1]['is_active'], True) self.assertEqual(zerver_userprofile[1]['is_active'], True)
self.assertEqual(zerver_userprofile[0]['is_mirror_dummy'], False) self.assertEqual(zerver_userprofile[0]['is_mirror_dummy'], False)
self.assertEqual(zerver_userprofile[2]['id'], test_added_users['U09TYF5Sk']) self.assertEqual(zerver_userprofile[2]['id'], test_slack_user_id_to_zulip_user_id['U09TYF5Sk'])
self.assertEqual(zerver_userprofile[2]['is_bot'], True) self.assertEqual(zerver_userprofile[2]['is_bot'], True)
self.assertEqual(zerver_userprofile[2]['is_active'], True) self.assertEqual(zerver_userprofile[2]['is_active'], True)
self.assertEqual(zerver_userprofile[2]['is_mirror_dummy'], False) self.assertEqual(zerver_userprofile[2]['is_mirror_dummy'], False)
@@ -291,7 +291,7 @@ class SlackImporter(ZulipTestCase):
self.assertEqual(zerver_userprofile[2]['bot_type'], 1) self.assertEqual(zerver_userprofile[2]['bot_type'], 1)
self.assertEqual(zerver_userprofile[2]['avatar_source'], 'U') self.assertEqual(zerver_userprofile[2]['avatar_source'], 'U')
self.assertEqual(zerver_userprofile[3]['id'], test_added_users['UHSG7OPQN']) self.assertEqual(zerver_userprofile[3]['id'], test_slack_user_id_to_zulip_user_id['UHSG7OPQN'])
self.assertEqual(zerver_userprofile[3]['is_realm_admin'], False) self.assertEqual(zerver_userprofile[3]['is_realm_admin'], False)
self.assertEqual(zerver_userprofile[3]['is_staff'], False) self.assertEqual(zerver_userprofile[3]['is_staff'], False)
self.assertEqual(zerver_userprofile[3]['is_active'], False) self.assertEqual(zerver_userprofile[3]['is_active'], False)
@@ -331,32 +331,35 @@ class SlackImporter(ZulipTestCase):
def test_build_subscription(self) -> None: def test_build_subscription(self) -> None:
channel_members = ["U061A1R2R", "U061A3E0G", "U061A5N1G", "U064KUGRJ"] channel_members = ["U061A1R2R", "U061A3E0G", "U061A5N1G", "U064KUGRJ"]
added_users = {"U061A1R2R": 1, "U061A3E0G": 8, "U061A5N1G": 7, "U064KUGRJ": 5} slack_user_id_to_zulip_user_id = {"U061A1R2R": 1, "U061A3E0G": 8, "U061A5N1G": 7, "U064KUGRJ": 5}
subscription_id_count = 0 subscription_id_count = 0
recipient_id = 12 recipient_id = 12
zerver_subscription = [] # type: List[Dict[str, Any]] zerver_subscription = [] # type: List[Dict[str, Any]]
final_subscription_id = get_subscription(channel_members, zerver_subscription, final_subscription_id = get_subscription(channel_members, zerver_subscription,
recipient_id, added_users, recipient_id, slack_user_id_to_zulip_user_id,
subscription_id_count) subscription_id_count)
# sanity checks # sanity checks
self.assertEqual(final_subscription_id, 4) self.assertEqual(final_subscription_id, 4)
self.assertEqual(zerver_subscription[0]['recipient'], 12) self.assertEqual(zerver_subscription[0]['recipient'], 12)
self.assertEqual(zerver_subscription[0]['id'], 0) self.assertEqual(zerver_subscription[0]['id'], 0)
self.assertEqual(zerver_subscription[0]['user_profile'], added_users[channel_members[0]]) self.assertEqual(zerver_subscription[0]['user_profile'],
self.assertEqual(zerver_subscription[2]['user_profile'], added_users[channel_members[2]]) slack_user_id_to_zulip_user_id[channel_members[0]])
self.assertEqual(zerver_subscription[2]['user_profile'],
slack_user_id_to_zulip_user_id[channel_members[2]])
self.assertEqual(zerver_subscription[3]['id'], 3) self.assertEqual(zerver_subscription[3]['id'], 3)
self.assertEqual(zerver_subscription[1]['recipient'], self.assertEqual(zerver_subscription[1]['recipient'],
zerver_subscription[3]['recipient']) zerver_subscription[3]['recipient'])
self.assertEqual(zerver_subscription[1]['pin_to_top'], False) self.assertEqual(zerver_subscription[1]['pin_to_top'], False)
def test_channels_to_zerver_stream(self) -> None: def test_channels_to_zerver_stream(self) -> None:
added_users = {"U061A1R2R": 1, "U061A3E0G": 8, "U061A5N1G": 7, "U064KUGRJ": 5} slack_user_id_to_zulip_user_id = {"U061A1R2R": 1, "U061A3E0G": 8, "U061A5N1G": 7, "U064KUGRJ": 5}
zerver_userprofile = [{'id': 1}, {'id': 8}, {'id': 7}, {'id': 5}] zerver_userprofile = [{'id': 1}, {'id': 8}, {'id': 7}, {'id': 5}]
realm_id = 3 realm_id = 3
realm, added_channels, added_mpims, dm_members, \ realm, added_channels, added_mpims, dm_members, \
added_recipient = channels_to_zerver_stream(self.fixture_file_name("", "slack_fixtures"), realm_id, added_recipient = channels_to_zerver_stream(self.fixture_file_name("", "slack_fixtures"), realm_id,
{"zerver_userpresence": []}, added_users, zerver_userprofile) {"zerver_userpresence": []}, slack_user_id_to_zulip_user_id,
zerver_userprofile)
test_added_channels = {'sharedchannel': ("C061A0HJG", 3), 'general': ("C061A0YJG", 1), test_added_channels = {'sharedchannel': ("C061A0HJG", 3), 'general': ("C061A0YJG", 1),
'general1': ("C061A0YJP", 2), 'random': ("C061A0WJG", 0)} 'general1': ("C061A0YJP", 2), 'random': ("C061A0WJG", 0)}
@@ -365,7 +368,8 @@ class SlackImporter(ZulipTestCase):
'mpdm-user4--user1--user5-1': ('G6N944JPL', 2)} 'mpdm-user4--user1--user5-1': ('G6N944JPL', 2)}
test_dm_members = {'DJ47BL849': ('U061A1R2R', 'U061A5N1G'), 'DHX1UP7EG': ('U061A5N1G', 'U064KUGRJ'), test_dm_members = {'DJ47BL849': ('U061A1R2R', 'U061A5N1G'), 'DHX1UP7EG': ('U061A5N1G', 'U064KUGRJ'),
'DK8HSJDHS': ('U061A1R2R', 'U064KUGRJ'), 'DRS3PSLDK': ('U064KUGRJ', 'U064KUGRJ')} 'DK8HSJDHS': ('U061A1R2R', 'U064KUGRJ'), 'DRS3PSLDK': ('U064KUGRJ', 'U064KUGRJ')}
test_added_recipient_keys = set(added_users.keys()) | set(test_added_channels.keys()) | set(test_added_mpims.keys()) test_added_recipient_keys = set(slack_user_id_to_zulip_user_id.keys()) | set(test_added_channels.keys()) \
| set(test_added_mpims.keys())
self.assertDictEqual(test_added_channels, added_channels) self.assertDictEqual(test_added_channels, added_channels)
# zerver defaultstream already tested in helper functions # zerver defaultstream already tested in helper functions
@@ -376,7 +380,7 @@ class SlackImporter(ZulipTestCase):
self.assertDictEqual(test_dm_members, dm_members) self.assertDictEqual(test_dm_members, dm_members)
# We can't do an assertDictEqual since during the construction of Personal # We can't do an assertDictEqual since during the construction of Personal
# recipients, added_users are iterated in diffrent order in Python 3.5 and 3.6. # recipients, slack_user_id_to_zulip_user_id are iterated in diffrent order in Python 3.5 and 3.6.
self.assertEqual(set(added_recipient.keys()), test_added_recipient_keys) self.assertEqual(set(added_recipient.keys()), test_added_recipient_keys)
self.assertEqual(set(added_recipient.values()), set(i for i in range(11))) self.assertEqual(set(added_recipient.values()), set(i for i in range(11)))
@@ -415,12 +419,14 @@ class SlackImporter(ZulipTestCase):
realm_id = 1 realm_id = 1
user_list = [] # type: List[Dict[str, Any]] user_list = [] # type: List[Dict[str, Any]]
realm, added_users, added_recipient, added_channels, added_mpims, dm_members, avatar_list, em = slack_workspace_to_realm( realm, slack_user_id_to_zulip_user_id, added_recipient, \
'testdomain', realm_id, user_list, 'test-realm', './random_path', {}) added_channels, added_mpims, dm_members, \
avatar_list, em = slack_workspace_to_realm('testdomain', realm_id, user_list, 'test-realm',
'./random_path', {})
test_zerver_realmdomain = [{'realm': realm_id, 'allow_subdomains': False, test_zerver_realmdomain = [{'realm': realm_id, 'allow_subdomains': False,
'domain': 'testdomain', 'id': realm_id}] 'domain': 'testdomain', 'id': realm_id}]
# Functioning already tests in helper functions # Functioning already tests in helper functions
self.assertEqual(added_users, {}) self.assertEqual(slack_user_id_to_zulip_user_id, {})
self.assertEqual(added_channels, {}) self.assertEqual(added_channels, {})
self.assertEqual(added_mpims, {}) self.assertEqual(added_mpims, {})
self.assertEqual(added_recipient, {}) self.assertEqual(added_recipient, {})
@@ -498,7 +504,7 @@ class SlackImporter(ZulipTestCase):
{"id": "U061A5N1G", "name": "jane doe", "deleted": False, "real_name": "Jane"}, {"id": "U061A5N1G", "name": "jane doe", "deleted": False, "real_name": "Jane"},
{"id": "U061A1R2R", "name": "jon", "deleted": False, "real_name": "Jon"}] {"id": "U061A1R2R", "name": "jon", "deleted": False, "real_name": "Jon"}]
added_users = {"U066MTL5U": 5, "U061A5N1G": 24, "U061A1R2R": 43} slack_user_id_to_zulip_user_id = {"U066MTL5U": 5, "U061A5N1G": 24, "U061A1R2R": 43}
reactions = [{"name": "grinning", "users": ["U061A5N1G"], "count": 1}] reactions = [{"name": "grinning", "users": ["U061A5N1G"], "count": 1}]
@@ -543,7 +549,7 @@ class SlackImporter(ZulipTestCase):
zerver_message, zerver_usermessage, attachment, uploads, reaction = \ zerver_message, zerver_usermessage, attachment, uploads, reaction = \
channel_message_to_zerver_message( channel_message_to_zerver_message(
1, user_data, added_users, added_recipient, 1, user_data, slack_user_id_to_zulip_user_id, added_recipient,
all_messages, [], subscriber_map, all_messages, [], subscriber_map,
added_channels, dm_members, 'domain', set()) added_channels, dm_members, 'domain', set())
# functioning already tested in helper function # functioning already tested in helper function
@@ -726,9 +732,9 @@ class SlackImporter(ZulipTestCase):
domain_name = 'example.com' domain_name = 'example.com'
realm_id = 5 realm_id = 5
message_id = 99 message_id = 99
user = 'alice' slack_user_id = 'alice'
users = [alice] users = [alice]
added_users = { slack_user_id_to_zulip_user_id = {
'alice': alice_id, 'alice': alice_id,
} }
@@ -740,9 +746,9 @@ class SlackImporter(ZulipTestCase):
domain_name=domain_name, domain_name=domain_name,
realm_id=realm_id, realm_id=realm_id,
message_id=message_id, message_id=message_id,
user=user, slack_user_id=slack_user_id,
users=users, users=users,
added_users=added_users, slack_user_id_to_zulip_user_id=slack_user_id_to_zulip_user_id,
zerver_attachment=zerver_attachment, zerver_attachment=zerver_attachment,
uploads_list=uploads_list, uploads_list=uploads_list,
) )