paulmataruso/zulip
1
0
Fork 0
mirror of https://github.com/zulip/zulip.git synced 2025-11-03 13:33:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

python: Normalize quotes with Black.

Browse Source 
Signed-off-by: Anders Kaseorg <anders@zulip.com>
This commit is contained in:
Anders Kaseorg
2021-02-11 23:20:45 -08:00
committed by Tim Abbott
parent 11741543da
commit 6e4c3e41dc
989 changed files with 32792 additions and 32792 deletions
Download Patch File Download Diff File Expand all files Collapse all files

324
zerver/data_import/slack.py
View File

@@ -93,7 +93,7 @@ def slack_workspace_to_realm(
"""
NOW = float(timezone_now().timestamp())
zerver_realm: List[ZerverFieldsT] = build_zerver_realm(realm_id, realm_subdomain, NOW, 'Slack')
zerver_realm: List[ZerverFieldsT] = build_zerver_realm(realm_id, realm_subdomain, NOW, "Slack")
realm = build_realm(zerver_realm, realm_id, domain_name)
(
@@ -114,14 +114,14 @@ def slack_workspace_to_realm(
)
zerver_realmemoji, emoji_url_map = build_realmemoji(custom_emoji_list, realm_id)
realm['zerver_realmemoji'] = zerver_realmemoji
realm["zerver_realmemoji"] = zerver_realmemoji
# See https://zulip.com/help/set-default-streams-for-new-users
# for documentation on zerver_defaultstream
realm['zerver_userprofile'] = zerver_userprofile
realm["zerver_userprofile"] = zerver_userprofile
realm['zerver_customprofilefield'] = zerver_customprofilefield
realm['zerver_customprofilefieldvalue'] = zerver_customprofilefield_value
realm["zerver_customprofilefield"] = zerver_customprofilefield
realm["zerver_customprofilefieldvalue"] = zerver_customprofilefield_value
return (
realm,
@@ -142,16 +142,16 @@ def build_realmemoji(
emoji_url_map = {}
emoji_id = 0
for emoji_name, url in custom_emoji_list.items():
if 'emoji.slack-edge.com' in url:
if "emoji.slack-edge.com" in url:
# Some of the emojis we get from the API have invalid links
# this is to prevent errors related to them
realmemoji = RealmEmoji(
name=emoji_name, id=emoji_id, file_name=os.path.basename(url), deactivated=False
)
realmemoji_dict = model_to_dict(realmemoji, exclude=['realm', 'author'])
realmemoji_dict['author'] = None
realmemoji_dict['realm'] = realm_id
realmemoji_dict = model_to_dict(realmemoji, exclude=["realm", "author"])
realmemoji_dict["author"] = None
realmemoji_dict["realm"] = realm_id
emoji_url_map[emoji_name] = url
zerver_realmemoji.append(realmemoji_dict)
@@ -177,7 +177,7 @@ def users_to_zerver_userprofile(
4. zerver_customprofilefield, which is a list of all custom profile fields
5. zerver_customprofilefield_values, which is a list of user profile fields
"""
logging.info('######### IMPORTING USERS STARTED #########\n')
logging.info("######### IMPORTING USERS STARTED #########\n")
zerver_userprofile = []
zerver_customprofilefield: List[ZerverFieldsT] = []
zerver_customprofilefield_values: List[ZerverFieldsT] = []
@@ -186,7 +186,7 @@ def users_to_zerver_userprofile(
# The user data we get from the Slack API does not contain custom profile data
# Hence we get it from the Slack zip file
slack_data_file_user_list = get_data_file(slack_data_dir + '/users.json')
slack_data_file_user_list = get_data_file(slack_data_dir + "/users.json")
slack_user_id_to_custom_profile_fields: ZerverFieldsT = {}
slack_custom_field_name_to_zulip_custom_field_id: ZerverFieldsT = {}
@@ -202,9 +202,9 @@ def users_to_zerver_userprofile(
user_id_count += 1
for user in users:
slack_user_id = user['id']
slack_user_id = user["id"]
if user.get('is_primary_owner', False):
if user.get("is_primary_owner", False):
user_id = primary_owner_id
else:
user_id = user_id_count
@@ -212,7 +212,7 @@ def users_to_zerver_userprofile(
email = get_user_email(user, domain_name)
# ref: https://zulip.com/help/change-your-profile-picture
avatar_url = build_avatar_url(
slack_user_id, user['team_id'], user['profile']['avatar_hash']
slack_user_id, user["team_id"], user["profile"]["avatar_hash"]
)
build_avatar(user_id, realm_id, email, avatar_url, timestamp, avatar_list)
role = UserProfile.ROLE_MEMBER
@@ -245,32 +245,32 @@ def users_to_zerver_userprofile(
userprofile = UserProfile(
full_name=get_user_full_name(user),
is_active=not user.get('deleted', False) and not user["is_mirror_dummy"],
is_active=not user.get("deleted", False) and not user["is_mirror_dummy"],
is_mirror_dummy=user["is_mirror_dummy"],
id=user_id,
email=email,
delivery_email=email,
avatar_source='U',
is_bot=user.get('is_bot', False),
avatar_source="U",
is_bot=user.get("is_bot", False),
role=role,
bot_type=1 if user.get('is_bot', False) else None,
bot_type=1 if user.get("is_bot", False) else None,
date_joined=timestamp,
timezone=timezone,
last_login=timestamp,
)
userprofile_dict = model_to_dict(userprofile)
# Set realm id separately as the corresponding realm is not yet a Realm model instance
userprofile_dict['realm'] = realm_id
userprofile_dict["realm"] = realm_id
zerver_userprofile.append(userprofile_dict)
slack_user_id_to_zulip_user_id[slack_user_id] = user_id
if not user.get('is_primary_owner', False):
if not user.get("is_primary_owner", False):
user_id_count += 1
logging.info("%s -> %s", user['name'], userprofile_dict['email'])
logging.info("%s -> %s", user["name"], userprofile_dict["email"])
process_customprofilefields(zerver_customprofilefield, zerver_customprofilefield_values)
logging.info('######### IMPORTING USERS FINISHED #########\n')
logging.info("######### IMPORTING USERS FINISHED #########\n")
return (
zerver_userprofile,
avatar_list,
@@ -292,7 +292,7 @@ def build_customprofile_field(
# Reference: https://api.slack.com/methods/users.profile.set
for field, value in fields.items():
if field not in slack_custom_field_name_to_zulip_custom_field_id:
slack_custom_fields = ['phone', 'skype']
slack_custom_fields = ["phone", "skype"]
if field in slack_custom_fields:
field_name = field
else:
@@ -304,8 +304,8 @@ def build_customprofile_field(
# Processing is done in the function 'process_customprofilefields'
)
customprofilefield_dict = model_to_dict(customprofilefield, exclude=['realm'])
customprofilefield_dict['realm'] = realm_id
customprofilefield_dict = model_to_dict(customprofilefield, exclude=["realm"])
customprofilefield_dict["realm"] = realm_id
slack_custom_field_name_to_zulip_custom_field_id[field] = custom_profile_field_id
custom_profile_field_id += 1
@@ -316,15 +316,15 @@ def build_customprofile_field(
def process_slack_custom_fields(
user: ZerverFieldsT, slack_user_id_to_custom_profile_fields: ZerverFieldsT
) -> None:
slack_user_id_to_custom_profile_fields[user['id']] = {}
if user['profile'].get('fields'):
slack_user_id_to_custom_profile_fields[user['id']] = user['profile']['fields']
slack_user_id_to_custom_profile_fields[user["id"]] = {}
if user["profile"].get("fields"):
slack_user_id_to_custom_profile_fields[user["id"]] = user["profile"]["fields"]
slack_custom_fields = ['phone', 'skype']
slack_custom_fields = ["phone", "skype"]
for field in slack_custom_fields:
if field in user['profile']:
slack_user_id_to_custom_profile_fields[user['id']][field] = {
'value': user['profile'][field]
if field in user["profile"]:
slack_user_id_to_custom_profile_fields[user["id"]][field] = {
"value": user["profile"][field]
}
@@ -336,15 +336,15 @@ def build_customprofilefields_values(
custom_field_values: List[ZerverFieldsT],
) -> int:
for field, value in fields.items():
if value['value'] == "":
if value["value"] == "":
continue
custom_field_value = CustomProfileFieldValue(id=custom_field_id, value=value['value'])
custom_field_value = CustomProfileFieldValue(id=custom_field_id, value=value["value"])
custom_field_value_dict = model_to_dict(
custom_field_value, exclude=['user_profile', 'field']
custom_field_value, exclude=["user_profile", "field"]
)
custom_field_value_dict['user_profile'] = user_id
custom_field_value_dict['field'] = slack_custom_field_name_to_zulip_custom_field_id[field]
custom_field_value_dict["user_profile"] = user_id
custom_field_value_dict["field"] = slack_custom_field_name_to_zulip_custom_field_id[field]
custom_field_values.append(custom_field_value_dict)
custom_field_id += 1
@@ -356,21 +356,21 @@ def process_customprofilefields(
) -> None:
for field in customprofilefield:
for field_value in customprofilefield_value:
if field_value['field'] == field['id'] and len(field_value['value']) > 50:
field['field_type'] = 2 # corresponding to Long text
if field_value["field"] == field["id"] and len(field_value["value"]) > 50:
field["field_type"] = 2 # corresponding to Long text
break
def get_user_email(user: ZerverFieldsT, domain_name: str) -> str:
if 'email' in user['profile']:
return user['profile']['email']
if user['is_mirror_dummy']:
if "email" in user["profile"]:
return user["profile"]["email"]
if user["is_mirror_dummy"]:
return "{}@{}.slack.com".format(user["name"], user["team_domain"])
if 'bot_id' in user['profile']:
if 'real_name_normalized' in user['profile']:
slack_bot_name = user['profile']['real_name_normalized']
elif 'first_name' in user['profile']:
slack_bot_name = user['profile']['first_name']
if "bot_id" in user["profile"]:
if "real_name_normalized" in user["profile"]:
slack_bot_name = user["profile"]["real_name_normalized"]
elif "first_name" in user["profile"]:
slack_bot_name = user["profile"]["first_name"]
else:
raise AssertionError("Could not identify bot type")
return slack_bot_name.replace("Bot", "").replace(" ", "") + f"-bot@{domain_name}"
@@ -385,20 +385,20 @@ def build_avatar_url(slack_user_id: str, team_id: str, avatar_hash: str) -> str:
def get_owner(user: ZerverFieldsT) -> bool:
owner = user.get('is_owner', False)
primary_owner = user.get('is_primary_owner', False)
owner = user.get("is_owner", False)
primary_owner = user.get("is_primary_owner", False)
return primary_owner or owner
def get_admin(user: ZerverFieldsT) -> bool:
admin = user.get('is_admin', False)
admin = user.get("is_admin", False)
return admin
def get_guest(user: ZerverFieldsT) -> bool:
restricted_user = user.get('is_restricted', False)
ultra_restricted_user = user.get('is_ultra_restricted', False)
restricted_user = user.get("is_restricted", False)
ultra_restricted_user = user.get("is_ultra_restricted", False)
# Slack's Single channel and multi channel guests both have
# is_restricted set to True. So assuming Slack doesn't change their
@@ -410,7 +410,7 @@ def get_guest(user: ZerverFieldsT) -> bool:
def get_user_timezone(user: ZerverFieldsT) -> str:
_default_timezone = "America/New_York"
timezone = user.get("tz", _default_timezone)
if timezone is None or '/' not in timezone:
if timezone is None or "/" not in timezone:
timezone = _default_timezone
return timezone
@@ -433,7 +433,7 @@ def channels_to_zerver_stream(
5. slack_recipient_name_to_zulip_recipient_id, which is a dictionary to map from Slack recipient
name(channel names, mpim names, usernames etc) to Zulip recipient_id
"""
logging.info('######### IMPORTING CHANNELS STARTED #########\n')
logging.info("######### IMPORTING CHANNELS STARTED #########\n")
added_channels = {}
added_mpims = {}
@@ -475,20 +475,20 @@ def channels_to_zerver_stream(
)
realm["zerver_stream"].append(stream)
slack_default_channels = ['general', 'random']
if channel['name'] in slack_default_channels and not stream['deactivated']:
slack_default_channels = ["general", "random"]
if channel["name"] in slack_default_channels and not stream["deactivated"]:
defaultstream = build_defaultstream(realm_id, stream_id, defaultstream_id)
realm["zerver_defaultstream"].append(defaultstream)
defaultstream_id += 1
added_channels[stream['name']] = (channel['id'], stream_id)
added_channels[stream["name"]] = (channel["id"], stream_id)
recipient = build_recipient(stream_id, recipient_id, Recipient.STREAM)
realm["zerver_recipient"].append(recipient)
slack_recipient_name_to_zulip_recipient_id[stream['name']] = recipient_id
slack_recipient_name_to_zulip_recipient_id[stream["name"]] = recipient_id
subscription_id_count = get_subscription(
channel['members'],
channel["members"],
realm["zerver_subscription"],
recipient_id,
slack_user_id_to_zulip_user_id,
@@ -497,7 +497,7 @@ def channels_to_zerver_stream(
stream_id_count += 1
recipient_id_count += 1
logging.info("%s -> created", channel['name'])
logging.info("%s -> created", channel["name"])
# TODO map Slack's pins to Zulip's stars
# There is the security model that Slack's pins are known to the team owner
@@ -512,11 +512,11 @@ def channels_to_zerver_stream(
# }
# ],
public_channels = get_data_file(slack_data_dir + '/channels.json')
public_channels = get_data_file(slack_data_dir + "/channels.json")
process_channels(public_channels)
try:
private_channels = get_data_file(slack_data_dir + '/groups.json')
private_channels = get_data_file(slack_data_dir + "/groups.json")
except FileNotFoundError:
private_channels = []
process_channels(private_channels, True)
@@ -531,14 +531,14 @@ def channels_to_zerver_stream(
huddle = build_huddle(huddle_id_count)
realm["zerver_huddle"].append(huddle)
added_mpims[mpim['name']] = (mpim['id'], huddle_id_count)
added_mpims[mpim["name"]] = (mpim["id"], huddle_id_count)
recipient = build_recipient(huddle_id_count, recipient_id_count, Recipient.HUDDLE)
realm["zerver_recipient"].append(recipient)
slack_recipient_name_to_zulip_recipient_id[mpim['name']] = recipient_id_count
slack_recipient_name_to_zulip_recipient_id[mpim["name"]] = recipient_id_count
subscription_id_count = get_subscription(
mpim['members'],
mpim["members"],
realm["zerver_subscription"],
recipient_id_count,
slack_user_id_to_zulip_user_id,
@@ -547,10 +547,10 @@ def channels_to_zerver_stream(
huddle_id_count += 1
recipient_id_count += 1
logging.info("%s -> created", mpim['name'])
logging.info("%s -> created", mpim["name"])
try:
mpims = get_data_file(slack_data_dir + '/mpims.json')
mpims = get_data_file(slack_data_dir + "/mpims.json")
except FileNotFoundError:
mpims = []
process_mpims(mpims)
@@ -571,12 +571,12 @@ def channels_to_zerver_stream(
dm_members[dm["id"]] = (user_a, user_b)
try:
dms = get_data_file(slack_data_dir + '/dms.json')
dms = get_data_file(slack_data_dir + "/dms.json")
except FileNotFoundError:
dms = []
process_dms(dms)
logging.info('######### IMPORTING STREAMS FINISHED #########\n')
logging.info("######### IMPORTING STREAMS FINISHED #########\n")
return (
realm,
added_channels,
@@ -623,7 +623,7 @@ def process_long_term_idle_users(
recent_senders: Set[str] = set()
NOW = float(timezone_now().timestamp())
for message in all_messages:
timestamp = float(message['ts'])
timestamp = float(message["ts"])
slack_user_id = get_message_sending_user(message)
if not slack_user_id:
continue
@@ -644,16 +644,16 @@ def process_long_term_idle_users(
for slack_user in users:
if slack_user["id"] in recent_senders:
continue
zulip_user_id = slack_user_id_to_zulip_user_id[slack_user['id']]
zulip_user_id = slack_user_id_to_zulip_user_id[slack_user["id"]]
long_term_idle.add(zulip_user_id)
for user_profile_row in zerver_userprofile:
if user_profile_row['id'] in long_term_idle:
user_profile_row['long_term_idle'] = True
if user_profile_row["id"] in long_term_idle:
user_profile_row["long_term_idle"] = True
# Setting last_active_message_id to 1 means the user, if
# imported, will get the full message history for the
# streams they were on.
user_profile_row['last_active_message_id'] = 1
user_profile_row["last_active_message_id"] = 1
return long_term_idle
@@ -692,7 +692,7 @@ def convert_slack_workspace_messages(
)
all_messages = get_messages_iterator(slack_data_dir, added_channels, added_mpims, dm_members)
logging.info('######### IMPORTING MESSAGES STARTED #########\n')
logging.info("######### IMPORTING MESSAGES STARTED #########\n")
total_reactions: List[ZerverFieldsT] = []
total_attachments: List[ZerverFieldsT] = []
@@ -701,7 +701,7 @@ def convert_slack_workspace_messages(
dump_file_id = 1
subscriber_map = make_subscriber_map(
zerver_subscription=realm['zerver_subscription'],
zerver_subscription=realm["zerver_subscription"],
)
while True:
@@ -747,7 +747,7 @@ def convert_slack_workspace_messages(
dump_file_id += 1
logging.info('######### IMPORTING MESSAGES FINISHED #########\n')
logging.info("######### IMPORTING MESSAGES FINISHED #########\n")
return total_reactions, total_uploads, total_attachments
@@ -778,7 +778,7 @@ def get_messages_iterator(
dir_name = os.path.basename(dir_path)
messages = []
for message in get_data_file(message_dir):
if message.get('user') == 'U00':
if message.get("user") == "U00":
# Skip messages involving the the "U00" user,
# which is apparently used in some channel rename
# messages. It's likely just the result of some
@@ -787,17 +787,17 @@ def get_messages_iterator(
# skipping those messages is simpler.
continue
if dir_name in added_channels:
message['channel_name'] = dir_name
message["channel_name"] = dir_name
elif dir_name in added_mpims:
message['mpim_name'] = dir_name
message["mpim_name"] = dir_name
elif dir_name in dm_members:
message['pm_name'] = dir_name
message["pm_name"] = dir_name
messages.append(message)
messages_for_one_day += messages
# we sort the messages according to the timestamp to show messages with
# the proper date order
yield from sorted(messages_for_one_day, key=lambda m: m['ts'])
yield from sorted(messages_for_one_day, key=lambda m: m["ts"])
def channel_message_to_zerver_message(
@@ -842,7 +842,7 @@ def channel_message_to_zerver_message(
# These are Sometimes produced by Slack
continue
subtype = message.get('subtype', False)
subtype = message.get("subtype", False)
if subtype in [
# Zulip doesn't have a pinned_item concept
"pinned_item",
@@ -856,7 +856,7 @@ def channel_message_to_zerver_message(
try:
content, mentioned_user_ids, has_link = convert_to_zulip_markdown(
message['text'], users, added_channels, slack_user_id_to_zulip_user_id
message["text"], users, added_channels, slack_user_id_to_zulip_user_id
)
except Exception:
print("Slack message unexpectedly missing text representation:")
@@ -866,14 +866,14 @@ def channel_message_to_zerver_message(
if "channel_name" in message:
is_private = False
recipient_id = slack_recipient_name_to_zulip_recipient_id[message['channel_name']]
recipient_id = slack_recipient_name_to_zulip_recipient_id[message["channel_name"]]
elif "mpim_name" in message:
is_private = True
recipient_id = slack_recipient_name_to_zulip_recipient_id[message['mpim_name']]
recipient_id = slack_recipient_name_to_zulip_recipient_id[message["mpim_name"]]
elif "pm_name" in message:
is_private = True
sender = get_message_sending_user(message)
members = dm_members[message['pm_name']]
members = dm_members[message["pm_name"]]
if sender == members[0]:
recipient_id = slack_recipient_name_to_zulip_recipient_id[members[1]]
sender_recipient_id = slack_recipient_name_to_zulip_recipient_id[members[0]]
@@ -881,12 +881,12 @@ def channel_message_to_zerver_message(
recipient_id = slack_recipient_name_to_zulip_recipient_id[members[0]]
sender_recipient_id = slack_recipient_name_to_zulip_recipient_id[members[1]]
message_id = NEXT_ID('message')
message_id = NEXT_ID("message")
if 'reactions' in message.keys():
if "reactions" in message.keys():
build_reactions(
reaction_list,
message['reactions'],
message["reactions"],
slack_user_id_to_zulip_user_id,
message_id,
zerver_realmemoji,
@@ -899,11 +899,11 @@ def channel_message_to_zerver_message(
# For example "sh_room_created" has the message 'started a call'
# which should be displayed as '/me started a call'
if subtype in ["bot_add", "sh_room_created", "me_message"]:
content = f'/me {content}'
if subtype == 'file_comment':
content = f"/me {content}"
if subtype == "file_comment":
# The file_comment message type only indicates the
# responsible user in a subfield.
message['user'] = message['comment']['user']
message["user"] = message["comment"]["user"]
file_info = process_message_files(
message=message,
@@ -917,17 +917,17 @@ def channel_message_to_zerver_message(
uploads_list=uploads_list,
)
content += file_info['content']
has_link = has_link or file_info['has_link']
content += file_info["content"]
has_link = has_link or file_info["has_link"]
has_attachment = file_info['has_attachment']
has_image = file_info['has_image']
has_attachment = file_info["has_attachment"]
has_image = file_info["has_image"]
topic_name = 'imported from Slack'
topic_name = "imported from Slack"
zulip_message = build_message(
topic_name,
float(message['ts']),
float(message["ts"]),
message_id,
content,
rendered_content,
@@ -987,35 +987,35 @@ def process_message_files(
has_image = False
has_link = False
files = message.get('files', [])
files = message.get("files", [])
subtype = message.get('subtype')
subtype = message.get("subtype")
if subtype == 'file_share':
if subtype == "file_share":
# In Slack messages, uploads can either have the subtype as 'file_share' or
# have the upload information in 'files' keyword
files = [message['file']]
files = [message["file"]]
markdown_links = []
for fileinfo in files:
if fileinfo.get('mode', '') in ['tombstone', 'hidden_by_limit']:
if fileinfo.get("mode", "") in ["tombstone", "hidden_by_limit"]:
# Slack sometimes includes tombstone mode files with no
# real data on the actual file (presumably in cases where
# the file was deleted). hidden_by_limit mode is for files
# that are hidden because of 10k cap in free plan.
continue
url = fileinfo['url_private']
url = fileinfo["url_private"]
if 'files.slack.com' in url:
if "files.slack.com" in url:
# For attachments with Slack download link
has_attachment = True
has_link = True
has_image = True if 'image' in fileinfo['mimetype'] else False
has_image = True if "image" in fileinfo["mimetype"] else False
file_user = [
iterate_user for iterate_user in users if message['user'] == iterate_user['id']
iterate_user for iterate_user in users if message["user"] == iterate_user["id"]
]
file_user_email = get_user_email(file_user[0], domain_name)
@@ -1043,13 +1043,13 @@ def process_message_files(
# For attachments with link not from Slack
# Example: Google drive integration
has_link = True
if 'title' in fileinfo:
file_name = fileinfo['title']
if "title" in fileinfo:
file_name = fileinfo["title"]
else:
file_name = fileinfo['name']
markdown_links.append('[{}]({})'.format(file_name, fileinfo['url_private']))
file_name = fileinfo["name"]
markdown_links.append("[{}]({})".format(file_name, fileinfo["url_private"]))
content = '\n'.join(markdown_links)
content = "\n".join(markdown_links)
return dict(
content=content,
@@ -1065,15 +1065,15 @@ def get_attachment_path_and_content(fileinfo: ZerverFieldsT, realm_id: int) -> T
s3_path = "/".join(
[
str(realm_id),
'SlackImportAttachment', # This is a special placeholder which should be kept
"SlackImportAttachment", # This is a special placeholder which should be kept
# in sync with 'exports.py' function 'import_message_data'
format(random.randint(0, 255), 'x'),
format(random.randint(0, 255), "x"),
secrets.token_urlsafe(18),
sanitize_name(fileinfo['name']),
sanitize_name(fileinfo["name"]),
]
)
attachment_path = f'/user_uploads/{s3_path}'
content = '[{}]({})'.format(fileinfo['title'], attachment_path)
attachment_path = f"/user_uploads/{s3_path}"
content = "[{}]({})".format(fileinfo["title"], attachment_path)
return s3_path, content
@@ -1087,12 +1087,12 @@ def build_reactions(
) -> None:
realmemoji = {}
for realm_emoji in zerver_realmemoji:
realmemoji[realm_emoji['name']] = realm_emoji['id']
realmemoji[realm_emoji["name"]] = realm_emoji["id"]
# For the Unicode emoji codes, we use equivalent of
# function 'emoji_name_to_emoji_code' in 'zerver/lib/emoji' here
for slack_reaction in reactions:
emoji_name = slack_reaction['name']
emoji_name = slack_reaction["name"]
if emoji_name in name_to_codepoint:
emoji_code = name_to_codepoint[emoji_name]
reaction_type = Reaction.UNICODE_EMOJI
@@ -1102,8 +1102,8 @@ def build_reactions(
else:
continue
for slack_user_id in slack_reaction['users']:
reaction_id = NEXT_ID('reaction')
for slack_user_id in slack_reaction["users"]:
reaction_id = NEXT_ID("reaction")
reaction = Reaction(
id=reaction_id,
emoji_code=emoji_code,
@@ -1111,9 +1111,9 @@ def build_reactions(
reaction_type=reaction_type,
)
reaction_dict = model_to_dict(reaction, exclude=['message', 'user_profile'])
reaction_dict['message'] = message_id
reaction_dict['user_profile'] = slack_user_id_to_zulip_user_id[slack_user_id]
reaction_dict = model_to_dict(reaction, exclude=["message", "user_profile"])
reaction_dict["message"] = message_id
reaction_dict["user_profile"] = slack_user_id_to_zulip_user_id[slack_user_id]
reaction_list.append(reaction_dict)
@@ -1127,23 +1127,23 @@ def build_uploads(
uploads_list: List[ZerverFieldsT],
) -> None:
upload = dict(
path=fileinfo['url_private'], # Save Slack's URL here, which is used later while processing
path=fileinfo["url_private"], # Save Slack's URL here, which is used later while processing
realm_id=realm_id,
content_type=None,
user_profile_id=user_id,
last_modified=fileinfo['timestamp'],
last_modified=fileinfo["timestamp"],
user_profile_email=email,
s3_path=s3_path,
size=fileinfo['size'],
size=fileinfo["size"],
)
uploads_list.append(upload)
def get_message_sending_user(message: ZerverFieldsT) -> Optional[str]:
if 'user' in message:
return message['user']
if message.get('file'):
return message['file'].get('user')
if "user" in message:
return message["user"]
if message.get("file"):
return message["file"].get("user")
return None
@@ -1158,9 +1158,9 @@ def fetch_shared_channel_users(
user["is_mirror_dummy"] = False
normal_user_ids.add(user["id"])
public_channels = get_data_file(slack_data_dir + '/channels.json')
public_channels = get_data_file(slack_data_dir + "/channels.json")
try:
private_channels = get_data_file(slack_data_dir + '/groups.json')
private_channels = get_data_file(slack_data_dir + "/groups.json")
except FileNotFoundError:
private_channels = []
for channel in public_channels + private_channels:
@@ -1218,7 +1218,7 @@ def fetch_team_icons(
os.makedirs(os.path.join(output_dir, str(realm_id)), exist_ok=True)
original_icon_output_path = os.path.join(output_dir, str(realm_id), "icon.original")
with open(original_icon_output_path, 'wb') as output_file:
with open(original_icon_output_path, "wb") as output_file:
shutil.copyfileobj(response_raw, output_file)
records.append(
{
@@ -1229,8 +1229,8 @@ def fetch_team_icons(
)
resized_icon_output_path = os.path.join(output_dir, str(realm_id), "icon.png")
with open(resized_icon_output_path, 'wb') as output_file:
with open(original_icon_output_path, 'rb') as original_file:
with open(resized_icon_output_path, "wb") as output_file:
with open(original_icon_output_path, "rb") as original_file:
resized_data = resize_logo(original_file.read())
output_file.write(resized_data)
records.append(
@@ -1254,15 +1254,15 @@ def do_convert_data(slack_zip_file: str, output_dir: str, token: str, threads: i
check_token_access(token)
slack_data_dir = slack_zip_file.replace('.zip', '')
slack_data_dir = slack_zip_file.replace(".zip", "")
if not os.path.exists(slack_data_dir):
os.makedirs(slack_data_dir)
os.makedirs(output_dir, exist_ok=True)
if os.listdir(output_dir):
raise Exception('Output directory should be empty!')
raise Exception("Output directory should be empty!")
subprocess.check_call(['unzip', '-q', slack_zip_file, '-d', slack_data_dir])
subprocess.check_call(["unzip", "-q", slack_zip_file, "-d", slack_data_dir])
# We get the user data from the legacy token method of Slack API, which is depreciated
# but we use it as the user email data is provided only in this method
@@ -1294,48 +1294,48 @@ def do_convert_data(slack_zip_file: str, output_dir: str, token: str, threads: i
added_mpims,
dm_members,
realm,
realm['zerver_userprofile'],
realm['zerver_realmemoji'],
realm["zerver_userprofile"],
realm["zerver_realmemoji"],
domain_name,
output_dir,
)
# Move zerver_reactions to realm.json file
realm['zerver_reaction'] = reactions
realm["zerver_reaction"] = reactions
emoji_folder = os.path.join(output_dir, 'emoji')
emoji_folder = os.path.join(output_dir, "emoji")
os.makedirs(emoji_folder, exist_ok=True)
emoji_records = process_emojis(realm['zerver_realmemoji'], emoji_folder, emoji_url_map, threads)
emoji_records = process_emojis(realm["zerver_realmemoji"], emoji_folder, emoji_url_map, threads)
avatar_folder = os.path.join(output_dir, 'avatars')
avatar_folder = os.path.join(output_dir, "avatars")
avatar_realm_folder = os.path.join(avatar_folder, str(realm_id))
os.makedirs(avatar_realm_folder, exist_ok=True)
avatar_records = process_avatars(
avatar_list, avatar_folder, realm_id, threads, size_url_suffix='-512'
avatar_list, avatar_folder, realm_id, threads, size_url_suffix="-512"
)
uploads_folder = os.path.join(output_dir, 'uploads')
uploads_folder = os.path.join(output_dir, "uploads")
os.makedirs(os.path.join(uploads_folder, str(realm_id)), exist_ok=True)
uploads_records = process_uploads(uploads_list, uploads_folder, threads)
attachment = {"zerver_attachment": zerver_attachment}
team_info_dict = get_slack_api_data("https://slack.com/api/team.info", "team", token=token)
realm_icons_folder = os.path.join(output_dir, 'realm_icons')
realm_icons_folder = os.path.join(output_dir, "realm_icons")
realm_icon_records = fetch_team_icons(
realm["zerver_realm"][0], team_info_dict, realm_icons_folder
)
create_converted_data_files(realm, output_dir, '/realm.json')
create_converted_data_files(emoji_records, output_dir, '/emoji/records.json')
create_converted_data_files(avatar_records, output_dir, '/avatars/records.json')
create_converted_data_files(uploads_records, output_dir, '/uploads/records.json')
create_converted_data_files(attachment, output_dir, '/attachment.json')
create_converted_data_files(realm_icon_records, output_dir, '/realm_icons/records.json')
create_converted_data_files(realm, output_dir, "/realm.json")
create_converted_data_files(emoji_records, output_dir, "/emoji/records.json")
create_converted_data_files(avatar_records, output_dir, "/avatars/records.json")
create_converted_data_files(uploads_records, output_dir, "/uploads/records.json")
create_converted_data_files(attachment, output_dir, "/attachment.json")
create_converted_data_files(realm_icon_records, output_dir, "/realm_icons/records.json")
rm_tree(slack_data_dir)
subprocess.check_call(["tar", "-czf", output_dir + '.tar.gz', output_dir, '-P'])
subprocess.check_call(["tar", "-czf", output_dir + ".tar.gz", output_dir, "-P"])
logging.info('######### DATA CONVERSION FINISHED #########\n')
logging.info("######### DATA CONVERSION FINISHED #########\n")
logging.info("Zulip data dump created at %s", output_dir)
@@ -1347,11 +1347,11 @@ def get_data_file(path: str) -> Any:
def check_token_access(token: str) -> None:
if token.startswith("xoxp-"):
logging.info('This is a Slack user token, which grants all rights the user has!')
logging.info("This is a Slack user token, which grants all rights the user has!")
elif token.startswith("xoxb-"):
data = requests.get("https://slack.com/api/team.info", {"token": token})
has_scopes = set(data.headers.get("x-oauth-scopes", "").split(","))
required_scopes = set(['emoji:read', 'users:read', 'users:read.email', 'team:read'])
required_scopes = set(["emoji:read", "users:read", "users:read.email", "team:read"])
missing_scopes = required_scopes - has_scopes
if missing_scopes:
raise ValueError(
@@ -1370,8 +1370,8 @@ def get_slack_api_data(slack_api_url: str, get_param: str, **kwargs: Any) -> Any
if data.status_code == requests.codes.ok:
result = data.json()
if not result['ok']:
raise Exception('Error accessing Slack API: {}'.format(result['error']))
if not result["ok"]:
raise Exception("Error accessing Slack API: {}".format(result["error"]))
return result[get_param]
raise Exception('HTTP error accessing the Slack API.')
raise Exception("HTTP error accessing the Slack API.")