paulmataruso/zulip
1
0
Fork 0
mirror of https://github.com/zulip/zulip.git synced 2025-11-02 21:13:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

slack import: Generate message_id/reaction_id with NEXT_ID.

Browse Source 
This avoids the need to pass tuples of ints around, which
is pretty brittle.
This commit is contained in:
Steve Howell
2018-10-28 17:55:32 +00:00
committed by Tim Abbott
parent 2f58eb1057
commit 30c493ed24
2 changed files with 21 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
zerver/data_import/slack.py
View File

@@ -25,6 +25,7 @@ from zerver.data_import.import_util import ZerverFieldsT, build_zerver_realm, \
build_defaultstream, build_attachment, process_avatars, process_uploads, \
process_emojis, build_realm, build_stream, build_message, \
create_converted_data_files, make_subscriber_map
from zerver.data_import.sequencer import NEXT_ID
from zerver.lib.parallel import run_parallel
from zerver.lib.upload import random_name, sanitize_name
from zerver.lib.export import MESSAGE_BATCH_CHUNK_SIZE
@@ -459,9 +460,6 @@ def convert_slack_workspace_messages(slack_data_dir: str, users: List[ZerverFiel
total_attachments = [] # type: List[ZerverFieldsT]
total_uploads = [] # type: List[ZerverFieldsT]
message_id = reaction_id = 0
id_list = (message_id, reaction_id)
# The messages are stored in batches
low_index = 0
upper_index = low_index + chunk_size
@@ -475,11 +473,11 @@ def convert_slack_workspace_messages(slack_data_dir: str, users: List[ZerverFiel
message_data = all_messages[low_index:upper_index]
if len(message_data) == 0:
break
zerver_message, zerver_usermessage, attachment, uploads, \
reactions, id_list = channel_message_to_zerver_message(
zerver_message, zerver_usermessage, attachment, uploads, reactions = \
channel_message_to_zerver_message(
realm_id, users, added_users, added_recipient, message_data,
zerver_realmemoji, subscriber_map, added_channels,
id_list, domain_name)
domain_name)
message_json = dict(
zerver_message=zerver_message,
@@ -522,13 +520,11 @@ def channel_message_to_zerver_message(realm_id: int,
zerver_realmemoji: List[ZerverFieldsT],
subscriber_map: Dict[int, Set[int]],
added_channels: AddedChannelsT,
id_list: Tuple[int, int],
domain_name: str) -> Tuple[List[ZerverFieldsT],
List[ZerverFieldsT],
List[ZerverFieldsT],
List[ZerverFieldsT],
List[ZerverFieldsT],
Tuple[int, int]]:
List[ZerverFieldsT]]:
"""
Returns:
1. zerver_message, which is a list of the messages
@@ -536,9 +532,7 @@ def channel_message_to_zerver_message(realm_id: int,
3. zerver_attachment, which is a list of the attachments
4. uploads_list, which is a list of uploads to be mapped in uploads records.json
5. reaction_list, which is a list of all user reactions
6. id_list, which is a tuple of max ids of messages, reactions
"""
message_id_count, reaction_id_count = id_list
zerver_message = []
zerver_usermessage = [] # type: List[ZerverFieldsT]
uploads_list = [] # type: List[ZerverFieldsT]
@@ -578,13 +572,13 @@ def channel_message_to_zerver_message(realm_id: int,
rendered_content = None
recipient_id = added_recipient[message['channel_name']]
message_id = message_id_count
message_id = NEXT_ID('message')
# Process message reactions
if 'reactions' in message.keys():
reaction_id_count = build_reactions(reaction_list, message['reactions'], added_users,
message_id, reaction_id_count, name_to_codepoint,
zerver_realmemoji)
build_reactions(reaction_list, message['reactions'], added_users,
message_id, name_to_codepoint,
zerver_realmemoji)
# Process different subtypes of slack messages
@@ -630,11 +624,8 @@ def channel_message_to_zerver_message(realm_id: int,
message_id=message_id,
)
message_id_count += 1
id_list = (message_id_count, reaction_id_count)
return zerver_message, zerver_usermessage, zerver_attachment, uploads_list, \
reaction_list, id_list
reaction_list
def process_message_files(message: ZerverFieldsT,
domain_name: str,
@@ -718,9 +709,9 @@ def get_attachment_path_and_content(fileinfo: ZerverFieldsT, realm_id: int) -> T
return s3_path, content
def build_reactions(reaction_list: List[ZerverFieldsT], reactions: List[ZerverFieldsT],
added_users: AddedUsersT, message_id: int, reaction_id: int,
added_users: AddedUsersT, message_id: int,
name_to_codepoint: ZerverFieldsT,
zerver_realmemoji: List[ZerverFieldsT]) -> int:
zerver_realmemoji: List[ZerverFieldsT]) -> None:
realmemoji = {}
for realm_emoji in zerver_realmemoji:
realmemoji[realm_emoji['name']] = realm_emoji['id']
@@ -741,6 +732,7 @@ def build_reactions(reaction_list: List[ZerverFieldsT], reactions: List[ZerverFi
continue
for user in slack_reaction['users']:
reaction_id = NEXT_ID('reaction')
reaction = Reaction(
id=reaction_id,
emoji_code=emoji_code,
@@ -752,9 +744,7 @@ def build_reactions(reaction_list: List[ZerverFieldsT], reactions: List[ZerverFi
reaction_dict['message'] = message_id
reaction_dict['user_profile'] = added_users[user]
reaction_id += 1
reaction_list.append(reaction_dict)
return reaction_id
def build_uploads(user_id: int, realm_id: int, email: str, fileinfo: ZerverFieldsT, s3_path: str,
uploads_list: List[ZerverFieldsT]) -> None:

18
zerver/tests/test_slack_importer.py
View File

@@ -441,11 +441,12 @@ class SlackImporter(ZulipTestCase):
zerver_usermessage = [] # type: List[Dict[str, Any]]
subscriber_map = dict() # type: Dict[int, Set[int]]
added_channels = {'random': ('c5', 1), 'general': ('c6', 2)} # type: Dict[str, Tuple[str, int]]
zerver_message, zerver_usermessage, attachment, uploads, \
reaction, id_list = channel_message_to_zerver_message(
zerver_message, zerver_usermessage, attachment, uploads, reaction = \
channel_message_to_zerver_message(
1, user_data, added_users, added_recipient,
all_messages, [], subscriber_map,
added_channels, (0, 0), 'domain')
added_channels, 'domain')
# functioning already tested in helper function
self.assertEqual(zerver_usermessage, [])
# subtype: channel_join is filtered
@@ -453,11 +454,9 @@ class SlackImporter(ZulipTestCase):
self.assertEqual(uploads, [])
self.assertEqual(attachment, [])
self.assertEqual(id_list, (5, 1))
# Test reactions
self.assertEqual(reaction[0]['user_profile'], 24)
self.assertEqual(reaction[0]['message'], 1)
self.assertEqual(reaction[0]['emoji_name'], reactions[0]['name'])
# Message conversion already tested in tests.test_slack_message_conversion
@@ -472,8 +471,8 @@ class SlackImporter(ZulipTestCase):
self.assertEqual(zerver_message[2]['subject'], 'imported from slack')
self.assertEqual(zerver_message[1]['recipient'], added_recipient['random'])
self.assertEqual(zerver_message[1]['id'], 1)
self.assertEqual(zerver_message[4]['id'], 4)
self.assertEqual(zerver_message[3]['id'], zerver_message[0]['id'] + 3)
self.assertEqual(zerver_message[4]['id'], zerver_message[0]['id'] + 4)
self.assertIsNone(zerver_message[3]['rendered_content'])
self.assertEqual(zerver_message[0]['has_image'], False)
@@ -496,15 +495,14 @@ class SlackImporter(ZulipTestCase):
user_list = [] # type: List[Dict[str, Any]]
reactions = [{"name": "grinning", "users": ["U061A5N1G"], "count": 1}]
attachments = uploads = [] # type: List[Dict[str, Any]]
id_list = (2, 0)
zerver_usermessage = [{'id': 3}, {'id': 5}, {'id': 6}, {'id': 9}]
mock_get_all_messages.side_effect = [zerver_message]
mock_message.side_effect = [[zerver_message[:1], zerver_usermessage[:2],
attachments, uploads, reactions[:1], id_list],
attachments, uploads, reactions[:1]],
[zerver_message[1:2], zerver_usermessage[2:5],
attachments, uploads, reactions[1:1], id_list]]
attachments, uploads, reactions[1:1]]]
test_reactions, uploads, zerver_attachment = convert_slack_workspace_messages(
'./random_path', user_list, 2, {}, {}, added_channels,
realm, [], 'domain', 'var/test-slack-import', chunk_size=1)