From b8e1e8b31d2973e909b8876038a508ebebcca40c Mon Sep 17 00:00:00 2001 From: Rhea Parekh Date: Wed, 1 Aug 2018 03:48:04 +0530 Subject: [PATCH] import: Add slack import files in zerver/data_import directory. --- mypy.ini | 2 -- tools/test-backend | 3 +- zerver/data_import/__init__.py | 0 .../slack.py} | 2 +- .../slack_message_conversion.py | 8 +++--- .../management/commands/convert_slack_data.py | 2 +- zerver/tests/test_slack_importer.py | 28 +++++++++---------- zerver/tests/test_slack_message_conversion.py | 2 +- 8 files changed, 23 insertions(+), 24 deletions(-) create mode 100644 zerver/data_import/__init__.py rename zerver/{lib/slack_data_to_zulip_data.py => data_import/slack.py} (99%) rename zerver/{lib => data_import}/slack_message_conversion.py (95%) diff --git a/mypy.ini b/mypy.ini index 1d156dbc1d..b86a896fab 100644 --- a/mypy.ini +++ b/mypy.ini @@ -397,8 +397,6 @@ strict_optional = False strict_optional = False [mypy-zerver.tornado.websocket_client] strict_optional = False -[mypy-zerver.lib.slack_message_conversion] -strict_optional = False [mypy-zerver.views.registration] strict_optional = False diff --git a/tools/test-backend b/tools/test-backend index 7edbeaed35..96981c9f7c 100755 --- a/tools/test-backend +++ b/tools/test-backend @@ -81,7 +81,6 @@ not_yet_fully_covered = { 'zerver/lib/profile.py', 'zerver/lib/queue.py', 'zerver/lib/rate_limiter.py', - 'zerver/lib/slack_data_to_zulip_data.py', 'zerver/lib/sqlalchemy_utils.py', 'zerver/lib/storage.py', 'zerver/lib/str_utils.py', @@ -97,6 +96,8 @@ not_yet_fully_covered = { 'zerver/lib/api_test_helpers.py', # Experimenatal 'zerver/lib/widget.py', + # Data import files + 'zerver/data_import/slack.py', # Webhook integrations with incomplete coverage 'zerver/webhooks/beanstalk/view.py', 'zerver/webhooks/bitbucket2/view.py', diff --git a/zerver/data_import/__init__.py b/zerver/data_import/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/zerver/lib/slack_data_to_zulip_data.py b/zerver/data_import/slack.py similarity index 99% rename from zerver/lib/slack_data_to_zulip_data.py rename to zerver/data_import/slack.py index 164b3b68e1..bdb012b1ac 100755 --- a/zerver/lib/slack_data_to_zulip_data.py +++ b/zerver/data_import/slack.py @@ -19,7 +19,7 @@ from django.forms.models import model_to_dict from typing import Any, Dict, List, Optional, Tuple from zerver.forms import check_subdomain_available from zerver.models import Reaction, RealmEmoji, Realm, UserProfile -from zerver.lib.slack_message_conversion import convert_to_zulip_markdown, \ +from zerver.data_import.slack_message_conversion import convert_to_zulip_markdown, \ get_user_full_name from zerver.lib.parallel import run_parallel from zerver.lib.avatar_hash import user_avatar_path_from_ids diff --git a/zerver/lib/slack_message_conversion.py b/zerver/data_import/slack_message_conversion.py similarity index 95% rename from zerver/lib/slack_message_conversion.py rename to zerver/data_import/slack_message_conversion.py index 9509e93e76..15e2e82235 100644 --- a/zerver/lib/slack_message_conversion.py +++ b/zerver/data_import/slack_message_conversion.py @@ -1,5 +1,5 @@ import re -from typing import Any, Dict, Tuple, List +from typing import Any, Dict, Tuple, List, Optional # stubs ZerverFieldsT = Dict[str, Any] @@ -113,10 +113,10 @@ def convert_to_zulip_markdown(text: str, users: List[ZerverFieldsT], return text, mentioned_users_id, message_has_link def get_user_mentions(token: str, users: List[ZerverFieldsT], - added_users: AddedUsersT) -> Tuple[str, int]: + added_users: AddedUsersT) -> Tuple[str, Optional[int]]: slack_usermention_match = re.search(SLACK_USERMENTION_REGEX, token, re.VERBOSE) - short_name = slack_usermention_match.group(4) - slack_id = slack_usermention_match.group(2) + short_name = slack_usermention_match.group(4) # type: ignore # slack_usermention_match exists and is not None + slack_id = slack_usermention_match.group(2) # type: ignore # slack_usermention_match exists and is not None for user in users: if (user['id'] == slack_id and user['name'] == short_name and short_name) or \ (user['id'] == slack_id and short_name is None): diff --git a/zerver/management/commands/convert_slack_data.py b/zerver/management/commands/convert_slack_data.py index 49b7e75308..07ebb0f5d1 100644 --- a/zerver/management/commands/convert_slack_data.py +++ b/zerver/management/commands/convert_slack_data.py @@ -8,7 +8,7 @@ from typing import Any from django.core.management.base import BaseCommand, CommandParser, CommandError -from zerver.lib.slack_data_to_zulip_data import do_convert_data +from zerver.data_import.slack import do_convert_data class Command(BaseCommand): help = """Convert the Slack data into Zulip data format.""" diff --git a/zerver/tests/test_slack_importer.py b/zerver/tests/test_slack_importer.py index a1d0ee1ec6..e800de5b52 100644 --- a/zerver/tests/test_slack_importer.py +++ b/zerver/tests/test_slack_importer.py @@ -2,7 +2,7 @@ from django.conf import settings from django.utils.timezone import now as timezone_now -from zerver.lib.slack_data_to_zulip_data import ( +from zerver.data_import.slack import ( rm_tree, get_slack_api_data, build_zerver_realm, @@ -123,7 +123,7 @@ class SlackImporter(ZulipTestCase): self.assertEqual(get_user_timezone(user_timezone_none), "America/New_York") self.assertEqual(get_user_timezone(user_no_timezone), "America/New_York") - @mock.patch("zerver.lib.slack_data_to_zulip_data.get_data_file") + @mock.patch("zerver.data_import.slack.get_data_file") def test_users_to_zerver_userprofile(self, mock_get_data_file: mock.Mock) -> None: custom_profile_field_user1 = {"Xf06054BBB": {"value": "random1"}, "Xf023DSCdd": {"value": "employee"}} @@ -251,7 +251,7 @@ class SlackImporter(ZulipTestCase): zerver_subscription[3]['recipient']) self.assertEqual(zerver_subscription[1]['pin_to_top'], False) - @mock.patch("zerver.lib.slack_data_to_zulip_data.get_data_file") + @mock.patch("zerver.data_import.slack.get_data_file") def test_channels_to_zerver_stream(self, mock_get_data_file: mock.Mock) -> None: added_users = {"U061A1R2R": 1, "U061A3E0G": 8, "U061A5N1G": 7, "U064KUGRJ": 5} @@ -321,10 +321,10 @@ class SlackImporter(ZulipTestCase): self.assertEqual(zerver_stream[2]['id'], test_added_channels[zerver_stream[2]['name']][1]) - @mock.patch("zerver.lib.slack_data_to_zulip_data.build_zerver_realm", return_value=[{}]) - @mock.patch("zerver.lib.slack_data_to_zulip_data.users_to_zerver_userprofile", + @mock.patch("zerver.data_import.slack.build_zerver_realm", return_value=[{}]) + @mock.patch("zerver.data_import.slack.users_to_zerver_userprofile", return_value=[[], [], {}, [], []]) - @mock.patch("zerver.lib.slack_data_to_zulip_data.channels_to_zerver_stream", + @mock.patch("zerver.data_import.slack.channels_to_zerver_stream", return_value=[[], [], {}, [], [], {}]) def test_slack_workspace_to_realm(self, mock_channels_to_zerver_stream: mock.Mock, mock_users_to_zerver_userprofile: mock.Mock, @@ -385,7 +385,7 @@ class SlackImporter(ZulipTestCase): self.assertEqual(zerver_usermessage[3]['id'], 3) self.assertEqual(zerver_usermessage[3]['message'], message_id) - @mock.patch("zerver.lib.slack_data_to_zulip_data.build_zerver_usermessage", return_value = 2) + @mock.patch("zerver.data_import.slack.build_zerver_usermessage", return_value = 2) def test_channel_message_to_zerver_message(self, mock_build_zerver_usermessage: mock.Mock) -> None: user_data = [{"id": "U066MTL5U", "name": "john doe", "deleted": False, "real_name": "John"}, @@ -462,8 +462,8 @@ class SlackImporter(ZulipTestCase): self.assertEqual(zerver_message[0]['sender'], 43) self.assertEqual(zerver_message[3]['sender'], 24) - @mock.patch("zerver.lib.slack_data_to_zulip_data.channel_message_to_zerver_message") - @mock.patch("zerver.lib.slack_data_to_zulip_data.get_all_messages") + @mock.patch("zerver.data_import.slack.channel_message_to_zerver_message") + @mock.patch("zerver.data_import.slack.get_all_messages") def test_convert_slack_workspace_messages(self, mock_get_all_messages: mock.Mock, mock_message: mock.Mock) -> None: os.makedirs('var/test-slack-import', exist_ok=True) @@ -504,12 +504,12 @@ class SlackImporter(ZulipTestCase): self.assertEqual(test_reactions, reactions) - @mock.patch("zerver.lib.slack_data_to_zulip_data.process_uploads", return_value = []) - @mock.patch("zerver.lib.slack_data_to_zulip_data.build_zerver_attachment", + @mock.patch("zerver.data_import.slack.process_uploads", return_value = []) + @mock.patch("zerver.data_import.slack.build_zerver_attachment", return_value = []) - @mock.patch("zerver.lib.slack_data_to_zulip_data.build_avatar_url") - @mock.patch("zerver.lib.slack_data_to_zulip_data.build_avatar") - @mock.patch("zerver.lib.slack_data_to_zulip_data.get_slack_api_data") + @mock.patch("zerver.data_import.slack.build_avatar_url") + @mock.patch("zerver.data_import.slack.build_avatar") + @mock.patch("zerver.data_import.slack.get_slack_api_data") def test_slack_import_to_existing_database(self, mock_get_slack_api_data: mock.Mock, mock_build_avatar_url: mock.Mock, mock_build_avatar: mock.Mock, diff --git a/zerver/tests/test_slack_message_conversion.py b/zerver/tests/test_slack_message_conversion.py index 2d492d5d72..9fadce4b9d 100644 --- a/zerver/tests/test_slack_message_conversion.py +++ b/zerver/tests/test_slack_message_conversion.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from django.conf import settings -from zerver.lib.slack_message_conversion import ( +from zerver.data_import.slack_message_conversion import ( convert_to_zulip_markdown, get_user_full_name )