From 4db7ea22960dbac60bf65a660c1fd2bc97b4ec00 Mon Sep 17 00:00:00 2001 From: PieterCK Date: Wed, 15 Jan 2025 13:02:43 +0700 Subject: [PATCH] migration_status: Add `parse_migration_status`. This commit adds `parse_migration_status`, which takes in the string output of `showmigrations` and parse it into key-value pair of installed apps and a list of its migration status. This is a prep commit to rework the check migrations function of import/export which will parse the output of `showmigrations` to write the `migration_status.json` file. --- zerver/lib/export.py | 8 +- zerver/lib/import_realm.py | 2 +- zerver/lib/migration_status.py | 46 ++++++++++- .../with_stale_migrations.txt | 76 +++++++++++++++++++ zerver/tests/test_import_export.py | 33 ++++++-- zerver/tests/test_migration_status.py | 61 +++++++++++++++ 6 files changed, 209 insertions(+), 17 deletions(-) create mode 100644 zerver/tests/fixtures/import_fixtures/showmigrations_fixtures/with_stale_migrations.txt create mode 100644 zerver/tests/test_migration_status.py diff --git a/zerver/lib/export.py b/zerver/lib/export.py index 69b0c86806..baa30ec3c2 100644 --- a/zerver/lib/export.py +++ b/zerver/lib/export.py @@ -33,6 +33,7 @@ from analytics.models import RealmCount, StreamCount, UserCount from scripts.lib.zulip_tools import overwrite_symlink from version import ZULIP_VERSION from zerver.lib.avatar_hash import user_avatar_base_path_from_ids +from zerver.lib.migration_status import AppMigrations, MigrationStatusJson from zerver.lib.pysa import mark_sanitized from zerver.lib.timestamp import datetime_to_timestamp from zerver.lib.upload.s3 import get_bucket @@ -98,8 +99,6 @@ SourceFilter: TypeAlias = Callable[[Record], bool] CustomFetch: TypeAlias = Callable[[TableData, Context], None] -AppMigrations: TypeAlias = dict[str, list[str]] - class MessagePartial(TypedDict): zerver_message: list[Record] @@ -107,11 +106,6 @@ class MessagePartial(TypedDict): realm_id: int -class MigrationStatusJson(TypedDict): - migrations_by_app: AppMigrations - zulip_version: str - - MESSAGE_BATCH_CHUNK_SIZE = 1000 ALL_ZULIP_TABLES = { diff --git a/zerver/lib/import_realm.py b/zerver/lib/import_realm.py index fb032e0a11..cc024bf523 100644 --- a/zerver/lib/import_realm.py +++ b/zerver/lib/import_realm.py @@ -31,7 +31,6 @@ from zerver.lib.bulk_create import bulk_set_users_or_streams_recipient_fields from zerver.lib.export import ( DATE_FIELDS, Field, - MigrationStatusJson, Path, Record, TableData, @@ -41,6 +40,7 @@ from zerver.lib.export import ( from zerver.lib.markdown import markdown_convert from zerver.lib.markdown import version as markdown_version from zerver.lib.message import get_last_message_id +from zerver.lib.migration_status import MigrationStatusJson from zerver.lib.mime_types import guess_type from zerver.lib.partial import partial from zerver.lib.push_notifications import sends_notifications_directly diff --git a/zerver/lib/migration_status.py b/zerver/lib/migration_status.py index 736db8bbee..56e6105fd0 100644 --- a/zerver/lib/migration_status.py +++ b/zerver/lib/migration_status.py @@ -2,7 +2,15 @@ import os import re from importlib import import_module from io import StringIO -from typing import Any +from typing import Any, TypeAlias, TypedDict + +AppMigrations: TypeAlias = dict[str, list[str]] + + +class MigrationStatusJson(TypedDict): + migrations_by_app: AppMigrations + zulip_version: str + STALE_MIGRATIONS = [ # Ignore django-guardian, which we installed until 1.7.0~3134 @@ -82,3 +90,39 @@ def get_migration_status(**options: Any) -> str: out.seek(0) output = out.read() return re.sub(r"\x1b\[[0-9;]*m", "", output) + + +def parse_migration_status( + migration_status_print: str, stale_migrations: list[tuple[str, str]] = STALE_MIGRATIONS +) -> AppMigrations: + lines = migration_status_print.strip().split("\n") + migrations_dict: AppMigrations = {} + current_app = None + line_prefix = ("[X]", "[ ]", "[-]", "(no migrations)") + + stale_migrations_dict: dict[str, list[str]] = {} + for app, migration in stale_migrations: + if app not in stale_migrations_dict: + stale_migrations_dict[app] = [] + stale_migrations_dict[app].append(migration) + + for line in lines: + line = line.strip() + if not line.startswith(line_prefix) and line: + current_app = line + migrations_dict[current_app] = [] + elif line.startswith(line_prefix): + assert current_app is not None + apps_stale_migrations = stale_migrations_dict.get(current_app) + if ( + apps_stale_migrations is not None + and line != "(no migrations)" + and line[4:] in apps_stale_migrations + ): + continue + migrations_dict[current_app].append(line) + + # Installed apps that have no migrations and we still use will have + # "(no migrations)" as its only "migrations" list. Ones that just + # have [] means it's just a left over stale app we can clean up. + return {app: migrations for app, migrations in migrations_dict.items() if migrations != []} diff --git a/zerver/tests/fixtures/import_fixtures/showmigrations_fixtures/with_stale_migrations.txt b/zerver/tests/fixtures/import_fixtures/showmigrations_fixtures/with_stale_migrations.txt new file mode 100644 index 0000000000..a5bbdd2d23 --- /dev/null +++ b/zerver/tests/fixtures/import_fixtures/showmigrations_fixtures/with_stale_migrations.txt @@ -0,0 +1,76 @@ +analytics + [X] 0001_squashed_0021_alter_fillstate_id (21 squashed migrations) +auth + [X] 0001_initial + [X] 0002_alter_permission_name_max_length + [X] 0003_alter_user_email_max_length +confirmation + [X] 0001_squashed_0014_confirmation_confirmatio_content_80155a_idx (14 squashed migrations) + [X] 0015_alter_confirmation_object_id +contenttypes + [X] 0001_initial + [X] 0002_remove_content_type_name +corporate + [X] 0001_squashed_0044_convert_ids_to_bigints (44 squashed migrations) +guardian + [X] 0001_initial +otp_static + [X] 0001_initial + [X] 0002_throttling + [X] 0003_add_timestamps +otp_totp + [X] 0001_initial + [X] 0002_auto_20190420_0723 + [X] 0003_add_timestamps +pgroonga + [X] 0001_enable + [X] 0002_html_escape_subject + [X] 0003_v2_api_upgrade +phonenumber + [X] 0001_squashed_0001_initial (10 squashed migrations) +sessions + [X] 0001_initial +social_django + [X] 0001_initial (2 squashed migrations) + [X] 0002_add_related_name (2 squashed migrations) + [X] 0003_alter_email_max_length (2 squashed migrations) +sites + [X] 0001_initial + [X] 0002_alter_domain_unique +two_factor + [X] 0001_squashed_0008_delete_phonedevice +zerver + [X] 0001_squashed_0569 (545 squashed migrations) + [X] 0002_django_1_8 + [X] 0003_custom_indexes + [X] 0004_userprofile_left_side_userlist + [X] 0005_auto_20150920_1340 + [X] 0006_zerver_userprofile_email_upper_idx + [X] 0007_userprofile_is_bot_active_indexes + [X] 0008_preregistrationuser_upper_email_idx + [X] 0009_add_missing_migrations + [X] 0010_delete_streamcolor + [X] 0011_remove_guardian + [X] 0012_remove_appledevicetoken + [X] 0013_realmemoji + [X] 0014_realm_emoji_url_length + [X] 0015_attachment + [X] 0016_realm_create_stream_by_admins_only + [X] 0017_userprofile_bot_type + [X] 0018_realm_emoji_message + [X] 0019_preregistrationuser_realm_creation + [X] 0020_add_tracking_attachment + [X] 0021_migrate_attachment_data + [X] 0022_subscription_pin_to_top + [X] 0023_userprofile_default_language + [X] 0024_realm_allow_message_editing + [X] 0025_realm_message_content_edit_limit + [X] 0026_delete_mituser + [X] 0027_realm_default_language + [X] 0028_userprofile_tos_version + [X] 0576_backfill_imageattachment + [X] 0622_backfill_imageattachment_again +default + [X] 0005_auto_20160727_2333 +zilencer + [X] 0001_squashed_0064_remotezulipserver_last_merge_base (64 squashed migrations) diff --git a/zerver/tests/test_import_export.py b/zerver/tests/test_import_export.py index 4e3df77e3e..0d5f71f1f6 100644 --- a/zerver/tests/test_import_export.py +++ b/zerver/tests/test_import_export.py @@ -44,15 +44,9 @@ from zerver.lib import upload from zerver.lib.avatar_hash import user_avatar_path from zerver.lib.bot_config import set_bot_config from zerver.lib.bot_lib import StateHandler -from zerver.lib.export import ( - AppMigrations, - MigrationStatusJson, - Record, - do_export_realm, - do_export_user, - export_usermessages_batch, -) +from zerver.lib.export import Record, do_export_realm, do_export_user, export_usermessages_batch from zerver.lib.import_realm import do_import_realm, get_incoming_message_ids +from zerver.lib.migration_status import STALE_MIGRATIONS, AppMigrations, MigrationStatusJson from zerver.lib.streams import create_stream_if_needed from zerver.lib.test_classes import ZulipTestCase from zerver.lib.test_helpers import ( @@ -381,6 +375,15 @@ class ExportFile(ZulipTestCase): exempt the fixture from this test.""", ) + # Make sure export doesn't produce a migration_status.json with stale + # migrations. + stale_migrations = [] + for app, stale_migration in STALE_MIGRATIONS: + installed_app = exported["migrations_by_app"].get(app) + if installed_app: + stale_migrations = [mig for mig in installed_app if mig.endswith(stale_migration)] + self.assert_length(stale_migrations, 0) + class RealmImportExportTest(ExportFile): def create_user_and_login(self, email: str, realm: Realm) -> None: @@ -2252,6 +2255,20 @@ class RealmImportExportTest(ExportFile): ) do_import_realm(get_output_dir(), "test-zulip") + def test_clean_up_migration_status_json(self) -> None: + user = self.example_user("hamlet") + with ( + patch("zerver.lib.export.get_migration_status") as mock_export, + ): + mock_export.return_value = self.fixture_data( + "with_stale_migrations.txt", "import_fixtures/showmigrations_fixtures" + ) + + realm = user.realm + self.export_realm_and_create_auditlog(realm) + + self.verify_migration_status_json() + class SingleUserExportTest(ExportFile): def do_files_test(self, is_s3: bool) -> None: diff --git a/zerver/tests/test_migration_status.py b/zerver/tests/test_migration_status.py new file mode 100644 index 0000000000..9edc79372b --- /dev/null +++ b/zerver/tests/test_migration_status.py @@ -0,0 +1,61 @@ +from zerver.lib.migration_status import ( + STALE_MIGRATIONS, + AppMigrations, + get_migration_status, + parse_migration_status, +) +from zerver.lib.test_classes import ZulipTestCase + + +class MigrationStatusTests(ZulipTestCase): + def test_parse_migration_status(self) -> None: + showmigrations_sample = """ +analytics + [X] 0001_squashed_0021_alter_fillstate_id (21 squashed migrations) +auth + [ ] 0012_alter_user_first_name_max_length +zerver + [-] 0015_alter_confirmation_object_id +two_factor + (no migrations) +""" + app_migrations = parse_migration_status(showmigrations_sample) + expected: AppMigrations = { + "analytics": ["[X] 0001_squashed_0021_alter_fillstate_id (21 squashed migrations)"], + "auth": ["[ ] 0012_alter_user_first_name_max_length"], + "zerver": ["[-] 0015_alter_confirmation_object_id"], + "two_factor": ["(no migrations)"], + } + self.assertDictEqual(app_migrations, expected) + + # Run one with the real showmigrations. A more thorough tests of these + # functions are done in the test_import_export.py as part of the import- + # export suite. + showmigrations = get_migration_status(app_label="zerver") + app_migrations = parse_migration_status(showmigrations) + zerver_migrations = app_migrations.get("zerver") + self.assertIsNotNone(zerver_migrations) + self.assertNotEqual(zerver_migrations, []) + + def test_parse_stale_migration_status(self) -> None: + assert ("guardian", "0001_initial") in STALE_MIGRATIONS + showmigrations_sample = """ +analytics + [X] 0001_squashed_0021_alter_fillstate_id (21 squashed migrations) +auth + [ ] 0012_alter_user_first_name_max_length +zerver + [-] 0015_alter_confirmation_object_id +two_factor + (no migrations) +guardian + [X] 0001_initial +""" + app_migrations = parse_migration_status(showmigrations_sample) + expected: AppMigrations = { + "analytics": ["[X] 0001_squashed_0021_alter_fillstate_id (21 squashed migrations)"], + "auth": ["[ ] 0012_alter_user_first_name_max_length"], + "zerver": ["[-] 0015_alter_confirmation_object_id"], + "two_factor": ["(no migrations)"], + } + self.assertDictEqual(app_migrations, expected)