migration_status: Add parse_migration_status.

This commit adds `parse_migration_status`, which takes in the string
output of `showmigrations` and parse it into key-value pair of installed
apps and a list of its migration status.

This is a prep commit to rework the check migrations function of
import/export which will parse the output of `showmigrations` to write
the `migration_status.json` file.

zerver/lib/export.py

@@ -33,6 +33,7 @@ from analytics.models import RealmCount, StreamCount, UserCount
 from scripts.lib.zulip_tools import overwrite_symlink
 from version import ZULIP_VERSION
 from zerver.lib.avatar_hash import user_avatar_base_path_from_ids
+from zerver.lib.migration_status import AppMigrations, MigrationStatusJson
 from zerver.lib.pysa import mark_sanitized
 from zerver.lib.timestamp import datetime_to_timestamp
 from zerver.lib.upload.s3 import get_bucket
@@ -98,8 +99,6 @@ SourceFilter: TypeAlias = Callable[[Record], bool]
 
 CustomFetch: TypeAlias = Callable[[TableData, Context], None]
 
-AppMigrations: TypeAlias = dict[str, list[str]]
-
 
 class MessagePartial(TypedDict):
     zerver_message: list[Record]
@@ -107,11 +106,6 @@ class MessagePartial(TypedDict):
     realm_id: int
 
 
-class MigrationStatusJson(TypedDict):
-    migrations_by_app: AppMigrations
-    zulip_version: str
-
-
 MESSAGE_BATCH_CHUNK_SIZE = 1000
 
 ALL_ZULIP_TABLES = {

zerver/lib/import_realm.py

@@ -31,7 +31,6 @@ from zerver.lib.bulk_create import bulk_set_users_or_streams_recipient_fields
 from zerver.lib.export import (
     DATE_FIELDS,
     Field,
-    MigrationStatusJson,
     Path,
     Record,
     TableData,
@@ -41,6 +40,7 @@ from zerver.lib.export import (
 from zerver.lib.markdown import markdown_convert
 from zerver.lib.markdown import version as markdown_version
 from zerver.lib.message import get_last_message_id
+from zerver.lib.migration_status import MigrationStatusJson
 from zerver.lib.mime_types import guess_type
 from zerver.lib.partial import partial
 from zerver.lib.push_notifications import sends_notifications_directly

zerver/lib/migration_status.py

@@ -2,7 +2,15 @@ import os
 import re
 from importlib import import_module
 from io import StringIO
-from typing import Any
+from typing import Any, TypeAlias, TypedDict
+
+AppMigrations: TypeAlias = dict[str, list[str]]
+
+
+class MigrationStatusJson(TypedDict):
+    migrations_by_app: AppMigrations
+    zulip_version: str
+
 
 STALE_MIGRATIONS = [
     # Ignore django-guardian, which we installed until 1.7.0~3134
@@ -82,3 +90,39 @@ def get_migration_status(**options: Any) -> str:
     out.seek(0)
     output = out.read()
     return re.sub(r"\x1b\[[0-9;]*m", "", output)
+
+
+def parse_migration_status(
+    migration_status_print: str, stale_migrations: list[tuple[str, str]] = STALE_MIGRATIONS
+) -> AppMigrations:
+    lines = migration_status_print.strip().split("\n")
+    migrations_dict: AppMigrations = {}
+    current_app = None
+    line_prefix = ("[X]", "[ ]", "[-]", "(no migrations)")
+
+    stale_migrations_dict: dict[str, list[str]] = {}
+    for app, migration in stale_migrations:
+        if app not in stale_migrations_dict:
+            stale_migrations_dict[app] = []
+        stale_migrations_dict[app].append(migration)
+
+    for line in lines:
+        line = line.strip()
+        if not line.startswith(line_prefix) and line:
+            current_app = line
+            migrations_dict[current_app] = []
+        elif line.startswith(line_prefix):
+            assert current_app is not None
+            apps_stale_migrations = stale_migrations_dict.get(current_app)
+            if (
+                apps_stale_migrations is not None
+                and line != "(no migrations)"
+                and line[4:] in apps_stale_migrations
+            ):
+                continue
+            migrations_dict[current_app].append(line)
+
+    # Installed apps that have no migrations and we still use will have
+    # "(no migrations)" as its only "migrations" list. Ones that just
+    # have [] means it's just a left over stale app we can clean up.
+    return {app: migrations for app, migrations in migrations_dict.items() if migrations != []}

zerver/tests/fixtures/import_fixtures/showmigrations_fixtures/with_stale_migrations.txt

@@ -0,0 +1,76 @@
+analytics
+ [X] 0001_squashed_0021_alter_fillstate_id (21 squashed migrations)
+auth
+ [X] 0001_initial
+ [X] 0002_alter_permission_name_max_length
+ [X] 0003_alter_user_email_max_length
+confirmation
+ [X] 0001_squashed_0014_confirmation_confirmatio_content_80155a_idx (14 squashed migrations)
+ [X] 0015_alter_confirmation_object_id
+contenttypes
+ [X] 0001_initial
+ [X] 0002_remove_content_type_name
+corporate
+ [X] 0001_squashed_0044_convert_ids_to_bigints (44 squashed migrations)
+guardian
+ [X] 0001_initial
+otp_static
+ [X] 0001_initial
+ [X] 0002_throttling
+ [X] 0003_add_timestamps
+otp_totp
+ [X] 0001_initial
+ [X] 0002_auto_20190420_0723
+ [X] 0003_add_timestamps
+pgroonga
+ [X] 0001_enable
+ [X] 0002_html_escape_subject
+ [X] 0003_v2_api_upgrade
+phonenumber
+ [X] 0001_squashed_0001_initial (10 squashed migrations)
+sessions
+ [X] 0001_initial
+social_django
+ [X] 0001_initial (2 squashed migrations)
+ [X] 0002_add_related_name (2 squashed migrations)
+ [X] 0003_alter_email_max_length (2 squashed migrations)
+sites
+ [X] 0001_initial
+ [X] 0002_alter_domain_unique
+two_factor
+ [X] 0001_squashed_0008_delete_phonedevice
+zerver
+ [X] 0001_squashed_0569 (545 squashed migrations)
+ [X] 0002_django_1_8
+ [X] 0003_custom_indexes
+ [X] 0004_userprofile_left_side_userlist
+ [X] 0005_auto_20150920_1340
+ [X] 0006_zerver_userprofile_email_upper_idx
+ [X] 0007_userprofile_is_bot_active_indexes
+ [X] 0008_preregistrationuser_upper_email_idx
+ [X] 0009_add_missing_migrations
+ [X] 0010_delete_streamcolor
+ [X] 0011_remove_guardian
+ [X] 0012_remove_appledevicetoken
+ [X] 0013_realmemoji
+ [X] 0014_realm_emoji_url_length
+ [X] 0015_attachment
+ [X] 0016_realm_create_stream_by_admins_only
+ [X] 0017_userprofile_bot_type
+ [X] 0018_realm_emoji_message
+ [X] 0019_preregistrationuser_realm_creation
+ [X] 0020_add_tracking_attachment
+ [X] 0021_migrate_attachment_data
+ [X] 0022_subscription_pin_to_top
+ [X] 0023_userprofile_default_language
+ [X] 0024_realm_allow_message_editing
+ [X] 0025_realm_message_content_edit_limit
+ [X] 0026_delete_mituser
+ [X] 0027_realm_default_language
+ [X] 0028_userprofile_tos_version
+ [X] 0576_backfill_imageattachment
+ [X] 0622_backfill_imageattachment_again
+default
+ [X] 0005_auto_20160727_2333
+zilencer
+ [X] 0001_squashed_0064_remotezulipserver_last_merge_base (64 squashed migrations)

zerver/tests/test_import_export.py

@@ -44,15 +44,9 @@ from zerver.lib import upload
 from zerver.lib.avatar_hash import user_avatar_path
 from zerver.lib.bot_config import set_bot_config
 from zerver.lib.bot_lib import StateHandler
-from zerver.lib.export import (
-    AppMigrations,
-    MigrationStatusJson,
-    Record,
-    do_export_realm,
-    do_export_user,
-    export_usermessages_batch,
-)
+from zerver.lib.export import Record, do_export_realm, do_export_user, export_usermessages_batch
 from zerver.lib.import_realm import do_import_realm, get_incoming_message_ids
+from zerver.lib.migration_status import STALE_MIGRATIONS, AppMigrations, MigrationStatusJson
 from zerver.lib.streams import create_stream_if_needed
 from zerver.lib.test_classes import ZulipTestCase
 from zerver.lib.test_helpers import (
@@ -381,6 +375,15 @@ class ExportFile(ZulipTestCase):
                     exempt the fixture from this test.""",
                 )
 
+        # Make sure export doesn't produce a migration_status.json with stale
+        # migrations.
+        stale_migrations = []
+        for app, stale_migration in STALE_MIGRATIONS:
+            installed_app = exported["migrations_by_app"].get(app)
+            if installed_app:
+                stale_migrations = [mig for mig in installed_app if mig.endswith(stale_migration)]
+        self.assert_length(stale_migrations, 0)
+
 
 class RealmImportExportTest(ExportFile):
     def create_user_and_login(self, email: str, realm: Realm) -> None:
@@ -2252,6 +2255,20 @@ class RealmImportExportTest(ExportFile):
             )
             do_import_realm(get_output_dir(), "test-zulip")
 
+    def test_clean_up_migration_status_json(self) -> None:
+        user = self.example_user("hamlet")
+        with (
+            patch("zerver.lib.export.get_migration_status") as mock_export,
+        ):
+            mock_export.return_value = self.fixture_data(
+                "with_stale_migrations.txt", "import_fixtures/showmigrations_fixtures"
+            )
+
+            realm = user.realm
+        self.export_realm_and_create_auditlog(realm)
+
+        self.verify_migration_status_json()
+
 
 class SingleUserExportTest(ExportFile):
     def do_files_test(self, is_s3: bool) -> None:

zerver/tests/test_migration_status.py

@@ -0,0 +1,61 @@
+from zerver.lib.migration_status import (
+    STALE_MIGRATIONS,
+    AppMigrations,
+    get_migration_status,
+    parse_migration_status,
+)
+from zerver.lib.test_classes import ZulipTestCase
+
+
+class MigrationStatusTests(ZulipTestCase):
+    def test_parse_migration_status(self) -> None:
+        showmigrations_sample = """
+analytics
+ [X] 0001_squashed_0021_alter_fillstate_id (21 squashed migrations)
+auth
+ [ ] 0012_alter_user_first_name_max_length
+zerver
+ [-] 0015_alter_confirmation_object_id
+two_factor
+ (no migrations)
+"""
+        app_migrations = parse_migration_status(showmigrations_sample)
+        expected: AppMigrations = {
+            "analytics": ["[X] 0001_squashed_0021_alter_fillstate_id (21 squashed migrations)"],
+            "auth": ["[ ] 0012_alter_user_first_name_max_length"],
+            "zerver": ["[-] 0015_alter_confirmation_object_id"],
+            "two_factor": ["(no migrations)"],
+        }
+        self.assertDictEqual(app_migrations, expected)
+
+        # Run one with the real showmigrations. A more thorough tests of these
+        # functions are done in the test_import_export.py as part of the import-
+        # export suite.
+        showmigrations = get_migration_status(app_label="zerver")
+        app_migrations = parse_migration_status(showmigrations)
+        zerver_migrations = app_migrations.get("zerver")
+        self.assertIsNotNone(zerver_migrations)
+        self.assertNotEqual(zerver_migrations, [])
+
+    def test_parse_stale_migration_status(self) -> None:
+        assert ("guardian", "0001_initial") in STALE_MIGRATIONS
+        showmigrations_sample = """
+analytics
+ [X] 0001_squashed_0021_alter_fillstate_id (21 squashed migrations)
+auth
+ [ ] 0012_alter_user_first_name_max_length
+zerver
+ [-] 0015_alter_confirmation_object_id
+two_factor
+ (no migrations)
+guardian
+ [X] 0001_initial
+"""
+        app_migrations = parse_migration_status(showmigrations_sample)
+        expected: AppMigrations = {
+            "analytics": ["[X] 0001_squashed_0021_alter_fillstate_id (21 squashed migrations)"],
+            "auth": ["[ ] 0012_alter_user_first_name_max_length"],
+            "zerver": ["[-] 0015_alter_confirmation_object_id"],
+            "two_factor": ["(no migrations)"],
+        }
+        self.assertDictEqual(app_migrations, expected)