From 459b4867f201314663bc152a1cab4247cd7b23e0 Mon Sep 17 00:00:00 2001 From: Vector73 Date: Wed, 2 Jul 2025 09:17:57 +0000 Subject: [PATCH] tools: Add support for viewing updated changelog for testing. Adds support for showing unmerged changelogs in "changelog.md" for testing purposes. --- docs/documentation/api.md | 4 + tools/merge-api-changelogs | 140 ++----------------------- tools/test-backend | 2 + zerver/lib/templates.py | 23 ++++ zerver/openapi/merge_api_changelogs.py | 140 +++++++++++++++++++++++++ 5 files changed, 180 insertions(+), 129 deletions(-) create mode 100644 zerver/openapi/merge_api_changelogs.py diff --git a/docs/documentation/api.md b/docs/documentation/api.md index 3a030a67eb..a4b7cbecb7 100644 --- a/docs/documentation/api.md +++ b/docs/documentation/api.md @@ -352,6 +352,10 @@ above. **Changes**: New in Zulip 11.0 (feature level ZF-1f4a39). ``` +1. Proofread your new documentation in its rendered HTML, including + all links! Unmerged changelog entries are conveniently previewed on + `/api/changelog`. + ## Why a custom system? Given that our documentation is written in large part using the diff --git a/tools/merge-api-changelogs b/tools/merge-api-changelogs index 8d40a6c4bb..546b104bf2 100755 --- a/tools/merge-api-changelogs +++ b/tools/merge-api-changelogs @@ -1,137 +1,19 @@ #!/usr/bin/env python3 -import glob import os -import re import subprocess import sys -from pathlib import Path +TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) +os.chdir(os.path.dirname(TOOLS_DIR)) +sys.path.insert(0, os.path.dirname(TOOLS_DIR)) -def get_changelog_files_list() -> list[str]: - dir_path = Path("api_docs/unmerged.d") - if os.path.exists(dir_path): - return [os.path.basename(path) for path in glob.glob(f"{dir_path}/ZF-??????.md")] - - return [] - - -def get_unmerged_changelogs() -> str: - changelogs = "" - dir_path = Path("api_docs/unmerged.d") - changelog_files_list = get_changelog_files_list() - if changelog_files_list: - print(f"Unmerged changelog files: {changelog_files_list}") - else: - print("No unmerged changelog files found.") - - for file_name in changelog_files_list: - file_path = Path(f"{dir_path}/{file_name}") - with open(file_path) as f: - changelogs += f.read().strip("\n") + "\n" - - return changelogs - - -def increment_and_get_feature_level() -> int: - new_feature_level = None - version_file_path = Path("version.py") - - with open(version_file_path) as file: - lines = file.readlines() - - new_feature_level = None - - with open(version_file_path, "w") as file: - for line in lines: - if line.startswith("API_FEATURE_LEVEL = "): - match = re.search(r"\d+", line) - if match: - new_feature_level = int(match.group()) + 1 - file.write(f"API_FEATURE_LEVEL = {new_feature_level}\n") - continue - - file.write(line) - - assert new_feature_level is not None - print(f"Updated API feature level: {new_feature_level - 1} -> {new_feature_level}") - return new_feature_level - - -def get_current_major_version() -> str | None: - changelog_path = Path("api_docs/changelog.md") - with open(changelog_path) as file: - for line in file: - match = re.search(r"## Changes in Zulip (\d+\.\d+)", line) - if match: - return match.group(1) - return None - - -def merge_changelogs(changelogs: str, new_feature_level: int) -> None: - changelogs_merged = False - changelog_path = Path("api_docs/changelog.md") - - with open(changelog_path) as file: - lines = file.readlines() - - changelogs_merged = False - - with open(changelog_path, "w") as file: - for line in lines: - file.write(line) - if changelogs_merged: - continue - if re.fullmatch(r"## Changes in Zulip \d+\.\d+\n", line): - changelogs_merged = True - file.write(f"\n**Feature level {new_feature_level}**\n") - file.write(f"\n{changelogs}") - - print(f"Changelogs merged to {changelog_path}.") - - -def update_feature_level_in_api_docs(new_feature_level: int) -> None: - changelog_files_list = get_changelog_files_list() - num_replaces = 0 - current_version = get_current_major_version() - - # Get all the markdown files in api_docs folder along with zulip.yaml. - api_docs_folder = Path("api_docs") - api_docs_paths = list(api_docs_folder.glob("*.md")) - api_docs_paths.append(Path("zerver/openapi/zulip.yaml")) - - for api_docs_path in api_docs_paths: - with open(api_docs_path) as file: - lines = file.readlines() - - num_replaces = 0 - - with open(api_docs_path, "w") as file: - for line in lines: - old_line = line - for file_name in changelog_files_list: - temporary_feature_level = file_name[: -len(".md")] - - pattern = rf"Zulip \d+\.\d+ \(feature level {temporary_feature_level}\)" - replacement = f"Zulip {current_version} (feature level {new_feature_level})" - line = re.sub(pattern, replacement, line) - - if old_line != line: - num_replaces += 1 - - file.write(line) - - if num_replaces: - print(f"Updated {api_docs_path}; {num_replaces} replaces were made.") - - -def remove_unmerged_changelog_files() -> None: - changelog_files_list = get_changelog_files_list() - for file_name in changelog_files_list: - os.remove(Path(f"api_docs/unmerged.d/{file_name}")) - - if changelog_files_list: - print("Removed all the unmerged changelog files.") - +from zerver.openapi.merge_api_changelogs import ( + get_feature_level, + get_unmerged_changelogs, + merge_changelogs, + remove_unmerged_changelog_files, + update_feature_level_in_api_docs, +) if __name__ == "__main__": ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -139,7 +21,7 @@ if __name__ == "__main__": changelogs = get_unmerged_changelogs() if changelogs: - new_feature_level = increment_and_get_feature_level() + new_feature_level = get_feature_level() merge_changelogs(changelogs, new_feature_level) update_feature_level_in_api_docs(new_feature_level) remove_unmerged_changelog_files() diff --git a/tools/test-backend b/tools/test-backend index 6c75c47b5c..5a63a99b96 100755 --- a/tools/test-backend +++ b/tools/test-backend @@ -108,6 +108,8 @@ not_yet_fully_covered = [ "zerver/openapi/javascript_examples.py", "zerver/openapi/python_examples.py", "zerver/openapi/test_curl_examples.py", + # Helper for tooling; doesn't need coverage + "zerver/openapi/merge_api_changelogs.py", # Tornado should ideally have full coverage, but we're not there. "zerver/tornado/descriptors.py", "zerver/tornado/django_api.py", diff --git a/zerver/lib/templates.py b/zerver/lib/templates.py index 73cb8480dd..6bd2328c93 100644 --- a/zerver/lib/templates.py +++ b/zerver/lib/templates.py @@ -1,4 +1,5 @@ import time +from pathlib import Path from typing import Any import markdown @@ -25,6 +26,11 @@ import zerver.lib.markdown.static import zerver.lib.markdown.tabbed_sections import zerver.openapi.markdown_extension from zerver.lib.cache import dict_to_items_tuple, ignore_unhashable_lru_cache, items_tuple_to_dict +from zerver.openapi.merge_api_changelogs import ( + get_feature_level, + get_unmerged_changelogs, + merge_changelogs, +) register = Library() @@ -38,6 +44,15 @@ def and_n_others(values: list[str], limit: int) -> str: ) +def get_updated_changelog() -> str | None: + unmerged_changelogs = get_unmerged_changelogs(verbose=False) + if not unmerged_changelogs: + return None + + new_feature_level = get_feature_level(update_feature_level=False) + return merge_changelogs(unmerged_changelogs, new_feature_level, False) + + @register.filter(name="display_list", is_safe=True) def display_list(values: list[str], display_limit: int) -> str: """ @@ -168,6 +183,14 @@ def render_markdown_path( else: markdown_string = jinja.env.loader.get_source(jinja.env, markdown_file_path)[0] + if ( + settings.DEVELOPMENT + and Path(markdown_file_path).resolve() == Path("api_docs/changelog.md").resolve() + ): + updated_changelog = get_updated_changelog() + if updated_changelog is not None: + markdown_string = updated_changelog + API_ENDPOINT_NAME = context.get("API_ENDPOINT_NAME", "") if context is not None else "" markdown_string = markdown_string.replace("API_ENDPOINT_NAME", API_ENDPOINT_NAME) diff --git a/zerver/openapi/merge_api_changelogs.py b/zerver/openapi/merge_api_changelogs.py new file mode 100644 index 0000000000..9dbce45767 --- /dev/null +++ b/zerver/openapi/merge_api_changelogs.py @@ -0,0 +1,140 @@ +import glob +import os +import re +from pathlib import Path + + +def get_changelog_files_list() -> list[str]: + dir_path = Path("api_docs/unmerged.d") + if os.path.exists(dir_path): + return [os.path.basename(path) for path in glob.glob(f"{dir_path}/ZF-??????.md")] + + return [] + + +def get_unmerged_changelogs(verbose: bool = True) -> str: + changelogs = "" + dir_path = Path("api_docs/unmerged.d") + changelog_files_list = get_changelog_files_list() + if verbose: + if changelog_files_list: + print(f"Unmerged changelog files: {changelog_files_list}") + else: + print("No unmerged changelog files found.") + + for file_name in changelog_files_list: + file_path = Path(f"{dir_path}/{file_name}") + with open(file_path) as f: + changelogs += f.read().strip("\n") + "\n" + + return changelogs + + +def get_feature_level(update_feature_level: bool = True) -> int: + new_feature_level = None + version_file_path = Path("version.py") + + with open(version_file_path) as file: + lines = file.readlines() + + new_feature_level = None + + with open(version_file_path, "w") as file: + for line in lines: + if line.startswith("API_FEATURE_LEVEL = "): + match = re.search(r"\d+", line) + if match: + new_feature_level = int(match.group()) + 1 + if update_feature_level: + file.write(f"API_FEATURE_LEVEL = {new_feature_level}\n") + continue + + file.write(line) + + assert new_feature_level is not None + if update_feature_level: + print(f"Updated API feature level: {new_feature_level - 1} -> {new_feature_level}") + return new_feature_level + + +def get_current_major_version() -> str | None: + changelog_path = Path("api_docs/changelog.md") + with open(changelog_path) as file: + for line in file: + match = re.search(r"## Changes in Zulip (\d+\.\d+)", line) + if match: + return match.group(1) + return None + + +def merge_changelogs(changelogs: str, new_feature_level: int, update_changelog: bool = True) -> str: + changelogs_merged = False + changelog_path = Path("api_docs/changelog.md") + + changelog_markdown_string = "" + + with open(changelog_path) as file: + lines = file.readlines() + + changelogs_merged = False + + with open(changelog_path, "w") as file: + for line in lines: + file.write(line) + changelog_markdown_string += line + if changelogs_merged: + continue + if re.fullmatch(r"## Changes in Zulip \d+\.\d+\n", line): + changelogs_merged = True + updates = f"\n**Feature level {new_feature_level}**\n\n{changelogs}" + changelog_markdown_string += updates + if update_changelog: + file.write(updates) + + if update_changelog: + print(f"Changelogs merged to {changelog_path}.") + return changelog_markdown_string + + +def update_feature_level_in_api_docs(new_feature_level: int) -> None: + changelog_files_list = get_changelog_files_list() + num_replaces = 0 + current_version = get_current_major_version() + + # Get all the markdown files in api_docs folder along with zulip.yaml. + api_docs_folder = Path("api_docs") + api_docs_paths = list(api_docs_folder.glob("*.md")) + api_docs_paths.append(Path("zerver/openapi/zulip.yaml")) + + for api_docs_path in api_docs_paths: + with open(api_docs_path) as file: + lines = file.readlines() + + num_replaces = 0 + + with open(api_docs_path, "w") as file: + for line in lines: + old_line = line + for file_name in changelog_files_list: + temporary_feature_level = file_name[: -len(".md")] + + pattern = rf"Zulip \d+\.\d+ \(feature level {temporary_feature_level}\)" + replacement = f"Zulip {current_version} (feature level {new_feature_level})" + line = re.sub(pattern, replacement, line) + + if old_line != line: + num_replaces += 1 + + file.write(line) + + if num_replaces: + print(f"Updated {api_docs_path}; {num_replaces} replaces were made.") + + +def remove_unmerged_changelog_files() -> None: + changelog_files_list = get_changelog_files_list() + for file_name in changelog_files_list: + os.remove(Path(f"api_docs/unmerged.d/{file_name}")) + + if changelog_files_list: + print("Removed all the unmerged changelog files.")