#!/usr/bin/env python3 import os import re import shutil import subprocess import sys from textwrap import indent from typing import TypedDict import django from django.template import engines from django.template.backends.jinja2 import Jinja2 from pydantic.alias_generators import to_pascal BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from scripts.lib.setup_path import setup_path setup_path() os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings" django.setup() from zerver.lib.markdown.tabbed_sections import generate_content_blocks, parse_tabs INDENT_SPACES = " " SETTINGS_LINK_PATTERN = re.compile(r"{settings_tab\|(?P.*?)}") RELATIVE_LINKS_PATTERN = re.compile(r"\{relative\|(?P.*?)\|(?P.*?)\}") class IncludeFileInfo(TypedDict): is_only_ordered_list: bool def convert_kebab_to_pascal(text: str) -> str: # to_pascal is a function for converting snake case to pascal. return to_pascal(text).replace("-", "") def replace_emoticon_translation_table(markdown_string: str, import_statement_set: set[str]) -> str: """ We will replace emoticon_translations custom syntax in Python with astro component. """ result = markdown_string.replace( "\\{emoticon_translations\\}", """ """, ) if result != markdown_string: import_statement_set.add( "import EmoticonTranslations from '../../components/EmoticonTranslations.astro';" ) return result def replace_setting_links( markdown_string: str, import_statement_set: set[str], import_relative_base_path: str ) -> str: setting_links_pattern = re.compile(r"{settings_tab\|(?P.*?)}") def replace_setting_links_match(match: re.Match[str]) -> str: import_statement_set.add( f'import NavigationSteps from "{import_relative_base_path}/NavigationSteps.astro"' ) setting_identifier = match.group("setting_identifier") return f'' return setting_links_pattern.sub(replace_setting_links_match, markdown_string) def replace_relative_links( markdown_string: str, import_statement_set: set[str], import_relative_base_path: str ) -> str: def replace_relative_links_match(match: re.Match[str]) -> str: import_statement_set.add( f'import NavigationSteps from "{import_relative_base_path}/NavigationSteps.astro"' ) link_type = match.group("link_type") key = match.group("key") return f'' return RELATIVE_LINKS_PATTERN.sub(replace_relative_links_match, markdown_string) def replace_image_path(markdown_string: str, replacement_path: str) -> str: """ We will point to the existing image folder till the cutover. After that, we will copy the images to src folder for starlight_help in order to take advantage of Astro's image optimization. See https://chat.zulip.org/#narrow/stream/6-frontend/topic/Handling.20images.20in.20help.20center.20starlight.20migration.2E/near/1915130 """ # We do not replace /static/images directly since there are a few # instances in the documentation where zulip.com links are # referenced with that blurb as a part of the url. result = markdown_string.replace("(/static/images/starlight_help", f"({replacement_path}") return result.replace('="/static/images/starlight_help', f'="{replacement_path}') def fix_file_imports( markdown_string: str, import_statement_set: set[str], import_relative_base_path: str ) -> str: def get_pascal_filename_without_extension(match_string: str) -> str: return convert_kebab_to_pascal(os.path.basename(match_string).split(".")[0]) def convert_to_astro_tag(match: re.Match[str]) -> str: # The space before < makes sure that the component is interpreted correctly # when it is not occupying it's own line, e.g. it is part of a list item. return " <" + get_pascal_filename_without_extension(match.group(1)) + " />" RE = re.compile(r" {,3}\{!([^!]+)!\} *$", re.MULTILINE) result = RE.sub(convert_to_astro_tag, markdown_string) matches = RE.findall(markdown_string) for match_string in matches: pascal_filename = get_pascal_filename_without_extension(match_string) import_statement_set.add( f'import {pascal_filename} from "{import_relative_base_path}/_{pascal_filename}.mdx"' ) return result def escape_curly_braces(markdown_string: str) -> str: """ MDX will treat curly braces as a JS expression, we need to escape it if we don't want it to be treated as such. """ result = markdown_string.replace("{", r"\{") return result.replace("}", r"\}") def fix_relative_path(markdown_string: str) -> str: """ Since the docs will live at the `starlight_help/` url until we migrate the project completely, we will replace `help/` with `starlight_help/` """ return markdown_string.replace("help/", "starlight_help/") def insert_string_at_line(text: str, destination_str: str, n: int) -> str: lines = destination_str.splitlines() if 1 <= n <= len(lines): lines.insert(n - 1, text) return "\n".join(lines) def replace_icon_with_unplugin_component( match: re.Match[str], icon_package_name: str, import_statement_set: set[str], ) -> str: icon_name = match.group(1) component_name = to_pascal(icon_name).replace("-", "") + "Icon" import_statement = f'import {component_name} from "~icons/{icon_package_name}/{icon_name}"' import_statement_set.add(import_statement) return f"<{component_name} />" def replace_icons(markdown_string: str, import_statement_set: set[str]) -> str: """ Write some examples here and some assumptions we made about the icon tags. """ font_awesome_pattern = re.compile( r']*class="(?:[^"]*\s)?fa(?:\s+fa-([a-z0-9\-]+))(?:\s[^"]*)?"[^>]*>(?:\s[^<]*)?', ) def replace_font_awesome_icon_with_unplugin_component(match: re.Match[str]) -> str: return replace_icon_with_unplugin_component(match, "fa", import_statement_set) result = re.sub( font_awesome_pattern, replace_font_awesome_icon_with_unplugin_component, markdown_string ) zulip_icon_pattern = re.compile( r']*class="(?:[^"]*\s)?zulip-icon(?:\s+zulip-icon-([a-z0-9\-]+))(?:\s[^"]*)?"[^>]*>(?:\s[^<]*)?', ) def replace_zulip_icon_with_unplugin_component(match: re.Match[str]) -> str: return replace_icon_with_unplugin_component(match, "zulip-icon", import_statement_set) result = re.sub(zulip_icon_pattern, replace_zulip_icon_with_unplugin_component, result) return result def convert_comments(markdown_string: str) -> str: return markdown_string.replace("", "*/}") def convert_tab_syntax(markdown_string: str, import_statement_set: set[str]) -> str: """ Convert our custom tab syntax to relevant MDX equivalent. This function is inspired from `tabbed_section.py`'s run method. """ TABBED_SECTION_TEMPLATE = """ {blocks} """ TAB_ITEM_TEMPLATE = """ {content} """ def indent_content_inside_blocks(content_block: str) -> str: """ The original function `generate_content_blocks` does no indenting, and we need the content in between our TabItems to be indented. This is bit of string manipulation, but it is a better alternative compared to duplicating the original function here just to indent. """ content_block_lines = content_block.splitlines() result_lines = [] for line in content_block_lines: if line.startswith(("")): result_lines.append(line) else: result_lines.append(INDENT_SPACES + line) return "\n".join(result_lines) lines = markdown_string.splitlines() tab_section = parse_tabs(lines) while tab_section: if "tabs" in tab_section: content_blocks = generate_content_blocks(tab_section, lines, TAB_ITEM_TEMPLATE) content_blocks = indent_content_inside_blocks(content_blocks) content_blocks = indent(content_blocks, INDENT_SPACES) tabs_mdx = TABBED_SECTION_TEMPLATE.format(blocks=content_blocks) import_statement_set.add( "import { Tabs, TabItem } from '@astrojs/starlight/components'" ) else: # This is the case where we don't have any tabs but we were # using the `start_tabs` and `end_tabs` syntax to create # a border around the instructions. We just put the content # as is in this case since we don't want to add that border # anymore. tabs_mdx = ("\n").join( lines[(tab_section["start_tabs_index"] + 1) : (tab_section["end_tabs_index"])] ) start = tab_section["start_tabs_index"] end = tab_section["end_tabs_index"] + 1 lines = [*lines[:start], tabs_mdx, *lines[end:]] tab_section = parse_tabs(lines) return "\n".join(lines) def detab(text: str) -> tuple[str, str]: """ Remove a tab from the front of each line of the given text. Taken directly from https://github.com/Python-Markdown/markdown/blob/64a3c0fbc00327fbfee1fd6b44da0e5453287fe4/markdown/blockprocessors.py#L85 We need this function for converting admonitions to asides, it is okay to be duplicating this code for this script. """ tab_length = 4 newtext = [] lines = text.split("\n") for line in lines: if line.startswith(" " * tab_length): newtext.append(line[tab_length:]) elif not line.strip(): newtext.append("") else: break return "\n".join(newtext), "\n".join(lines[len(newtext) :]) def is_include_only_ordered_list(markdown_string: str) -> bool: """ Check if a given markdown string is only an ordered list and does not contain other components. After stripping down whitespace, the string should start with `1.`. There can be a lot of other components in the markdown string, but we keep our criteria small since almost all of the cases of include files needing a flatten list around them start with `1.`. """ markdown_string = markdown_string.strip() return markdown_string.startswith("1.") def is_line_part_of_an_ordered_list(line: str) -> bool: """ Everywhere is our markdown, we use `1.` for our lists instead of explicit numbers, so we only check for that here. A single item in a list can be spread across multiple lines with some indentation. So if the line starts with at least two spaces, we consider it part of the list for this conversion script. Newlines can be part of a list, so we return true for those too. """ return line.startswith((" ", "1.")) or line.strip() == "" def insert_flattened_steps_component( markdown_string: str, include_files_info: dict[str, IncludeFileInfo], import_statement_set: set[str], components_dir_path: str, ) -> str: """ We insert FlattenedSteps components where include files are being treated as part of ordered lists. Astro renders included files as it's own component, which would result in multiple ordered lists instead of a single list if we did not use this component. See the astro component file itself to know more how FlattenedSteps works. """ file_include_pattern = re.compile(r"^ {,3}\{!([^!]+)!\} *$", re.MULTILINE) lines = markdown_string.splitlines() def traverse_to_boundary(start: int, step: int) -> int: index = start while 0 <= index < len(lines): line = lines[index] if is_line_part_of_an_ordered_list(line): index += step continue file_match = file_include_pattern.match(line) if file_match: filename = file_match.group(1) if include_files_info[filename]["is_only_ordered_list"]: index += step continue settings_link_match = SETTINGS_LINK_PATTERN.match(line) relative_links_match = RELATIVE_LINKS_PATTERN.match(line) if settings_link_match or relative_links_match: index += step continue break return index # If a file with `is_only_ordered_list` set to True is followed # immediately by a similar file with it set to true, our loop # will try to insert the same text at the same position twice # resulting in two opening one after the other. # Using a set avoids this problem. insertions = set() def insert_flatten_list(match: re.Match[str]) -> None: match_line_index = markdown_string[: match.start()].count("\n") upper_bound = traverse_to_boundary(match_line_index - 1, step=-1) insertions.add((upper_bound + 1, "")) lower_bound = traverse_to_boundary(match_line_index + 1, step=1) insertions.add((lower_bound, "")) for match in SETTINGS_LINK_PATTERN.finditer(markdown_string): insert_flatten_list(match) for match in RELATIVE_LINKS_PATTERN.finditer(markdown_string): insert_flatten_list(match) for match in file_include_pattern.finditer(markdown_string): filename = match.group(1) if not include_files_info[filename]["is_only_ordered_list"]: continue insert_flatten_list(match) if insertions: import_statement_set.add( f"import FlattenedSteps from '{components_dir_path}/FlattenedSteps.astro';" ) # Insert tags in reverse order to avoid index shifting for index, tag in sorted(insertions, reverse=True): lines.insert(index, tag) return "\n".join(lines) def remove_blank_lines_from_steps(markdown_string: str) -> str: lines = markdown_string.split("\n") output_lines = [] inside_flattened_steps = False inside_steps = False for line in lines: if "" in line: inside_flattened_steps = True elif "" in line: inside_flattened_steps = False if "" in line: inside_steps = True elif "" in line: inside_steps = False if (inside_flattened_steps or inside_steps) and line.strip() == "": continue output_lines.append(line) return "\n".join(output_lines) def insert_steps_component_for_ordered_lists( markdown_string: str, import_statements: set[str] ) -> str: list_item_pattern = re.compile(r"^\s*1\.\s+") lines = markdown_string.split("\n") output_lines = [] inside_flattened_steps = False i = 0 while i < len(lines): line = lines[i] if "" in line: inside_flattened_steps = True elif "" in line: inside_flattened_steps = False if not inside_flattened_steps and list_item_pattern.match(line): list_block = [] while i < len(lines): current = lines[i] if ( list_item_pattern.match(current) or current.strip() == "" or current.startswith((" ", "\t")) ): list_block.append(current) i += 1 else: break import_statements.add("import { Steps } from '@astrojs/starlight/components';") output_lines.append("") output_lines.extend(list_block) output_lines.append("") else: output_lines.append(line) i += 1 return "\n".join(output_lines) def convert_admonitions_to_asides( markdown_string: str, import_statement_set: set[str], components_dir_path: str ) -> str: """ Lots of code in this function is taken from https://github.com/Python-Markdown/markdown/blob/64a3c0fbc00327fbfee1fd6b44da0e5453287fe4/markdown/extensions/admonition.py `(?:^|\n)!!!` has been changed to `(?:^|\n)( *)!!!` to allow for indented admonitions to be converted. """ RE = re.compile(r'(?:^|\n)( *)!!! ?([\w\-]+(?: +[\w\-]+)*)(?: +"(.*?)")? *(?:\n|$)') RE_SPACES = re.compile(" +") def get_admonition_class_and_title(match: re.Match[str]) -> tuple[str, str | None]: klass, title = match.group(2).lower(), match.group(3) klass = RE_SPACES.sub(" ", klass) if title is None: # no title was provided, use the capitalized class name as title title = klass.split(" ", 1)[0].capitalize() elif title == "": # an explicit blank title should not be rendered title = None return klass, title def replace_with_mdx_syntax(text: str) -> str: match = RE.search(text) if match: pre_admonition_declaration_text = text[: match.start()] post_admonition_declaration_text = text[match.end() :] # removes the first line admonition_content, post_admonition_content_text = detab( post_admonition_declaration_text ) # We strip newline since we add explicit newlines before # and after the component in the conversion code that # follows this. Extra blank lines in between the components # will make a tight list loose, which we do not desire. admonition_content = indent(admonition_content, INDENT_SPACES).strip("\n") klass, title = get_admonition_class_and_title(match) # We ignore the title obtained above in each of the if # block since in our current help center files, we do not # specify the title anywhere. This script only handles cases # that exist in our help center files, nothing more than that # is handled. if klass == "warn": # We have converted `warn` to `note` since that was the # translation that remains most faithful to how we # display `warn` admonitions in our current help center # implementation. # See https://chat.zulip.org/#narrow/channel/19-documentation/topic/Stage.202.3A.20New.20syntax.20for.20!!!tip.20in.20help-beta/near/2174415 # for more details. replacement = f"\n{match.group(1)}\n{admonition_content}\n{match.group(1)}\n" import_statement_set.add( f"import ZulipNote from '{components_dir_path}/ZulipNote.astro';" ) elif klass == "tip": replacement = f"\n{match.group(1)}\n{admonition_content}\n{match.group(1)}\n" import_statement_set.add( f"import ZulipTip from '{components_dir_path}/ZulipTip.astro';" ) elif klass == "keyboard_tip": replacement = f"\n{match.group(1)}\n{admonition_content}\n{match.group(1)}\n" import_statement_set.add( f"import KeyboardTip from '{components_dir_path}/KeyboardTip.astro';" ) else: raise Exception(f"Unexpected admonition class during conversion: {klass}") text = pre_admonition_declaration_text + replacement + post_admonition_content_text return replace_with_mdx_syntax(text) else: return text return replace_with_mdx_syntax(markdown_string) def convert_env_variables(markdown_string: str, import_statement_set: set[str]) -> str: # We run this step after we've escaped braces. if r"\{\{ support_email \}\}" in markdown_string: # This variable has already been declared in astro.config.mjs. import_statement_set.add('import {SUPPORT_EMAIL} from "astro:env/client";') markdown_string = markdown_string.replace( r"\{\{ support_email \}\}", "<>{SUPPORT_EMAIL}" ) return markdown_string def insert_imports(markdown_string: str, import_statement_set: set[str], line_number: int) -> str: if len(import_statement_set) == 0: return markdown_string # This function is called when the frontmatter has not yet been # inserted. First line of the file is always the heading/title of # the file. We rely on the heading being the first line later in # the conversion when inserting frontmatter. For this reason, we # add the imports to the second line. for import_statement in import_statement_set: markdown_string = insert_string_at_line(import_statement, markdown_string, line_number) # Add empty line at the end of import statement list. markdown_string = insert_string_at_line( "", markdown_string, line_number + len(import_statement_set) ) return markdown_string def insert_frontmatter(markdown_string: str) -> str: """ We use the heading in the first line for the existing files to extract the document title. We are not adding a description to the frontmatter yet. """ heading = markdown_string.partition("\n")[0].lstrip("#").strip() title = f"---\ntitle: {heading}\n---\n" # Remove the first line since starlight will display the # `title` as `H1` anyways. return title + markdown_string.split("\n", 1)[-1] def get_markdown_string_from_file(markdown_file_path: str) -> str: jinja = engines["Jinja2"] assert isinstance(jinja, Jinja2) if markdown_file_path.startswith("/"): with open(markdown_file_path) as fp: return fp.read() return jinja.env.loader.get_source(jinja.env, markdown_file_path)[0] def convert_help_center_file_to_mdx( markdown_file_path: str, include_files_info: dict[str, IncludeFileInfo] ) -> str: """ Given a path to a Markdown file, return the equivalent MDX file. """ result = get_markdown_string_from_file(markdown_file_path) # All imports inserted during conversion should be tracked here. import_statement_set: set[str] = set() result = fix_relative_path(result) # All unordered lists at the time of writing this comment are # standalone components and we do not need to do any transformation # for them. result = insert_flattened_steps_component( result, include_files_info, import_statement_set, "../../components" ) result = insert_steps_component_for_ordered_lists(result, import_statement_set) result = remove_blank_lines_from_steps(result) result = fix_file_imports(result, import_statement_set, "./include") result = convert_admonitions_to_asides(result, import_statement_set, "../../components") result = convert_tab_syntax(result, import_statement_set) result = replace_setting_links(result, import_statement_set, "../../components") result = replace_relative_links(result, import_statement_set, "../../components") result = escape_curly_braces(result) result = replace_emoticon_translation_table(result, import_statement_set) result = replace_image_path(result, "../../../../static/images/help") result = replace_icons(result, import_statement_set) result = convert_comments(result) result = convert_env_variables(result, import_statement_set) result = insert_imports(result, import_statement_set, 2) result = insert_frontmatter(result) return result def convert_include_file_to_mdx( markdown_file_path: str, include_files_info: dict[str, IncludeFileInfo], ) -> str: """ Given a path to a Markdown file, return the equivalent MDX file. We do not do certain operations that we do on a normal help file since these files are not to be served standalone but instead as macros in other files. - replace_emoticon_translation_table is skipped since that function only applies to one file, and that file is not an include file. - insert_frontmatter is skipped since frontmatter is not needed in files that are not served standalone. """ result = get_markdown_string_from_file(markdown_file_path) # All imports inserted during conversion should be tracked here. import_statement_set: set[str] = set() result = fix_relative_path(result) # All unordered lists at the time of writing this comment are # standalone components and we do not need to do any transformation # for them. result = insert_flattened_steps_component( result, include_files_info, import_statement_set, "../../../components" ) # If the file is only an ordered list, it will already have been # wrapped by FlattenedSteps and does not need any changes inside # the file itself. if not include_files_info[os.path.basename(markdown_file_path)]["is_only_ordered_list"]: result = insert_steps_component_for_ordered_lists(result, import_statement_set) else: # We just remove all blank lines except the trailing blank line # at the end of the file if the file is only an ordered list. result = "\n".join(line for line in result.splitlines() if line.strip()) + "\n" result = remove_blank_lines_from_steps(result) result = fix_file_imports(result, import_statement_set, ".") result = convert_admonitions_to_asides(result, import_statement_set, "../../../components") result = convert_tab_syntax(result, import_statement_set) result = replace_setting_links(result, import_statement_set, "../../../components") result = replace_relative_links(result, import_statement_set, "../../../components") result = escape_curly_braces(result) result = replace_image_path(result, "../../../../../static/images/help") result = replace_icons(result, import_statement_set) result = convert_comments(result) result = convert_env_variables(result, import_statement_set) result = insert_imports(result, import_statement_set, 1) return result def get_include_files_info(include_input_dir: str) -> dict[str, IncludeFileInfo]: def is_include_only_ordered_list_recursive(markdown_string: str) -> bool: if markdown_string.startswith("{!"): nested_file_name = ( markdown_string.splitlines()[0].strip().replace("{!", "").replace("!}", "") ) nested_file_path = os.path.join(include_input_dir, nested_file_name) nested_markdown_string = get_markdown_string_from_file(nested_file_path) return is_include_only_ordered_list_recursive(nested_markdown_string) else: return is_include_only_ordered_list(markdown_string) include_files_info: dict[str, IncludeFileInfo] = {} for name in os.listdir(include_input_dir): markdown_file_path = os.path.join(include_input_dir, name) if name.endswith(".md") and os.path.isfile(markdown_file_path): markdown_string = get_markdown_string_from_file(markdown_file_path) include_files_info[name] = { "is_only_ordered_list": is_include_only_ordered_list_recursive(markdown_string) } return include_files_info def run() -> None: input_dir = os.path.join(BASE_DIR, "help") output_dir = os.path.join(BASE_DIR, "starlight_help/src/content/docs") include_input_dir = os.path.join(input_dir, "include") include_output_dir = os.path.join(output_dir, "include") print("Starting the conversion from MD to MDX...") # We delete the directory first to remove any stale files that # might have been deleted in the `help` folder but their converted # mdx files stay around. We create it first just in case to avoid # rmtree throwing exceptions. os.makedirs(output_dir, exist_ok=True) shutil.rmtree(output_dir) os.makedirs(output_dir, exist_ok=True) converted_count = 0 os.makedirs(include_output_dir, exist_ok=True) include_files_info: dict[str, IncludeFileInfo] = get_include_files_info(include_input_dir) for name in os.listdir(include_input_dir): if name.endswith(".md") and os.path.isfile(os.path.join(include_input_dir, name)): converted_count += 1 mdx = convert_include_file_to_mdx( os.path.join(include_input_dir, name), include_files_info ) with open( os.path.join( include_output_dir, "_" + convert_kebab_to_pascal(os.path.basename(name).split(".")[0]) + ".mdx", ), "w", ) as mdx_file: mdx_file.write(mdx) print( f"Converted {converted_count} include files. Proceeding to the conversion of main help files ..." ) converted_count = 0 for name in os.listdir(input_dir): if name.endswith(".md") and os.path.isfile(os.path.join(input_dir, name)): converted_count += 1 mdx = convert_help_center_file_to_mdx(os.path.join(input_dir, name), include_files_info) with open( os.path.join( output_dir, os.path.basename(name).split(".")[0] + ".mdx", ), "w", ) as mdx_file: mdx_file.write(mdx) print(f"Converted {converted_count} main help files. Conversion completed.") # We use format-silent here, since running format on the converted # files gives us tons of warnings while fixing those warnings. We # also remove the frail flag when compared to the standard format # since the first run will always give us warnings. subprocess.check_call( ["/usr/local/bin/corepack", "pnpm", "run", "format-silent"], cwd="starlight_help", ) run()