#!/usr/bin/env python3 import os import re import shutil import sys import django from django.template import engines from django.template.backends.jinja2 import Jinja2 from pydantic.alias_generators import to_pascal BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) from scripts.lib.setup_path import setup_path setup_path() os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings" django.setup() def replace_emoticon_translation_table(markdown_string: str) -> str: """ We will replace emoticon_translations custom syntax in Python with astro component. """ return markdown_string.replace( "\\{emoticon_translations\\}", """ import EmoticonTranslations from '../../components/EmoticonTranslations.astro'; """, ) def replace_image_path(markdown_string: str) -> str: """ We will point to the existing image folder till the cutover. After that, we will copy the images to src folder for help-beta in order to take advantage of Astro's image optimization. See https://chat.zulip.org/#narrow/stream/6-frontend/topic/Handling.20images.20in.20help.20center.20starlight.20migration.2E/near/1915130 """ # We do not replace /static/images directly since there are a few # instances in the documentation where zulip.com links are # referenced with that blurb as a part of the url. result = markdown_string.replace("(/static/images/help-beta", "(../../../../static/images/help") return result.replace('="/static/images/help-beta', '="../../../../static/images/help') def fix_file_imports(markdown_string: str) -> str: def convert_to_pascal(text: str) -> str: return to_pascal(text).replace("-", "").replace(".Md", "") def convert_to_astro_tag(match: re.Match[str]) -> str: return "<" + convert_to_pascal(match.group(1)) + " />" def append_str_to_line(text: str, destination_str: str, n: int) -> str: lines = destination_str.splitlines() if 1 <= n <= len(lines): lines[n - 1] += "\n" + text + "\n" return "\n".join(lines) RE = re.compile(r"^ {,3}\{!([^!]+)!\} *$", re.MULTILINE) result = RE.sub(convert_to_astro_tag, markdown_string) matches = RE.findall(markdown_string) import_statement_set = { f'import {convert_to_pascal(match)} from "./include/_{match}"' for match in matches } for import_statement in import_statement_set: # First line of the file is always the heading/title of the # file. We rely on the heading being the first line later # in the function inserting frontmatter and thus we add # this to the second line. result = append_str_to_line(import_statement, result, 2) return result def escape_curly_braces(markdown_string: str) -> str: """ MDX will treat curly braces as a JS expression, we need to escape it if we don't want it to be treated as such. """ result = markdown_string.replace("{", r"\{") return result.replace("}", r"\}") def fix_relative_path(markdown_string: str) -> str: """ Since the docs will live at the `help-beta/` url until we migrate the project completely, we will replace `help/` with `help-beta/` """ return markdown_string.replace("help/", "help-beta/") def insert_frontmatter(markdown_string: str) -> str: """ We use the heading in the first line for the existing files to extract the document title. We are not adding a description to the frontmatter yet. """ heading = markdown_string.partition("\n")[0].lstrip("#").strip() title = f"---\ntitle: {heading}\n---\n" # Remove the first line since starlight will display the # `title` as `H1` anyways. return title + markdown_string.split("\n", 1)[-1] def convert_string_to_mdx(markdown_string: str) -> str: result = markdown_string result = fix_file_imports(result) result = escape_curly_braces(result) result = fix_relative_path(result) result = replace_emoticon_translation_table(result) result = replace_image_path(result) result = insert_frontmatter(result) return result def convert_file_to_mdx( markdown_file_path: str, ) -> str: """ Given a path to a Markdown file, return the equivalent MDX file. """ jinja = engines["Jinja2"] assert isinstance(jinja, Jinja2) if markdown_file_path.startswith("/"): with open(markdown_file_path) as fp: markdown_string = fp.read() else: markdown_string = jinja.env.loader.get_source(jinja.env, markdown_file_path)[0] return convert_string_to_mdx(markdown_string) def run() -> None: input_dir = os.path.join(BASE_DIR, "help") output_dir = os.path.join(BASE_DIR, "help-beta/src/content/docs") print("Starting the conversion from MD to MDX...") converted_count = 0 # We delete the directory first to remove any stale files # that might have been deleted in the `help` folder but # their converted mdx files stay around shutil.rmtree(output_dir) os.makedirs(output_dir, exist_ok=True) for name in os.listdir(input_dir): if os.path.isfile(os.path.join(input_dir, name)): converted_count += 1 mdx = convert_file_to_mdx(os.path.join(input_dir, name)) with open( os.path.join( BASE_DIR, output_dir, os.path.basename(name).split(".")[0] + ".mdx", ), "w", ) as mdx_file: mdx_file.write(mdx) print(f"Converted {converted_count} files. Conversion completed.") # All files in the `include` folder will only be imports and not # standalone files. Therefore we do not do any manipulation or # them to mdx. include_source_dir = os.path.join(BASE_DIR, "help/include") include_destination_dir = os.path.join(BASE_DIR, "help-beta/src/content/docs/include") shutil.copytree(include_source_dir, include_destination_dir) # We do not want Astro to render these include files as standalone # files, prefixing them with an underscore accomplishes that. # https://docs.astro.build/en/guides/routing/#excluding-pages for name in os.listdir(include_destination_dir): os.rename( os.path.join(include_destination_dir, name), os.path.join(include_destination_dir, "_" + name), ) run()