zulip/tools/convert-help-center-docs-to-mdx

#!/usr/bin/env python3

import os
import re
import shutil
import sys

import django
from django.template import engines
from django.template.backends.jinja2 import Jinja2
from pydantic.alias_generators import to_pascal

BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))

from scripts.lib.setup_path import setup_path

setup_path()

os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings"
django.setup()


def replace_emoticon_translation_table(markdown_string: str) -> str:
    """
    We will replace emoticon_translations custom syntax in Python with
    <EmoticonTranslations> astro component.
    """
    return markdown_string.replace(
        "\\{emoticon_translations\\}",
        """
import EmoticonTranslations from '../../components/EmoticonTranslations.astro';

<EmoticonTranslations />
""",
    )


def replace_image_path(markdown_string: str) -> str:
    """
    We will point to the existing image folder till
    the cutover. After that, we will copy the images
    to src folder for help-beta in order to take
    advantage of Astro's image optimization.
    See https://chat.zulip.org/#narrow/stream/6-frontend/topic/Handling.20images.20in.20help.20center.20starlight.20migration.2E/near/1915130
    """
    # We do not replace /static/images directly since there are a few
    # instances in the documentation where zulip.com links are
    # referenced with that blurb as a part of the url.
    result = markdown_string.replace("(/static/images/help-beta", "(../../../../static/images/help")
    return result.replace('="/static/images/help-beta', '="../../../../static/images/help')


def fix_file_imports(markdown_string: str) -> str:
    def convert_to_pascal(text: str) -> str:
        return to_pascal(text).replace("-", "").replace(".Md", "")

    def convert_to_astro_tag(match: re.Match[str]) -> str:
        return "<" + convert_to_pascal(match.group(1)) + " />"

    def append_str_to_line(text: str, destination_str: str, n: int) -> str:
        lines = destination_str.splitlines()
        if 1 <= n <= len(lines):
            lines[n - 1] += "\n" + text + "\n"

        return "\n".join(lines)

    RE = re.compile(r"^ {,3}\{!([^!]+)!\} *$", re.MULTILINE)
    result = RE.sub(convert_to_astro_tag, markdown_string)
    matches = RE.findall(markdown_string)

    import_statement_set = {
        f'import {convert_to_pascal(match)} from "./include/_{match}"' for match in matches
    }
    for import_statement in import_statement_set:
        # First line of the file is always the heading/title of the
        # file. We rely on the heading being the first line later
        # in the function inserting frontmatter and thus we add
        # this to the second line.
        result = append_str_to_line(import_statement, result, 2)

    return result


def escape_curly_braces(markdown_string: str) -> str:
    """
    MDX will treat curly braces as a JS expression,
    we need to escape it if we don't want it to be
    treated as such.
    """
    result = markdown_string.replace("{", r"\{")
    return result.replace("}", r"\}")


def fix_relative_path(markdown_string: str) -> str:
    """
    Since the docs will live at the `help-beta/` url
    until we migrate the project completely, we will
    replace `help/` with `help-beta/`
    """
    return markdown_string.replace("help/", "help-beta/")


def insert_frontmatter(markdown_string: str) -> str:
    """
    We use the heading in the first line for the
    existing files to extract the document title.
    We are not adding a description to the frontmatter
    yet.
    """
    heading = markdown_string.partition("\n")[0].lstrip("#").strip()
    title = f"---\ntitle: {heading}\n---\n"
    # Remove the first line since starlight will display the
    # `title` as `H1` anyways.
    return title + markdown_string.split("\n", 1)[-1]


def convert_string_to_mdx(markdown_string: str) -> str:
    result = markdown_string
    result = fix_file_imports(result)
    result = escape_curly_braces(result)
    result = fix_relative_path(result)
    result = replace_emoticon_translation_table(result)
    result = replace_image_path(result)
    result = insert_frontmatter(result)
    return result


def convert_file_to_mdx(
    markdown_file_path: str,
) -> str:
    """
    Given a path to a Markdown file, return the equivalent MDX file.
    """
    jinja = engines["Jinja2"]
    assert isinstance(jinja, Jinja2)
    if markdown_file_path.startswith("/"):
        with open(markdown_file_path) as fp:
            markdown_string = fp.read()
    else:
        markdown_string = jinja.env.loader.get_source(jinja.env, markdown_file_path)[0]

    return convert_string_to_mdx(markdown_string)


def run() -> None:
    input_dir = os.path.join(BASE_DIR, "help")
    output_dir = os.path.join(BASE_DIR, "help-beta/src/content/docs")
    print("Starting the conversion from MD to MDX...")

    converted_count = 0

    # We delete the directory first to remove any stale files
    # that might have been deleted in the `help` folder but
    # their converted mdx files stay around
    shutil.rmtree(output_dir)
    os.makedirs(output_dir, exist_ok=True)
    for name in os.listdir(input_dir):
        if os.path.isfile(os.path.join(input_dir, name)):
            converted_count += 1
            mdx = convert_file_to_mdx(os.path.join(input_dir, name))
            with open(
                os.path.join(
                    BASE_DIR,
                    output_dir,
                    os.path.basename(name).split(".")[0] + ".mdx",
                ),
                "w",
            ) as mdx_file:
                mdx_file.write(mdx)
    print(f"Converted {converted_count} files. Conversion completed.")

    # All files in the `include` folder will only be imports and not
    # standalone files. Therefore we do not do any manipulation or
    # them to mdx.
    include_source_dir = os.path.join(BASE_DIR, "help/include")
    include_destination_dir = os.path.join(BASE_DIR, "help-beta/src/content/docs/include")
    shutil.copytree(include_source_dir, include_destination_dir)

    # We do not want Astro to render these include files as standalone
    # files, prefixing them with an underscore accomplishes that.
    # https://docs.astro.build/en/guides/routing/#excluding-pages
    for name in os.listdir(include_destination_dir):
        os.rename(
            os.path.join(include_destination_dir, name),
            os.path.join(include_destination_dir, "_" + name),
        )


run()