markdown: Rewrite include plugin without markdown-include.

markdown-include is GPL licensed.

Also, rewrite it as a block processor, so that it works correctly
inside indented blocks.

Signed-off-by: Anders Kaseorg <anders@zulip.com>

zerver/lib/markdown/include.py

@@ -1,70 +1,54 @@
 import os
 import re
-from typing import Any, List
+from typing import List, Match
+from xml.etree.ElementTree import Element
 
-import markdown
-from markdown_include.include import IncludePreprocessor, MarkdownInclude
+from markdown import Extension, Markdown
+from markdown.blockparser import BlockParser
+from markdown.blockprocessors import BlockProcessor
 
 from zerver.lib.exceptions import InvalidMarkdownIncludeStatement
-from zerver.lib.markdown.priorities import PREPROCESSOR_PRIORITES
-
-INC_SYNTAX = re.compile(r"\{!\s*(.+?)\s*!\}")
+from zerver.lib.markdown.priorities import BLOCK_PROCESSOR_PRIORITIES
 
 
-class MarkdownIncludeCustom(MarkdownInclude):
-    def extendMarkdown(self, md: markdown.Markdown) -> None:
-        md.preprocessors.register(
-            IncludeCustomPreprocessor(md, self.getConfigs()),
-            "include_wrapper",
-            PREPROCESSOR_PRIORITES["include"],
+class IncludeExtension(Extension):
+    def __init__(self, base_path: str) -> None:
+        super().__init__()
+        self.base_path = base_path
+
+    def extendMarkdown(self, md: Markdown) -> None:
+        md.parser.blockprocessors.register(
+            IncludeBlockProcessor(md.parser, self.base_path),
+            "include",
+            BLOCK_PROCESSOR_PRIORITIES["include"],
         )
 
 
-class IncludeCustomPreprocessor(IncludePreprocessor):
-    """
-    This is a custom implementation of the markdown_include
-    extension that checks for include statements and if the included
-    macro file does not exist or can't be opened, raises a custom
-    JsonableError exception. The rest of the functionality is identical
-    to the original markdown_include extension.
-    """
+class IncludeBlockProcessor(BlockProcessor):
+    RE = re.compile(r"^ {,3}\{!([^!]+)!\} *$", re.M)
 
-    def run(self, lines: List[str]) -> List[str]:
-        done = False
-        while not done:
-            for line in lines:
-                loc = lines.index(line)
-                m = INC_SYNTAX.search(line)
+    def __init__(self, parser: BlockParser, base_path: str) -> None:
+        super().__init__(parser)
+        self.base_path = base_path
 
-                if m:
-                    filename = m.group(1)
-                    filename = os.path.expanduser(filename)
-                    if not os.path.isabs(filename):
-                        filename = os.path.normpath(
-                            os.path.join(self.base_path, filename),
-                        )
-                    try:
-                        with open(filename, encoding=self.encoding) as r:
-                            text = r.readlines()
-                    except Exception as e:
-                        print(f"Warning: could not find file {filename}. Error: {e}")
-                        lines[loc] = INC_SYNTAX.sub("", line)
-                        raise InvalidMarkdownIncludeStatement(m.group(0).strip())
+    def test(self, parent: Element, block: str) -> bool:  # type: ignore[override] # https://github.com/python/typeshed/pull/8166
+        return bool(self.RE.search(block))
 
-                    line_split = INC_SYNTAX.split(line)
-                    if len(text) == 0:
-                        text.append("")
-                    for i in range(len(text)):
-                        text[i] = text[i].rstrip("\r\n")
-                    text[0] = line_split[0] + text[0]
-                    text[-1] = text[-1] + line_split[2]
-                    lines = lines[:loc] + text + lines[loc + 1 :]
-                    break
-            else:
-                done = True
+    def expand_include(self, m: Match[str]) -> str:
+        try:
+            with open(os.path.normpath(os.path.join(self.base_path, m[1]))) as f:
+                lines = f.read().splitlines()
+        except OSError as e:
+            raise InvalidMarkdownIncludeStatement(m[0].strip()) from e
 
-        return lines
+        for prep in self.parser.md.preprocessors:
+            lines = prep.run(lines)
+
+        return "\n".join(lines)
+
+    def run(self, parent: Element, blocks: List[str]) -> None:
+        blocks[:1] = self.RE.sub(self.expand_include, blocks[0]).split("\n\n")
 
 
-def makeExtension(*args: Any, **kwargs: str) -> MarkdownIncludeCustom:
-    return MarkdownIncludeCustom(kwargs)
+def makeExtension(base_path: str) -> IncludeExtension:
+    return IncludeExtension(base_path=base_path)