diff --git a/templates/zerver/help/include/empty.md b/templates/zerver/help/include/empty.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/templates/zerver/tests/markdown/test_custom_include_extension.md b/templates/zerver/tests/markdown/test_custom_include_extension.md new file mode 100644 index 0000000000..c81c4783a9 --- /dev/null +++ b/templates/zerver/tests/markdown/test_custom_include_extension.md @@ -0,0 +1 @@ +{!nonexistent-macro.md!} diff --git a/templates/zerver/tests/markdown/test_custom_include_extension_empty.md b/templates/zerver/tests/markdown/test_custom_include_extension_empty.md new file mode 100644 index 0000000000..587f43d64c --- /dev/null +++ b/templates/zerver/tests/markdown/test_custom_include_extension_empty.md @@ -0,0 +1 @@ +{!empty.md!} diff --git a/zerver/lib/bugdown/include.py b/zerver/lib/bugdown/include.py new file mode 100644 index 0000000000..e924546085 --- /dev/null +++ b/zerver/lib/bugdown/include.py @@ -0,0 +1,67 @@ +from __future__ import print_function +import re +import os +from typing import Any, Dict, Optional, List + +import markdown +from markdown_include.include import MarkdownInclude, IncludePreprocessor + +from zerver.lib.exceptions import InvalidMarkdownIncludeStatement + +INC_SYNTAX = re.compile(r'\{!\s*(.+?)\s*!\}') + + +class MarkdownIncludeCustom(MarkdownInclude): + def extendMarkdown(self, md: markdown.Markdown, md_globals: Dict[str, Any]) -> None: + md.preprocessors.add( + 'include_wrapper', + IncludeCustomPreprocessor(md, self.getConfigs()), + '_begin' + ) + +class IncludeCustomPreprocessor(IncludePreprocessor): + """ + This is a custom implementation of the markdown_include + extension that checks for include statements and if the included + macro file does not exist or can't be opened, raises a custom + JsonableError exception. The rest of the functionality is identical + to the original markdown_include extension. + """ + def run(self, lines: List[str]) -> List[str]: + done = False + while not done: + for line in lines: + loc = lines.index(line) + m = INC_SYNTAX.search(line) + + if m: + filename = m.group(1) + filename = os.path.expanduser(filename) + if not os.path.isabs(filename): + filename = os.path.normpath( + os.path.join(self.base_path, filename) + ) + try: + with open(filename, 'r', encoding=self.encoding) as r: + text = r.readlines() + except Exception as e: + print('Warning: could not find file {}. Error: {}'.format(filename, e)) + lines[loc] = INC_SYNTAX.sub('', line) + raise InvalidMarkdownIncludeStatement(m.group(0).strip()) + + line_split = INC_SYNTAX.split(line) + if len(text) == 0: + text.append('') + for i in range(len(text)): + text[i] = text[i].rstrip('\r\n') + text[0] = line_split[0] + text[0] + text[-1] = text[-1] + line_split[2] + lines = lines[:loc] + text + lines[loc+1:] + break + else: + done = True + + return lines + +def makeExtension(*args: Any, **kwargs: str) -> MarkdownIncludeCustom: + return MarkdownIncludeCustom(kwargs) diff --git a/zerver/lib/exceptions.py b/zerver/lib/exceptions.py index 9bcaba20e8..71c7f3c3d1 100644 --- a/zerver/lib/exceptions.py +++ b/zerver/lib/exceptions.py @@ -41,6 +41,7 @@ class ErrorCode(AbstractEnum): CSRF_FAILED = () INVITATION_FAILED = () INVALID_ZULIP_SERVER = () + INVALID_MARKDOWN_INCLUDE_STATEMENT = () REQUEST_CONFUSING_VAR = () class JsonableError(Exception): @@ -152,6 +153,17 @@ class CannotDeactivateLastUserError(JsonableError): def msg_format() -> str: return _("Cannot deactivate the only {entity}.") +class InvalidMarkdownIncludeStatement(JsonableError): + code = ErrorCode.INVALID_MARKDOWN_INCLUDE_STATEMENT + data_fields = ['include_statement'] + + def __init__(self, include_statement: str) -> None: + self.include_statement = include_statement + + @staticmethod + def msg_format() -> str: + return _("Invalid markdown include statement: {include_statement}") + class RateLimited(PermissionDenied): def __init__(self, msg: str="") -> None: super().__init__(msg) diff --git a/zerver/templatetags/app_filters.py b/zerver/templatetags/app_filters.py index 3cad1d07ea..fe59148a5a 100644 --- a/zerver/templatetags/app_filters.py +++ b/zerver/templatetags/app_filters.py @@ -7,7 +7,6 @@ import markdown.extensions.admonition import markdown.extensions.codehilite import markdown.extensions.extra import markdown.extensions.toc -import markdown_include.include from django.conf import settings from django.template import Library, engines, loader from django.utils.safestring import mark_safe @@ -21,6 +20,7 @@ import zerver.lib.bugdown.tabbed_sections import zerver.lib.bugdown.help_settings_links import zerver.lib.bugdown.help_relative_links import zerver.lib.bugdown.help_emoticon_translations_table +import zerver.lib.bugdown.include from zerver.context_processors import zulip_default_context from zerver.lib.cache import ignore_unhashable_lru_cache @@ -115,7 +115,7 @@ def render_markdown_path(markdown_file_path: str, zerver.lib.bugdown.help_emoticon_translations_table.makeExtension(), ] if md_macro_extension is None: - md_macro_extension = markdown_include.include.makeExtension( + md_macro_extension = zerver.lib.bugdown.include.makeExtension( base_path='templates/zerver/help/include/') if any(doc in markdown_file_path for doc in docs_without_macros): diff --git a/zerver/tests/test_templates.py b/zerver/tests/test_templates.py index f24de3ce4d..7161a96f2f 100644 --- a/zerver/tests/test_templates.py +++ b/zerver/tests/test_templates.py @@ -11,6 +11,7 @@ from django.template import Template, Context from django.template.loader import get_template from django.test.client import RequestFactory +from zerver.lib.exceptions import InvalidMarkdownIncludeStatement from zerver.lib.test_helpers import get_all_templates from zerver.lib.test_classes import ( ZulipTestCase, @@ -312,6 +313,25 @@ footer 'non-indentedcodeblockwithmultiplelinesfooter') self.assertEqual(content_sans_whitespace, expected) + def test_custom_markdown_include_extension(self) -> None: + template = get_template("tests/test_markdown.html") + context = { + 'markdown_test_file': "zerver/tests/markdown/test_custom_include_extension.md" + } + + with self.assertRaisesRegex(InvalidMarkdownIncludeStatement, "Invalid markdown include statement"): + template.render(context) + + def test_custom_markdown_include_extension_empty_macro(self) -> None: + template = get_template("tests/test_markdown.html") + context = { + 'markdown_test_file': "zerver/tests/markdown/test_custom_include_extension_empty.md" + } + content = template.render(context) + content_sans_whitespace = content.replace(" ", "").replace('\n', '') + expected = 'headerfooter' + self.assertEqual(content_sans_whitespace, expected) + def test_custom_tos_template(self) -> None: response = self.client_get("/terms/")