markdown: Render nested multi-line code blocks correctly.

This commit adds a Markdown tree-processor extension that renders
multi-line code blocks that are nested inside lists with the
formatting. Note that the code block could be nested inside multiple
list levels and would still get rendered correctly.

Tim: This fixes the need for unpleasant workarounds like
f5bfa4e793 and makes nested code blocks
in our documentation look exactly how users would expect them to.

static/styles/portico.scss

@@ -315,6 +315,7 @@ body {
     margin-left: 0;
 }
 
+ol > li > div.codehilite,
 .markdown ol > li > p:not(:first-child),
 .portico-landing.integrations ol > li > p:not(:first-child) {
     padding-left: 24px;
@@ -326,6 +327,10 @@ body {
     display: block;
 }
 
+li div.codehilite {
+    background-color: #ffffff;
+}
+
 .portico-landing.integrations ol ul {
     padding-left: 40px;
     margin-left: 5px;
@@ -1678,10 +1683,6 @@ input.new-organization-button {
     margin: 5px 25px 5px;
 }
 
-.markdown .content code {
-    display: inline-block;
-}
-
 .markdown ol p {
     margin: 0 0 2px;
 }

templates/zerver/tests/markdown/test_nested_code_blocks.md

@@ -0,0 +1,13 @@
+# This is a heading.
+
+1. A list item with an indented code block:
+
+    ```
+    indented code block
+    with multiple lines
+    ```
+
+```
+non-indented code block
+with multiple lines
+```

zerver/lib/bugdown/nested_code_blocks.py

@@ -0,0 +1,74 @@
+from markdown.extensions import Extension
+from markdown.treeprocessors import Treeprocessor
+from typing import Any, Dict, Optional, List, Tuple
+import markdown
+from xml.etree.cElementTree import Element
+
+from zerver.lib.bugdown import walk_tree_with_family, ResultWithFamily
+
+class NestedCodeBlocksRenderer(Extension):
+    def extendMarkdown(self, md: markdown.Markdown, md_globals: Dict[str, Any]) -> None:
+        md.treeprocessors.add(
+            'nested_code_blocks',
+            NestedCodeBlocksRendererTreeProcessor(md, self.getConfigs()),
+            '_end'
+        )
+
+class NestedCodeBlocksRendererTreeProcessor(markdown.treeprocessors.Treeprocessor):
+    def __init__(self, md: markdown.Markdown, config: Dict[str, Any]) -> None:
+        super(NestedCodeBlocksRendererTreeProcessor, self).__init__(md)
+
+    def run(self, root: Element) -> None:
+        code_tags = walk_tree_with_family(root, self.get_code_tags)
+        nested_code_blocks = self.get_nested_code_blocks(code_tags)
+        for block in nested_code_blocks:
+            tag, text = block.result
+            codehilite_block = self.get_codehilite_block(text)
+            self.replace_element(block.family.grandparent,
+                                 codehilite_block,
+                                 block.family.parent)
+
+    def get_code_tags(self, e: Element) -> Optional[Tuple[str, Optional[str]]]:
+        if e.tag == "code":
+            return (e.tag, e.text)
+        return None
+
+    def get_nested_code_blocks(
+            self, code_tags: List[ResultWithFamily]
+    ) -> List[ResultWithFamily]:
+        nested_code_blocks = []
+        for code_tag in code_tags:
+            parent = code_tag.family.parent  # type: Any
+            grandparent = code_tag.family.grandparent  # type: Any
+            if parent.tag == "p" and grandparent.tag == "li":
+                # if the parent (<p>) has no text, that means that the <code>
+                # element inside is its only child and thus, we can confidently
+                # say that this is a nested code block
+                if parent.text is None:
+                    nested_code_blocks.append(code_tag)
+
+        return nested_code_blocks
+
+    def get_codehilite_block(self, code_block_text: str) -> Element:
+        div = markdown.util.etree.Element("div")
+        div.set("class", "codehilite")
+        pre = markdown.util.etree.SubElement(div, "pre")
+        pre.text = code_block_text
+        return div
+
+    def replace_element(
+            self, parent: Optional[Element],
+            replacement: markdown.util.etree.Element,
+            element_to_replace: Element
+    ) -> None:
+        if parent is None:
+            return
+
+        children = parent.getchildren()
+        for index, child in enumerate(children):
+            if child is element_to_replace:
+                parent.insert(index, replacement)
+                parent.remove(element_to_replace)
+
+def makeExtension(*args: Any, **kwargs: str) -> NestedCodeBlocksRenderer:
+    return NestedCodeBlocksRenderer(kwargs)

zerver/templatetags/app_filters.py

@@ -14,6 +14,7 @@ from django.utils.safestring import mark_safe
 import zerver.lib.bugdown.fenced_code
 import zerver.lib.bugdown.api_arguments_table_generator
 import zerver.lib.bugdown.api_code_examples
+import zerver.lib.bugdown.nested_code_blocks
 import zerver.lib.bugdown.help_settings_links
 from zerver.context_processors import zulip_default_context
 from zerver.lib.cache import ignore_unhashable_lru_cache
@@ -99,6 +100,7 @@ def render_markdown_path(markdown_file_path: str, context: Optional[Dict[Any, An
             zerver.lib.bugdown.api_arguments_table_generator.makeExtension(
                 base_path='templates/zerver/api/'),
             zerver.lib.bugdown.api_code_examples.makeExtension(),
+            zerver.lib.bugdown.nested_code_blocks.makeExtension(),
             zerver.lib.bugdown.help_settings_links.makeExtension(),
         ]
     if md_macro_extension is None:

zerver/tests/test_templates.py

@@ -213,6 +213,21 @@ class TemplateTestCase(ZulipTestCase):
         self.assertEqual(content_sans_whitespace,
                          'header<h1id="hello">Hello!</h1><p>Thisissome<em>boldtext</em>.</p>footer')
 
+    def test_markdown_nested_code_blocks(self) -> None:
+        template = get_template("tests/test_markdown.html")
+        context = {
+            'markdown_test_file': "zerver/tests/markdown/test_nested_code_blocks.md"
+        }
+        content = template.render(context)
+
+        content_sans_whitespace = content.replace(" ", "").replace('\n', '')
+        expected = ('header<h1id="this-is-a-heading">Thisisaheading.</h1><ol>'
+                    '<li><p>Alistitemwithanindentedcodeblock:</p><divclass="codehilite">'
+                    '<pre>indentedcodeblockwithmultiplelines</pre></div></li></ol>'
+                    '<divclass="codehilite"><pre><span></span>'
+                    'non-indentedcodeblockwithmultiplelines</pre></div>footer')
+        self.assertEqual(content_sans_whitespace, expected)
+
     def test_custom_tos_template(self) -> None:
         response = self.client_get("/terms/")