openapi: Combine two similar openapi markdown extensions.

Previously api_description and api_code_examples were two independent
markdown extensions for displaying OpenAPI content used in the same
places.  We combine them into a single markdown extension (with two
processors) and move them to the openapi folder to make the codebase
more readable and better group the openapi code in the same place.

zerver/lib/bugdown/api_description.py

@@ -1,63 +0,0 @@
-import re
-from markdown.extensions import Extension
-from markdown.preprocessors import Preprocessor
-from typing import Any, Dict, Optional, List
-import markdown
-
-from zerver.openapi.openapi import get_openapi_description
-
-MACRO_REGEXP = re.compile(r'\{generate_api_description(\(\s*(.+?)\s*\))}')
-
-class APIDescriptionGenerator(Extension):
-    def __init__(self, api_url: Optional[str]) -> None:
-        self.config = {
-            'api_url': [
-                api_url,
-                'API URL to use when rendering api links'
-            ]
-        }
-
-    def extendMarkdown(self, md: markdown.Markdown, md_globals: Dict[str, Any]) -> None:
-        md.preprocessors.add(
-            'generate_api_description', APIDescriptionPreprocessor(md, self.getConfigs()), '_begin'
-        )
-
-class APIDescriptionPreprocessor(Preprocessor):
-    def __init__(self, md: markdown.Markdown, config: Dict[str, Any]) -> None:
-        super().__init__(md)
-        self.api_url = config['api_url']
-
-    def run(self, lines: List[str]) -> List[str]:
-        done = False
-        while not done:
-            for line in lines:
-                loc = lines.index(line)
-                match = MACRO_REGEXP.search(line)
-
-                if match:
-                    function = match.group(2)
-                    text = self.render_description(function)
-                    # The line that contains the directive to include the macro
-                    # may be preceded or followed by text or tags, in that case
-                    # we need to make sure that any preceding or following text
-                    # stays the same.
-                    line_split = MACRO_REGEXP.split(line, maxsplit=0)
-                    preceding = line_split[0]
-                    following = line_split[-1]
-                    text = [preceding] + text + [following]
-                    lines = lines[:loc] + text + lines[loc+1:]
-                    break
-            else:
-                done = True
-        return lines
-
-    def render_description(self, function: str) -> List[str]:
-        description: List[str] = []
-        path, method = function.rsplit(':', 1)
-        description_dict = get_openapi_description(path, method)
-        description_dict = description_dict.replace('{{api_url}}', self.api_url)
-        description.extend(description_dict.splitlines())
-        return description
-
-def makeExtension(*args: Any, **kwargs: str) -> APIDescriptionGenerator:
-    return APIDescriptionGenerator(*args, **kwargs)

zerver/openapi/markdown_extension.py

@@ -10,10 +10,12 @@ from typing import Any, Dict, Optional, List, Tuple, Pattern
 import markdown
 
 import zerver.openapi.python_examples
-from zerver.openapi.openapi import get_openapi_fixture, openapi_spec
+from zerver.openapi.openapi import get_openapi_fixture, openapi_spec, get_openapi_description
 
-MACRO_REGEXP = re.compile(r'\{generate_code_example(\(\s*(.+?)\s*\))*\|\s*(.+?)\s*\|\s*(.+?)\s*(\(\s*(.+)\s*\))?\}')
+MACRO_REGEXP = re.compile(
+    r'\{generate_code_example(\(\s*(.+?)\s*\))*\|\s*(.+?)\s*\|\s*(.+?)\s*(\(\s*(.+)\s*\))?\}')
 PYTHON_EXAMPLE_REGEX = re.compile(r'\# \{code_example\|\s*(.+?)\s*\}')
+MACRO_REGEXP_DESC = re.compile(r'\{generate_api_description(\(\s*(.+?)\s*\))}')
 
 PYTHON_CLIENT_CONFIG = """
 #!/usr/bin/env python3
@@ -196,12 +198,14 @@ def generate_curl_example(endpoint: str, method: str,
         if global_security == [{'basicAuth': []}]:
             authentication_required = True
         else:
-            raise AssertionError("Unhandled global securityScheme. Please update the code to handle this scheme.")
+            raise AssertionError("Unhandled global securityScheme."
+                                 + " Please update the code to handle this scheme.")
     elif operation_security == []:
         if operation in insecure_operations:
             authentication_required = False
         else:
-            raise AssertionError("Unknown operation without a securityScheme. Please update insecure_operations.")
+            raise AssertionError("Unknown operation without a securityScheme. "
+                                 + "Please update insecure_operations.")
     else:
         raise AssertionError("Unhandled securityScheme. Please update the code to handle this scheme.")
 
@@ -263,7 +267,7 @@ SUPPORTED_LANGUAGES: Dict[str, Any] = {
     }
 }
 
-class APICodeExamplesGenerator(Extension):
+class APIMarkdownExtension(Extension):
     def __init__(self, api_url: Optional[str]) -> None:
         self.config = {
             'api_url': [
@@ -276,6 +280,9 @@ class APICodeExamplesGenerator(Extension):
         md.preprocessors.add(
             'generate_code_example', APICodeExamplesPreprocessor(md, self.getConfigs()), '_begin'
         )
+        md.preprocessors.add(
+            'generate_api_description', APIDescriptionPreprocessor(md, self.getConfigs()), '_begin'
+        )
 
 class APICodeExamplesPreprocessor(Preprocessor):
     def __init__(self, md: markdown.Markdown, config: Dict[str, Any]) -> None:
@@ -335,5 +342,42 @@ class APICodeExamplesPreprocessor(Preprocessor):
 
         return fixture
 
-def makeExtension(*args: Any, **kwargs: str) -> APICodeExamplesGenerator:
-    return APICodeExamplesGenerator(*args, **kwargs)
+class APIDescriptionPreprocessor(Preprocessor):
+    def __init__(self, md: markdown.Markdown, config: Dict[str, Any]) -> None:
+        super().__init__(md)
+        self.api_url = config['api_url']
+
+    def run(self, lines: List[str]) -> List[str]:
+        done = False
+        while not done:
+            for line in lines:
+                loc = lines.index(line)
+                match = MACRO_REGEXP_DESC.search(line)
+
+                if match:
+                    function = match.group(2)
+                    text = self.render_description(function)
+                    # The line that contains the directive to include the macro
+                    # may be preceded or followed by text or tags, in that case
+                    # we need to make sure that any preceding or following text
+                    # stays the same.
+                    line_split = MACRO_REGEXP_DESC.split(line, maxsplit=0)
+                    preceding = line_split[0]
+                    following = line_split[-1]
+                    text = [preceding] + text + [following]
+                    lines = lines[:loc] + text + lines[loc+1:]
+                    break
+            else:
+                done = True
+        return lines
+
+    def render_description(self, function: str) -> List[str]:
+        description: List[str] = []
+        path, method = function.rsplit(':', 1)
+        description_dict = get_openapi_description(path, method)
+        description_dict = description_dict.replace('{{api_url}}', self.api_url)
+        description.extend(description_dict.splitlines())
+        return description
+
+def makeExtension(*args: Any, **kwargs: str) -> APIMarkdownExtension:
+    return APIMarkdownExtension(*args, **kwargs)

zerver/openapi/test_curl_examples.py

@@ -6,7 +6,7 @@ import markdown
 import html
 
 from zulip import Client
-from zerver.lib.bugdown import api_code_examples
+from zerver.openapi import markdown_extension
 from zerver.models import get_realm
 from zerver.openapi.curl_param_value_generators import REGISTERED_GENERATOR_FUNCTIONS, CALLED_GENERATOR_FUNCTIONS
 
@@ -14,7 +14,7 @@ def test_generated_curl_examples_for_success(client: Client) -> None:
     authentication_line = "{}:{}".format(client.email, client.api_key)
     # A limited markdown engine that just processes the code example syntax.
     realm = get_realm("zulip")
-    md_engine = markdown.Markdown(extensions=[api_code_examples.makeExtension(
+    md_engine = markdown.Markdown(extensions=[markdown_extension.makeExtension(
         api_url=realm.uri + "/api")])
 
     # We run our curl tests in alphabetical order, since we depend

zerver/templatetags/app_filters.py

@@ -11,8 +11,7 @@ from jinja2.exceptions import TemplateNotFound
 
 import zerver.lib.bugdown.fenced_code
 import zerver.lib.bugdown.api_arguments_table_generator
-import zerver.lib.bugdown.api_code_examples
-import zerver.lib.bugdown.api_description
+import zerver.openapi.markdown_extension
 import zerver.lib.bugdown.nested_code_blocks
 import zerver.lib.bugdown.tabbed_sections
 import zerver.lib.bugdown.help_settings_links
@@ -124,10 +123,7 @@ def render_markdown_path(markdown_file_path: str,
         #
         # TODO: Convert this to something more efficient involving
         # passing the API URL as a direct parameter.
-        extensions = extensions + [zerver.lib.bugdown.api_code_examples.makeExtension(
-            api_url=context["api_url"],
-        )]
-        extensions = extensions + [zerver.lib.bugdown.api_description.makeExtension(
+        extensions = extensions + [zerver.openapi.markdown_extension.makeExtension(
             api_url=context["api_url"],
         )]
     if not any(doc in markdown_file_path for doc in docs_without_macros):

zerver/tests/test_openapi.py

@@ -8,7 +8,7 @@ from unittest.mock import patch, MagicMock
 from django.http import HttpResponse
 
 import zerver.openapi.openapi as openapi
-from zerver.lib.bugdown.api_code_examples import generate_curl_example, \
+from zerver.openapi.markdown_extension import generate_curl_example, \
     render_curl_example, parse_language_and_options
 from zerver.lib.request import _REQ
 from zerver.lib.test_classes import ZulipTestCase