api docs: Read parameters and response fixtures from OpenAPI files.

mypy.ini

@@ -254,6 +254,9 @@ ignore_missing_imports = True
 [mypy-two_factor,two_factor.*]
 ignore_missing_imports = True
 
+[mypy-yamole]
+ignore_missing_imports = True
+
 
 
 

templates/zerver/api/fixtures.json

@@ -360,10 +360,6 @@
         "msg": "",
         "result": "success"
     },
-    "update-message": {
-        "msg": "",
-        "result": "success"
-    },
     "update-message-edit-permission-error": {
         "code": "BAD_REQUEST",
         "msg": "You don't have permission to edit this message",

templates/zerver/api/update-message.md

@@ -27,7 +27,7 @@ curl -X "PATCH" {{ api_url }}/v1/messages/<msg_id> \
 
 <div data-language="python" markdown="1">
 
-{generate_code_example(python)|update-message|example}
+{generate_code_example(python)|/messages/{message_id}:patch|example}
 
 </div>
 
@@ -67,7 +67,7 @@ You only have permission to edit a message if:
 
 ## Arguments
 
-{generate_api_arguments_table|arguments.json|update-message.md}
+{generate_api_arguments_table|zulip.yaml|/messages/{message_id}:patch}
 
 ## Response
 
@@ -75,9 +75,9 @@ You only have permission to edit a message if:
 
 A typical successful JSON response may look like:
 
-{generate_code_example|update-message|fixture}
+{generate_code_example|/messages/{message_id}:patch|fixture(200)}
 
 A typical JSON response for when one doesn't have the permission to
 edit a particular message:
 
-{generate_code_example|update-message-edit-permission-error|fixture}
+{generate_code_example|/messages/{message_id}:patch|fixture(400)}

version.py

@@ -8,4 +8,4 @@ ZULIP_VERSION = "1.8.1+git"
 # Typically, adding a dependency only requires a minor version bump, and
 # removing a dependency requires a major version bump.
 
-PROVISION_VERSION = '20.3'
+PROVISION_VERSION = '20.4'

zerver/lib/api_test_helpers.py

@@ -1,10 +1,12 @@
 from typing import Dict, Any, Optional, Iterable
 from io import StringIO
+from yamole import YamoleParser
 
 import json
 import os
 
 from zerver.lib import mdiff
+from zerver.lib.openapi import get_openapi_fixture
 
 if False:
     from zulip import Client
@@ -383,7 +385,8 @@ def update_message(client, message_id):
     result = client.update_message(request)
     # {code_example|end}
 
-    fixture = FIXTURES['update-message']
+    fixture = get_openapi_fixture('/messages/{message_id}', 'patch', '200')
+
     test_against_fixture(result, fixture)
 
     # test it was actually updated
@@ -491,7 +494,7 @@ TEST_FUNCTIONS = {
     'render-message': render_message,
     'stream-message': stream_message,
     'private-message': private_message,
-    'update-message': update_message,
+    '/messages/{message_id}:patch': update_message,
     'get-stream-id': get_stream_id,
     'get-subscribed-streams': list_subscriptions,
     'get-all-streams': get_streams,

zerver/lib/bugdown/api_arguments_table_generator.py

@@ -4,10 +4,11 @@ import ujson
 
 from markdown.extensions import Extension
 from markdown.preprocessors import Preprocessor
+from zerver.lib.openapi import get_openapi_parameters
 from typing import Any, Dict, Optional, List
 import markdown
 
-REGEXP = re.compile(r'\{generate_api_arguments_table\|\s*(.+?)\s*\|\s*(.+?)\s*\}')
+REGEXP = re.compile(r'\{generate_api_arguments_table\|\s*(.+?)\s*\|\s*(.+)\s*\}')
 
 
 class MarkdownArgumentsTableGenerator(Extension):
@@ -16,8 +17,6 @@ class MarkdownArgumentsTableGenerator(Extension):
             configs = {}
         self.config = {
             'base_path': ['.', 'Default location from which to evaluate relative paths for the JSON files.'],
-            'openapi_path': ['../../../zerver/openapi',
-                             'Default location from which to evaluate relative paths for the YAML files.']
         }
         for key, value in configs.items():
             self.setConfig(key, value)
@@ -41,19 +40,29 @@ class APIArgumentsTablePreprocessor(Preprocessor):
                 match = REGEXP.search(line)
 
                 if match:
-                    json_filename = match.group(1)
+                    filename = match.group(1)
-                    doc_filename = match.group(2)
+                    doc_name = match.group(2)
-                    json_filename = os.path.expanduser(json_filename)
+                    filename = os.path.expanduser(filename)
-                    if not os.path.isabs(json_filename):
+
-                        json_filename = os.path.normpath(os.path.join(self.base_path, json_filename))
+                    is_openapi_format = filename.endswith('.yaml')
+
+                    if not os.path.isabs(filename):
+                        parent_dir = self.base_path
+                        filename = os.path.normpath(os.path.join(parent_dir, filename))
+
                     try:
-                        with open(json_filename, 'r') as fp:
+                        if is_openapi_format:
-                            json_obj = ujson.load(fp)
+                            endpoint, method = doc_name.rsplit(':', 1)
-                            arguments = json_obj[doc_filename]
+                            arguments = get_openapi_parameters(endpoint, method)
-                            text = self.render_table(arguments)
+                        else:
+                            with open(filename, 'r') as fp:
+                                json_obj = ujson.load(fp)
+                                arguments = json_obj[doc_name]
+
+                        text = self.render_table(arguments)
                     except Exception as e:
                         print('Warning: could not find file {}. Ignoring '
-                              'statement. Error: {}'.format(json_filename, e))
+                              'statement. Error: {}'.format(filename, e))
                         # If the file cannot be opened, just substitute an empty line
                         # in place of the macro include line
                         lines[loc] = REGEXP.sub('', line)
@@ -101,11 +110,19 @@ class APIArgumentsTablePreprocessor(Preprocessor):
         md_engine = markdown.Markdown(extensions=[])
 
         for argument in arguments:
+            oneof = ['`' + item + '`'
+                     for item in argument.get('schema', {}).get('enum', [])]
+            description = argument['description']
+            if oneof:
+                description += '\nMust be one of: {}.'.format(', '.join(oneof))
+            # TODO: Swagger allows indicating where the argument goes
+            # (path, querystring, form data...). A column in the table should
+            # be added for this.
             table.append(tr.format(
-                argument=argument['argument'],
+                argument=argument.get('argument') or argument.get('name'),
                 example=argument['example'],
                 required='Yes' if argument.get('required') else 'No',
-                description=md_engine.convert(argument['description']),
+                description=md_engine.convert(description),
             ))
 
         table.append("</tbody>")

zerver/lib/bugdown/api_code_examples.py

@@ -7,11 +7,13 @@ import inspect
 from markdown.extensions import Extension
 from markdown.preprocessors import Preprocessor
 from typing import Any, Dict, Optional, List
+from yamole import YamoleParser
 import markdown
 
 import zerver.lib.api_test_helpers
+from zerver.lib.openapi import get_openapi_fixture
 
-MACRO_REGEXP = re.compile(r'\{generate_code_example(\(\s*(.+?)\s*\))*\|\s*(.+?)\s*\|\s*(.+?)\s*(\(\s*(.+?)\s*\))?\}')
+MACRO_REGEXP = re.compile(r'\{generate_code_example(\(\s*(.+?)\s*\))*\|\s*(.+?)\s*\|\s*(.+?)\s*(\(\s*(.+)\s*\))?\}')
 CODE_EXAMPLE_REGEX = re.compile(r'\# \{code_example\|\s*(.+?)\s*\}')
 
 PYTHON_CLIENT_CONFIG = """
@@ -138,8 +140,12 @@ class APICodeExamplesPreprocessor(Preprocessor):
     def render_fixture(self, function: str, name: Optional[str]=None) -> List[str]:
         fixture = []
 
-        if name:
+        # We assume that if the function we're rendering starts with a slash
-            fixture_dict = zerver.lib.api_test_helpers.FIXTURES[function][name]
+        # it's a path in the endpoint and therefore it uses the new OpenAPI
+        # format.
+        if function.startswith('/'):
+            path, method = function.rsplit(':', 1)
+            fixture_dict = get_openapi_fixture(path, method, name)
         else:
             fixture_dict = zerver.lib.api_test_helpers.FIXTURES[function]
 

zerver/lib/openapi.py

@@ -0,0 +1,26 @@
+# Set of helper functions to manipulate the OpenAPI files that define our REST
+# API's specification.
+import os
+from typing import Any, Dict, List, Optional
+
+from yamole import YamoleParser
+
+OPENAPI_SPEC_PATH = os.path.abspath(os.path.join(
+    os.path.dirname(__file__),
+    '../openapi/zulip.yaml'))
+
+with open(OPENAPI_SPEC_PATH) as file:
+    yaml_parser = YamoleParser(file)
+
+OPENAPI_SPEC = yaml_parser.data
+
+
+def get_openapi_fixture(endpoint: str, method: str,
+                        response: Optional[str]='200') -> Dict[str, Any]:
+    return (OPENAPI_SPEC['paths'][endpoint][method.lower()]['responses']
+            [response]['content']['application/json']['schema']
+            ['example'])
+
+def get_openapi_parameters(endpoint: str,
+                           method: str) -> List[Dict[str, Any]]:
+    return (OPENAPI_SPEC['paths'][endpoint][method]['parameters'])

zerver/openapi/zulip.yaml

@@ -60,7 +60,8 @@ paths:
         example: change_all
       - name: content
         in: query
-        description: Message's new body.
+        description: |
+          The content of the message. Maximum message size of 10000 bytes.
         schema:
           type: string
         example: Hello
@@ -68,7 +69,11 @@ paths:
       - basicAuth: []
       responses:
         '200':
-          $ref: '#/components/responses/SimpleSuccess'
+          description: Success
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/JsonSuccess'
         '400':
           description: Bad request.
           content:
@@ -84,6 +89,12 @@ paths:
                       - The time limit for editing this message has past
                       - Nothing to change
                       - Topic can't be empty
+                - example:
+                    {
+                      "code": "BAD_REQUEST",
+                      "msg": "You don't have permission to edit this message",
+                      "result": "error"
+                    }
 components:
   #######################
   # Security definitions
@@ -96,13 +107,13 @@ components:
         Basic authentication, with the user's email as the username, and the
         API key as the password. The API key can be fetched using the
         `/fetch_api_key` or `/dev_fetch_api_key` endpoints.
+
   schemas:
     JsonResponse:
       type: object
       properties:
         result:
           type: string
-
     JsonSuccess:
       allOf:
       - $ref: '#/components/schemas/JsonResponse'
@@ -115,7 +126,11 @@ components:
             - success
           msg:
             type: string
-
+      - example:
+          {
+            "msg": "",
+            "result": "success"
+          }
     JsonError:
       allOf:
       - $ref: '#/components/schemas/JsonResponse'

zerver/templatetags/app_filters.py

@@ -96,8 +96,7 @@ def render_markdown_path(markdown_file_path: str, context: Optional[Dict[Any, An
             ),
             zerver.lib.bugdown.fenced_code.makeExtension(),
             zerver.lib.bugdown.api_arguments_table_generator.makeExtension(
-                base_path='templates/zerver/api/',
+                base_path='templates/zerver/api/'),
-                openapi_path='zerver/openapi'),
             zerver.lib.bugdown.api_code_examples.makeExtension(),
             zerver.lib.bugdown.help_settings_links.makeExtension(),
         ]