tests: Allow testing our REST API against the OpenAPI docs.

requirements/common.in

@@ -195,4 +195,4 @@ django-sendfile==0.3.11
 disposable-email-domains==0.0.28
 
 # Needed for parsing YAML with JSON references from the REST API spec files
-yamole==2.1.1
+yamole==2.1.2

requirements/dev.txt

@@ -180,5 +180,5 @@ websocket-client==0.44.0  # via docker
 werkzeug==0.12.2          # via moto
 wrapt==1.10.11            # via aws-xray-sdk
 xmltodict==0.11.0         # via moto
-yamole==2.1.1
+yamole==2.1.2
 zope.interface==4.4.3     # via twisted

requirements/prod.txt

@@ -122,4 +122,4 @@ urllib3==1.22             # via requests
 uwsgi==2.0.17
 virtualenv-clone==0.3.0
 wcwidth==0.1.7            # via prompt-toolkit
-yamole==2.1.1
+yamole==2.1.2

version.py

@@ -8,4 +8,4 @@ ZULIP_VERSION = "1.8.1+git"
 # Typically, adding a dependency only requires a minor version bump, and
 # removing a dependency requires a major version bump.
 
-PROVISION_VERSION = '20.5'
+PROVISION_VERSION = '20.6'

zerver/lib/api_test_helpers.py

@@ -6,7 +6,7 @@ import json
 import os
 
 from zerver.lib import mdiff
-from zerver.lib.openapi import get_openapi_fixture
+from zerver.lib.openapi import validate_against_openapi_schema
 
 if False:
     from zulip import Client
@@ -385,9 +385,7 @@ def update_message(client, message_id):
     result = client.update_message(request)
     # {code_example|end}
 
-    fixture = get_openapi_fixture('/messages/{message_id}', 'patch', '200')
+    validate_against_openapi_schema(result, '/messages/{message_id}', 'patch')
-
-    test_against_fixture(result, fixture)
 
     # test it was actually updated
     url = 'messages/' + str(message_id)

zerver/lib/openapi.py

@@ -14,13 +14,58 @@ with open(OPENAPI_SPEC_PATH) as file:
 
 OPENAPI_SPEC = yaml_parser.data
 
+class SchemaError(Exception):
+    pass
 
 def get_openapi_fixture(endpoint: str, method: str,
                         response: Optional[str]='200') -> Dict[str, Any]:
+    """Fetch a fixture from the full spec object.
+    """
     return (OPENAPI_SPEC['paths'][endpoint][method.lower()]['responses']
             [response]['content']['application/json']['schema']
             ['example'])
 
 def get_openapi_parameters(endpoint: str,
                            method: str) -> List[Dict[str, Any]]:
-    return (OPENAPI_SPEC['paths'][endpoint][method]['parameters'])
+    return (OPENAPI_SPEC['paths'][endpoint][method.lower()]['parameters'])
+
+def validate_against_openapi_schema(content: Dict[str, Any], endpoint: str,
+                                    method: str) -> None:
+    """Compare a "content" dict with the defined schema for a specific method
+    in an endpoint.
+    """
+    schema = (OPENAPI_SPEC['paths'][endpoint][method.lower()]['responses']
+              ['200']['content']['application/json']['schema'])
+
+    for key, value in content.items():
+        # Check that the key is defined in the schema
+        if key not in schema['properties']:
+            raise SchemaError('Extraneous key "{}" in the response\'s'
+                              'content'.format(key))
+
+        # Check that the types match
+        expected_type = to_python_type(schema['properties'][key]['type'])
+        actual_type = type(value)
+        if expected_type is not actual_type:
+            raise SchemaError('Expected type {} for key "{}", but actually '
+                              'got {}'.format(expected_type, key, actual_type))
+
+    # Check that at least all the required keys are present
+    for req_key in schema['required']:
+        if req_key not in content.keys():
+            raise SchemaError('Expected to find the "{}" required key')
+
+def to_python_type(py_type: str) -> type:
+    """Transform an OpenAPI-like type to a Pyton one.
+    https://swagger.io/docs/specification/data-models/data-types
+    """
+    TYPES = {
+        'string': str,
+        'number': float,
+        'integer': int,
+        'boolean': bool,
+        'array': list,
+        'object': dict
+    }
+
+    return TYPES[py_type]

zerver/tests/test_openapi.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+
+from typing import Dict, Any
+
+from zerver.lib.test_classes import ZulipTestCase
+from zerver.lib.openapi import (
+    get_openapi_fixture, get_openapi_parameters,
+    validate_against_openapi_schema, to_python_type, SchemaError
+)
+
+TEST_ENDPOINT = '/messages/{message_id}'
+TEST_METHOD = 'patch'
+TEST_RESPONSE = '400'
+
+
+class OpenAPIToolsTest(ZulipTestCase):
+    """Make sure that the tools we use to handle our OpenAPI specification
+    (located in zerver/lib/openapi.py) work as expected.
+
+    These tools are mostly dedicated to fetching parts of the -already parsed-
+    specification, and comparing them to objects returned by our REST API.
+    """
+    def test_get_openapi_fixture(self) -> None:
+        actual = get_openapi_fixture(TEST_ENDPOINT, TEST_METHOD, TEST_RESPONSE)
+        expected = {
+            'code': 'BAD_REQUEST',
+            'msg': 'You don\'t have permission to edit this message',
+            'result': 'error'
+        }
+        self.assertEqual(actual, expected)
+
+    def test_get_openapi_parameters(self) -> None:
+        actual = get_openapi_parameters(TEST_ENDPOINT, TEST_METHOD)
+        expected_item = {
+            'name': 'message_id',
+            'in': 'path',
+            'description':
+                'The ID of the message that you wish to edit/update.',
+            'example': 42,
+            'required': True,
+            'schema': {'type': 'integer'}
+        }
+        assert(expected_item in actual)
+
+    def test_validate_against_openapi_schema(self) -> None:
+        with self.assertRaises(SchemaError,
+                               msg=('Extraneous key "foo" in '
+                                    'the response\'scontent')):
+            bad_content = {
+                'msg': '',
+                'result': 'success',
+                'foo': 'bar'
+            }  # type: Dict[str, Any]
+            validate_against_openapi_schema(bad_content,
+                                            TEST_ENDPOINT,
+                                            TEST_METHOD)
+
+        with self.assertRaises(SchemaError,
+                               msg=("Expected type <class 'str'> for key "
+                                    "\"msg\", but actually got "
+                                    "<class 'int'>")):
+            bad_content = {
+                'msg': 42,
+                'result': 'success',
+            }
+            validate_against_openapi_schema(bad_content,
+                                            TEST_ENDPOINT,
+                                            TEST_METHOD)
+
+        with self.assertRaises(SchemaError,
+                               msg='Expected to find the "msg" required key'):
+            bad_content = {
+                'result': 'success',
+            }
+            validate_against_openapi_schema(bad_content,
+                                            TEST_ENDPOINT,
+                                            TEST_METHOD)
+
+        # No exceptions should be raised here.
+        good_content = {
+            'msg': '',
+            'result': 'success',
+        }
+        validate_against_openapi_schema(good_content,
+                                        TEST_ENDPOINT,
+                                        TEST_METHOD)
+
+    def test_to_python_type(self) -> None:
+        TYPES = {
+            'string': str,
+            'number': float,
+            'integer': int,
+            'boolean': bool,
+            'array': list,
+            'object': dict
+        }
+
+        for oa_type, py_type in TYPES.items():
+            self.assertEqual(to_python_type(oa_type), py_type)