test-backend: Use a fixed whitelist of shallow-tested templates.

This enforces the requirement that all templates not on this list are
tested.  Part of #1677.

tools/test-backend

@@ -65,7 +65,7 @@ if __name__ == "__main__":
     parser.add_option('--no-shallow', dest='no_shallow',
                       action="store_true",
                       default=False,
-                      help="Don't allow shallow testing of templates")
+                      help="Don't allow shallow testing of templates (deprecated)")
     parser.add_option('--verbose', dest='verbose',
                       action="store_true",
                       default=False,
@@ -174,15 +174,11 @@ if __name__ == "__main__":
     templates_not_rendered = test_runner.get_shallow_tested_templates()
     if templates_not_rendered:
         missed_count = len(templates_not_rendered)
-        if options.no_shallow or options.verbose:
-            print("*** Shallow tested templates: {}".format(missed_count))
-
         if options.verbose:
+            print("*** Shallow tested templates: {}".format(missed_count))
             for template in templates_not_rendered:
                 print('--- {}'.format(template))
-
+        failures = True
-        if options.no_shallow:
-            failures = True
 
     if options.coverage:
         cov.stop()

zerver/tests/test_templates.py

@@ -14,6 +14,8 @@ from zerver.lib.test_helpers import get_all_templates
 from zerver.lib.test_classes import (
     ZulipTestCase,
 )
+from zerver.context_processors import common_context
+
 
 class get_form_value(object):
     def __init__(self, value):
@@ -46,7 +48,68 @@ class TemplateTestCase(ZulipTestCase):
         # Just add the templates whose context has a conflict with other
         # templates' context in `defer`.
         defer = ['analytics/activity.html']
-        skip = defer + ['tests/test_markdown.html', 'zerver/terms.html']
+        email = [
+            'zerver/emails/invitation/invitation_reminder_email.html',
+            'zerver/emails/invitation/invitation_reminder_email.subject',
+            'zerver/emails/invitation/invitation_reminder_email.text',
+        ]
+        logged_out = [
+            '404.html',
+            '500.html',
+            'confirmation/confirm.html',
+            'confirmation/confirm_mituser.html',
+            'zerver/reset_confirm.html',
+            'zerver/reset_done.html',
+            'zerver/reset_emailed.html',
+            'zerver/reset.html',
+            'zerver/unsubscribe_link_error.html',
+            'zerver/portico.html',
+            'zerver/portico_signup.html',
+            'zerver/register.html',
+        ]
+        logged_in = [
+            'zerver/home.html',
+            'zerver/invite_user.html',
+            'zerver/keyboard_shortcuts.html',
+            'zerver/left-sidebar.html',
+            'zerver/logout.html',
+            'zerver/markdown_help.html',
+            'zerver/navbar.html',
+            'zerver/right-sidebar.html',
+            'zerver/search_operators.html',
+            'zerver/stream_creation_prompt.html',
+            'zerver/subscriptions.html',
+            'zerver/tutorial_finale.html',
+        ]
+        unusual = [
+            'confirmation/mituser_confirmation_email_body.txt',
+            'confirmation/mituser_confirmation_email_subject.txt',
+            'confirmation/mituser_invite_email_body.txt',
+            'confirmation/mituser_invite_email_subject.txt',
+            'corporate/mit.html',
+            'corporate/privacy.html',
+            'corporate/terms-enterprise.html',
+            'corporate/zephyr.html',
+            'corporate/zephyr-mirror.html',
+            'pipeline/css.jinja',
+            'pipeline/inline_js.jinja',
+            'pipeline/js.jinja',
+            'zilencer/enterprise_tos_accept_body.txt',
+            'zerver/zulipchat_migration_tos.html',
+            'zilencer/enterprise_tos_accept_body.txt',
+            'zerver/help/index.md',
+            'zerver/help/missing.md',
+            'zerver/closed_realm.html',
+            'zerver/topic_is_muted.html',
+            'zerver/bankruptcy.html',
+            'zerver/image-overlay.html',
+            'zerver/invalid_realm.html',
+            'zerver/compose.html',
+            'zerver/debug.html',
+            'zerver/base.html',
+            'zerver/api_content.json',
+        ]
+        skip = defer + email + logged_out + logged_in + unusual + ['tests/test_markdown.html', 'zerver/terms.html']
         templates = [t for t in get_all_templates() if t not in skip]
         self.render_templates(templates, self.get_context())
 
@@ -55,9 +118,9 @@ class TemplateTestCase(ZulipTestCase):
         self.render_templates(defer, self.get_context(**update))
 
     def render_templates(self, templates, context):
-        # type: (Iterable[Template], Dict[str, Any]) -> None
+        # type: (Iterable[str], Dict[str, Any]) -> None
-        for template in templates:
+        for template_name in templates:
-            template = get_template(template)
+            template = get_template(template_name)
             try:
                 template.render(context)
             except Exception: