Add tests for template_parser.tokenize.

tools/tests/test_template_parser.py

@@ -7,6 +7,7 @@ import unittest
 try:
     from tools.lib.template_parser import (
         is_django_block_tag,
+        tokenize,
         validate,
     )
 except ImportError:
@@ -32,3 +33,44 @@ class ParserTest(unittest.TestCase):
                 </tr>
             </table>'''
         validate(text=my_html)
+
+    def test_tokenize(self):
+        # type: () -> None
+        tag = '<meta whatever>bla'
+        token = tokenize(tag)[0]
+        self.assertEqual(token.kind, 'html_special')
+
+        tag = '<a>bla'
+        token = tokenize(tag)[0]
+        self.assertEqual(token.kind, 'html_start')
+        self.assertEqual(token.tag, 'a')
+
+        tag = '<br />bla'
+        token = tokenize(tag)[0]
+        self.assertEqual(token.kind, 'html_singleton')
+        self.assertEqual(token.tag, 'br')
+
+        tag = '</a>bla'
+        token = tokenize(tag)[0]
+        self.assertEqual(token.kind, 'html_end')
+        self.assertEqual(token.tag, 'a')
+
+        tag = '{{#with foo}}bla'
+        token = tokenize(tag)[0]
+        self.assertEqual(token.kind, 'handlebars_start')
+        self.assertEqual(token.tag, 'with')
+
+        tag = '{{/with}}bla'
+        token = tokenize(tag)[0]
+        self.assertEqual(token.kind, 'handlebars_end')
+        self.assertEqual(token.tag, 'with')
+
+        tag = '{% if foo %}bla'
+        token = tokenize(tag)[0]
+        self.assertEqual(token.kind, 'django_start')
+        self.assertEqual(token.tag, 'if')
+
+        tag = '{% endif %}bla'
+        token = tokenize(tag)[0]
+        self.assertEqual(token.kind, 'django_end')
+        self.assertEqual(token.tag, 'if')