Add Django and html singleton tags support to pretty print.

In this commit we are modifying pretty print tool to support
Django and html singleton tags. For Addition of html singleton
tags template parser was modified to emit psudeo
html singleton end tags to accompany html singleton tags and
token class was updated to have line_span field.

tools/lib/pretty_print.py

@@ -38,39 +38,50 @@ def pretty_print_html(html, num_spaces=4):
     # we proceed, we will push/pop info dictionaries on/off a stack.
     for token in tokens:
 
-        if token.kind in ('html_start', 'handlebars_start'):
+        if token.kind in ('html_start', 'handlebars_start',
+                          'html_singleton', 'django_start'):
             # An HTML start tag should only cause a new indent if we
             # are on a new line.
-            is_block = token.line > stack[-1]['line']
+            if token.tag not in ('extends', 'include', 'else', 'elif'):
+                is_block = token.line > stack[-1]['line']
 
-            if is_block:
+                if is_block:
-                if token.kind == 'handlebars_start' and stack[-1]['token_kind'] == 'handlebars_start':
+                    if (token.kind == 'handlebars_start' and
-                    info = stack.pop()
+                            stack[-1]['token_kind'] == 'handlebars_start' and
-                    info['depth'] = info['depth'] + 1
+                            not stack[-1]['indenting']):
-                    stack.append(info)
+                        info = stack.pop()
-                new_depth = stack[-1]['depth'] + 1
+                        info['depth'] = info['depth'] + 1
-                extra_indent = stack[-1]['extra_indent']
+                        info['indenting'] = True
-                line = lines[token.line - 1]
+                        stack.append(info)
-                adjustment = len(line)-len(line.lstrip()) + 1
+                    new_depth = stack[-1]['depth'] + 1
-                offset = (1 + extra_indent + new_depth * num_spaces) - adjustment
+                    extra_indent = stack[-1]['extra_indent']
-                info = dict(
+                    line = lines[token.line - 1]
-                    block=True,
+                    adjustment = len(line)-len(line.lstrip()) + 1
-                    depth=new_depth,
+                    offset = (1 + extra_indent + new_depth * num_spaces) - adjustment
-                    actual_depth=new_depth,
+                    info = dict(
-                    line=token.line,
+                        block=True,
-                    token_kind=token.kind,
+                        depth=new_depth,
-                    offset=offset,
+                        actual_depth=new_depth,
-                    extra_indent=token.col - adjustment + extra_indent
+                        line=token.line,
-                )
+                        token_kind=token.kind,
-                if token.kind == 'handlebars_start':
+                        offset=offset,
-                    info.update(dict(depth=new_depth - 1))
+                        extra_indent=token.col - adjustment + extra_indent,
-            else:
+                        indenting=True
-                info = dict(
+                    )
-                    block=False,
+                    if token.kind == 'handlebars_start':
-                    line=token.line
+                        info.update(dict(depth=new_depth - 1, indenting=False))
-                )
+                else:
-            stack.append(info)
+                    info = dict(
-        elif token.kind in ('html_end', 'handlebars_end'):
+                        block=False,
+                        depth=stack[-1]['depth'],
+                        actual_depth=stack[-1]['depth'],
+                        line=token.line,
+                        token_kind=token.kind,
+                        extra_indent=stack[-1]['extra_indent']
+                    )
+                stack.append(info)
+        elif token.kind in ('html_end', 'handlebars_end',
+                            'html_singleton_end', 'django_end'):
             info = stack.pop()
             if info['block']:
                 # We are at the end of an indentation block.  We
@@ -81,14 +92,16 @@ def pretty_print_html(html, num_spaces=4):
                 end_line = token.line
                 offsets[start_line] = info['offset']
                 offsets[end_line] = info['offset']
-                if token.tag != 'pre':
+                if token.tag != 'pre' and token.kind != 'html_singleton_end' and token.tag != 'script':
                     for line_num in range(start_line + 1, end_line):
                         # Be careful not to override offsets that happened
                         # deeper in the HTML within our block.
                         if line_num not in offsets:
                             line = lines[line_num - 1]
                             new_depth = info['depth'] + 1
-                            if line.lstrip().startswith('{{else}}'):
+                            if (line.lstrip().startswith('{{else}}') or
+                                    line.lstrip().startswith('{% else %}') or
+                                    line.lstrip().startswith('{% elif')):
                                 new_depth = info['actual_depth']
                             extra_indent = info['extra_indent']
                             adjustment = len(line)-len(line.lstrip()) + 1

tools/lib/template_parser.py

@@ -27,13 +27,14 @@ class TokenizerState(object):
         self.col = 1
 
 class Token(object):
-    def __init__(self, kind, s, tag, line, col):
+    def __init__(self, kind, s, tag, line, col, line_span):
-        # type: (str, str, str, int, int) -> None
+        # type: (str, str, str, int, int, int) -> None
         self.kind = kind
         self.s = s
         self.tag = tag
         self.line = line
         self.col = col
+        self.line_span = line_span
 
 def tokenize(text):
     # type: (str) -> List[Token]
@@ -131,15 +132,30 @@ def tokenize(text):
                                           (e.message, state.line, state.col,
                                            e.line_content))
 
+        line_span = len(s.split('\n'))
         token = Token(
             kind=kind,
             s=s,
             tag=tag,
             line=state.line,
             col=state.col,
+            line_span=line_span
         )
         tokens.append(token)
         advance(len(s))
+        if kind == 'html_singleton':
+            # Here we insert a Pseudo html_singleton_end tag so as to have
+            # ease of detection of end of singleton html tags which might be
+            # needed in some cases as with our html pretty printer.
+            token = Token(
+                kind='html_singleton_end',
+                s='</' + tag + '>',
+                tag=tag,
+                line=state.line,
+                col=state.col,
+                line_span=1
+            )
+            tokens.append(token)
 
     return tokens
 

tools/tests/test_pretty_print.py

@@ -188,6 +188,48 @@ GOOD_HTML6 = """
     <p> <strong> <span class = "whatever">foobar </span> </strong></p>
 </div>
 """
+
+BAD_HTML7 = """
+<div class="foobar">
+<input type="foobar" name="temp" value="{{dyn_name}}"
+       {{#unless invite_only}}checked="checked"{{/unless}} /> {{dyn_name}}
+{{#if invite_only}}<i class="icon-vector-lock"></i>{{/if}}
+</div>
+"""
+
+GOOD_HTML7 = """
+<div class="foobar">
+    <input type="foobar" name="temp" value="{{dyn_name}}"
+           {{#unless invite_only}}checked="checked"{{/unless}} /> {{dyn_name}}
+    {{#if invite_only}}<i class="icon-vector-lock"></i>{{/if}}
+</div>
+"""
+
+BAD_HTML8 = """
+{{#each test}}
+  {{#with this}}
+  {{#if foobar}}
+    <div class="anything">{{{test}}}</div>
+  {{/if}}
+  {{#if foobar2}}
+  {{partial "teststuff"}}
+  {{/if}}
+  {{/with}}
+{{/each}}
+"""
+
+GOOD_HTML8 = """
+{{#each test}}
+    {{#with this}}
+        {{#if foobar}}
+        <div class="anything">{{{test}}}</div>
+        {{/if}}
+        {{#if foobar2}}
+        {{partial "teststuff"}}
+        {{/if}}
+    {{/with}}
+{{/each}}
+"""
 class TestPrettyPrinter(unittest.TestCase):
     def compare(self, a, b):
         # type: (str, str) -> None
@@ -203,3 +245,5 @@ class TestPrettyPrinter(unittest.TestCase):
         self.compare(pretty_print_html(BAD_HTML4), GOOD_HTML4)
         self.compare(pretty_print_html(BAD_HTML5), GOOD_HTML5)
         self.compare(pretty_print_html(BAD_HTML6), GOOD_HTML6)
+        self.compare(pretty_print_html(BAD_HTML7), GOOD_HTML7)
+        self.compare(pretty_print_html(BAD_HTML8), GOOD_HTML8)