paulmataruso/zulip
1
0
Fork 0
mirror of https://github.com/zulip/zulip.git synced 2025-11-09 08:26:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

check-templates: Avoid duplicate tokenizing step.

Browse Source 
Now we only tokenize the file once, and we pass
**validated** tokens to the pretty printer.

There are a few reasons for this:

    * It obviously saves a lot of extra computation
      just in terms of tokenization.

    * It allows our validator to add fields
      to the Token objects that help the pretty
      printer.

I also removed/tweaked a lot of legacy tests for
pretty_print.py that were exercising bizarrely
formatted HTML that we now simply ban during the
validation phase.
This commit is contained in:
Steve Howell
2021-12-02 12:19:19 +00:00
committed by Tim Abbott
parent 0decfa8da0
commit c0d72ba236
4 changed files with 68 additions and 236 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
tools/check-templates
View File

@@ -115,10 +115,8 @@ def check_html_templates(templates: Iterable[str], all_dups: bool, fix: bool) ->
sys.exit(1) sys.exit(1)
for fn in templates: for fn in templates:
validate(fn) tokens = validate(fn)
if not validate_indent_html(fn, tokens, fix):
for fn in templates:
if not validate_indent_html(fn, fix):
sys.exit(1) sys.exit(1)
@@ -127,10 +125,8 @@ def check_handlebar_templates(templates: Iterable[str], fix: bool) -> None:
templates = [fn for fn in templates if fn.endswith(".hbs")] templates = [fn for fn in templates if fn.endswith(".hbs")]
for fn in templates: for fn in templates:
validate(fn) tokens = validate(fn)
if not validate_indent_html(fn, tokens, fix):
for fn in templates:
if not validate_indent_html(fn, fix):
sys.exit(1) sys.exit(1)

10
tools/lib/pretty_print.py
View File

@@ -3,7 +3,7 @@ from typing import List, Optional, Set
from zulint.printer import ENDC, GREEN from zulint.printer import ENDC, GREEN
from .template_parser import Token, is_django_block_tag, tokenize from .template_parser import Token, is_django_block_tag
def requires_indent(line: str) -> bool: def requires_indent(line: str) -> bool:
@@ -122,9 +122,7 @@ def get_exempted_lines(tokens: List[Token]) -> Set[int]:
return exempted return exempted
def pretty_print_html(html: str) -> str: def pretty_print_html(html: str, tokens: List[Token]) -> str:
tokens = tokenize(html)
exempted_lines = get_exempted_lines(tokens) exempted_lines = get_exempted_lines(tokens)
tokens.reverse() tokens.reverse()
@@ -207,10 +205,10 @@ def pretty_print_html(html: str) -> str:
return "\n".join(formatted_lines) return "\n".join(formatted_lines)
def validate_indent_html(fn: str, fix: bool) -> bool: def validate_indent_html(fn: str, tokens: List[Token], fix: bool) -> bool:
with open(fn) as f: with open(fn) as f:
html = f.read() html = f.read()
phtml = pretty_print_html(html) phtml = pretty_print_html(html, tokens)
if not html.split("\n") == phtml.split("\n"): if not html.split("\n") == phtml.split("\n"):
if fix: if fix:
print(GREEN + f"Automatically fixing indentation for {fn}" + ENDC) print(GREEN + f"Automatically fixing indentation for {fn}" + ENDC)

4
tools/lib/template_parser.py
View File

@@ -316,7 +316,7 @@ def tag_flavor(token: Token) -> Optional[str]:
raise AssertionError(f"tools programmer neglected to handle {kind} tokens") raise AssertionError(f"tools programmer neglected to handle {kind} tokens")
def validate(fn: Optional[str] = None, text: Optional[str] = None) -> None: def validate(fn: Optional[str] = None, text: Optional[str] = None) -> List[Token]:
assert fn or text assert fn or text
if fn is None: if fn is None:
@@ -445,6 +445,8 @@ def validate(fn: Optional[str] = None, text: Optional[str] = None) -> None:
ensure_matching_indentation(fn, tokens, lines) ensure_matching_indentation(fn, tokens, lines)
return tokens
def ensure_matching_indentation(fn: str, tokens: List[Token], lines: List[str]) -> None: def ensure_matching_indentation(fn: str, tokens: List[Token], lines: List[str]) -> None:
for token in tokens: for token in tokens:

208
tools/tests/test_pretty_print.py
View File

@@ -1,6 +1,7 @@
import unittest import unittest
from tools.lib.pretty_print import pretty_print_html from tools.lib.pretty_print import pretty_print_html
from tools.lib.template_parser import validate
# Note that GOOD_HTML isn't necessarily beautiful HTML. Apart # Note that GOOD_HTML isn't necessarily beautiful HTML. Apart
# from adjusting indentation, we mostly leave things alone to # from adjusting indentation, we mostly leave things alone to
@@ -10,8 +11,6 @@ BAD_HTML = """
<!-- test --> <!-- test -->
<!DOCTYPE html> <!DOCTYPE html>
<html> <html>
<!-- test --> <!-- test -->
<head> <head>
@@ -32,7 +31,8 @@ BAD_HTML = """
</pre> </pre>
<div class = "foo" <div class = "foo"
id = "bar" id = "bar"
role = "whatever">{{ bla }}</div> role = "whatever">{{ bla }}
</div>
</body> </body>
</html> </html>
<!-- test --> <!-- test -->
@@ -42,8 +42,6 @@ GOOD_HTML = """
<!-- test --> <!-- test -->
<!DOCTYPE html> <!DOCTYPE html>
<html> <html>
<!-- test --> <!-- test -->
<head> <head>
@@ -64,7 +62,8 @@ GOOD_HTML = """
</pre> </pre>
<div class = "foo" <div class = "foo"
id = "bar" id = "bar"
role = "whatever">{{ bla }}</div> role = "whatever">{{ bla }}
</div>
</body> </body>
</html> </html>
<!-- test --> <!-- test -->
@@ -91,7 +90,7 @@ BAD_HTML2 = """
<body> <body>
{{# foobar area}} {{# foobar area}}
foobarfoobarfoo<b>bar</b> foobarfoobarfoo<b>bar</b>
{{/ foobar area}} {{/ foobar}}
</body> </body>
</html> </html>
""" """
@@ -101,7 +100,7 @@ GOOD_HTML2 = """
<body> <body>
{{# foobar area}} {{# foobar area}}
foobarfoobarfoo<b>bar</b> foobarfoobarfoo<b>bar</b>
{{/ foobar area}} {{/ foobar}}
</body> </body>
</html> </html>
""" """
@@ -280,167 +279,10 @@ GOOD_HTML11 = """
</div> </div>
""" """
BAD_HTML12 = """
<div class="test1">
<pre>
<div class="test2">
foobar
<div class="test2">
</div>
</div>
</pre>
</div>
"""
GOOD_HTML12 = """ def pretty_print(html: str) -> str:
<div class="test1"> tokens = validate(fn=None, text=html)
<pre> return pretty_print_html(html, tokens)
<div class="test2">
foobar
<div class="test2">
</div>
</div>
</pre>
</div>
"""
BAD_HTML13 = """
<div>
{{#if this.code}}
<div>&nbsp:{{this.name}}:</div>
{{else}}
{{#if this.is_realm_emoji}}
<img src="{{this.url}}" class="emoji" />
{{else}}
<br />
{{/if}}
{{/if}}
<div>{{this.count}}</div>
</div>
"""
GOOD_HTML13 = """
<div>
{{#if this.code}}
<div>&nbsp:{{this.name}}:</div>
{{else}}
{{#if this.is_realm_emoji}}
<img src="{{this.url}}" class="emoji" />
{{else}}
<br />
{{/if}}
{{/if}}
<div>{{this.count}}</div>
</div>
"""
BAD_HTML14 = """
<div>
{{#if this.code}}
<pre>Here goes some cool code.</pre>
{{else}}
<div>
content of first div
<div>
content of second div.
</div>
</div>
{{/if}}
</div>
"""
GOOD_HTML14 = """
<div>
{{#if this.code}}
<pre>Here goes some cool code.</pre>
{{else}}
<div>
content of first div
<div>
content of second div.
</div>
</div>
{{/if}}
</div>
"""
BAD_HTML15 = """
<div>
<img alt=":thumbs_up:"
class="emoji"
src="/path/to/png"
title=":thumbs_up:"/>
<img alt=":thumbs_up:"
class="emoji"
src="/path/to/png"
title=":thumbs_up:"/>
<img alt=":thumbs_up:"
title=":thumbs_up:"/>
</div>
"""
GOOD_HTML15 = """
<div>
<img alt=":thumbs_up:"
class="emoji"
src="/path/to/png"
title=":thumbs_up:"/>
<img alt=":thumbs_up:"
class="emoji"
src="/path/to/png"
title=":thumbs_up:"/>
<img alt=":thumbs_up:"
title=":thumbs_up:"/>
</div>
"""
BAD_HTML16 = """
<div>
{{> settings_checkbox
setting_name="realm_name_in_notifications"
is_checked=user_settings.realm_name_in_notifications
label=settings_label.realm_name_in_notifications}}
</div>
"""
GOOD_HTML16 = """
<div>
{{> settings_checkbox
setting_name="realm_name_in_notifications"
is_checked=user_settings.realm_name_in_notifications
label=settings_label.realm_name_in_notifications}}
</div>
"""
BAD_HTML17 = """
<div>
<button type="button"
class="btn btn-primary btn-small">{{t "Yes" }}</button>
<button type="button"
id="confirm_btn"
class="btn btn-primary btn-small">{{t "Yes" }}</button>
<div class = "foo"
id = "bar"
role = "whatever">
{{ bla }}
</div>
</div>
"""
GOOD_HTML17 = """
<div>
<button type="button"
class="btn btn-primary btn-small">{{t "Yes" }}</button>
<button type="button"
id="confirm_btn"
class="btn btn-primary btn-small">{{t "Yes" }}</button>
<div class = "foo"
id = "bar"
role = "whatever">
{{ bla }}
</div>
</div>
"""
class TestPrettyPrinter(unittest.TestCase): class TestPrettyPrinter(unittest.TestCase):
@@ -448,21 +290,15 @@ class TestPrettyPrinter(unittest.TestCase):
self.assertEqual(a.split("\n"), b.split("\n")) self.assertEqual(a.split("\n"), b.split("\n"))
def test_pretty_print(self) -> None: def test_pretty_print(self) -> None:
self.compare(pretty_print_html(GOOD_HTML), GOOD_HTML) self.compare(pretty_print(GOOD_HTML), GOOD_HTML)
self.compare(pretty_print_html(BAD_HTML), GOOD_HTML) self.compare(pretty_print(BAD_HTML), GOOD_HTML)
self.compare(pretty_print_html(BAD_HTML1), GOOD_HTML1) self.compare(pretty_print(BAD_HTML1), GOOD_HTML1)
self.compare(pretty_print_html(BAD_HTML2), GOOD_HTML2) self.compare(pretty_print(BAD_HTML2), GOOD_HTML2)
self.compare(pretty_print_html(BAD_HTML4), GOOD_HTML4) self.compare(pretty_print(BAD_HTML4), GOOD_HTML4)
self.compare(pretty_print_html(BAD_HTML5), GOOD_HTML5) self.compare(pretty_print(BAD_HTML5), GOOD_HTML5)
self.compare(pretty_print_html(BAD_HTML6), GOOD_HTML6) self.compare(pretty_print(BAD_HTML6), GOOD_HTML6)
self.compare(pretty_print_html(BAD_HTML7), GOOD_HTML7) self.compare(pretty_print(BAD_HTML7), GOOD_HTML7)
self.compare(pretty_print_html(BAD_HTML8), GOOD_HTML8) self.compare(pretty_print(BAD_HTML8), GOOD_HTML8)
self.compare(pretty_print_html(BAD_HTML9), GOOD_HTML9) self.compare(pretty_print(BAD_HTML9), GOOD_HTML9)
self.compare(pretty_print_html(BAD_HTML10), GOOD_HTML10) self.compare(pretty_print(BAD_HTML10), GOOD_HTML10)
self.compare(pretty_print_html(BAD_HTML11), GOOD_HTML11) self.compare(pretty_print(BAD_HTML11), GOOD_HTML11)
self.compare(pretty_print_html(BAD_HTML12), GOOD_HTML12)
self.compare(pretty_print_html(BAD_HTML13), GOOD_HTML13)
self.compare(pretty_print_html(BAD_HTML14), GOOD_HTML14)
self.compare(pretty_print_html(BAD_HTML15), GOOD_HTML15)
self.compare(pretty_print_html(BAD_HTML16), GOOD_HTML16)
self.compare(pretty_print_html(BAD_HTML17), GOOD_HTML17)