mirror of
https://github.com/zulip/zulip.git
synced 2025-11-15 19:31:58 +00:00
7bc3fa034e
Make sure that the periods are part of the translatable strings. Tweaked by tabbott to properly scope the rules.
719 lines
30 KiB
Python
Executable File
719 lines
30 KiB
Python
Executable File
#!/usr/bin/env python
|
|
from __future__ import print_function
|
|
from __future__ import absolute_import
|
|
from contextlib import contextmanager
|
|
import logging
|
|
import os
|
|
import re
|
|
import sys
|
|
import optparse
|
|
import subprocess
|
|
import traceback
|
|
|
|
# check for the venv
|
|
from lib import sanity_check
|
|
sanity_check.check_venv(__file__)
|
|
|
|
import lister
|
|
from typing import cast, Any, Callable, Dict, Iterator, List, Optional, Tuple
|
|
|
|
# Exclude some directories and files from lint checking
|
|
EXCLUDED_FILES = [
|
|
# Third-party code that doesn't match our style
|
|
"api/integrations/perforce/git_p4.py",
|
|
"puppet/apt/.forge-release",
|
|
"puppet/apt/README.md",
|
|
"static/third",
|
|
# Transifex syncs translation.json files without trailing
|
|
# newlines; there's nothing other than trailing newlines we'd be
|
|
# checking for in these anyway.
|
|
"static/locale",
|
|
]
|
|
|
|
@contextmanager
|
|
def bright_red_output():
|
|
# type: () -> Iterator[None]
|
|
# Make the lint output bright red
|
|
sys.stdout.write('\x1B[1;31m')
|
|
sys.stdout.flush()
|
|
try:
|
|
yield
|
|
finally:
|
|
# Restore normal terminal colors
|
|
sys.stdout.write('\x1B[0m')
|
|
|
|
|
|
def check_pyflakes(options, by_lang):
|
|
# type: (Any, Dict[str, List[str]]) -> bool
|
|
if not by_lang['py']:
|
|
return False
|
|
failed = False
|
|
pyflakes = subprocess.Popen(['pyflakes'] + by_lang['py'],
|
|
stdout = subprocess.PIPE,
|
|
stderr = subprocess.PIPE,
|
|
universal_newlines = True)
|
|
|
|
# pyflakes writes some output (like syntax errors) to stderr. :/
|
|
for pipe in (pyflakes.stdout, pyflakes.stderr):
|
|
for ln in pipe:
|
|
if options.full or not (
|
|
('imported but unused' in ln or
|
|
'redefinition of unused' in ln or
|
|
# Our ipython startup pythonrc file intentionally imports *
|
|
("scripts/lib/pythonrc.py" in ln and
|
|
" import *' used; unable to detect undefined names" in ln) or
|
|
# Special dev_settings.py import
|
|
"from .prod_settings_template import *" in ln or
|
|
("settings.py" in ln and
|
|
("settings import *' used; unable to detect undefined names" in ln or
|
|
"may be undefined, or defined from star imports" in ln)) or
|
|
("zerver/tornado/ioloop_logging.py" in ln and
|
|
"redefinition of function 'instrument_tornado_ioloop'" in ln) or
|
|
("zephyr_mirror_backend.py:" in ln and
|
|
"redefinition of unused 'simplejson' from line" in ln))):
|
|
sys.stdout.write(ln)
|
|
failed = True
|
|
return failed
|
|
|
|
|
|
def check_pep8(files):
|
|
# type: (List[str]) -> bool
|
|
failed = False
|
|
ignored_rules = [
|
|
# Each of these rules are ignored for the explained reason.
|
|
|
|
# "multiple spaces before operator"
|
|
# There are several typos here, but also several instances that are
|
|
# being used for alignment in dict keys/values using the `dict`
|
|
# constructor. We could fix the alignment cases by switching to the `{}`
|
|
# constructor, but it makes fixing this rule a little less
|
|
# straightforward.
|
|
'E221',
|
|
|
|
# 'missing whitespace around arithmetic operator'
|
|
# This should possibly be cleaned up, though changing some of
|
|
# these may make the code less readable.
|
|
'E226',
|
|
|
|
# "unexpected spaces around keyword / parameter equals"
|
|
# Many of these should be fixed, but many are also being used for
|
|
# alignment/making the code easier to read.
|
|
'E251',
|
|
|
|
# 'at least two spaces before inline comment'
|
|
# This we should probably clean up, but has 800+ instances in
|
|
# conflict (many of them type comments), so fixing this will
|
|
# be a lot of churn.
|
|
'E261',
|
|
|
|
# "block comment should start with '#'"
|
|
# These serve to show which lines should be changed in files customized
|
|
# by the user. We could probably resolve one of E265 or E266 by
|
|
# standardizing on a single style for lines that the user might want to
|
|
# change.
|
|
'E265',
|
|
|
|
# "too many leading '#' for block comment"
|
|
# Most of these are there for valid reasons.
|
|
'E266',
|
|
|
|
# "expected 2 blank lines after class or function definition"
|
|
# Zulip only uses 1 blank line after class/function
|
|
# definitions; the PEP-8 recommendation results in super sparse code.
|
|
'E302', 'E305',
|
|
|
|
# "module level import not at top of file"
|
|
# Most of these are there for valid reasons, though there might be a
|
|
# few that could be eliminated.
|
|
'E402',
|
|
|
|
# "line too long"
|
|
# Zulip is a bit less strict about line length, and has its
|
|
# own check for this (see max_length)
|
|
'E501',
|
|
|
|
# "do not assign a lambda expression, use a def"
|
|
# Fixing these would probably reduce readability in most cases.
|
|
'E731',
|
|
]
|
|
if len(files) == 0:
|
|
return False
|
|
pep8 = subprocess.Popen(
|
|
['pycodestyle'] + files + ['--ignore={rules}'.format(rules=','.join(ignored_rules))],
|
|
stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True)
|
|
for pipe in (pep8.stdout, pep8.stderr):
|
|
for ln in pipe:
|
|
sys.stdout.write(ln)
|
|
failed = True
|
|
return failed
|
|
|
|
|
|
def run_parallel(lint_functions):
|
|
# type: (Dict[str, Callable[[], int]]) -> bool
|
|
pids = []
|
|
for name, func in lint_functions.items():
|
|
pid = os.fork()
|
|
if pid == 0:
|
|
logging.info("start " + name)
|
|
result = func()
|
|
logging.info("finish " + name)
|
|
sys.stdout.flush()
|
|
sys.stderr.flush()
|
|
os._exit(result)
|
|
pids.append(pid)
|
|
failed = False
|
|
|
|
for pid in pids:
|
|
(_, status) = os.waitpid(pid, 0)
|
|
if status != 0:
|
|
failed = True
|
|
return failed
|
|
|
|
def build_custom_checkers(by_lang):
|
|
# type: (Dict[str, List[str]]) -> Tuple[Callable[[], bool], Callable[[], bool]]
|
|
RuleList = List[Dict[str, Any]]
|
|
|
|
def custom_check_file(fn, rules, skip_rules=None, max_length=None):
|
|
# type: (str, RuleList, Optional[Any], Optional[int]) -> bool
|
|
failed = False
|
|
lineFlag = False
|
|
for i, line in enumerate(open(fn)):
|
|
line_newline_stripped = line.strip('\n')
|
|
line_fully_stripped = line_newline_stripped.strip()
|
|
skip = False
|
|
lineFlag = True
|
|
for rule in skip_rules or []:
|
|
if re.match(rule, line):
|
|
skip = True
|
|
if skip:
|
|
continue
|
|
for rule in rules:
|
|
exclude_list = rule.get('exclude', set())
|
|
if fn in exclude_list or os.path.dirname(fn) in exclude_list:
|
|
continue
|
|
exclude_list = rule.get('exclude_line', set())
|
|
if (fn, line_fully_stripped) in exclude_list:
|
|
continue
|
|
if rule.get("include_only"):
|
|
found = False
|
|
for item in rule.get("include_only", set()):
|
|
if item in fn:
|
|
found = True
|
|
if not found:
|
|
continue
|
|
try:
|
|
line_to_check = line_fully_stripped
|
|
if rule.get('strip') is not None:
|
|
if rule['strip'] == '\n':
|
|
line_to_check = line_newline_stripped
|
|
else:
|
|
raise Exception("Invalid strip rule")
|
|
if re.search(rule['pattern'], line_to_check):
|
|
sys.stdout.write(rule['description'] + ' at %s line %s:\n' % (fn, i+1))
|
|
print(line)
|
|
failed = True
|
|
except Exception:
|
|
print("Exception with %s at %s line %s" % (rule['pattern'], fn, i+1))
|
|
traceback.print_exc()
|
|
if isinstance(line, bytes):
|
|
line_length = len(line.decode("utf-8"))
|
|
else:
|
|
line_length = len(line)
|
|
if (max_length is not None and line_length > max_length and
|
|
'# type' not in line and 'test' not in fn and 'example' not in fn and
|
|
not re.match("\[[ A-Za-z0-9_:,&()-]*\]: http.*", line) and
|
|
"#ignorelongline" not in line and 'migrations' not in fn):
|
|
print("Line too long (%s) at %s line %s: %s" % (len(line), fn, i+1, line_newline_stripped))
|
|
failed = True
|
|
lastLine = line
|
|
if lineFlag and '\n' not in lastLine:
|
|
print("No newline at the end of file. Fix with `sed -i '$a\\' %s`" % (fn,))
|
|
failed = True
|
|
return failed
|
|
|
|
whitespace_rules = [
|
|
# This linter should be first since bash_rules depends on it.
|
|
{'pattern': '\s+$',
|
|
'strip': '\n',
|
|
'description': 'Fix trailing whitespace'},
|
|
{'pattern': '\t',
|
|
'strip': '\n',
|
|
'exclude': set(['zerver/lib/bugdown/codehilite.py',
|
|
'tools/travis/success-http-headers.txt']),
|
|
'description': 'Fix tab-based whitespace'},
|
|
] # type: RuleList
|
|
markdown_whitespace_rules = list([rule for rule in whitespace_rules if rule['pattern'] != '\s+$']) + [
|
|
# Two spaces trailing a line with other content is okay--it's a markdown line break.
|
|
# This rule finds one space trailing a non-space, three or more trailing spaces, and
|
|
# spaces on an empty line.
|
|
{'pattern': '((?<!\s)\s$)|(\s\s\s+$)|(^\s+$)',
|
|
'strip': '\n',
|
|
'description': 'Fix trailing whitespace'},
|
|
{'pattern': '^#+[A-Za-z0-9]',
|
|
'strip': '\n',
|
|
'description': 'Missing space after # in heading'},
|
|
] # type: RuleList
|
|
js_rules = cast(RuleList, [
|
|
{'pattern': '[^_]function\(',
|
|
'description': 'The keyword "function" should be followed by a space'},
|
|
{'pattern': '.*blueslip.warning\(.*',
|
|
'description': 'The module blueslip has no function warning, try using blueslip.warn'},
|
|
{'pattern': '[)]{$',
|
|
'description': 'Missing space between ) and {'},
|
|
{'pattern': '["\']json/',
|
|
'description': 'Relative URL for JSON route not supported by i18n'},
|
|
# This rule is constructed with + to avoid triggering on itself
|
|
{'pattern': " =" + '[^ =>~"]',
|
|
'description': 'Missing whitespace after "="'},
|
|
{'pattern': '^[ ]*//[A-Za-z0-9]',
|
|
'description': 'Missing space after // in comment'},
|
|
{'pattern': 'if[(]',
|
|
'description': 'Missing space between if and ('},
|
|
{'pattern': 'else{$',
|
|
'description': 'Missing space between else and {'},
|
|
{'pattern': '^else {$',
|
|
'description': 'Write JS else statements on same line as }'},
|
|
{'pattern': '^else if',
|
|
'description': 'Write JS else statements on same line as }'},
|
|
{'pattern': 'console[.][a-z]',
|
|
'exclude': set(['static/js/blueslip.js',
|
|
'frontend_tests/zjsunit',
|
|
'frontend_tests/casper_lib/common.js',
|
|
'frontend_tests/node_tests',
|
|
'static/js/debug.js']),
|
|
'description': 'console.log and similar should not be used in webapp'},
|
|
{'pattern': '[.]text\(["\'][a-zA-Z]',
|
|
'description': 'Strings passed to $().text should be wrapped in i18n.t() for internationalization'},
|
|
{'pattern': 'compose_error\(["\']',
|
|
'exclude': set(['tools/lint-all']),
|
|
'description': 'Argument to compose_error should be a literal string enclosed '
|
|
'by i18n.t()'},
|
|
{'pattern': 'report_success\(["\']',
|
|
'exclude': set(['tools/lint-all']),
|
|
'description': 'Argument to report_success should be a literal string enclosed '
|
|
'by i18n.t()'},
|
|
{'pattern': 'report_error\(["\']',
|
|
'exclude': set(['tools/lint-all']),
|
|
'description': 'Argument to report_error should be a literal string enclosed '
|
|
'by i18n.t()'},
|
|
]) + whitespace_rules
|
|
python_rules = cast(RuleList, [
|
|
{'pattern': '^(?!#)@login_required',
|
|
'description': '@login_required is unsupported; use @zulip_login_required'},
|
|
{'pattern': '".*"%\([a-z_].*\)?$',
|
|
'description': 'Missing space around "%"'},
|
|
{'pattern': "'.*'%\([a-z_].*\)?$",
|
|
'exclude': set(['tools/lint-all',
|
|
'analytics/lib/counts.py',
|
|
'analytics/tests/test_counts.py',
|
|
]),
|
|
'exclude_line': set([
|
|
('zerver/views/users.py',
|
|
"return json_error(_(\"Email '%(email)s' does not belong to domain '%(domain)s'\") %"),
|
|
('zproject/settings.py',
|
|
"'format': '%(asctime)s %(levelname)-8s %(message)s'"),
|
|
]),
|
|
'description': 'Missing space around "%"'},
|
|
# This rule is constructed with + to avoid triggering on itself
|
|
{'pattern': " =" + '[^ =>~"]',
|
|
'description': 'Missing whitespace after "="'},
|
|
{'pattern': '":\w[^"]*$',
|
|
'description': 'Missing whitespace after ":"'},
|
|
{'pattern': "':\w[^']*$",
|
|
'description': 'Missing whitespace after ":"'},
|
|
{'pattern': "^\s+[#]\w",
|
|
'strip': '\n',
|
|
'description': 'Missing whitespace after "#"'},
|
|
{'pattern': "assertEquals[(]",
|
|
'description': 'Use assertEqual, not assertEquals (which is deprecated).'},
|
|
{'pattern': "== None",
|
|
'exclude': 'tools/lint-all',
|
|
'description': 'Use `is None` to check whether something is None'},
|
|
{'pattern': "type:[(]",
|
|
'description': 'Missing whitespace after ":" in type annotation'},
|
|
{'pattern': "# type [(]",
|
|
'description': 'Missing : after type in type annotation'},
|
|
{'pattern': "#type",
|
|
'exclude': 'tools/lint-all',
|
|
'description': 'Missing whitespace after "#" in type annotation'},
|
|
{'pattern': 'if[(]',
|
|
'description': 'Missing space between if and ('},
|
|
{'pattern': ", [)]",
|
|
'description': 'Unnecessary whitespace between "," and ")"'},
|
|
{'pattern': "% [(]",
|
|
'description': 'Unnecessary whitespace between "%" and "("'},
|
|
# This next check could have false positives, but it seems pretty
|
|
# rare; if we find any, they can be added to the exclude list for
|
|
# this rule.
|
|
{'pattern': ' % [a-zA-Z0-9_.]*\)?$',
|
|
'exclude_line': set([
|
|
('tools/tests/test_template_parser.py', '{% foo'),
|
|
]),
|
|
'description': 'Used % comprehension without a tuple'},
|
|
{'pattern': '.*%s.* % \([a-zA-Z0-9_.]*\)$',
|
|
'description': 'Used % comprehension without a tuple'},
|
|
{'pattern': 'django.utils.translation',
|
|
'include_only': set(['test']),
|
|
'description': 'Test strings should not be tagged for translationx'},
|
|
{'pattern': 'json_success\({}\)',
|
|
'description': 'Use json_success() to return nothing'},
|
|
# To avoid json_error(_variable) and json_error(_(variable))
|
|
{'pattern': '\Wjson_error\(_\(?\w+\)',
|
|
'exclude': set(['tools/lint-all', 'zerver/tests']),
|
|
'description': 'Argument to json_error should be a literal string enclosed by _()'},
|
|
{'pattern': '\Wjson_error\([\'"].+[),]$',
|
|
'exclude': set(['tools/lint-all', 'zerver/tests']),
|
|
'exclude_line': set([
|
|
# We don't want this string tagged for translation.
|
|
('zerver/views/compatibility.py', 'return json_error("Client is too old")'),
|
|
]),
|
|
'description': 'Argument to json_error should a literal string enclosed by _()'},
|
|
# To avoid JsonableError(_variable) and JsonableError(_(variable))
|
|
{'pattern': '\WJsonableError\(_\(?\w.+\)',
|
|
'exclude': set(['tools/lint-all', 'zerver/tests']),
|
|
'description': 'Argument to JsonableError should be a literal string enclosed by _()'},
|
|
{'pattern': '\WJsonableError\(["\'].+\)',
|
|
'exclude': set(['tools/lint-all', 'zerver/tests']),
|
|
'description': 'Argument to JsonableError should be a literal string enclosed by _()'},
|
|
{'pattern': '([a-zA-Z0-9_]+)=REQ\([\'"]\\1[\'"]',
|
|
'description': 'REQ\'s first argument already defaults to parameter name'},
|
|
{'pattern': 'self\.client\.(get|post|patch|put|delete)',
|
|
'exclude': set(['zilencer/tests.py']),
|
|
'description': \
|
|
'''Do not call self.client directly for put/patch/post/get.
|
|
See WRAPPER_COMMENT in test_helpers.py for details.
|
|
'''},
|
|
# Directly fetching Message objects in e.g. views code is often a security bug.
|
|
{'pattern': '[^r][M]essage.objects.get',
|
|
'exclude': set(["zerver/tests", "zerver/worker/queue_processors.py"]),
|
|
'description': 'Please use access_message() to fetch Message objects',
|
|
},
|
|
{'pattern': '[S]tream.objects.get',
|
|
'include_only': set(["zerver/views/"]),
|
|
'description': 'Please use access_stream_by_*() to fetch Stream objects',
|
|
},
|
|
{'pattern': 'get_stream[(]',
|
|
'include_only': set(["zerver/views/", "zerver/lib/actions.py"]),
|
|
# messages.py needs to support accessing invite-only streams
|
|
# that you are no longer subscribed to, so need get_stream.
|
|
'exclude': set(['zerver/views/messages.py']),
|
|
'exclude_line': set([
|
|
# This is a check for whether a stream rename is invalid because it already exists
|
|
('zerver/lib/actions.py', 'existing_deactivated_stream = get_stream(new_name, stream.realm)'),
|
|
# This one in check_message is kinda terrible, since it's
|
|
# how most instances are written, but better to exclude something than nothing
|
|
('zerver/lib/actions.py', 'stream = get_stream(stream_name, realm)'),
|
|
]),
|
|
'description': 'Please use access_stream_by_*() to fetch Stream objects',
|
|
},
|
|
{'pattern': '[S]tream.objects.filter',
|
|
'include_only': set(["zerver/views/"]),
|
|
'description': 'Please use access_stream_by_*() to fetch Stream objects',
|
|
},
|
|
{'pattern': 'from zerver.models import',
|
|
'include_only': set(["/migrations/"]),
|
|
'description': "Don't import models in migrations; see docs/schema-migrations.md for solution",
|
|
},
|
|
{'pattern': 'datetime[.](now|utcnow)',
|
|
'include_only': set(["zerver/", "analytics/"]),
|
|
'description': "Don't use datetime in backend code.\n"
|
|
"See https://zulip.readthedocs.io/en/latest/code-style.html#naive-datetime-objects",
|
|
},
|
|
# This rule might give false positives in virtualenv setup files which should be excluded,
|
|
# and comments which should be rewritten to avoid use of "python2", "python3", etc.
|
|
{'pattern': 'python[23]',
|
|
'exclude': set(['tools/lib/provision.py',
|
|
'tools/setup/setup_venvs.py',
|
|
'scripts/lib/setup_venv.py',
|
|
'tools/lint-all']),
|
|
'description': 'Explicit python invocations should not include a version'}
|
|
]) + whitespace_rules
|
|
bash_rules = [
|
|
{'pattern': '#!.*sh [-xe]',
|
|
'description': 'Fix shebang line with proper call to /usr/bin/env for Bash path, change -x|-e switches'
|
|
' to set -x|set -e'},
|
|
] + whitespace_rules[0:1] # type: RuleList
|
|
css_rules = cast(RuleList, [
|
|
{'pattern': '^[^:]*:\S[^:]*;$',
|
|
'description': "Missing whitespace after : in CSS"},
|
|
{'pattern': '[a-z]{',
|
|
'description': "Missing whitespace before '{' in CSS."},
|
|
{'pattern': 'https://',
|
|
'description': "Zulip CSS should have no dependencies on external resources"},
|
|
{'pattern': '^[ ][ ][a-zA-Z0-9]',
|
|
'description': "Incorrect 2-space indentation in CSS",
|
|
'exclude': set(['static/styles/thirdparty-fonts.css']),
|
|
'strip': '\n'},
|
|
{'pattern': '{\w',
|
|
'description': "Missing whitespace after '{' in CSS (should be newline)."},
|
|
]) + whitespace_rules # type: RuleList
|
|
prose_style_rules = [
|
|
{'pattern': '[^\/\#\-\"]([jJ]avascript)', # exclude usage in hrefs/divs
|
|
'description': "javascript should be spelled JavaScript"},
|
|
{'pattern': '[^\/\-\.\"\'\_\=\>]([gG]ithub)[^\.\-\_\"\<]', # exclude usage in hrefs/divs
|
|
'description': "github should be spelled GitHub"},
|
|
{'pattern': '[oO]rganisation', # exclude usage in hrefs/divs
|
|
'description': "Organization is spelled with a z"},
|
|
{'pattern': '!!! warning',
|
|
'description': "!!! warning is invalid; it's spelled '!!! warn'"},
|
|
] # type: RuleList
|
|
html_rules = whitespace_rules + prose_style_rules + [
|
|
{'pattern': 'placeholder="[^{]',
|
|
'description': "`placeholder` value should be translatable.",
|
|
'exclude_line': [('templates/zerver/register.html', 'placeholder="acme"'),
|
|
('templates/zerver/register.html', 'placeholder="Acme"'),
|
|
('static/templates/settings/realm-domains-modal.handlebars',
|
|
'<td><input type="text" class="new-alias-domain" placeholder="acme.com"></input></td>')],
|
|
'exclude': set(["static/templates/settings/emoji-settings-admin.handlebars",
|
|
"static/templates/settings/realm-filter-settings-admin.handlebars",
|
|
"static/templates/settings/bot-settings.handlebars"])},
|
|
{'pattern': "placeholder='[^{]",
|
|
'description': "`placeholder` value should be translatable."},
|
|
{'pattern': 'script src="http',
|
|
'description': "Don't directly load dependencies from CDNs. See docs/front-end-build-process.md"},
|
|
{'pattern': "title='[^{]",
|
|
'description': "`title` value should be translatable."},
|
|
{'pattern': 'title="[^{\:]',
|
|
'exclude_line': set([
|
|
('templates/zerver/markdown_help.html',
|
|
'<td><img alt=":heart:" class="emoji" src="/static/generated/emoji/images/emoji/heart.png" title=":heart:" /></td>')
|
|
]),
|
|
'description': "`title` value should be translatable."},
|
|
] # type: RuleList
|
|
handlebars_rules = html_rules + [
|
|
{'pattern': "[<]script",
|
|
'description': "Do not use inline <script> tags here; put JavaScript in static/js instead."},
|
|
{'pattern': "{{t '.*' }}[\.\?!]",
|
|
'description': "Period should be part of the translatable string."},
|
|
{'pattern': '{{t ".*" }}[\.\?!]',
|
|
'description': "Period should be part of the translatable string."},
|
|
{'pattern': "{{/tr}}[\.\?!]",
|
|
'description': "Period should be part of the translatable string."},
|
|
]
|
|
jinja2_rules = html_rules + [
|
|
{'pattern': "{% endtrans %}[\.\?!]",
|
|
'description': "Period should be part of the translatable string."},
|
|
{'pattern': "{{ _(.+) }}[\.\?!]",
|
|
'description': "Period should be part of the translatable string."},
|
|
]
|
|
json_rules = [] # type: RuleList # fix newlines at ends of files
|
|
# It is okay that json_rules is empty, because the empty list
|
|
# ensures we'll still check JSON files for whitespace.
|
|
markdown_rules = markdown_whitespace_rules + prose_style_rules
|
|
help_markdown_rules = markdown_rules + [
|
|
{'pattern': '[a-z][.][A-Z]',
|
|
'description': "Likely missing space after end of sentence"},
|
|
{'pattern': '[rR]ealm',
|
|
'description': "Realms are referred to as Organizations in user-facing docs."},
|
|
]
|
|
txt_rules = whitespace_rules
|
|
|
|
def check_custom_checks_py():
|
|
# type: () -> bool
|
|
failed = False
|
|
|
|
for fn in by_lang['py']:
|
|
if custom_check_file(fn, python_rules, max_length=140):
|
|
failed = True
|
|
return failed
|
|
|
|
def check_custom_checks_nonpy():
|
|
# type: () -> bool
|
|
failed = False
|
|
|
|
for fn in by_lang['js']:
|
|
if custom_check_file(fn, js_rules):
|
|
failed = True
|
|
|
|
for fn in by_lang['sh']:
|
|
if custom_check_file(fn, bash_rules):
|
|
failed = True
|
|
|
|
for fn in by_lang['css']:
|
|
if custom_check_file(fn, css_rules):
|
|
failed = True
|
|
|
|
for fn in by_lang['handlebars']:
|
|
if custom_check_file(fn, handlebars_rules):
|
|
failed = True
|
|
|
|
for fn in by_lang['html']:
|
|
if custom_check_file(fn, jinja2_rules):
|
|
failed = True
|
|
|
|
for fn in by_lang['json']:
|
|
if custom_check_file(fn, json_rules):
|
|
failed = True
|
|
|
|
markdown_docs_length_exclude = {
|
|
"contrib_bots/bots/converter/readme.md",
|
|
"docs/bots-guide.md",
|
|
"docs/dev-env-first-time-contributors.md",
|
|
"docs/webhook-walkthrough.md",
|
|
"docs/life-of-a-request.md",
|
|
"docs/logging.md",
|
|
"docs/migration-renumbering.md",
|
|
"docs/readme-symlink.md",
|
|
"README.md",
|
|
}
|
|
for fn in by_lang['md']:
|
|
max_length = None
|
|
if fn not in markdown_docs_length_exclude:
|
|
max_length = 120
|
|
rules = markdown_rules
|
|
if fn.startswith("templates/zerver/help"):
|
|
rules = help_markdown_rules
|
|
if custom_check_file(fn, rules, max_length=max_length):
|
|
failed = True
|
|
|
|
for fn in by_lang['txt'] + by_lang['text']:
|
|
if custom_check_file(fn, txt_rules):
|
|
failed = True
|
|
|
|
for fn in by_lang['yaml']:
|
|
if custom_check_file(fn, txt_rules):
|
|
failed = True
|
|
|
|
return failed
|
|
|
|
return (check_custom_checks_py, check_custom_checks_nonpy)
|
|
|
|
def run():
|
|
# type: () -> None
|
|
parser = optparse.OptionParser()
|
|
parser.add_option('--force', default=False,
|
|
action="store_true",
|
|
help='Run tests despite possible problems.')
|
|
parser.add_option('--full',
|
|
action='store_true',
|
|
help='Check some things we typically ignore')
|
|
parser.add_option('--pep8',
|
|
action='store_true',
|
|
help='Run the pep8 checker')
|
|
parser.add_option('--modified', '-m',
|
|
action='store_true',
|
|
help='Only check modified files')
|
|
parser.add_option('--verbose', '-v',
|
|
action='store_true',
|
|
help='Print verbose timing output')
|
|
(options, args) = parser.parse_args()
|
|
|
|
tools_dir = os.path.dirname(os.path.abspath(__file__))
|
|
root_dir = os.path.dirname(tools_dir)
|
|
sys.path.insert(0, root_dir)
|
|
|
|
from tools.lib.test_script import (
|
|
get_provisioning_status,
|
|
)
|
|
|
|
os.chdir(root_dir)
|
|
|
|
if not options.force:
|
|
ok, msg = get_provisioning_status()
|
|
if not ok:
|
|
print(msg)
|
|
print('If you really know what you are doing, use --force to run anyway.')
|
|
sys.exit(1)
|
|
|
|
by_lang = cast(Dict[str, List[str]],
|
|
lister.list_files(args, modified_only=options.modified,
|
|
ftypes=['py', 'sh', 'js', 'pp', 'css', 'handlebars',
|
|
'html', 'json', 'md', 'txt', 'text', 'yaml'],
|
|
use_shebang=True, group_by_ftype=True, exclude=EXCLUDED_FILES))
|
|
|
|
# Invoke the appropriate lint checker for each language,
|
|
# and also check files for extra whitespace.
|
|
|
|
logging.basicConfig(format="%(asctime)s %(message)s")
|
|
logger = logging.getLogger()
|
|
if options.verbose:
|
|
logger.setLevel(logging.INFO)
|
|
else:
|
|
logger.setLevel(logging.WARNING)
|
|
|
|
check_custom_checks_py, check_custom_checks_nonpy = build_custom_checkers(by_lang)
|
|
|
|
lint_functions = {} # type: Dict[str, Callable[[], int]]
|
|
|
|
def lint(func):
|
|
# type: (Callable[[], int]) -> Callable[[], int]
|
|
lint_functions[func.__name__] = func
|
|
return func
|
|
|
|
with bright_red_output():
|
|
@lint
|
|
def check_urls():
|
|
# type: () -> int
|
|
result = subprocess.call(['tools/check-urls'])
|
|
return result
|
|
|
|
@lint
|
|
def templates():
|
|
# type: () -> int
|
|
args = ['tools/check-templates']
|
|
if options.modified:
|
|
args.append('-m')
|
|
result = subprocess.call(args)
|
|
return result
|
|
|
|
@lint
|
|
def add_class():
|
|
# type: () -> int
|
|
result = subprocess.call(['tools/find-add-class'])
|
|
return result
|
|
|
|
@lint
|
|
def css():
|
|
# type: () -> int
|
|
result = subprocess.call(['tools/check-css'])
|
|
return result
|
|
|
|
@lint
|
|
def eslint():
|
|
# type: () -> int
|
|
if len(by_lang['js']) == 0:
|
|
return 0
|
|
result = subprocess.call(['node', 'node_modules/.bin/eslint', '--quiet'] +
|
|
by_lang['js'])
|
|
return result
|
|
|
|
@lint
|
|
def puppet():
|
|
# type: () -> int
|
|
if not by_lang['pp']:
|
|
return 0
|
|
result = subprocess.call(['puppet', 'parser', 'validate'] + by_lang['pp'])
|
|
return result
|
|
|
|
@lint
|
|
def custom_py():
|
|
# type: () -> int
|
|
failed = check_custom_checks_py()
|
|
return 1 if failed else 0
|
|
|
|
@lint
|
|
def custom_nonpy():
|
|
# type: () -> int
|
|
failed = check_custom_checks_nonpy()
|
|
return 1 if failed else 0
|
|
|
|
@lint
|
|
def pyflakes():
|
|
# type: () -> int
|
|
failed = check_pyflakes(options, by_lang)
|
|
return 1 if failed else 0
|
|
|
|
if options.pep8:
|
|
@lint
|
|
def pep8():
|
|
# type: () -> int
|
|
failed = check_pep8(by_lang['py'])
|
|
return 1 if failed else 0
|
|
|
|
failed = run_parallel(lint_functions)
|
|
|
|
sys.exit(1 if failed else 0)
|
|
|
|
if __name__ == '__main__':
|
|
run()
|