mirror of
https://github.com/zulip/zulip.git
synced 2025-10-27 10:03:56 +00:00
bf9f9c8b5d
mypy daemon performs significantly better than running the regular mypy cli tool when we type check the entire codebase multiple times locally. This adds running mypy daemon as an option for both `tools/run-mypy` and `tools/lint`. To ensure daemon messages like "Daemon started", "Daemon stopped" won't get printed we filter any output that starts with "Daemon". Signed-off-by: Zixuan James Li <p359101898@gmail.com>
253 lines
7.8 KiB
Python
Executable File
253 lines
7.8 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import os
|
|
import random
|
|
import re
|
|
import sys
|
|
|
|
tools_dir = os.path.dirname(os.path.abspath(__file__))
|
|
root_dir = os.path.join(tools_dir, "..")
|
|
sys.path.insert(0, root_dir)
|
|
|
|
# check for the venv
|
|
from tools.lib import sanity_check
|
|
|
|
sanity_check.check_venv(__file__)
|
|
|
|
from zulint.command import LinterConfig, add_default_linter_arguments
|
|
|
|
from tools.linter_lib.custom_check import non_py_rules, python_rules
|
|
|
|
|
|
def run() -> None:
|
|
from tools.lib.test_script import (
|
|
add_provision_check_override_param,
|
|
assert_provisioning_status_ok,
|
|
)
|
|
from tools.linter_lib.exclude import EXCLUDED_FILES, PUPPET_CHECK_RULES_TO_EXCLUDE
|
|
from tools.linter_lib.pep8 import check_pep8
|
|
from tools.linter_lib.pyflakes import check_pyflakes
|
|
|
|
parser = argparse.ArgumentParser()
|
|
add_provision_check_override_param(parser)
|
|
parser.add_argument("--full", action="store_true", help="Check some things we typically ignore")
|
|
parser.add_argument("--use-mypy-daemon", action="store_true", help="Run mypy daemon instead")
|
|
add_default_linter_arguments(parser)
|
|
args = parser.parse_args()
|
|
|
|
os.chdir(root_dir)
|
|
|
|
assert_provisioning_status_ok(args.skip_provision_check)
|
|
|
|
# Invoke the appropriate lint checker for each language,
|
|
# and also check files for extra whitespace.
|
|
|
|
linter_config = LinterConfig(args)
|
|
|
|
by_lang = linter_config.list_files(
|
|
groups={
|
|
"backend": [
|
|
"bash",
|
|
"json",
|
|
"md",
|
|
"pp",
|
|
"py",
|
|
"pyi",
|
|
"rst",
|
|
"sh",
|
|
"text",
|
|
"txt",
|
|
"yaml",
|
|
"yml",
|
|
],
|
|
"frontend": [
|
|
"css",
|
|
"flow",
|
|
"hbs",
|
|
"html",
|
|
"js",
|
|
"lock",
|
|
"ts",
|
|
],
|
|
},
|
|
exclude=EXCLUDED_FILES,
|
|
)
|
|
|
|
linter_config.external_linter(
|
|
"css",
|
|
["node", "node_modules/.bin/stylelint"],
|
|
["css"],
|
|
fix_arg="--fix",
|
|
description="Standard CSS style and formatting linter (config: stylelint.config.js)",
|
|
)
|
|
linter_config.external_linter(
|
|
"eslint",
|
|
["node", "node_modules/.bin/eslint", "--max-warnings=0", "--cache", "--ext", ".js,.ts"],
|
|
["js", "ts"],
|
|
fix_arg="--fix",
|
|
description="Standard JavaScript style and formatting linter (config: .eslintrc).",
|
|
)
|
|
linter_config.external_linter(
|
|
"puppet",
|
|
["env", "RUBYOPT=-W0", "puppet", "parser", "validate"],
|
|
["pp"],
|
|
description="Runs the puppet parser validator, checking for syntax errors.",
|
|
)
|
|
linter_config.external_linter(
|
|
"puppet-lint",
|
|
["puppet-lint", "--fail-on-warnings", *PUPPET_CHECK_RULES_TO_EXCLUDE],
|
|
["pp"],
|
|
fix_arg="--fix",
|
|
description="Standard puppet linter (config: tools/linter_lib/exclude.py)",
|
|
)
|
|
linter_config.external_linter(
|
|
"templates",
|
|
["tools/check-templates"],
|
|
["hbs", "html"],
|
|
description="Custom linter checks whitespace formatting of HTML templates",
|
|
fix_arg="--fix",
|
|
)
|
|
linter_config.external_linter(
|
|
"openapi",
|
|
["node", "tools/check-openapi"],
|
|
["yaml"],
|
|
description="Validates our OpenAPI/Swagger API documentation "
|
|
"(zerver/openapi/zulip.yaml) ",
|
|
fix_arg="--fix",
|
|
)
|
|
linter_config.external_linter(
|
|
"shellcheck",
|
|
["shellcheck", "-x", "-P", "SCRIPTDIR"],
|
|
["bash", "sh"],
|
|
description="Standard shell script linter",
|
|
)
|
|
linter_config.external_linter(
|
|
"shfmt",
|
|
["shfmt"],
|
|
["bash", "sh"],
|
|
check_arg="-d",
|
|
fix_arg="-w",
|
|
description="Formats shell scripts",
|
|
)
|
|
command = ["tools/run-mypy", "--quiet"]
|
|
if args.skip_provision_check:
|
|
command.append("--skip-provision-check")
|
|
if args.use_mypy_daemon:
|
|
command.append("--use-daemon")
|
|
linter_config.external_linter(
|
|
"mypy",
|
|
command,
|
|
["py", "pyi"],
|
|
pass_targets=False,
|
|
description="Static type checker for Python (config: pyproject.toml)",
|
|
suppress_line=(
|
|
lambda line: line.startswith("Daemon") or line == "Restarting: configuration changed"
|
|
)
|
|
if args.use_mypy_daemon
|
|
else lambda _: False,
|
|
)
|
|
linter_config.external_linter(
|
|
"tsc",
|
|
["tools/run-tsc"],
|
|
["ts"],
|
|
pass_targets=False,
|
|
description="TypeScript compiler (config: tsconfig.json)",
|
|
)
|
|
linter_config.external_linter(
|
|
"yarn-deduplicate",
|
|
["tools/run-yarn-deduplicate"],
|
|
["lock"],
|
|
pass_targets=False,
|
|
description="Shares duplicate packages in yarn.lock",
|
|
)
|
|
linter_config.external_linter(
|
|
"gitlint",
|
|
["tools/commit-message-lint"],
|
|
description="Checks commit messages for common formatting errors (config: .gitlint)",
|
|
)
|
|
linter_config.external_linter(
|
|
"isort",
|
|
["isort"],
|
|
["py", "pyi"],
|
|
description="Sorts Python import statements",
|
|
check_arg=["--check-only", "--diff"],
|
|
)
|
|
linter_config.external_linter(
|
|
"prettier",
|
|
["node_modules/.bin/prettier", "--check", "--loglevel=warn"],
|
|
["css", "flow", "js", "json", "md", "ts", "yaml", "yml"],
|
|
fix_arg=["--write"],
|
|
description="Formats CSS, JavaScript, YAML",
|
|
)
|
|
linter_config.external_linter(
|
|
"black",
|
|
["black"],
|
|
["py", "pyi"],
|
|
description="Reformats Python code",
|
|
check_arg=["--check"],
|
|
suppress_line=lambda line: line == "All done! ✨ 🍰 ✨\n"
|
|
or re.fullmatch(r"\d+ files? would be left unchanged\.\n", line) is not None,
|
|
)
|
|
|
|
semgrep_command = [
|
|
"semgrep",
|
|
"--config=./tools/semgrep.yml",
|
|
"--error",
|
|
"--disable-version-check",
|
|
"--quiet",
|
|
]
|
|
linter_config.external_linter(
|
|
"semgrep-py",
|
|
[*semgrep_command, "--lang=python"],
|
|
["py"],
|
|
fix_arg="--autofix",
|
|
description="Syntactic grep (semgrep) code search tool (config: ./tools/semgrep.yml)",
|
|
)
|
|
|
|
linter_config.external_linter(
|
|
"thirdparty",
|
|
["tools/check-thirdparty"],
|
|
description="Check docs/THIRDPARTY copyright file syntax",
|
|
)
|
|
|
|
@linter_config.lint
|
|
def custom_py() -> int:
|
|
"""Runs custom checks for python files (config: tools/linter_lib/custom_check.py)"""
|
|
failed = python_rules.check(by_lang, verbose=args.verbose)
|
|
return 1 if failed else 0
|
|
|
|
@linter_config.lint
|
|
def custom_nonpy() -> int:
|
|
"""Runs custom checks for non-python files (config: tools/linter_lib/custom_check.py)"""
|
|
failed = False
|
|
for rule in non_py_rules:
|
|
failed = failed or rule.check(by_lang, verbose=args.verbose)
|
|
return 1 if failed else 0
|
|
|
|
@linter_config.lint
|
|
def pyflakes() -> int:
|
|
"""Standard Python bug and code smell linter (config: tools/linter_lib/pyflakes.py)"""
|
|
failed = check_pyflakes(by_lang["py"], args)
|
|
return 1 if failed else 0
|
|
|
|
python_part1 = {x for x in by_lang["py"] + by_lang["pyi"] if random.randint(0, 1) == 0}
|
|
python_part2 = {y for y in by_lang["py"] + by_lang["pyi"] if y not in python_part1}
|
|
|
|
@linter_config.lint
|
|
def pep8_1of2() -> int:
|
|
"""Standard Python style linter on 50% of files (config: setup.cfg)"""
|
|
failed = check_pep8(list(python_part1))
|
|
return 1 if failed else 0
|
|
|
|
@linter_config.lint
|
|
def pep8_2of2() -> int:
|
|
"""Standard Python style linter on other 50% of files (config: setup.cfg)"""
|
|
failed = check_pep8(list(python_part2))
|
|
return 1 if failed else 0
|
|
|
|
linter_config.do_lint()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
run()
|