From 684dad8145860b8a24ea9d4b9df72743650a2610 Mon Sep 17 00:00:00 2001 From: Anders Kaseorg Date: Fri, 2 Jul 2021 23:22:44 -0700 Subject: [PATCH] tools: Use root-based absolute import for tools.lib, etc. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mypy can’t follow absolute imports based on directories other than the root. This was hiding some type errors due to ignore_missing_imports. Signed-off-by: Anders Kaseorg --- scripts/zulip-puppet-apply | 8 +++++--- tools/build-docs | 11 ++++++----- tools/check-capitalization | 11 ++++++----- tools/check-frontend-i18n | 14 +++++++------- tools/check-templates | 12 ++++++++---- tools/create-test-api-docs | 13 ++++++++----- tools/generate-integration-docs-screenshot | 22 +++++++++++----------- tools/lint | 12 ++++++------ tools/pretty-print-html | 5 ++++- tools/run-dev.py | 10 ++++------ tools/test-api | 10 +++++----- tools/test-backend | 18 +++++++++--------- tools/test-help-documentation | 14 ++++++-------- tools/test-queue-worker-reload | 4 +++- tools/test-run-dev | 4 +++- tools/test-tools | 19 ++++++++----------- tools/tests/test_zulint_custom_rules.py | 7 ++++--- tools/update-prod-static | 8 +++++--- 18 files changed, 108 insertions(+), 94 deletions(-) diff --git a/scripts/zulip-puppet-apply b/scripts/zulip-puppet-apply index 44c83b8aa3..9767604838 100755 --- a/scripts/zulip-puppet-apply +++ b/scripts/zulip-puppet-apply @@ -9,11 +9,13 @@ import tempfile import yaml -from lib.puppet_cache import setup_puppet_modules -from lib.zulip_tools import assert_running_as_root, parse_os_release +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, BASE_DIR) + +from scripts.lib.puppet_cache import setup_puppet_modules +from scripts.lib.zulip_tools import assert_running_as_root, parse_os_release assert_running_as_root() -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) parser = argparse.ArgumentParser(description="Run Puppet") parser.add_argument( diff --git a/tools/build-docs b/tools/build-docs index 0e50923e39..17a67cf2b2 100755 --- a/tools/build-docs +++ b/tools/build-docs @@ -1,16 +1,17 @@ #!/usr/bin/env python3 -# check for the venv -from lib import sanity_check - -sanity_check.check_venv(__file__) - import argparse import os import sys ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(ZULIP_PATH) + +# check for the venv +from tools.lib import sanity_check + +sanity_check.check_venv(__file__) + from scripts.lib.zulip_tools import ENDC, WARNING, run diff --git a/tools/check-capitalization b/tools/check-capitalization index 8ce03c7dc5..35b19e3f3e 100755 --- a/tools/check-capitalization +++ b/tools/check-capitalization @@ -1,10 +1,5 @@ #!/usr/bin/env python3 -# check for the venv -from lib import sanity_check - -sanity_check.check_venv(__file__) - import argparse import json import os @@ -13,6 +8,12 @@ import subprocess import sys sys.path.append(os.path.join(os.path.dirname(__file__), "..")) + +# check for the venv +from tools.lib import sanity_check + +sanity_check.check_venv(__file__) + from scripts.lib.zulip_tools import ENDC, FAIL, WARNING from tools.lib.capitalization import check_capitalization diff --git a/tools/check-frontend-i18n b/tools/check-frontend-i18n index 5e7f6a92ee..f146ebdb0a 100755 --- a/tools/check-frontend-i18n +++ b/tools/check-frontend-i18n @@ -1,18 +1,18 @@ #!/usr/bin/env python3 -from typing import List - -# check for the venv -from lib import sanity_check - -sanity_check.check_venv(__file__) - import argparse import json import os import subprocess import sys +from typing import List sys.path.append(os.path.join(os.path.dirname(__file__), "..")) + +# check for the venv +from tools.lib import sanity_check + +sanity_check.check_venv(__file__) + from scripts.lib.zulip_tools import ENDC, FAIL, WARNING diff --git a/tools/check-templates b/tools/check-templates index 00afba928b..0910d2594f 100755 --- a/tools/check-templates +++ b/tools/check-templates @@ -1,13 +1,13 @@ #!/usr/bin/env python3 import argparse import logging +import os import sys +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + # check for the venv -from lib import sanity_check -from lib.html_branches import build_id_dict -from lib.pretty_print import validate_indent_html -from lib.template_parser import validate +from tools.lib import sanity_check sanity_check.check_venv(__file__) @@ -15,6 +15,10 @@ from typing import Dict, Iterable, List from zulint import lister +from tools.lib.html_branches import build_id_dict +from tools.lib.pretty_print import validate_indent_html +from tools.lib.template_parser import validate + EXCLUDED_FILES = [ ## Test data Files for testing modules in tests "tools/tests/test_template_data", diff --git a/tools/create-test-api-docs b/tools/create-test-api-docs index 6a95634f81..f0fc6351c8 100755 --- a/tools/create-test-api-docs +++ b/tools/create-test-api-docs @@ -1,16 +1,19 @@ #!/usr/bin/env python3 -# check for the venv -from lib import sanity_check - -sanity_check.check_venv(__file__) - import html import os import pprint +import sys from collections import defaultdict from typing import Any, Dict, List, Set +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +# check for the venv +from tools.lib import sanity_check + +sanity_check.check_venv(__file__) + import orjson Call = Dict[str, Any] diff --git a/tools/generate-integration-docs-screenshot b/tools/generate-integration-docs-screenshot index 5020bb9fbc..b116c8697a 100755 --- a/tools/generate-integration-docs-screenshot +++ b/tools/generate-integration-docs-screenshot @@ -1,17 +1,23 @@ #!/usr/bin/env python3 """Create or update a webhook integration screenshot using a test fixture.""" -# check for the venv -from lib import sanity_check - -sanity_check.check_venv(__file__) +import argparse +import base64 import os +import subprocess import sys +from typing import Any, Dict, Optional, Tuple +from urllib.parse import parse_qsl, urlencode TOOLS_DIR = os.path.abspath(os.path.dirname(__file__)) ROOT_DIR = os.path.dirname(TOOLS_DIR) - sys.path.insert(0, ROOT_DIR) + +# check for the venv +from tools.lib import sanity_check + +sanity_check.check_venv(__file__) + from scripts.lib.setup_path import setup_path setup_path() @@ -21,12 +27,6 @@ import django django.setup() -import argparse -import base64 -import subprocess -from typing import Any, Dict, Optional, Tuple -from urllib.parse import parse_qsl, urlencode - import orjson import requests import zulip diff --git a/tools/lint b/tools/lint index 427da71a74..a67fb51a9b 100755 --- a/tools/lint +++ b/tools/lint @@ -5,21 +5,21 @@ import random import re import sys +tools_dir = os.path.dirname(os.path.abspath(__file__)) +root_dir = os.path.join(tools_dir, "..") +sys.path.insert(0, root_dir) + # check for the venv -from lib import sanity_check +from tools.lib import sanity_check sanity_check.check_venv(__file__) from zulint.command import LinterConfig, add_default_linter_arguments -from linter_lib.custom_check import non_py_rules, python_rules +from tools.linter_lib.custom_check import non_py_rules, python_rules def run() -> None: - tools_dir = os.path.dirname(os.path.abspath(__file__)) - root_dir = os.path.dirname(tools_dir) - sys.path.insert(0, root_dir) - from tools.lib.test_script import ( add_provision_check_override_param, assert_provisioning_status_ok, diff --git a/tools/pretty-print-html b/tools/pretty-print-html index dc5c1b7753..e87701cf0f 100755 --- a/tools/pretty-print-html +++ b/tools/pretty-print-html @@ -1,8 +1,11 @@ #!/usr/bin/env python3 +import os import sys from typing import List -from lib.pretty_print import pretty_print_html +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from tools.lib.pretty_print import pretty_print_html def clean_html(filenames: List[str]) -> None: diff --git a/tools/run-dev.py b/tools/run-dev.py index 9b06e9d31d..1e3990c8f3 100755 --- a/tools/run-dev.py +++ b/tools/run-dev.py @@ -8,17 +8,17 @@ import sys from typing import Any, Callable, Generator, List, Sequence from urllib.parse import urlunparse +TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.dirname(TOOLS_DIR)) + # check for the venv -from lib import sanity_check +from tools.lib import sanity_check sanity_check.check_venv(__file__) from tornado import gen, httpclient, httputil, web from tornado.ioloop import IOLoop -TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, os.path.dirname(TOOLS_DIR)) - from tools.lib.test_script import add_provision_check_override_param, assert_provisioning_status_ok if "posix" in os.name and os.geteuid() == 0: @@ -90,8 +90,6 @@ else: manage_args = [f"--settings={settings_module}"] os.environ["DJANGO_SETTINGS_MODULE"] = settings_module -sys.path.append(os.path.join(os.path.dirname(__file__), "..")) - from scripts.lib.zulip_tools import CYAN, ENDC, FAIL proxy_port = base_port diff --git a/tools/test-api b/tools/test-api index fd4e88da8e..32f8f16b55 100755 --- a/tools/test-api +++ b/tools/test-api @@ -5,15 +5,15 @@ import sys os.environ["RUNNING_OPENAPI_CURL_TEST"] = "1" -# check for the venv -from lib import sanity_check - -sanity_check.check_venv(__file__) - ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, ZULIP_PATH) os.chdir(ZULIP_PATH) +# check for the venv +from tools.lib import sanity_check + +sanity_check.check_venv(__file__) + from zulip import Client from tools.lib.test_script import add_provision_check_override_param, assert_provisioning_status_ok diff --git a/tools/test-backend b/tools/test-backend index 643d27033c..60917224aa 100755 --- a/tools/test-backend +++ b/tools/test-backend @@ -1,10 +1,5 @@ #!/usr/bin/env python3 -# check for the venv -from lib import sanity_check - -sanity_check.check_venv(__file__) - import argparse import contextlib import glob @@ -16,6 +11,15 @@ import tempfile from typing import Iterator, List from unittest import mock +TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) +os.chdir(os.path.dirname(TOOLS_DIR)) +sys.path.insert(0, os.path.dirname(TOOLS_DIR)) + +# check for the venv +from tools.lib import sanity_check + +sanity_check.check_venv(__file__) + import django import orjson import responses @@ -185,10 +189,6 @@ class ZulipInternetBlockedError(Exception): def main() -> None: - TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) - os.chdir(os.path.dirname(TOOLS_DIR)) - sys.path.insert(0, os.path.dirname(TOOLS_DIR)) - default_parallel = os.cpu_count() # Remove proxy settings for running backend tests diff --git a/tools/test-help-documentation b/tools/test-help-documentation index 41ffb4399a..baf1b25446 100755 --- a/tools/test-help-documentation +++ b/tools/test-help-documentation @@ -7,16 +7,16 @@ import sys from typing import Iterator ZULIP_PATH = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - -# check for the venv -from lib import sanity_check - -sanity_check.check_venv(__file__) - os.chdir(ZULIP_PATH) sys.path.insert(0, ZULIP_PATH) +# check for the venv +from tools.lib import sanity_check + +sanity_check.check_venv(__file__) + from tools.lib.test_script import add_provision_check_override_param +from tools.lib.test_server import test_server_running parser = argparse.ArgumentParser() add_provision_check_override_param(parser) @@ -28,8 +28,6 @@ parser.add_argument( ) options = parser.parse_args() -from tools.lib.test_server import test_server_running - os.makedirs("var/help-documentation", exist_ok=True) LOG_FILE = "var/help-documentation/server.log" diff --git a/tools/test-queue-worker-reload b/tools/test-queue-worker-reload index 5d58a8fb89..8c189391c1 100755 --- a/tools/test-queue-worker-reload +++ b/tools/test-queue-worker-reload @@ -6,8 +6,10 @@ import sys import time import types +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + # check for the venv -from lib import sanity_check +from tools.lib import sanity_check sanity_check.check_venv(__file__) diff --git a/tools/test-run-dev b/tools/test-run-dev index 19a2cef0f8..204ca83a42 100755 --- a/tools/test-run-dev +++ b/tools/test-run-dev @@ -6,7 +6,9 @@ import sys import time from typing import Tuple -from lib import sanity_check +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) + +from tools.lib import sanity_check sanity_check.check_venv(__file__) diff --git a/tools/test-tools b/tools/test-tools index cc9ce79cba..3baa31309f 100755 --- a/tools/test-tools +++ b/tools/test-tools @@ -4,8 +4,14 @@ import os import sys import unittest +tools_dir = os.path.dirname(os.path.abspath(__file__)) +root_dir = os.path.abspath(os.path.join(tools_dir, "..")) +tools_test_dir = os.path.join(tools_dir, "tests") + +sys.path.insert(0, root_dir) + # check for the venv -from lib import sanity_check +from tools.lib import sanity_check sanity_check.check_venv(__file__) @@ -14,22 +20,13 @@ if __name__ == "__main__": parser.add_argument("--coverage", action="store_true", help="compute test coverage") args = parser.parse_args() - def dir_join(dir1: str, dir2: str) -> str: - return os.path.abspath(os.path.join(dir1, dir2)) - - tools_dir = os.path.dirname(os.path.abspath(__file__)) - root_dir = dir_join(tools_dir, "..") - tools_test_dir = dir_join(tools_dir, "tests") - - sys.path.insert(0, root_dir) - loader = unittest.TestLoader() if args.coverage: import coverage cov = coverage.Coverage( - branch=True, omit=["*/zulip-venv-cache/*", dir_join(tools_test_dir, "*")] + branch=True, omit=["*/zulip-venv-cache/*", os.path.join(tools_test_dir, "*")] ) cov.start() diff --git a/tools/tests/test_zulint_custom_rules.py b/tools/tests/test_zulint_custom_rules.py index 393c32f495..6e875bbfc2 100644 --- a/tools/tests/test_zulint_custom_rules.py +++ b/tools/tests/test_zulint_custom_rules.py @@ -5,7 +5,7 @@ from unittest.mock import patch from zulint.custom_rules import RuleList -from linter_lib.custom_check import non_py_rules, python_rules +from tools.linter_lib.custom_check import non_py_rules, python_rules ROOT_DIR = os.path.abspath(os.path.join(__file__, "..", "..", "..")) CHECK_MESSAGE = "Fix the corresponding rule in `tools/linter_lib/custom_check.py`." @@ -13,9 +13,10 @@ CHECK_MESSAGE = "Fix the corresponding rule in `tools/linter_lib/custom_check.py class TestRuleList(TestCase): def setUp(self) -> None: - self.all_rules = python_rules.rules + all_rules = list(python_rules.rules) for rule in non_py_rules: - self.all_rules.extend(rule.rules) + all_rules.extend(rule.rules) + self.all_rules = all_rules def test_paths_in_rules(self) -> None: """Verifies that the paths mentioned in linter rules actually exist""" diff --git a/tools/update-prod-static b/tools/update-prod-static index 3e0b342673..6d7816853f 100755 --- a/tools/update-prod-static +++ b/tools/update-prod-static @@ -10,17 +10,19 @@ from scripts.lib.setup_path import setup_path setup_path() +# check for the venv +from tools.lib import sanity_check + +sanity_check.check_venv(__file__) + os.environ["DJANGO_SETTINGS_MODULE"] = "zproject.settings" from django.conf import settings -# check for the venv -from lib import sanity_check from scripts.lib.node_cache import setup_node_modules from scripts.lib.zulip_tools import assert_not_running_as_root, run assert_not_running_as_root() -sanity_check.check_venv(__file__) os.chdir(settings.DEPLOY_ROOT)