diff --git a/Makefile b/Makefile index 810873a5..4f6f6fa3 100644 --- a/Makefile +++ b/Makefile @@ -32,14 +32,8 @@ INTEGRATION_SKIP := clients/aws-sdk-qbusiness static-analysis: $(MAKE) meta-package-check - @set -e; for p in $(PROJECTS); do \ - dirs=$$(ls -d $$p/src $$p/tests 2>/dev/null || true); \ - group=$$(grep -q '^test = \[' $$p/pyproject.toml && echo --group test || true); \ - echo "== $$p"; \ - uvx ruff@$(RUFF_VERSION) check --config $$p/pyproject.toml $$dirs; \ - uvx ruff@$(RUFF_VERSION) format --check --config $$p/pyproject.toml $$dirs; \ - $(UV_RUN) $$p $$group --with pyright==$(PYRIGHT_VERSION) pyright -p $$p $$dirs; \ - done + uv run --no-project --python $(PYTHON_VERSION) python scripts/project-checks/project_checks.py \ + $(RUFF_VERSION) $(PYRIGHT_VERSION) $(PYTHON_VERSION) $(PROJECTS) $(MAKE) aio-check $(MAKE) check-imports $(MAKE) check-conventions diff --git a/scripts/project-checks/project_checks.py b/scripts/project-checks/project_checks.py new file mode 100644 index 00000000..9c4904ae --- /dev/null +++ b/scripts/project-checks/project_checks.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Lints and type-checks each client and package against only its own dependencies. + +Each project resolves its own venv, so a missing dependency surfaces here even +when a sibling would have supplied it in a shared venv. Projects run in +parallel, but each one's output is printed whole and in project order. + +Usage: project_checks.py RUFF_VERSION PYRIGHT_VERSION PYTHON_VERSION PROJECT... +""" + +import os +import subprocess +import sys +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + + +def main() -> int: + ruff, pyright, python, *projects = sys.argv[1:] + failed: list[str] = [] + with ThreadPoolExecutor(max_workers=os.cpu_count() or 4) as pool: + futures = [pool.submit(_check, p, ruff, pyright, python) for p in projects] + for project, future in zip(projects, futures, strict=True): + ok, output = future.result() + if ok: + print(f"ok {project}", flush=True) + else: + failed.append(project) + print(f"FAIL {project}\n{output}", flush=True) + if failed: + print( + f"\n{len(failed)} of {len(projects)} project(s) failed: {' '.join(failed)}" + ) + return 1 + print(f"\n{len(projects)} project(s) passed.") + return 0 + + +def _check(project: str, ruff: str, pyright: str, python: str) -> tuple[bool, str]: + dirs = [d for d in (f"{project}/src", f"{project}/tests") if (ROOT / d).is_dir()] + pyproject = ROOT / project / "pyproject.toml" + group = ["--group", "test"] if "\ntest = [" in pyproject.read_text() else [] + commands = [ + ["uvx", f"ruff@{ruff}", "check", "--config", str(pyproject), *dirs], + ["uvx", f"ruff@{ruff}", "format", "--check", "--config", str(pyproject), *dirs], + [ + "uv", + "run", + "--python", + python, + "--project", + project, + *group, + "--with", + f"pyright=={pyright}", + "pyright", + "-p", + project, + *dirs, + ], + ] + output: list[str] = [] + ok = True + # Every command runs, so one failing project reports lint and type errors together. + for command in commands: + proc = subprocess.run( + command, cwd=ROOT, capture_output=True, text=True, check=False + ) + if proc.returncode != 0: + ok = False + output.append(f"$ {' '.join(command)}\n{proc.stdout}{proc.stderr}") + return ok, "".join(output) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/verifications/_common.py b/scripts/verifications/_common.py index 218961f7..40cd191a 100644 --- a/scripts/verifications/_common.py +++ b/scripts/verifications/_common.py @@ -5,13 +5,15 @@ import ast import functools import importlib.metadata +import itertools import json +import multiprocessing import os import subprocess import sys import tomllib from collections.abc import Callable, Iterable, Iterator -from concurrent.futures import ThreadPoolExecutor +from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor from dataclasses import dataclass, replace from pathlib import Path from types import ModuleType @@ -186,6 +188,10 @@ def model_data(self) -> bool: def imports(self) -> list[Import]: if self.model_data: return [] + if not _IN_WORKER: + _load_all_imports() + if "imports" in self.__dict__: + return self.__dict__["imports"] return list(_ImportCollector(self).collect()) @functools.cached_property @@ -302,6 +308,36 @@ def analyzed_modules() -> list[Module]: return [m for m in runtime_modules() if not m.model_data] +def map_modules[T](fn: Callable[[Module], T], modules: Iterable[Module]) -> list[T]: + """Applies ``fn`` to each module in forked worker processes, in module order. + + ``fn`` must be a module-level function and its result picklable. + """ + names = [m.name for m in modules] + index = module_index() + if _IN_WORKER or len(names) < 2: + return [fn(index[n]) for n in names] + workers = os.cpu_count() or 4 + with ProcessPoolExecutor( + max_workers=workers, mp_context=_FORK, initializer=_mark_worker + ) as pool: + return list( + pool.map( + _apply, + itertools.repeat(fn), + names, + chunksize=max(1, len(names) // (workers * 4)), + ) + ) + + +def module_findings( + fn: Callable[[Module], list[Finding]], modules: Iterable[Module] +) -> list[Finding]: + """Runs a per-module check in parallel, keeping findings in module order.""" + return [f for found in map_modules(fn, modules) for f in found] + + @functools.cache def module_index() -> dict[str, Module]: return {m.name: m for m in runtime_modules()} @@ -726,3 +762,36 @@ def _soften(finding: Finding) -> Finding: ): return replace(finding, severity="warning") return finding + + +# CAUTION: fork, so workers inherit the parent's module index and the check's +# own functions; spawn would re-import __main__ and rebuild everything. +_FORK = multiprocessing.get_context("fork") +_IN_WORKER = False + + +def _mark_worker() -> None: + global _IN_WORKER + _IN_WORKER = True + + +def _apply[T](fn: Callable[[Module], T], name: str) -> T: + return fn(module_index()[name]) + + +type _ImportRow = tuple[str, str | None, str | None, int, Context, bool] + + +def _import_rows(module: Module) -> list[_ImportRow]: + return [ + (i.target, i.name, i.asname, i.line, i.context, i.guarded) + for i in module.imports + ] + + +@functools.cache +def _load_all_imports() -> None: + """Collects every runtime module's imports in parallel the first time any is read.""" + modules = [m for m in runtime_modules() if not m.model_data] + for module, rows in zip(modules, map_modules(_import_rows, modules), strict=True): + module.__dict__["imports"] = [Import(module, *row) for row in rows] diff --git a/scripts/verifications/docstrings.py b/scripts/verifications/docstrings.py index 308f31e4..1fded50c 100644 --- a/scripts/verifications/docstrings.py +++ b/scripts/verifications/docstrings.py @@ -15,18 +15,24 @@ """ import ast +import functools import importlib import inspect +import os import re import sys +from concurrent.futures import ThreadPoolExecutor from pathlib import Path from _common import ( ROOT, Finding, + Module, + Package, analyzed_modules, main, model_data_globs, + module_findings, packages, run, runtime_modules, @@ -42,51 +48,50 @@ def check() -> list[Finding]: return [ *(f for f in _ruff_findings() if not _inherits_docstring(f)), - *_constant_findings(), + *module_findings(_constant_findings, analyzed_modules()), ] def _ruff_findings() -> list[Finding]: + with ThreadPoolExecutor(max_workers=os.cpu_count() or 4) as pool: + return [ + f for found in pool.map(_package_ruff_findings, packages()) for f in found + ] + + +def _package_ruff_findings(package: Package) -> list[Finding]: ruff = str(Path(sys.executable).parent / "ruff") - findings: list[Finding] = [] - for package in packages(): - proc = run( - [ - ruff, - "check", - "--config", - str(package.dir / "pyproject.toml"), - "--select", - _SELECT, - "--output-format", - "concise", - "--no-cache", - *( - arg - for glob in model_data_globs(package) - for arg in ("--extend-exclude", glob) - ), - str(package.src), - ] - ) - for line in proc.stdout.splitlines(): - if (match := _LOCATION.match(line)) is not None: - findings.append( - Finding( - match["message"], - (ROOT / match["path"]).resolve(), - int(match["line"]), - ) - ) - return findings + proc = run( + [ + ruff, + "check", + "--config", + str(package.dir / "pyproject.toml"), + "--select", + _SELECT, + "--output-format", + "concise", + "--no-cache", + *( + arg + for glob in model_data_globs(package) + for arg in ("--extend-exclude", glob) + ), + str(package.src), + ] + ) + return [ + Finding(match["message"], (ROOT / match["path"]).resolve(), int(match["line"])) + for line in proc.stdout.splitlines() + if (match := _LOCATION.match(line)) is not None + ] def _inherits_docstring(finding: Finding) -> bool: """Whether a D102 method overrides one whose docstring editors and docs show.""" if not finding.message.startswith("D102 ") or finding.path is None: return False - path = finding.path - module = next((m for m in runtime_modules() if m.path.resolve() == path), None) + module = _modules_by_path().get(finding.path) if module is None: return False qualname = _method_qualname(module.tree, finding.line) @@ -104,6 +109,11 @@ def _inherits_docstring(finding: Finding) -> bool: return inspect.getdoc(obj) is not None +@functools.cache +def _modules_by_path() -> dict[Path, Module]: + return {m.path.resolve(): m for m in runtime_modules()} + + def _method_qualname(tree: ast.Module, line: int) -> list[str] | None: def walk(body: list[ast.stmt], prefix: list[str]) -> list[str] | None: for node in body: @@ -121,35 +131,34 @@ def walk(body: list[ast.stmt], prefix: list[str]) -> list[str] | None: return walk(tree.body, []) -def _constant_findings() -> list[Finding]: +def _constant_findings(module: Module) -> list[Finding]: findings: list[Finding] = [] - for module in analyzed_modules(): - if not module.public: + if not module.public: + return findings + body = module.tree.body + for index, node in enumerate(body): + if isinstance(node, ast.Assign) and len(node.targets) == 1: + target = node.targets[0] + elif isinstance(node, ast.AnnAssign): + target = node.target + else: continue - body = module.tree.body - for index, node in enumerate(body): - if isinstance(node, ast.Assign) and len(node.targets) == 1: - target = node.targets[0] - elif isinstance(node, ast.AnnAssign): - target = node.target - else: - continue - if not isinstance(target, ast.Name) or not _CONSTANT.match(target.id): - continue - following = body[index + 1] if index + 1 < len(body) else None - documented = ( - isinstance(following, ast.Expr) - and isinstance(following.value, ast.Constant) - and isinstance(following.value.value, str) - ) - if not documented: - findings.append( - Finding( - f"public constant {target.id} has no docstring", - module.path, - node.lineno, - ) + if not isinstance(target, ast.Name) or not _CONSTANT.match(target.id): + continue + following = body[index + 1] if index + 1 < len(body) else None + documented = ( + isinstance(following, ast.Expr) + and isinstance(following.value, ast.Constant) + and isinstance(following.value.value, str) + ) + if not documented: + findings.append( + Finding( + f"public constant {target.id} has no docstring", + module.path, + node.lineno, ) + ) return findings diff --git a/scripts/verifications/dunder_all.py b/scripts/verifications/dunder_all.py index 4cf132ad..499c720f 100644 --- a/scripts/verifications/dunder_all.py +++ b/scripts/verifications/dunder_all.py @@ -19,8 +19,10 @@ from _common import ( Finding, + Module, analyzed_modules, main, + module_findings, module_index, owned_roots, resolve_from, @@ -31,56 +33,57 @@ def check() -> list[Finding]: + return module_findings(_module_findings, analyzed_modules()) + + +def _module_findings(module: Module) -> list[Finding]: findings: list[Finding] = [] index = module_index() - for module in analyzed_modules(): - exported = module.dunder_all - if exported is None: + exported = module.dunder_all + if exported is None: + return findings + exported_set = set(exported) + for name in sorted(module.defined - _NOT_API): + if not name.startswith("_") and name not in exported_set: + findings.append( + Finding(f"public {name} is missing from __all__", module.path) + ) + if not module.is_package: + return findings + used = _used_names(module.tree) + for name, imp in sorted(module.imported.items()): + if name.startswith("_") or imp.top not in owned_roots() or imp.name is None: continue - exported_set = set(exported) - for name in sorted(module.defined - _NOT_API): - if not name.startswith("_") and name not in exported_set: - findings.append( - Finding(f"public {name} is missing from __all__", module.path) - ) - if not module.is_package: + # A used import is a private re-export, and a TYPE_CHECKING one is unbound. + if name not in exported_set and ( + imp.context == "type_checking" or name in used + ): continue - used = _used_names(module.tree) - for name, imp in sorted(module.imported.items()): - if name.startswith("_") or imp.top not in owned_roots() or imp.name is None: - continue - # A used import is a private re-export, and a TYPE_CHECKING one is unbound. - if name not in exported_set and ( - imp.context == "type_checking" or name in used - ): - continue - target, attr = resolve_from(imp) - if attr is None: - continue - if name not in exported_set: - findings.append( - Finding( - f"{name} (from {target}) is imported only to re-export but " - "is missing from __all__", - module.path, - imp.line, - ) + target, attr = resolve_from(imp) + if attr is None: + continue + if name not in exported_set: + findings.append( + Finding( + f"{name} (from {target}) is imported only to re-export but " + "is missing from __all__", + module.path, + imp.line, ) - continue - source = index.get(target) - source_all = ( - source.dunder_all - if source is not None and not source.model_data - else None ) - if source_all is not None and attr not in source_all: - findings.append( - Finding( - f"re-exports {attr} from {target}, which does not list it in __all__", - module.path, - imp.line, - ) + continue + source = index.get(target) + source_all = ( + source.dunder_all if source is not None and not source.model_data else None + ) + if source_all is not None and attr not in source_all: + findings.append( + Finding( + f"re-exports {attr} from {target}, which does not list it in __all__", + module.path, + imp.line, ) + ) return findings diff --git a/scripts/verifications/dynamic_imports.py b/scripts/verifications/dynamic_imports.py index 86964343..0b53dc41 100644 --- a/scripts/verifications/dynamic_imports.py +++ b/scripts/verifications/dynamic_imports.py @@ -14,29 +14,34 @@ from _common import ( Finding, + Module, analyzed_modules, import_call_aliases, is_import_call, main, + module_findings, ) def check() -> list[Finding]: + return module_findings(_module_findings, analyzed_modules()) + + +def _module_findings(module: Module) -> list[Finding]: findings: list[Finding] = [] - for module in analyzed_modules(): - aliases = import_call_aliases(module.tree) - for node in ast.walk(module.tree): - if not isinstance(node, ast.Call) or not is_import_call(node.func, aliases): - continue - first = node.args[0] if node.args else None - if not (isinstance(first, ast.Constant) and isinstance(first.value, str)): - findings.append( - Finding( - f"dynamic import with non-literal target: {ast.unparse(node)[:120]}", - module.path, - node.lineno, - ) + aliases = import_call_aliases(module.tree) + for node in ast.walk(module.tree): + if not isinstance(node, ast.Call) or not is_import_call(node.func, aliases): + continue + first = node.args[0] if node.args else None + if not (isinstance(first, ast.Constant) and isinstance(first.value, str)): + findings.append( + Finding( + f"dynamic import with non-literal target: {ast.unparse(node)[:120]}", + module.path, + node.lineno, ) + ) return findings diff --git a/scripts/verifications/exception_hierarchy.py b/scripts/verifications/exception_hierarchy.py index 799b0c7c..ce45030b 100644 --- a/scripts/verifications/exception_hierarchy.py +++ b/scripts/verifications/exception_hierarchy.py @@ -24,9 +24,11 @@ from _common import ( Finding, + Module, analyzed_modules, main, members, + module_findings, packages, runtime_modules, ) @@ -38,56 +40,46 @@ def check() -> list[Finding]: for package in packages(): sys.path.insert(0, str(package.src)) - return [*_class_findings(), *_raise_findings()] + return [ + *module_findings(_class_findings, runtime_modules()), + *module_findings(_raise_findings, analyzed_modules()), + ] -def _class_findings() -> list[Finding]: - findings: list[Finding] = [] +def _class_findings(module: Module) -> list[Finding]: + if "testing" in module.name.split("."): + return [] base = _resolve(_BASE) - for module in runtime_modules(): - if "testing" in module.name.split("."): - continue - try: - loaded = importlib.import_module(module.name) - except Exception as e: - findings.append( - Finding( - f"cannot import {module.name}: {e}", module.path, severity="warning" - ) + try: + loaded = importlib.import_module(module.name) + except Exception as e: + return [ + Finding( + f"cannot import {module.name}: {e}", module.path, severity="warning" ) - continue - for name, obj in members(module, loaded): - if ( - inspect.isclass(obj) - and issubclass(obj, BaseException) - and not issubclass(obj, Warning) - and obj.__module__ == module.name - and not issubclass(obj, base) - ): - findings.append( - Finding( - f"{name} does not derive from {base.__qualname__}", module.path - ) - ) - return findings + ] + return [ + Finding(f"{name} does not derive from {base.__qualname__}", module.path) + for name, obj in members(module, loaded) + if inspect.isclass(obj) + and issubclass(obj, BaseException) + and not issubclass(obj, Warning) + and obj.__module__ == module.name + and not issubclass(obj, base) + ] -def _raise_findings() -> list[Finding]: +def _raise_findings(module: Module) -> list[Finding]: findings: list[Finding] = [] - for module in analyzed_modules(): - shadowed = set(module.defined) | set(module.imported) - for node in ast.walk(module.tree): - if not isinstance(node, ast.Raise) or node.exc is None: - continue - exc = node.exc.func if isinstance(node.exc, ast.Call) else node.exc - if ( - isinstance(exc, ast.Name) - and exc.id in _GENERIC - and exc.id not in shadowed - ): - findings.append( - Finding(f"raises generic {exc.id}", module.path, node.lineno) - ) + shadowed = set(module.defined) | set(module.imported) + for node in ast.walk(module.tree): + if not isinstance(node, ast.Raise) or node.exc is None: + continue + exc = node.exc.func if isinstance(node.exc, ast.Call) else node.exc + if isinstance(exc, ast.Name) and exc.id in _GENERIC and exc.id not in shadowed: + findings.append( + Finding(f"raises generic {exc.id}", module.path, node.lineno) + ) return findings diff --git a/scripts/verifications/source_order.py b/scripts/verifications/source_order.py index 5d316e83..c5836f47 100644 --- a/scripts/verifications/source_order.py +++ b/scripts/verifications/source_order.py @@ -18,25 +18,23 @@ import ast from pathlib import Path -from _common import Finding, analyzed_modules, main +from _common import Finding, Module, analyzed_modules, main, module_findings type _Def = ast.FunctionDef | ast.AsyncFunctionDef | ast.ClassDef def check() -> list[Finding]: - findings: list[Finding] = [] - for module in analyzed_modules(): - lazy = _lazy_annotations(module.tree) - findings.extend( - _order(module.path, module.tree.body, scope="module", lazy=lazy) - ) - for node in ast.walk(module.tree): - if isinstance(node, ast.ClassDef): - findings.extend( - _order( - module.path, node.body, scope=f"class {node.name}", lazy=lazy - ) - ) + return module_findings(_module_findings, analyzed_modules()) + + +def _module_findings(module: Module) -> list[Finding]: + lazy = _lazy_annotations(module.tree) + findings = _order(module.path, module.tree.body, scope="module", lazy=lazy) + for node in ast.walk(module.tree): + if isinstance(node, ast.ClassDef): + findings.extend( + _order(module.path, node.body, scope=f"class {node.name}", lazy=lazy) + ) return findings