Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 2 additions & 8 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,8 @@ INTEGRATION_SKIP := clients/aws-sdk-qbusiness

static-analysis:
$(MAKE) meta-package-check
@set -e; for p in $(PROJECTS); do \
dirs=$$(ls -d $$p/src $$p/tests 2>/dev/null || true); \
group=$$(grep -q '^test = \[' $$p/pyproject.toml && echo --group test || true); \
echo "== $$p"; \
uvx ruff@$(RUFF_VERSION) check --config $$p/pyproject.toml $$dirs; \
uvx ruff@$(RUFF_VERSION) format --check --config $$p/pyproject.toml $$dirs; \
$(UV_RUN) $$p $$group --with pyright==$(PYRIGHT_VERSION) pyright -p $$p $$dirs; \
done
uv run --no-project --python $(PYTHON_VERSION) python scripts/project-checks/project_checks.py \
$(RUFF_VERSION) $(PYRIGHT_VERSION) $(PYTHON_VERSION) $(PROJECTS)
$(MAKE) aio-check
$(MAKE) check-imports
$(MAKE) check-conventions
Expand Down
80 changes: 80 additions & 0 deletions scripts/project-checks/project_checks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
#!/usr/bin/env python
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0
"""Lints and type-checks each client and package against only its own dependencies.

Each project resolves its own venv, so a missing dependency surfaces here even
when a sibling would have supplied it in a shared venv. Projects run in
parallel, but each one's output is printed whole and in project order.

Usage: project_checks.py RUFF_VERSION PYRIGHT_VERSION PYTHON_VERSION PROJECT...
"""

import os
import subprocess
import sys
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

ROOT = Path(__file__).resolve().parents[2]


def main() -> int:
ruff, pyright, python, *projects = sys.argv[1:]
failed: list[str] = []
with ThreadPoolExecutor(max_workers=os.cpu_count() or 4) as pool:
futures = [pool.submit(_check, p, ruff, pyright, python) for p in projects]
for project, future in zip(projects, futures, strict=True):
ok, output = future.result()
if ok:
print(f"ok {project}", flush=True)
else:
failed.append(project)
print(f"FAIL {project}\n{output}", flush=True)
if failed:
print(
f"\n{len(failed)} of {len(projects)} project(s) failed: {' '.join(failed)}"
)
return 1
print(f"\n{len(projects)} project(s) passed.")
return 0


def _check(project: str, ruff: str, pyright: str, python: str) -> tuple[bool, str]:
dirs = [d for d in (f"{project}/src", f"{project}/tests") if (ROOT / d).is_dir()]
pyproject = ROOT / project / "pyproject.toml"
group = ["--group", "test"] if "\ntest = [" in pyproject.read_text() else []
commands = [
["uvx", f"ruff@{ruff}", "check", "--config", str(pyproject), *dirs],
["uvx", f"ruff@{ruff}", "format", "--check", "--config", str(pyproject), *dirs],
[
"uv",
"run",
"--python",
python,
"--project",
project,
*group,
"--with",
f"pyright=={pyright}",
"pyright",
"-p",
project,
*dirs,
],
]
output: list[str] = []
ok = True
# Every command runs, so one failing project reports lint and type errors together.
for command in commands:
proc = subprocess.run(
command, cwd=ROOT, capture_output=True, text=True, check=False
)
if proc.returncode != 0:
ok = False
output.append(f"$ {' '.join(command)}\n{proc.stdout}{proc.stderr}")
return ok, "".join(output)


if __name__ == "__main__":
sys.exit(main())
71 changes: 70 additions & 1 deletion scripts/verifications/_common.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,15 @@
import ast
import functools
import importlib.metadata
import itertools
import json
import multiprocessing
import os
import subprocess
import sys
import tomllib
from collections.abc import Callable, Iterable, Iterator
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
from dataclasses import dataclass, replace
from pathlib import Path
from types import ModuleType
Expand Down Expand Up @@ -186,6 +188,10 @@ def model_data(self) -> bool:
def imports(self) -> list[Import]:
if self.model_data:
return []
if not _IN_WORKER:
_load_all_imports()
if "imports" in self.__dict__:
return self.__dict__["imports"]
return list(_ImportCollector(self).collect())

@functools.cached_property
Expand Down Expand Up @@ -302,6 +308,36 @@ def analyzed_modules() -> list[Module]:
return [m for m in runtime_modules() if not m.model_data]


def map_modules[T](fn: Callable[[Module], T], modules: Iterable[Module]) -> list[T]:
"""Applies ``fn`` to each module in forked worker processes, in module order.

``fn`` must be a module-level function and its result picklable.
"""
names = [m.name for m in modules]
index = module_index()
if _IN_WORKER or len(names) < 2:
return [fn(index[n]) for n in names]
workers = os.cpu_count() or 4
with ProcessPoolExecutor(
max_workers=workers, mp_context=_FORK, initializer=_mark_worker
) as pool:
return list(
pool.map(
_apply,
itertools.repeat(fn),
names,
chunksize=max(1, len(names) // (workers * 4)),
)
)


def module_findings(
fn: Callable[[Module], list[Finding]], modules: Iterable[Module]
) -> list[Finding]:
"""Runs a per-module check in parallel, keeping findings in module order."""
return [f for found in map_modules(fn, modules) for f in found]


@functools.cache
def module_index() -> dict[str, Module]:
return {m.name: m for m in runtime_modules()}
Expand Down Expand Up @@ -726,3 +762,36 @@ def _soften(finding: Finding) -> Finding:
):
return replace(finding, severity="warning")
return finding


# CAUTION: fork, so workers inherit the parent's module index and the check's
# own functions; spawn would re-import __main__ and rebuild everything.
_FORK = multiprocessing.get_context("fork")
_IN_WORKER = False


def _mark_worker() -> None:
global _IN_WORKER
_IN_WORKER = True


def _apply[T](fn: Callable[[Module], T], name: str) -> T:
return fn(module_index()[name])


type _ImportRow = tuple[str, str | None, str | None, int, Context, bool]


def _import_rows(module: Module) -> list[_ImportRow]:
return [
(i.target, i.name, i.asname, i.line, i.context, i.guarded)
for i in module.imports
]


@functools.cache
def _load_all_imports() -> None:
"""Collects every runtime module's imports in parallel the first time any is read."""
modules = [m for m in runtime_modules() if not m.model_data]
for module, rows in zip(modules, map_modules(_import_rows, modules), strict=True):
module.__dict__["imports"] = [Import(module, *row) for row in rows]
129 changes: 69 additions & 60 deletions scripts/verifications/docstrings.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,24 @@
"""

import ast
import functools
import importlib
import inspect
import os
import re
import sys
from concurrent.futures import ThreadPoolExecutor
from pathlib import Path

from _common import (
ROOT,
Finding,
Module,
Package,
analyzed_modules,
main,
model_data_globs,
module_findings,
packages,
run,
runtime_modules,
Expand All @@ -42,51 +48,50 @@
def check() -> list[Finding]:
return [
*(f for f in _ruff_findings() if not _inherits_docstring(f)),
*_constant_findings(),
*module_findings(_constant_findings, analyzed_modules()),
]


def _ruff_findings() -> list[Finding]:
with ThreadPoolExecutor(max_workers=os.cpu_count() or 4) as pool:
return [
f for found in pool.map(_package_ruff_findings, packages()) for f in found
]


def _package_ruff_findings(package: Package) -> list[Finding]:
ruff = str(Path(sys.executable).parent / "ruff")
findings: list[Finding] = []
for package in packages():
proc = run(
[
ruff,
"check",
"--config",
str(package.dir / "pyproject.toml"),
"--select",
_SELECT,
"--output-format",
"concise",
"--no-cache",
*(
arg
for glob in model_data_globs(package)
for arg in ("--extend-exclude", glob)
),
str(package.src),
]
)
for line in proc.stdout.splitlines():
if (match := _LOCATION.match(line)) is not None:
findings.append(
Finding(
match["message"],
(ROOT / match["path"]).resolve(),
int(match["line"]),
)
)
return findings
proc = run(
[
ruff,
"check",
"--config",
str(package.dir / "pyproject.toml"),
"--select",
_SELECT,
"--output-format",
"concise",
"--no-cache",
*(
arg
for glob in model_data_globs(package)
for arg in ("--extend-exclude", glob)
),
str(package.src),
]
)
return [
Finding(match["message"], (ROOT / match["path"]).resolve(), int(match["line"]))
for line in proc.stdout.splitlines()
if (match := _LOCATION.match(line)) is not None
]


def _inherits_docstring(finding: Finding) -> bool:
"""Whether a D102 method overrides one whose docstring editors and docs show."""
if not finding.message.startswith("D102 ") or finding.path is None:
return False
path = finding.path
module = next((m for m in runtime_modules() if m.path.resolve() == path), None)
module = _modules_by_path().get(finding.path)
if module is None:
return False
qualname = _method_qualname(module.tree, finding.line)
Expand All @@ -104,6 +109,11 @@ def _inherits_docstring(finding: Finding) -> bool:
return inspect.getdoc(obj) is not None


@functools.cache
def _modules_by_path() -> dict[Path, Module]:
return {m.path.resolve(): m for m in runtime_modules()}


def _method_qualname(tree: ast.Module, line: int) -> list[str] | None:
def walk(body: list[ast.stmt], prefix: list[str]) -> list[str] | None:
for node in body:
Expand All @@ -121,35 +131,34 @@ def walk(body: list[ast.stmt], prefix: list[str]) -> list[str] | None:
return walk(tree.body, [])


def _constant_findings() -> list[Finding]:
def _constant_findings(module: Module) -> list[Finding]:
findings: list[Finding] = []
for module in analyzed_modules():
if not module.public:
if not module.public:
return findings
body = module.tree.body
for index, node in enumerate(body):
if isinstance(node, ast.Assign) and len(node.targets) == 1:
target = node.targets[0]
elif isinstance(node, ast.AnnAssign):
target = node.target
else:
continue
body = module.tree.body
for index, node in enumerate(body):
if isinstance(node, ast.Assign) and len(node.targets) == 1:
target = node.targets[0]
elif isinstance(node, ast.AnnAssign):
target = node.target
else:
continue
if not isinstance(target, ast.Name) or not _CONSTANT.match(target.id):
continue
following = body[index + 1] if index + 1 < len(body) else None
documented = (
isinstance(following, ast.Expr)
and isinstance(following.value, ast.Constant)
and isinstance(following.value.value, str)
)
if not documented:
findings.append(
Finding(
f"public constant {target.id} has no docstring",
module.path,
node.lineno,
)
if not isinstance(target, ast.Name) or not _CONSTANT.match(target.id):
continue
following = body[index + 1] if index + 1 < len(body) else None
documented = (
isinstance(following, ast.Expr)
and isinstance(following.value, ast.Constant)
and isinstance(following.value.value, str)
)
if not documented:
findings.append(
Finding(
f"public constant {target.id} has no docstring",
module.path,
node.lineno,
)
)
return findings


Expand Down
Loading
Loading