diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f064dbc..a109f55 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -36,6 +36,20 @@ uv run pytest -q tests/ uv run --no-project --with hatchling python tests/sdist_smoke.py ``` +## Performance Comparisons + +Benchmark full directory analysis, including entrypoint discovery, with: + +```bash +uv run --locked python -m tests.benchmark_analysis /path/to/corpus --language python --repeat 7 +``` + +Use the same unchanged corpus, path, Python version, and dependencies in both +checkouts, without concurrent tests or mutation workers. The runner warms up +grammar loading, reports every timing and the median, and fingerprints the +complete graph output with `graph_sha256`. Compare the fingerprints as well as +the timings; a speedup should preserve or explicitly account for graph changes. + ## Mutation Testing Trailmark uses a pinned pytest-gremlins runner to verify test suite quality. @@ -98,6 +112,9 @@ their existing files; `tests/mutation_order.py` defines the ordering, and unknown components keep collection order. Use `--test-order=collection` to compare against pytest's original order. +Private helpers use their component test names without the leading underscore; +Python decorator helpers start with `test_python_decorator_bindings.py`. + **Focusing a campaign.** Pass `--targets` with comma-separated source paths, followed by the relevant test files. diff --git a/src/trailmark/analysis/_python_decorators.py b/src/trailmark/analysis/_python_decorators.py index 1d8b7af..ce1468a 100644 --- a/src/trailmark/analysis/_python_decorators.py +++ b/src/trailmark/analysis/_python_decorators.py @@ -20,10 +20,11 @@ import ast from dataclasses import dataclass, field, replace +from functools import cached_property, lru_cache from pathlib import Path from typing import Literal -from tree_sitter import Node, Parser +from tree_sitter import Node, Parser, Query, QueryCursor from tree_sitter_language_pack import get_language from trailmark.models.nodes import SourceLocation @@ -147,7 +148,20 @@ def python_decorators(path: str) -> list[DecoratorFact]: except OSError: return [] root = Parser(get_language("python")).parse(source).root_node - return _Analyzer(path, root).facts() + decorated = QueryCursor(_decorator_query()).captures(root).get("decorated", []) + if not decorated: + return [] + return _Analyzer(path, root).facts(sorted(decorated, key=lambda node: node.start_byte)) + + +@lru_cache(maxsize=1) +def _decorator_query() -> Query: + # Select functions before building lexical scopes: class decorators alone + # need no binding analysis. The full tree still supplies binding evidence. + return Query( + get_language("python"), + "(decorated_definition definition: (function_definition)) @decorated", + ) class _Analyzer: @@ -156,18 +170,17 @@ def __init__(self, path: str, root: Node) -> None: self._root = root self._scopes = _build_scopes(root) self._module = self._scopes[root.id] - self._stored, self._dynamic = _attribute_writes(root) - def facts(self) -> list[DecoratorFact]: + @cached_property + def _writes(self) -> tuple[frozenset[str], bool]: + # Only local identity proofs need the file-wide replacement scan. + return _attribute_writes(self._root) + + def facts(self, decorated: list[Node]) -> list[DecoratorFact]: facts: list[DecoratorFact] = [] - pending = [self._root] - while pending: - node = pending.pop() - pending.extend(reversed(node.named_children)) + for node in decorated: definition = node.child_by_field_name("definition") - if node.type != "decorated_definition" or definition is None: - continue - if definition.type != "function_definition": + if definition is None: continue scope = self._owner(node) position = node.start_byte if _statement_block(node) == scope.body else None @@ -240,7 +253,8 @@ def _disposition( def _unreplaced(self, function: Node, callee: _Value) -> bool: """Whether nothing visible in the file can replace a local identity decorator.""" - if self._dynamic: + stored, dynamic = self._writes + if dynamic: return False names = {_text(function.child_by_field_name("name") or function), callee.method} block = function.parent @@ -250,7 +264,7 @@ def _unreplaced(self, function: Node, callee: _Value) -> bool: if scope is None or not _plain_class(owner, scope): return False names.add(_text(owner.child_by_field_name("name") or owner)) - return not names & self._stored + return not names & stored def _literal( self, node: Node, scope: _Scope, position: int | None, depth: int diff --git a/tests/benchmark_analysis.py b/tests/benchmark_analysis.py new file mode 100644 index 0000000..b711901 --- /dev/null +++ b/tests/benchmark_analysis.py @@ -0,0 +1,59 @@ +"""Measure full directory analysis and fingerprint its complete graph output. + +Run on the same unchanged corpus in both checkouts, without concurrent tests: +``uv run python -m tests.benchmark_analysis /path/to/corpus --language python``. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import platform +import statistics +import time + +from trailmark.query import QueryEngine + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directory") + parser.add_argument("--language", default="auto") + parser.add_argument("--repeat", type=int, default=7) + args = parser.parse_args() + if args.repeat < 1: + parser.error("--repeat must be positive") + + elapsed = [] + digests = set() + # Warm up grammar loading and filesystem caches before timing. + QueryEngine.from_directory(args.directory, language=args.language) + for _ in range(args.repeat): + start = time.perf_counter() + engine = QueryEngine.from_directory(args.directory, language=args.language) + elapsed.append(time.perf_counter() - start) + graph = json.loads(engine.to_json()) + serialized = json.dumps(graph, sort_keys=True, separators=(",", ":")) + digests.add(hashlib.sha256(serialized.encode()).hexdigest()) + del engine + if len(digests) != 1: + raise RuntimeError("Graph output changed between benchmark repetitions") + print( + json.dumps( + { + "directory": args.directory, + "language": args.language, + "python": platform.python_version(), + "platform": platform.platform(), + "seconds": elapsed, + "median_seconds": statistics.median(elapsed), + "graph_sha256": digests.pop(), + }, + indent=2, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/tests/mutation_order.py b/tests/mutation_order.py index bc6599b..ea3a5f0 100644 --- a/tests/mutation_order.py +++ b/tests/mutation_order.py @@ -31,8 +31,10 @@ def prioritize_tests(source: str, node_ids: list[str]) -> list[str]: preferred = ["test_storage.py", "test_storage_edge_cases.py"] elif "query" in parts: preferred = ["test_query.py", "test_query_edge_cases.py", "test_hidden_apis.py"] + elif path.name == "_python_decorators.py": + preferred = ["test_python_decorator_bindings.py", "test_entrypoints.py"] else: - preferred = [f"test_{path.stem}.py"] + preferred = [f"test_{path.stem.removeprefix('_')}.py"] ranks = {name: rank for rank, name in enumerate(preferred)} return sorted( node_ids, key=lambda node: ranks.get(Path(node.split("::", 1)[0]).name, len(ranks)) diff --git a/tests/test_callable_references.py b/tests/test_callable_references.py index 3ef4721..8a2611f 100644 --- a/tests/test_callable_references.py +++ b/tests/test_callable_references.py @@ -532,6 +532,147 @@ def test_php_namespace_imports_resolve_class_constants(tmp_path: Path) -> None: assert {e.confidence for e in edges} == {EdgeConfidence.INFERRED} +def test_python_method_references_are_case_sensitive(tmp_path: Path) -> None: + graph = _discover( + tmp_path, + { + "app.py": ( + "class Handler:\n" + " def Run(self): pass\n" + " def run(self): pass\n" + "def setup():\n attach(Handler.Run, Handler.run)\n" + ) + }, + ) + facts = _evidence(graph) + assert [(f.node_id, f.binding, dict(f.details)["path"]) for f in facts] == [ + ("app:Handler.Run", "resolved", "args[0]"), + ("app:Handler.run", "resolved", "args[1]"), + ] + + +def test_python_package_initializer_resolves_sibling_imports(tmp_path: Path) -> None: + graph = _discover( + tmp_path, + { + "pkg/__init__.py": ( + "from .handlers import h as handler\ndef setup():\n attach(handler)\n" + ), + "pkg/handlers.py": "def h(): pass\n", + }, + ) + assert _references(graph) == {("pkg:setup", "pkg.handlers:h")} + (fact,) = _evidence(graph, "handler") + assert (fact.binding, fact.source_id) == ("resolved", "pkg:setup") + + +def test_python_duplicate_module_suffixes_keep_all_candidates_ambiguous(tmp_path: Path) -> None: + graph = _discover( + tmp_path, + { + "left/pkg/handlers.py": "def h(): pass\n", + "right/pkg/handlers.py": "def h(): pass\n", + "app.py": "from pkg import handlers\ndef setup():\n attach(handlers.h)\n", + }, + ) + assert _references(graph) == set() + assert {(f.node_id, f.binding) for f in _evidence(graph, "handlers.h")} == { + ("left.pkg.handlers:h", "ambiguous"), + ("right.pkg.handlers:h", "ambiguous"), + } + + +def test_php_grouped_imports_and_absolute_callable_names(tmp_path: Path) -> None: + graph = _discover( + tmp_path, + { + "lib.php": ( + r" None: + graph = _discover( + tmp_path, + { + "app.php": """ None: + graph = _discover( + tmp_path, + { + "app.php": """attach(h(...)); + attach(["Missing", "h"], [0 => "h", 1 => "h"]); +} +""" + }, + "php", + ) + facts = _evidence(graph) + assert {f.node_id for f in facts} == {"app:h"} + assert {f.binding for f in facts} == {"resolved"} + assert [(dict(f.details)["consumer"], dict(f.details)["path"]) for f in facts] == [ + ("Type::attach", "args[0]"), + ("new Registry", "args[0]"), + ("$r->attach", "args[0]"), + ("attach", "args[0][1]"), + ("attach", "args[1][0]"), + ("attach", "args[1][1]"), + ] + + # --------------------------------------------------------------------------- Stability diff --git a/tests/test_docs.py b/tests/test_docs.py index 80f5718..369732c 100644 --- a/tests/test_docs.py +++ b/tests/test_docs.py @@ -145,6 +145,8 @@ def pyproject_data() -> dict[str, object]: ("docs/installation.md", "Circom and Move use ABI 14; Verus uses ABI 15"), ("docs/installation.md", "installing published packages needs neither"), # Mutation testing + ("CONTRIBUTING.md", "tests.benchmark_analysis /path/to/corpus --language python --repeat 7"), + ("CONTRIBUTING.md", "complete graph output with `graph_sha256`"), ("CONTRIBUTING.md", "tests.run_mutations --reviewed --workers=4"), ("CONTRIBUTING.md", "tests.run_mutations --workers=4"), ("CONTRIBUTING.md", "tests.mutation_gate"), diff --git a/tests/test_mutation_order.py b/tests/test_mutation_order.py index 421b521..694e8f1 100644 --- a/tests/test_mutation_order.py +++ b/tests/test_mutation_order.py @@ -24,6 +24,9 @@ ("storage/graph_store.py", "test_storage.py"), ("query/api.py", "test_query.py"), ("analysis/entrypoints.py", "test_entrypoints.py"), + ("analysis/_callable_references.py", "test_callable_references.py"), + ("analysis/_reference_literals.py", "test_reference_literals.py"), + ("analysis/_python_decorators.py", "test_python_decorator_bindings.py"), ], ) def test_component_first_retains_every_test(source: str, focused: str) -> None: @@ -40,6 +43,23 @@ def test_unknown_component_keeps_collection_order() -> None: assert prioritize_tests("src/trailmark/new_feature.py", ids) == ids +def test_decorator_units_precede_framework_tests_and_keep_fallbacks() -> None: + ids = [ + "tests/test_integration.py::test_end_to_end", + "tests/test_entrypoints.py::test_route", + "tests/test_python_decorator_bindings.py::test_alias", + "tests/test_callable_references.py::test_reference", + "tests/test_python_decorator_bindings.py::test_shadow", + ] + assert prioritize_tests("src/trailmark/analysis/_python_decorators.py", ids) == [ + ids[2], + ids[4], + ids[1], + ids[0], + ids[3], + ] + + def test_parameter_ids_are_preserved_exactly(tmp_path: Path) -> None: ids = [ f"{tmp_path}/tests/test_values.py::test_value[ONE]", diff --git a/tests/test_python_decorator_bindings.py b/tests/test_python_decorator_bindings.py index 5575e09..c71586a 100644 --- a/tests/test_python_decorator_bindings.py +++ b/tests/test_python_decorator_bindings.py @@ -700,3 +700,137 @@ def deep(): ... assert _only(facts, "cyclic").receiver_binding == "ambiguous" assert _only(facts, "cyclic").disposition == "unknown" assert _only(facts, "deep").consumer_path is None + + +def test_only_function_decorators_are_reported_in_source_order(tmp_path: Path) -> None: + path = tmp_path / "subject.py" + path.write_text( + "@class_marker\nclass Views:\n" + " @outer('/a')\n @inner('/b')\n async def method(self):\n" + " @nested('/c')\n def handler(): pass\n" + "@last('/d')\ndef final(): pass\n" + ) + facts = python_decorators(str(path)) + assert [(fact.method, fact.handler_line, fact.handler_col) for fact in facts] == [ + ("outer", 5, 4), + ("inner", 5, 4), + ("nested", 7, 8), + ("last", 9, 0), + ] + assert [fact.location.start_line for fact in facts] == [3, 4, 6, 8] + assert [fact.path for fact in facts] == ["/a", "/b", "/c", "/d"] + + # Reusing the syntax query must not reuse facts from an earlier file version. + path.write_text("@class_marker\nclass Views:\n def method(self): pass\n") + assert python_decorators(str(path)) == [] + path.write_text("@replacement('/new')\ndef handler(): pass\n") + assert [(f.method, f.path) for f in python_decorators(str(path))] == [("replacement", "/new")] + + +def test_parenthesized_and_compound_receivers_keep_binding_precision(tmp_path: Path) -> None: + facts = _facts( + tmp_path, + """ + from vendor import Router + app = Router() + + @((app)).get('/known') + def known(): ... + + @(app or fallback).get('/unknown') + def unknown(): ... + """, + ) + assert _summary(_only(facts, "known")) == ( + "vendor.Router().get", + "call_result", + "get", + "/known", + "path_decorator", + "literal_path", + ) + assert _summary(_only(facts, "unknown")) == ( + None, + "ambiguous", + "get", + "/unknown", + "unknown", + "ambiguous_binding", + ) + + +@pytest.mark.parametrize("returned", ["lambda fn: other", "lambda fn, extra: fn"]) +def test_nonidentity_factories_are_not_suppressed(tmp_path: Path, returned: str) -> None: + facts = _facts( + tmp_path, + f"def lookup(path):\n return {returned}\n@lookup('/x')\ndef handler(): pass\n", + ) + assert _summary(_only(facts, "handler")) == ( + "lookup", + "enclosing_local", + "lookup", + "/x", + "path_decorator", + "literal_path", + ) + + +def test_nonlocal_writes_invalidate_the_enclosing_receiver(tmp_path: Path) -> None: + facts = _facts( + tmp_path, + """ + from vendor import Router + def setup(): + app = Router() + def replace(): + nonlocal app + app = other + @app.get('/x') + def handler(): ... + """, + ) + assert _summary(_only(facts, "handler")) == ( + None, + "ambiguous", + "get", + "/x", + "unknown", + "ambiguous_binding", + ) + + +def test_pattern_capture_shadows_the_global_receiver(tmp_path: Path) -> None: + facts = _facts( + tmp_path, + """ + from vendor import Router + app = Router() + def setup(value): + match value: + case app: + pass + @app.get('/x') + def handler(): ... + """, + ) + assert _summary(_only(facts, "handler")) == ( + None, + "ambiguous", + "get", + "/x", + "unknown", + "ambiguous_binding", + ) + + +@pytest.mark.parametrize("length", [118, 120, 121, 200]) +def test_long_argument_evidence_is_bounded_without_truncating_paths( + tmp_path: Path, length: int +) -> None: + path = "/" + "x" * (length - 3) + argument = repr(path) + assert len(argument) == length + facts = _facts(tmp_path, f"@app.get({argument})\ndef handler(): pass\n") + fact = _only(facts, "handler") + assert fact.path == path + assert fact.arguments == (argument if length <= 120 else argument[:117] + "...",)