diff --git a/Cargo.lock b/Cargo.lock index dc59707839..b4f801c2a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2547,6 +2547,49 @@ dependencies = [ "vec1", ] +[[package]] +name = "pyrefly_static_analysis" +version = "0.0.0" +dependencies = [ + "pyrefly_python", + "pyrefly_util", + "ruff_python_ast", + "ruff_python_parser", + "ruff_source_file", + "ruff_text_size", + "starlark_map", +] + +[[package]] +name = "pyrefly_static_analysis_exports" +version = "0.0.0" +dependencies = [ + "pyrefly", + "pyrefly_python", + "pyrefly_static_analysis", + "pyrefly_types", + "ruff_python_ast", + "ruff_text_size", + "starlark_map", +] + +[[package]] +name = "pyrefly_static_analysis_lifeguard" +version = "0.0.0" +dependencies = [ + "pyrefly_static_analysis", + "starlark_map", +] + +[[package]] +name = "pyrefly_static_analysis_spmd" +version = "0.0.0" +dependencies = [ + "pyrefly_python", + "pyrefly_static_analysis", + "serde_json", +] + [[package]] name = "pyrefly_types" version = "1.4.0" diff --git a/Cargo.toml b/Cargo.toml index 40a01e0ad5..ef6e86e8c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,10 @@ members = [ "crates/pyrefly_types", "crates/pyrefly_util", "crates/tsp_types", + "static_analysis", + "static_analysis/pyrefly_exports", + "static_analysis/spmd", + "static_analysis/lifeguard", "pyrefly", "pyrefly_wasm", ] diff --git a/pyrefly/lib/lib.rs b/pyrefly/lib/lib.rs index b40693e200..1d34729383 100644 --- a/pyrefly/lib/lib.rs +++ b/pyrefly/lib/lib.rs @@ -34,7 +34,9 @@ mod compat; #[doc(hidden)] pub mod embed; mod error; -mod export; +// Experimental export-only adapter for the static_analysis spike. This is not a stable API. +#[doc(hidden)] +pub mod export; #[doc(hidden)] pub mod lsp; mod memory_project; diff --git a/static_analysis/Cargo.toml b/static_analysis/Cargo.toml new file mode 100644 index 0000000000..611c4726d0 --- /dev/null +++ b/static_analysis/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "pyrefly_static_analysis" +version = "0.0.0" +edition = "2024" +license.workspace = true +repository.workspace = true +publish = false +description = "Experimental neutral Python frontend (scopes, operation CFG) and generic dataflow engine" + +[dependencies] +pyrefly_python = { path = "../crates/pyrefly_python" } +pyrefly_util = { path = "../crates/pyrefly_util" } +ruff_python_ast = "0.0.14" +ruff_python_parser = "0.0.14" +ruff_source_file = "0.0.14" +ruff_text_size = "0.0.14" +starlark_map = "0.14.2" + +[lints] +workspace = true diff --git a/static_analysis/DESIGN.md b/static_analysis/DESIGN.md new file mode 100644 index 0000000000..fdafabfed3 --- /dev/null +++ b/static_analysis/DESIGN.md @@ -0,0 +1,48 @@ +# Shared static-analysis frontend: experimental design + +This branch is an open-source feasibility spike. The question is whether two substantially different analyses can share Python scope, evaluation-order, and flow machinery—not whether this branch replaces either production checker. + +## Dependency direction + +```text +Python syntax and lexical ownership + | +ordered operations, regions, CFG, symbol history + | + +----------+--------------+ + | | | +SPMD Lifeguard history queries + domain domain without a domain +``` + +Pyrefly's type-specific `Bindings` and `Answers` are not the shared intermediate representation. A future Pyrefly binding emitter could consume this frontend and add inference dependencies downstream. The experimental export adapter depends on the existing checker crate but the core does not. + +## Shared work versus domain work + +The structured driver owns statement-level structure over the parsed AST: suite order, the decomposition of `if`/`elif`/`else` and loops, which arms run given a backend's static truth for their tests, and the loop that a `break` or `continue` leaves. Lowering is a backend of the driver that emits blocks and terminators. A backend owns its values and its policy for deciding tests; the driver never evaluates a test itself. + +The frontend owns declarations, scope visibility, evaluation order, deferred versus immediate code regions, and explicit flow edges. Statement ranges provide a coarse grouping of ordered expression operations. Consumers do not reparse source or reconstruct scopes by walking the AST. + +The engine owns frame bookkeeping, branch/loop propagation, joins, convergence limits, and deterministic diagnostic collection. A domain defines abstract values plus arbitrary state: heap/typestate facts, capability state, ambient meshes, effects, or other context. An operation may change existing facts; it need not have a useful return value. + +Assignment preserves a value reference, not a link that follows later rebinding of another name. Lexical symbol ownership, reaching definitions, and object aliasing are distinct. Domains that track mutable objects must preserve object identities and account for escapes. Captured bindings and globals are activation-time cells, not permanently frozen definition-time values. + +Ambient context is carried and joined along paths, rather than kept as a process-global mode. Scoped entry/exit and call contracts belong in analysis semantics. Unknown context or differing predecessor contexts must not be resolved by choosing one predecessor arbitrarily. + +The reusable declaration scanner and lexical navigation engine are separate from a backend's value tables. Runtime ownership requests unpruned syntax; configured/type-aware declaration views can select positions and branches through explicit policy hooks. Name occurrences carry a syntactic role and do not stand in for runtime reads. Candidate payloads, initialization and type-specific filtering stay in the consumer. + +Optional pre-check feedback is a separate interface keyed by source-qualified structural points. The facts interface carries explicit assumptions, diagnostic coverage and observed coverage; a conditional typing result is not unconditional execution proof. Point IDs are not CFG-block identities, and providers must reject stale source while consumers arrange fresh dependency/configuration snapshots. The shared frontend does not choose narrowing or merge policies when applying such facts. + +## Expression ordering and inference ordering + +Expression-flow consumption is useful for custom type systems, but runtime evaluation order is not type-inference dependency order. Ordinary assignment evaluates RHS before target expressions; augmented assignment evaluates its target/read before the RHS. Short-circuit operators and conditional expressions contain control flow. + +A downstream type checker may introduce contextual/deferred dependencies without changing the shared execution structure. A statement-level client may summarize expression behavior; it must not treat a statement as effect-free merely because its interior is unexpanded. + +## Evidence needed for generality + +The consumers must both use the same neutral operations and engine. SPMD exercises tensor/gradient contracts and active-mesh context. Lifeguard exercises imported-object provenance, mutation, deferred function effects, and call summaries. A history-only client and a small typestate test probe additional reuse without adding a third production analysis. + +Unsupported Python features and unconverged analyses are explicit incomplete outcomes. Passing a selected fixture subset is not full parity, and neither domain provides universal formal verification of arbitrary Python or custom kernels. + +Temporary adapters into complete Pyrefly/Lifeguard forks are possible follow-up experiments; production migration is outside this spike. diff --git a/static_analysis/PYREFLY_ADAPTER.md b/static_analysis/PYREFLY_ADAPTER.md new file mode 100644 index 0000000000..d3177c9bee --- /dev/null +++ b/static_analysis/PYREFLY_ADAPTER.md @@ -0,0 +1,108 @@ +# Pyrefly bindings as a downstream consumer + +## Goal and evidence boundary + +The long-term capability target is an exact match or superset of today's binding pass, without making `Bindings` or `Answers` the shared frontend API. Adoption by Pyrefly remains optional: a substantial performance cost could make sharing undesirable even if it is technically possible. + +**The current spike does not establish that capability target.** Its neutral scopes, regions, ordered operations and CFG provide a plausible substrate, but its runtime-focused IR discards information required by the actual bindings pass. This document is an implementation sketch and gap map, not a completed migration or a performance result. + +Existing Pyrefly references below are against upstream commit `0424eebb19345ca0b40727a15cfca9e759c05d6b`. The spike's files are the working implementation in this branch. + +## Dependency direction + +```text +one parse + shared declarations / scope / flow construction + | + neutral Program + optional lossless syntax view + | + Pyrefly-specific binding emitter + | + actual Bindings -> existing solver +``` + +The emitter belongs in Pyrefly or an optional adapter, not the generic core. Its job is to retain Pyrefly's binding tables, inference policy and result packaging while replacing frontend-facing work. Today's [Bindings::new](https://github.com/facebook/pyrefly/blob/0424eebb19345ca0b40727a15cfca9e759c05d6b/pyrefly/lib/binding/bindings.rs#L702-L897) combines that work with AST traversal, scope state, deferred name processing and exports. + +Binding emission should **not** happen as side effects of `Domain::transfer`: the forward engine repeats transfers during fixpoint iteration and diagnostic replay. An emitter should reserve graph identities, obtain neutral flow relationships, and emit each computation once. It need not use the generic worklist engine as its dependency solver. + +## Identity mapping + +`SymbolId` denotes one name owned by one scope. It is not one definition or one typing computation. The emitter needs separate mappings: + +- declaration/write occurrence -> `Key::Definition`; +- read occurrence -> `Key::BoundName` plus its selected value/narrowing dependency; +- scope/symbol -> annotation, capture and anywhere-summary bookkeeping; +- producing operation and merge port -> the computation supplying a value. + +The spike's `Reg` is not SSA: both arms of a conditional expression can write the same temporary. A simple `(RegionId, Reg) -> Key` map would therefore lose producers. Use operation identities and predecessor ports. + +Nor should every runtime operation become a separate `Binding::Expr`. Existing [NameAssign](https://github.com/facebook/pyrefly/blob/0424eebb19345ca0b40727a15cfca9e759c05d6b/pyrefly/lib/binding/binding.rs#L2327-L2357) retains a whole RHS expression so the existing solver can apply contextual typing. Runtime decomposition and typing-computation ownership must remain distinct. + +## Typing dependencies stay downstream + +Execution order is not inference order. An annotation or receiver can supply an expected type to the RHS; callable parameters constrain arguments; predicates generate narrowing dependencies; first-use inference can connect a definition to a later use. + +The shared frontend should preserve source occurrences, expression relationships and declaration records. Pyrefly decides whether a particular occurrence is a type, metadata, ordinary value, or non-pinning read. This includes `cast`, type-like subscripts, and `Annotated` metadata. Retaining syntax does not require a second scope/control-flow walker: shared traversal supplies occurrence records and expression roots, while the existing solver retains its expression inference. + +First-use resolution belongs in graph finalization. Pyrefly records uses with their computation owners, fills phis and then finalizes reads in insertion order. Runtime earliest-use order is insufficient; static/non-pinning reads and solve-time feedback differ. See [deferred name-use finalization](https://github.com/facebook/pyrefly/blob/0424eebb19345ca0b40727a15cfca9e759c05d6b/pyrefly/lib/binding/bindings.rs#L2027-L2117). + +## Flow, termination and deferred references + +CFG predecessors are inputs to Pyrefly phis, not finished phis. A branch's value can be paired with a typing computation that establishes whether that branch terminates. A dependency solving to `Never` may remove a branch from a typing merge; missing definitions generate deferred initialization checks. + +Expose incoming-edge identities and definitions before generic joins discard them. The adapter adds branch tests/end computations, narrowing, exhaustiveness and loop-merge policies. Explicit runtime return/raise structure helps, but does not resolve type-dependent continuation. + +Reserve identities before filling computations to support recursion and forward references. Runtime closure-cell reads are invocation-time facts; Pyrefly's static/anywhere lookup and forward typing references are separate adapter policies. Domain-free `History::reaching` marks possible writes from other regions with `Def::External`; such a set is not an exact local-only value history. It does not resolve invocation-time cross-region reaching definitions. + +## Concrete example + +```python +from typing import Callable + + +def pick(flag: bool, f: Callable[[int], int]) -> int: + if flag: + x: int = f(1) + else: + x = 0 + return x +``` + +An experimental emitter would: + +1. Reserve function, parameter, annotation and return identities. +2. Bind the condition's `flag` occurrence and add downstream narrowing/branch computations. +3. Emit two definition/`NameAssign` pairs for `x`. Keep `f(1)` as the true RHS computation; attach its `f` read to that assignment's usage context. Forward the canonical annotation to the other assignment under Pyrefly's policy. +4. Emit the merge with predecessor/continuation dependencies, then connect the return's `x` read to it. +5. Emit return-type and explicit-return computations and finalize deferred uses. + +This maps to existing statement/function constructor seams, but **requires additional syntax preservation**: current runtime lowering does not retain the `x: int` declaration sufficiently to reproduce its complete binding output. + +## Capability gap map + +The gap map is not limited to constructs already marked with a runtime `Gap`: + +- **Annotation syntax and occurrences:** annotation-only declarations can emit no runtime operation. `StaticExpr::Other` retains only a range; structured static forms lose some occurrence ranges and collection distinctions. Preserve declaration records and lossless annotation/type-parameter syntax. +- **Expression payloads and computation grouping:** bytes, complex/oversized numerical values and formatted-string structure are compressed. Target groups, keyword identifier ranges and enclosing RHS ownership are incomplete. Source slices are not a replacement for a parsed syntax view. +- **Scope/capture provenance:** declaration occurrences (`global`/`nonlocal`), static capture anchors and diagnostic ranges need fuller records. Operations now carry their lexical evaluation scope, and class/module loads have shared binding-presence/fallback semantics; those runtime facts still differ from Pyrefly's typing lookup policy. +- **Flow ports:** structural CFG edges exist, but joined frames/history sets discard incoming-edge provenance. Raised exits now retain source range, cause and state. Return terminators still need explicit/implicit origin and range metadata for exact binding parity. +- **Function/class metadata:** headers, suites, decorator roles, overload adjacency, annotation-defined fields and synthesized `__class__` plumbing are incomplete. Shared generic declaration structure belongs upstream; overload, `Self`, dataclass/framework synthesis and shape-DSL policy stay downstream. +- **Typing declarations:** PEP 695 scopes, alias RHS syntax, bounds/constraints/defaults and recursive alias anchors are outside v0. Ordinary function regions are not enough to reproduce them. +- **Patterns, exceptions, imports and packaging:** approximation gaps are not lossless representations. Exact binding/IDE parity also includes suppression ranges, scope traces, unused-symbol data and other result metadata. + +## Smallest next experiment + +Three frontend additions would make the seam testable: + +1. An **optional lossless syntax view**, with occurrence IDs, original expression roots, exact ranges, declaration/header/suite records and separately represented typing syntax. +2. A **neutral predecessor-port query**, including incoming definitions of symbols and temporary values plus entry/unbound paths. +3. A **parse-reuse entry point** accepting the already parsed module. Optional collectors should not allocate when absent. + +Then implement an opt-in emitter for names, assignments, one function, calls and branches. Compare canonicalized actual binding tables with `Bindings::new`: source identities, kinds, dependency edges, expectations and metadata, not merely final inferred types. Add first-use, annotation-only, loop/unbound, recursion/capture, termination, decoration, class and type-parameter fixtures incrementally. Unsupported constructs must fail coverage checks rather than fabricate parity. + +## Performance decision + +Benchmark optimized, comparable configurations: parse only; parse plus neutral frontend; frontend plus history; frontend plus syntax view/emitter; existing bindings. Separate parsing-included and parsing-excluded results, allocations and peak memory, with comparable AST lifetimes. + +Preserve existing skipped-body/lazy-work optimizations where possible. Measure the no-domain/no-history/no-syntax-overlay path explicitly. Do not adopt a percentage threshold without a baseline and measured noise; capability success and performance acceptability are independent decisions. + +A small parity-tested adapter would increase confidence in reusability. Until then, the two implemented domains establish a narrower result: shared analysis machinery for their supported subsets, not replacement capability for the full Pyrefly binding pass. diff --git a/static_analysis/README.md b/static_analysis/README.md new file mode 100644 index 0000000000..7ee1638e75 --- /dev/null +++ b/static_analysis/README.md @@ -0,0 +1,70 @@ +# Static-analysis sharing spike + +This is an **AI-assisted experimental branch**, not a production checker or an upstream change proposal. It tests whether Python scope, evaluation-order and flow machinery can support two different analyses without making Pyrefly's `Bindings`/`Answers` their API. Human review is needed before production use or upstream submission. + +## Packages + +- **`pyrefly_static_analysis`** (`src/`): lexical declaration/scope analysis, neutral operations and CFG, deferred code regions, full-state dataflow engine, and domain-free history queries. +- **`pyrefly_static_analysis_spmd`** (`spmd/`): tensor/gradient contracts, sharding descriptors, explicit collective transitions and an ambient active mesh. +- **`pyrefly_static_analysis_lifeguard`** (`lifeguard/`): a subset of lazy-import safety analysis, tracking value provenance, mutation, deferred effects and function summaries. +- **`pyrefly_static_analysis_exports`** (`pyrefly_exports/`): an optional adapter to Pyrefly's real export calculation. The core does not depend on the checker crate; this adapter does, without building Bindings or solving Answers. + +The consumers do not parse Python or walk its AST. They use the same resolved symbols, ordered operations, frames and engine. Their domain-specific values, effects, summaries and mesh rules remain outside the core. + +## Try it + +From the repository root, with the repository's Rust toolchain: + +```sh +cargo test -p pyrefly_static_analysis \ + -p pyrefly_static_analysis_spmd \ + -p pyrefly_static_analysis_lifeguard \ + -p pyrefly_static_analysis_exports + +cargo run -p pyrefly_static_analysis_spmd -- \ + static_analysis/spmd/examples/tp_mlp.py \ + static_analysis/spmd/examples/tp_mlp.json + +E=static_analysis/lifeguard/examples +cargo run -p pyrefly_static_analysis_lifeguard --bin lifeguard-spike -- \ + registry=$E/registry.py plugin=$E/plugin.py helpers=$E/helpers.py settings=$E/settings.py +``` + +The Lifeguard demo intentionally exits nonzero because it contains unsafe and unknown examples. The SPMD example validates only the supported contracts under its trusted JSON sidecar assumptions. Neither command executes the input Python or starts distributed training. + +See [SPMD usage and boundaries](spmd/README.md) and [Lifeguard usage, parity matrix and boundaries](lifeguard/README.md) for details and exit codes. + +## What is shared + +The frontend takes inspiration from the public Pyrefly binding/declaration pass and Lifeguard implementation, but separates Python execution structure from inference policy. It resolves static ownership before reachability pruning; records source scope, statements and expression operations; separates immediate code from call/iteration/await-triggered regions; and supplies control-flow joins and definition/use queries. + +A domain can change an existing name's facts or object/typestate information and carry arbitrary ambient state. It is not limited to returning a type for an expression. Tests include a small resource/lock domain, alongside the two primary consumers. + +Statement structure is sequenced by a structured driver (`structured.rs`) that owns suite order, `if`/loop decomposition, the application of a backend's static truth for tests, and `break`/`continue` attribution. Lowering is one backend of that driver; a backend that keeps its own values (for example a type checker's flow environments) can reuse it over the same parsed AST without lowering. + +Declaration collection is shared through a per-scope event scanner (`declarations.rs`). The runtime scope builder consumes its declaration and nested-scope events without pruning dead code. Other sinks can select configured views and interpret metadata, while traversal and ownership rules remain in the shared walker. `NameSite` occurrences are syntactic, not an execution-read stream; the operation IR and history remain the runtime-use interfaces. + +Lexical navigation is shared through `lexical.rs`: backends supply parent/category/barrier information and candidate probes, while visible-scope and nonlocal-search structure is reusable. Found values, initialization, error recovery and type-specific selection remain backend-owned. Runtime annotation/type-parameter evaluation still has explicit gaps. + +Source flow-point observation is optional (`analyze_python_with_flow_points`), with IDs stable for the same module/source across backend pruning. Points identify structural occurrences, not allocated CFG blocks; a block mapping is still needed for automatic engine pruning. The optional `FlowFacts` interface includes assumptions/coverage metadata and conditional completion facts without requiring a type-analysis provider. Missing facts stay unknown, and source/dependency freshness is part of the provider contract. + +Execution order is not type-inference dependency order. A downstream Pyrefly adapter could add contextual typing, narrowing, first-use dependencies and synthesized bindings without putting those policies in the common frontend. + +## What this does not establish + +- **Full Pyrefly binding-pass capability is not demonstrated.** The runtime-focused IR is lossy for typing syntax and declaration provenance. [The adapter sketch and gap map](PYREFLY_ADAPTER.md) describe the additions and parity experiment needed to test that goal. +- **Layered/lazy expression CFG construction is not implemented.** Statement ranges provide coarse provenance/grouping, but expression operations are currently lowered eagerly. History and capture queries are optional; this is not yet a zero-cost coarse-only frontend. +- **Python safety and formal verification are not universal claims.** Unknown effects, supported-library assumptions, incomplete control-flow modeling and user-supplied contracts delimit what each verdict means. The consumers cover selected reference scenarios, not complete upstream parity. +- **Performance adoption is undecided.** Passing functional tests does not establish acceptable overhead for replacing Pyrefly's binding phase. The engine's sparse live-temporary representation improves the branch probe, but History still has a blocks-by-symbols cost (about 24/96/352 ms for 250/500/1000 branches in one release probe). These are prototype measurements, not Pyrefly phase benchmarks. Optimized, allocation-aware comparisons and skipped-body/no-consumer measurements are necessary. +- **Nested cleanup expansion is bounded, not fully optimized.** A per-region budget limits duplicated `finally` bodies; even ten nested levels can hit it. The resulting `FinallyLimit` gap explicitly marks the analysis incomplete rather than risking unbounded expansion. +- **Cross-platform execution is unverified.** Validation for this spike was performed on Linux; macOS and Windows need separate runs. + +The [spike outcome and review evidence](SPIKE_REPORT.md) summarize the experiment and its validation. [DESIGN.md](DESIGN.md) explains the dependency direction and intended abstraction boundaries. Both the implementation and its scope are experimental; unsupported cases must not be presented as validated. + +## Public references + +- Pyrefly base: `facebook/pyrefly` at `0424eebb19345ca0b40727a15cfca9e759c05d6b`. +- SPMD semantics: `meta-pytorch/spmd_types` at `e5ebc6d840fa2399bcd10306664faf4a2b5feaf6`. +- Lifeguard scenarios: `facebook/Lifeguard` at `c04fde1358c467d7ec84244ddfb066186381f066`. + +The SPMD implementation is based on public semantics, not exact parity with a separate unavailable prototype. Each consumer documents intentional restrictions and differences from its reference. diff --git a/static_analysis/SPIKE_REPORT.md b/static_analysis/SPIKE_REPORT.md new file mode 100644 index 0000000000..cfb1d3351b --- /dev/null +++ b/static_analysis/SPIKE_REPORT.md @@ -0,0 +1,84 @@ +# Spike outcome and review evidence + +## What this experiment establishes + +Two substantially different analyses can share the new Python frontend and full-state +engine without using Pyrefly's `Bindings`/`Answers` as their API or adding their own AST +walkers. SPMD uses tensor/gradient descriptors and scoped mesh context; Lifeguard uses +value origins, mutation effects and invocation-time summaries. Both drove changes to +the shared base rather than receiving separate scope/flow workarounds. + +The shared pieces include static ownership, dynamic class/module fallback, presence-aware +slots and outer cells, ordered operations, deferred regions, branch/loop flow, cleanup +and exception provenance, free-symbol queries, and history queries. The tests also use +a resource/lock domain independent of either consumer. + +This is **not** a capability match for the full Pyrefly binding pass, full upstream +Lifeguard/SPMD parity, or a production/formal-safety claim for arbitrary Python. The +[Pyrefly consumer sketch](PYREFLY_ADAPTER.md) identifies the lossless syntax, declaration +records, predecessor provenance and performance experiments needed to test that goal. + +## Independent review + +Fresh Claude and Codex reviews assessed the integrated implementation, not only the +individual consumers. Both supported the central dependency direction and reuse claim, +but the first passing suite was insufficient: independent reproductions found false +safe/validated results, a panic, incorrect history, orphan operations and scaling issues. + +Those findings led to regression tests and changes including: + +- runtime evaluation of annotation-only targets; +- binding presence carried across cells, summaries and builtin fallback; +- global tensor partition validation independent of an optional return contract; +- recursive substitution of returned class/instance values and descriptor dispatch; +- provenance-preserving `cast` and conservative callback/generator handling; +- query-scoped export incompleteness and module-valued re-export resolution; +- once-only lowering of deferred bodies, external-write markers in history, eager first + generator iteration and cleanup of handler names; +- sparse live temporaries, ordered worklists, restricted capture keys and a bounded + `finally` expansion budget. + +Focused rechecks cleared the original blocking findings. This means the prototype is +ready to be examined and extended under its stated assumptions; it is not proof that +no other bugs remain. + +## Verification + +The targeted suite contains **174 passing tests**: 36 shared-core, 31 SPMD, 98 Lifeguard +and 9 export-bridge tests. The timing probe is ignored by default. The Lifeguard suite +includes 67 reference-adapted parity fixtures plus regression and spike-semantics cases; +its README records their provenance and intentional differences. + +The repository formatting/lint script, package checks and Ruff checks on Python examples +were run. Both documented CLI demonstrations were executed: the SPMD example validates +its supplied contracts; the Lifeguard example intentionally reports unsafe/unknown cases +and exits nonzero. Validation was performed on Linux only. + +Public reference commits are listed in [README.md](README.md). The SPMD consumer follows +public semantics rather than claiming exact parity with a separate unavailable prototype. + +## Costs and remaining design questions + +- **History remains quadratic.** A release probe over 250/500/1000 conditional statements + measured about 24/96/352 ms. Sparse/persistent per-symbol representations remain work. +- **The engine's branch probe improved** after sparse live-temporary storage; one release + probe measured about 0.28/1.9/7.5 ms for 100/800/3200 branches. These are isolated + prototype measurements, not a comparison with Pyrefly's binding phase. +- **Capture-key cost improved**: a 2000-decorated-function fixture used roughly 13–15 MB + and 0.01–0.02 s after restricting cells, versus a roughly 1.6 GB earlier snapshot. + Module-wide function-global unions can still be costly in other shapes of program. +- **Cleanup expansion is capped**, and roughly 8–10 nested `try`/`finally` levels can + already produce `FinallyLimit`, explicitly making results incomplete. +- **Invocation semantics remain partly consumer-owned.** A future reusable argument, + descriptor and activation layer is a promising seam; it should not absorb Lifeguard + effect policy into the core. +- **Expression lowering is eager**, despite statement provenance/grouping. A truly + layered, opt-in expression CFG and optional lossless typing syntax view are unbuilt. +- **Export integration is unstable and optional.** It exposes the existing export module + on this branch and currently parses again for module-value classification. That is not + a supported upstream API or replacement-performance result. +- **macOS/Windows execution and full binding-emitter parity are unverified.** Adoption + requires separate cross-platform, capability and optimized allocation-aware checks. + +No upstream pull request or trunk landing is part of this experiment. Human review is +required before proposing a production migration. diff --git a/static_analysis/lifeguard/Cargo.toml b/static_analysis/lifeguard/Cargo.toml new file mode 100644 index 0000000000..dacf0004df --- /dev/null +++ b/static_analysis/lifeguard/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "pyrefly_static_analysis_lifeguard" +version = "0.0.0" +edition = "2024" +license.workspace = true +repository.workspace = true +publish = false +description = "Experimental lazy-import safety analysis built on pyrefly_static_analysis" + +[[bin]] +name = "lifeguard-spike" +path = "src/main.rs" + +[dependencies] +pyrefly_static_analysis = { path = ".." } +starlark_map = "0.14.2" + +[lints] +workspace = true diff --git a/static_analysis/lifeguard/README.md b/static_analysis/lifeguard/README.md new file mode 100644 index 0000000000..cde3cd9709 --- /dev/null +++ b/static_analysis/lifeguard/README.md @@ -0,0 +1,257 @@ +# Lazy-import safety consumer (spike) + +`pyrefly_static_analysis_lifeguard` is an experimental reimplementation of a subset of +[Lifeguard](https://github.com/facebook/Lifeguard)'s lazy-import safety analysis. It is a +consumer of the neutral `pyrefly_static_analysis` frontend and engine, written to test +whether that shared core can carry an effect analysis as well as the SPMD type +consumer. It is not a replacement for Lifeguard and does not reproduce Lifeguard's +output format, policy layer (`lazy_eligible`, import chains), stubs, or incremental cache. + +## Running it + +```sh +cargo run -p pyrefly_static_analysis_lifeguard --bin lifeguard-spike -- NAME=PATH [NAME=PATH ...] +``` + +Every module is named explicitly, and nothing is discovered. A path ending in +`__init__.py` marks a package `__init__`. Imports of modules that are not listed are +treated as external objects. The output has one line per diagnostic +(`module:line:column: kind: message`) followed by one verdict line per module: +`safe`, `unsafe` or `unknown`, plus `load-imports-eagerly` when the module's own imports +must run eagerly. The exit status is 0 when every module is safe, 1 when any module is +unsafe, 2 when none is unsafe but some module could not be fully analyzed, and 64 for +usage or I/O errors. An unknown result never exits as verified. + +The `examples/` directory holds a small project: + +```sh +E=static_analysis/lifeguard/examples +cargo run -p pyrefly_static_analysis_lifeguard --bin lifeguard-spike -- \ + registry=$E/registry.py plugin=$E/plugin.py helpers=$E/helpers.py settings=$E/settings.py +``` + +```text +plugin:9:1: unsafe-call: registry:10: calls `append` on an object of module `registry` +settings:11:10: unknown-call: settings:11: calls unresolved `json.loads` +settings:15:5: sys-modules-access: calls `sys.modules.pop` +helpers: safe +plugin: unsafe +registry: safe +settings: unsafe load-imports-eagerly +``` + +`registry` mutates its own list while it is being imported, which is safe. `plugin` +applies a decorator that mutates `registry`'s state, which is an import-time effect on +another module. `settings` calls a library function the analysis does not model, which +is unknown rather than safe. It also touches `sys.modules` in a function that import +time never calls, and that still forces its imports to run eagerly. + +## How it uses the shared core + +The consumer never parses or walks syntax. Each module is lowered once by +`analyze_python`, and everything else is computed from the shared IR: + +- **Scopes and ownership.** Name operations carry the owning symbol after the core's + Python scope rules (`global`, `nonlocal`, class-scope visibility, and the class-body + fallback to globals and builtins). +- **Evaluation order and regions.** Operations arrive in runtime order, with the + right-hand side evaluated before targets. Decorators and defaults run in the defining + region, and function, lambda, generator and coroutine bodies are separate deferred + regions whose trigger says when they run. +- **Engine and frames.** `domain.rs` implements `Domain` with a set-of-origins value + lattice. The engine owns frames, joins at branches, loop fixpoints, reachability, the + replay of reachable operations for diagnostics, and the bindings (presence and + value) of symbols owned by other regions, including `global` and `nonlocal` writes + and deletions visible in the exit frame. Loads use the engine's default, including + the fallback from class scope to globals and from globals to builtins; the consumer + only seeds builtin cells with builtin values. +- **Function summaries.** A call to a project function runs the same engine over the + callee's region with symbolic parameters and the caller's cells as `Entry::outer`. + The cells are the callee's `Program::free_symbols` plus the globals that any function + of its module reads (which further calls may observe). Summaries are memoized by the + function, the importing module, and those cell bindings at invocation, so closures and + globals are read when the call happens, not when the function is defined. The + interprocedural driver in `analyzer.rs` solves recursive cycles by iterating from an + empty summary at the head of each strongly connected component. A summary that + depends on a call cut off by the depth limit is not cached. +- **Exports.** `from m import n` goes through the generic `ExportProvider`. The CLI and + tests use `SyntacticExports`, and re-exports between analyzed modules are followed + through each module's own exit values. + +The effect vocabulary (origins, ownership, mutation kinds, builtin summaries) and the +summary policy are consumer-owned. Porting this consumer led to three core utilities: +raise sites with a source range and cause, `Program::free_symbols` for the closure +cells a region needs, and `Program::module_symbols`. + +## Semantics + +A value is a set of origins: a constant, an object created while importing a given +module, an object reached through an external import (named by its dotted path), a +project module, a builtin, a project function, generator, property, class or instance, +a member (attribute or item) of another origin, a parameter of the current function, or +unknown. + +Whether an operation is an effect depends on who owns the object, not on the syntax of +the target: + +- Rebinding a name is a local write. `from foo import bar; bar = 1` is safe. +- Storing into or deleting an attribute or item of an object owned by another module, + or calling a known mutating method on it, is an `imported-mutation`. This includes + aliases (`x = mod.A; x[0] = 1`) and rebinding another module's global from one of + its functions. +- Mutating an object owned by the module being imported is safe, even inside a + function. The same function becomes unsafe when another module's import calls it. +- A function mutating a parameter records that in its summary. The caller reports + `imported-argument` when it passes an imported object, or forwards the mutation to + its own parameter. Values returned from a call, including classes, instances, bound + methods and generators that capture parameters, have the arguments substituted. +- There is no heap: the contents of attributes and items are not tracked. Mutating an + object itself is judged by its owner, but mutating an object reached through an + attribute or item (`self.items.append(x)`, `lst[0].append(x)`) is `unknown-mutation` + unless the chain starts at a parameter or an imported object. +- Methods bind as descriptors: functions bind the instance, `staticmethod` binds + nothing, and `classmethod` binds the class. Reading a property runs its getter, and + writing or deleting it runs its setter or deleter. A class's `__setattr__` and + `__delattr__` run on instance attribute writes. Class values are snapshots of the + class body, so writing an attribute on a project class, shadowing a method on an + instance, or writing globals through the module object is `unknown-mutation`. +- A generator or coroutine keeps the arguments it was created with. Its body runs when + it is iterated, awaited, delegated to with `yield from`, or passed to a builtin that + iterates its arguments (`list`, `next`, `sorted`, ...) or to a project function that + iterates the parameter. Lazy builtins (`map`, `zip`, `enumerate`, ...) carry it to + whatever iterates their result. Storing one in a collection or attribute, yielding it, + or passing it to a library function is `unknown-call`. +- Builtins run their eager callbacks (`key=` of `sorted`, `min`, `max` and + `list.sort`). Any callback passed to a lazy builtin (`map`, `filter`, + `iter(callable, sentinel)`) is `unknown-call`, because its calls happen during a + later iteration that the analysis does not connect to it. A builtin used as an eager + callback is `unknown-call` too, since it may run special methods of the untracked + items (`str` runs `__str__`), unless it has effects of its own to report (`open`) or + calls none (`id`). +- A class's namespace holds the class body's own bindings. A name the body deleted is + absent, and a name it may not have bound is partly unknown. +- Special methods that a project class defines run where Python would call them: + operators, comparisons, iteration, subscripts, truth tests, `__getattr__` and + `__getattribute__`, context managers (including `async with`), and the conversion + methods builtins call on their arguments. +- Cells carry presence as well as values, so deleting a global, in the module or in a + called function, exposes the builtin of the same name to later reads. A module's + globals, as importers see them, join its normal exit with its raising exits. +- Function bodies count only when import-time code calls them (including through + decorators, properties, constructors, iterated generators and awaited coroutines). + Defaults, decorators, class bases and class bodies count where they are defined. + `exec`, `sys.modules` access, `builtins.__import__` overrides and `__del__` force + eager imports wherever they occur. +- An unresolved callee, an unresolved method, a mutation of an unattributable value, an + escaped closure's cells, a reachable frontend gap, non-convergence, and recursion are + reported as `unknown-call`, `unknown-mutation` or `unsupported`. A module with only + such diagnostics has the verdict `unknown`, never `safe`. + +### Diagnostic kinds + +| Kind | Meaning | Verdict | +| --- | --- | --- | +| `imported-mutation` | import-time mutation of another module's object | unsafe | +| `imported-argument` | imported object passed to a parameter the callee mutates | unsafe | +| `unsafe-call` | the callee's body has an import-time effect (the message names its root cause) | unsafe | +| `prohibited-call` | `input`, `open`, `__import__`, `os.system`, computed `getattr`/`setattr`, ... | unsafe | +| `unhandled-raise` | import-time code reaches a `raise` | unsafe | +| `custom-finalizer`, `exec-call`, `sys-modules-access`, `builtins-import-override` | also set `load-imports-eagerly` | unsafe | +| `unknown-call`, `unknown-mutation`, `unsupported` | something could not be analyzed | unknown | + +### Assumptions shared with upstream + +These are heuristics rather than proofs, and Lifeguard makes the same assumptions: +attribute and item reads on external objects, operators, comparisons and truthiness +tests do not run code with effects unless a project class defines the special method. + +## Parity with the public test suite + +The fixtures in `tests/parity.rs` are adapted from facebook/Lifeguard at commit +`c04fde1358c467d7ec84244ddfb066186381f066`. That commit is the only commit in the +reference clone. Each test names the upstream file and the tests it adapts; several +upstream tests are often combined into one fixture. Together they adapt cases from +roughly 140 upstream test functions in eight of the 35 upstream test files. That is a +meaningful subset, not a full port. + +| Upstream file | Covered | Not covered | +| --- | --- | --- | +| `assign.rs` | module, attribute, subscript, annotated, augmented and destructuring stores; aliases; rebinding as safe; `builtins.__import__` overrides | thread-local stores | +| `basic.rs` | literal and computed `getattr`/`setattr`; own-module state; imported subscripts; raising callees; `__del__` | | +| `calls.rs` | cross-module calls, package re-exports, re-exported decorators, recursion, methods and constructors, literal methods, class aliases, prohibited builtins, `eval`, lambdas, `importlib` | `super()` dispatch, annotated return types from stubs, MRO edge cases, `import_module` targets | +| `sys_modules.rs` | reads that cannot fail, writes, deletes, method calls, computed keys | membership and `KeyError` probes | +| `nested_functions.rs` | called and uncalled nested functions, closure factories, decorator factories, class bodies inside functions, `functools.wraps` | | +| `decorators.rs` | local, imported, unresolved, parameterized and subscript decorators; decorator arguments; properties | decorator argument overflow, safelisted decorators | +| `param_mutation.rs` | attribute, item and method mutation of parameters; precise positional and keyword matching; transitive, diamond and cyclic forwarding; aliases; closures; copies | unbound methods called through the class, `*args` forwarding edge cases, the 64-argument limit | +| `blocks.rs` | defaults, class bases, `for` targets | per-statement expression coverage table | + +Not attempted: stubs and stub effects, implicit imports, star-import semantics, import +cycles as a policy, `__main__` handling, the subclass-registration rule, output policy, +the cache and map/reduce path, and the remaining upstream test files. + +### Intentional differences from upstream + +- **Unknown is never safe.** Upstream treats some unresolved calls as safe and reports + others as unsafe. Here they are always `unknown-call`, so unresolved externals, + unresolved methods on parameters, recursion, and classes with unresolved bases (whose + metaclass or `__init_subclass__` could run code) make a module `unknown`. +- **Flow-sensitive rebinding.** Reaching values come from the engine, so two documented + upstream false positives are not reported: `test_reassigned_param_not_forwarded` and + `test_param_reassigned_then_aliased_is_safe`. String constants also flow, so + `x = "foo"; getattr(f, x)` is a literal access. +- **Value-flow resolution.** Calls through aliases and lambdas run the callee instead of + being unknown, and hazards are found through aliases (`e = exec; e(...)`, + `m = sys.modules; m[k] = v`). +- **Import-time semantics for closures.** A returned closure that would mutate its + argument is not an effect until something calls it, and calling it is unknown because + its cells escaped. Upstream reports `imported-var-argument` at the factory call. +- **In-place operators.** `bar += 1` on an imported name may call `__iadd__`, so it is + an `unknown-mutation` rather than safe. +- **No reparsing.** `eval` of a string literal is unknown because the literal is not + parsed. +- **Locations.** `custom-finalizer` is reported at the class statement. + +## Known limitations + +- `try` statements, `match`, star imports and other frontend gaps make the enclosing + module `unknown`. Only explicit `raise` statements are reported as raising. A failing + `assert` is treated like any other operation that may raise, and the re-raise path + out of a `try` is covered by that `try`'s gap. Handler matching (upstream's caught + `raise`) is not modeled. +- Function objects do not carry their default values, so a parameter left to its + default is unknown. +- Class and instance values are built from class bodies in project modules. There is no + MRO linearization, `super()` support, or metaclass modeling, and no heap for + attribute and container contents, which makes many classes that mutate their own + attributes `unknown` rather than `safe`. +- Builtin, method and library summaries are a short hand-written table in `known.rs`. +- Tests ran on Linux only. The analysis has no platform-specific code, and the CLI test + uses portable temporary paths, but macOS and Windows were not validated. + +## Tests + +`cargo test -p pyrefly_static_analysis_lifeguard` runs: + +- `tests/parity.rs`: 67 fixtures adapted from the upstream suite. +- `tests/regressions.rs`: 16 tests for cases that earlier versions reported as safe, + panicked, or used too much memory. They cover: + - parameters captured by returned classes; + - `staticmethod` and `classmethod` binding, property setters and deleters, + `__setattr__`, and dynamic class writes; + - `typing.cast` and `max` provenance; + - generator arguments, `next`, generators passed to functions, and generator + expressions; + - builtin callbacks; + - special methods run by operators, iteration, subscripts, truth tests and builtins; + - `functools.cached_property`, dataclass default factories, and `async with`; + - deleted class attributes, and globals deleted so that a builtin becomes visible; + - module globals on raising paths; + - a module with a thousand decorated functions. +- `tests/semantics.rs`: 15 tests. They cover local rebinding versus external mutation, + `global` writes flowing back from calls, invocation-time closure cells, escaped + closures, generator and coroutine bodies, property getters, constructor arguments, + hazards through aliases, unresolved bases, frontend gaps (including `try` re-raise + and `assert` paths), unreachable code and caught raises, class-body name fallback, + import cycles, and the CLI's output and exit status. + +Expected diagnostics are written as `# E: ` comments on the line they apply to. diff --git a/static_analysis/lifeguard/examples/helpers.py b/static_analysis/lifeguard/examples/helpers.py new file mode 100644 index 0000000000..74af1238d5 --- /dev/null +++ b/static_analysis/lifeguard/examples/helpers.py @@ -0,0 +1,19 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import functools + + +def cached(func): + @functools.wraps(func) + def wrapper(*args): + return func(*args) + + return wrapper + + +@cached +def double(x): + return [x, x] diff --git a/static_analysis/lifeguard/examples/plugin.py b/static_analysis/lifeguard/examples/plugin.py new file mode 100644 index 0000000000..3930f5dd3f --- /dev/null +++ b/static_analysis/lifeguard/examples/plugin.py @@ -0,0 +1,11 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from registry import register + + +@register +def handler(event): + return event diff --git a/static_analysis/lifeguard/examples/registry.py b/static_analysis/lifeguard/examples/registry.py new file mode 100644 index 0000000000..ea6d95b098 --- /dev/null +++ b/static_analysis/lifeguard/examples/registry.py @@ -0,0 +1,14 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +REGISTRY = [] + + +def register(handler): + REGISTRY.append(handler) + return handler + + +register(print) diff --git a/static_analysis/lifeguard/examples/settings.py b/static_analysis/lifeguard/examples/settings.py new file mode 100644 index 0000000000..879cdf5888 --- /dev/null +++ b/static_analysis/lifeguard/examples/settings.py @@ -0,0 +1,15 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import json +import sys + +DEFAULTS = {"debug": False} +DEFAULTS["debug"] = True +CONFIG = json.loads("{}") + + +def reload_plugins(): + sys.modules.pop("plugin", None) diff --git a/static_analysis/lifeguard/src/analyzer.rs b/static_analysis/lifeguard/src/analyzer.rs new file mode 100644 index 0000000000..5b7f289544 --- /dev/null +++ b/static_analysis/lifeguard/src/analyzer.rs @@ -0,0 +1,555 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Drives the lazy-import domain over a set of modules: module bodies in import order, +//! memoized function summaries for calls, and a scan of every function body for the +//! effects that force a module's imports to run eagerly. + +use std::cell::RefCell; +use std::collections::BTreeMap; +use std::collections::BTreeSet; + +use pyrefly_static_analysis::Binding; +use pyrefly_static_analysis::Convergence; +use pyrefly_static_analysis::Engine; +use pyrefly_static_analysis::Entry; +use pyrefly_static_analysis::ExportProvider; +use pyrefly_static_analysis::ExportTarget; +use pyrefly_static_analysis::ModuleName; +use pyrefly_static_analysis::Name; +use pyrefly_static_analysis::Presence; +use pyrefly_static_analysis::Program; +use pyrefly_static_analysis::RaiseCause; +use pyrefly_static_analysis::RegionId; +use pyrefly_static_analysis::RegionResult; +use pyrefly_static_analysis::SymbolId; +use pyrefly_static_analysis::TextRange; +use pyrefly_static_analysis::analyze_python; + +use crate::Diagnostic; +use crate::Kind; +use crate::ModuleReport; +use crate::domain::Event; +use crate::domain::Finding; +use crate::domain::LifeguardDomain; +use crate::domain::Mode; +use crate::domain::builtin_binding; +use crate::value::FuncRef; +use crate::value::Origin; +use crate::value::Value; + +/// Calls nested deeper than this are reported as unknown. +const MAX_CALL_DEPTH: usize = 32; +/// Re-analyses of a recursive cycle before its summary is reported as unknown. +const MAX_CYCLE_ITERATIONS: usize = 8; + +/// The effect of one function activation, as seen by its caller. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Summary { + /// Import-time effects, each described by its root location and cause. + pub unsafe_reasons: BTreeSet, + /// Operations that could not be analyzed. + pub unknown_reasons: BTreeSet, + /// Parameter origins (of this or an enclosing activation) whose objects are mutated. + pub mutated: BTreeSet, + /// Parameter origins that are iterated or awaited. + pub consumed: BTreeSet, + /// Cells of the caller's symbols that the activation rebinds or deletes, with their + /// bindings at exit. + pub writes: Vec<(SymbolId, Binding)>, + pub returned: Value, +} + +impl Summary { + fn unknown(reason: &str) -> Self { + Self { + unknown_reasons: BTreeSet::from([reason.to_owned()]), + returned: Value::unknown(), + ..Self::default() + } + } +} + +/// A function activation: the body, the module whose import runs it, and the cells +/// (closure cells, globals and builtins) it sees at invocation time. +#[derive(Clone)] +pub struct Activation { + pub func: FuncRef, + pub importer: ModuleName, + pub cells: Vec<(SymbolId, Binding)>, +} + +type Cells = Vec<(SymbolId, Binding)>; +type SummaryTable = BTreeMap<(FuncRef, ModuleName), Vec<(Cells, Slot)>>; + +/// Bookkeeping for a summary under construction. +struct Pending { + /// The lowest call depth of an in-progress summary that this one depends on. + lowlink: usize, + /// True if this summary depends on a call cut off by the depth limit, which makes it + /// specific to the current call stack. + truncated: bool, +} + +enum Slot { + /// Being computed at call depth `depth`; recursive calls see `approx`. + InProgress { + approx: Summary, + depth: usize, + used: bool, + }, + Done(Summary), +} + +struct Module { + program: Program, + /// The module globals that some function of the module reads: the globals a call + /// into the module may observe, directly or through further calls. + function_globals: Vec, +} + +impl Module { + fn new(program: Program) -> Self { + let mut function_globals: Vec = (1..program.regions.len()) + .flat_map(|r| program.free_symbols(RegionId(r as u32)).iter().copied()) + .filter(|s| program.owner(*s) == Some(Program::MODULE)) + .collect(); + function_globals.sort(); + function_globals.dedup(); + Self { + program, + function_globals, + } + } +} + +struct ModuleOutcome { + /// Global bindings at the end of the module body; `None` while the body is running. + globals: Option>>, + diagnostics: Vec, +} + +pub struct Analyzer<'a> { + modules: BTreeMap, + exports: &'a dyn ExportProvider, + outcomes: RefCell>, + /// Summaries by function and importer, then by the cells of the activation. + summaries: RefCell, + /// The summaries under construction, innermost last. + pending: RefCell>, +} + +impl<'a> Analyzer<'a> { + pub fn new(modules: &[(ModuleName, String, bool)], exports: &'a dyn ExportProvider) -> Self { + Self { + modules: modules + .iter() + .map(|(name, source, is_init)| { + (*name, Module::new(analyze_python(source, *name, *is_init))) + }) + .collect(), + exports, + outcomes: RefCell::new(BTreeMap::new()), + summaries: RefCell::new(BTreeMap::new()), + pending: RefCell::new(Vec::new()), + } + } + + pub fn run(&self) -> Vec { + let mut names: Vec = self.modules.keys().copied().collect(); + names.sort_by(|a, b| a.as_str().cmp(b.as_str())); + names + .into_iter() + .map(|name| { + self.ensure_module(name); + let mut diagnostics = self.outcomes.borrow()[&name].diagnostics.clone(); + diagnostics.sort(); + diagnostics.dedup(); + ModuleReport { name, diagnostics } + }) + .collect() + } + + pub fn program(&self, module: ModuleName) -> &Program { + &self.modules[&module].program + } + + pub fn function_globals(&self, module: ModuleName) -> &[SymbolId] { + &self.modules[&module].function_globals + } + + pub fn contains(&self, module: ModuleName) -> bool { + self.modules.contains_key(&module) + } + + /// The global values of a project module after its body ran, or `None` if the module + /// is still running (an import cycle). + pub fn exit_globals(&self, module: ModuleName) -> Option>> { + self.ensure_module(module); + self.outcomes.borrow()[&module].globals.clone() + } + + /// The value of `module.attr` for a project module: a global, else a submodule. + pub fn module_attr(&self, module: ModuleName, attr: &Name) -> Value { + let program = self.program(module); + if let Some(symbol) = program.global(attr.as_str()) + && let Some(globals) = self.exit_globals(module) + { + let binding = &globals[&symbol]; + return match binding.presence { + Presence::Bound => binding.value.clone(), + Presence::Maybe => binding.value.join(&Value::unknown()), + Presence::Unbound => Value::unknown(), + }; + } + let submodule = module.append(attr); + if self.contains(submodule) { + Value::of(Origin::Module(submodule)) + } else { + Value::unknown() + } + } + + /// The value bound by `from module import name`. + pub fn imported_name(&self, module: ModuleName, name: &Name) -> Value { + let project_or_external = |m: ModuleName| { + Value::of(if self.contains(m) { + Origin::Module(m) + } else { + Origin::Imported(m) + }) + }; + match self.exports.resolve(module, name) { + ExportTarget::Definition(m, n) if self.contains(m) => self.module_attr(m, &n), + ExportTarget::Definition(m, n) => project_or_external(m.append(&n)), + ExportTarget::Module(m) => project_or_external(m), + ExportTarget::Unknown => Value::unknown(), + } + } + + fn ensure_module(&self, module: ModuleName) { + if self.outcomes.borrow().contains_key(&module) { + return; + } + self.outcomes.borrow_mut().insert( + module, + ModuleOutcome { + globals: None, + diagnostics: Vec::new(), + }, + ); + let program = self.program(module); + let domain = LifeguardDomain::new(self, program, module, Mode::Execute); + let builtins: Cells = (0..program.symbols.len()) + .filter_map(|i| { + let s = SymbolId(i as u32); + builtin_binding(program, s).map(|b| (s, b)) + }) + .collect(); + let mut entry = Entry::new(()); + entry.outer.extend(builtins.iter().cloned()); + let result = Engine::new(program, &domain).analyze(Program::MODULE, entry); + // A caller that catches the module's exception can still see its globals, so the + // normal and raising exits are joined. A module with neither exit reachable has + // unknown globals. + let unknown = Binding { + presence: Presence::Maybe, + value: Value::unknown(), + }; + let globals: BTreeMap> = program + .module_symbols() + .map(|s| { + let ends = [&result.exit, &result.raised]; + let bindings: Vec<&Binding> = ends + .into_iter() + .flatten() + .map(|f| f.binding(program, s).unwrap_or(&unknown)) + .collect(); + let binding = bindings + .iter() + .copied() + .cloned() + .reduce(|a, b| Binding { + presence: if a.presence == b.presence { + a.presence + } else { + Presence::Maybe + }, + value: a.value.join(&b.value), + }) + .unwrap_or_else(|| unknown.clone()); + (s, binding) + }) + .collect(); + + let diagnostic = |range: TextRange, kind: Kind, message: String| { + let (line, column) = program.line_col(range); + Diagnostic { + line: line.get(), + column: column.get(), + kind, + message, + } + }; + let mut diagnostics = Vec::new(); + for Finding { start, end, event } in &result.diagnostics { + let range = TextRange::new((*start).into(), (*end).into()); + diagnostics.push(match event { + Event::Report { kind, message } => diagnostic(range, *kind, message.clone()), + // Callers substitute parameters, so none should reach a module body; if + // one does, the mutation is reported rather than dropped. + Event::MutatesParam(_) | Event::ConsumesParam(_) => diagnostic( + range, + Kind::Unsupported, + "uses a parameter value that escaped its function".to_owned(), + ), + }); + } + for (range, message) in incompleteness(program, &result) { + diagnostics.push(diagnostic(range, Kind::Unsupported, message)); + } + for range in explicit_raises(&result) { + let message = "import-time code raises an exception".to_owned(); + diagnostics.push(diagnostic(range, Kind::UnhandledRaise, message)); + } + if let Some(outcome) = self.outcomes.borrow_mut().get_mut(&module) { + outcome.globals = Some(globals.clone()); + } + + // Function bodies count toward the module only when import-time code calls + // them, except for the effects that force eager imports wherever they occur. + let scan = LifeguardDomain::new(self, program, module, Mode::Scan); + let engine = Engine::new(program, &scan); + for index in 1..program.regions.len() { + let region = RegionId(index as u32); + let mut entry = Entry::new(()); + for s in program.free_symbols(region) { + let binding = builtin_binding(program, *s).or_else(|| globals.get(s).cloned()); + if let Some(binding) = binding { + entry.outer.insert(*s, binding); + } + } + let result = engine.analyze(region, entry); + for Finding { start, end, event } in result.diagnostics { + if let Event::Report { kind, message } = event + && kind.forces_eager_imports() + { + let range = TextRange::new(start.into(), end.into()); + diagnostics.push(diagnostic(range, kind, message)); + } + } + } + if let Some(outcome) = self.outcomes.borrow_mut().get_mut(&module) { + outcome.diagnostics = diagnostics; + } + } + + /// The summary of one activation. Recursive cycles are solved by iterating from an + /// empty summary at the head of each strongly connected component; summaries inside + /// a cycle are only cached once the head has converged. + pub fn summarize(&self, activation: Activation) -> Summary { + let key = (activation.func, activation.importer); + if let Some(found) = self.find(&activation, |slot| match slot { + Slot::Done(summary) => summary.clone(), + Slot::InProgress { + approx, + depth, + used, + } => { + *used = true; + let mut pending = self.pending.borrow_mut(); + let top = pending + .last_mut() + .expect("an in-progress summary has a caller on the stack"); + top.lowlink = top.lowlink.min(*depth); + approx.clone() + } + }) { + return found; + } + let depth = self.pending.borrow().len(); + if depth >= MAX_CALL_DEPTH { + if let Some(top) = self.pending.borrow_mut().last_mut() { + top.truncated = true; + } + return Summary::unknown("call depth limit reached"); + } + self.summaries.borrow_mut().entry(key).or_default().push(( + activation.cells.clone(), + Slot::InProgress { + approx: Summary::default(), + depth, + used: false, + }, + )); + self.pending.borrow_mut().push(Pending { + lowlink: depth, + truncated: false, + }); + let mut iterations = 0; + loop { + let mut summary = self.activate(&activation); + let (lowlink, truncated) = { + let pending = self.pending.borrow(); + let top = pending.last().expect("pushed above"); + (top.lowlink, top.truncated) + }; + if lowlink < depth || truncated { + // An enclosing activation heads this cycle and will recompute us, or the + // result depends on where the depth limit fell: do not cache it. + if let Some(slots) = self.summaries.borrow_mut().get_mut(&key) { + slots.retain(|(c, _)| *c != activation.cells); + } + let mut pending = self.pending.borrow_mut(); + pending.pop(); + if let Some(parent) = pending.last_mut() { + parent.lowlink = parent.lowlink.min(lowlink); + parent.truncated |= truncated; + } + return summary; + } + let finished = self + .find(&activation, |slot| { + let Slot::InProgress { approx, used, .. } = slot else { + unreachable!("only this activation finishes its own slot") + }; + if *used { + summary + .unknown_reasons + .insert("recursive call cycle; termination is not analyzed".to_owned()); + } + let converged = !*used || summary == *approx; + if converged || iterations == MAX_CYCLE_ITERATIONS { + if !converged { + summary + .unknown_reasons + .insert("recursive summary did not converge".to_owned()); + } + *slot = Slot::Done(summary.clone()); + true + } else { + *approx = summary.clone(); + *used = false; + false + } + }) + .expect("the slot was inserted above"); + if finished { + self.pending.borrow_mut().pop(); + return summary; + } + iterations += 1; + } + } + + /// Apply `f` to the slot of `activation`, if there is one. + fn find(&self, activation: &Activation, f: impl FnOnce(&mut Slot) -> T) -> Option { + let mut summaries = self.summaries.borrow_mut(); + let slots = summaries.get_mut(&(activation.func, activation.importer))?; + let (_, slot) = slots.iter_mut().find(|(c, _)| *c == activation.cells)?; + Some(f(slot)) + } + + /// Run the callee's region with symbolic parameters and the key's cells. + fn activate(&self, activation: &Activation) -> Summary { + let Activation { + func, + importer, + cells, + } = activation; + let program = self.program(func.module); + let region = program.region(func.region); + let domain = LifeguardDomain::new(self, program, *importer, Mode::Execute); + let mut entry = Entry::new(()); + entry.args = (0..region.params.len()) + .map(|i| Value::of(Origin::Param(*func, i))) + .collect(); + entry.outer.extend(cells.iter().cloned()); + let result = Engine::new(program, &domain).analyze(func.region, entry); + + let location = |start: u32| { + let range = TextRange::empty(start.into()); + format!("{}:{}", func.module, program.line_col(range).0) + }; + let mut summary = Summary::default(); + for Finding { start, event, .. } in &result.diagnostics { + match event { + Event::Report { kind, message } => { + // Calls already carry the root location of the effect they report. + let reason = match kind { + Kind::UnsafeCall | Kind::UnknownCall => message.clone(), + _ => format!("{}: {message}", location(*start)), + }; + if kind.is_unknown() { + summary.unknown_reasons.insert(reason); + } else { + summary.unsafe_reasons.insert(reason); + } + } + Event::MutatesParam(param) => { + summary.mutated.insert(param.clone()); + } + Event::ConsumesParam(param) => { + summary.consumed.insert(param.clone()); + } + } + } + for (range, message) in incompleteness(program, &result) { + let start = u32::from(range.start()); + summary + .unknown_reasons + .insert(format!("{}: {message}", location(start))); + } + for range in explicit_raises(&result) { + let start = u32::from(range.start()); + summary + .unsafe_reasons + .insert(format!("{}: raises an exception", location(start))); + } + if let Some(exit) = &result.exit { + let entry: BTreeMap<&SymbolId, &Binding> = + cells.iter().map(|(s, b)| (s, b)).collect(); + summary.writes = exit + .outer() + .iter() + .filter(|(s, v)| entry.get(s) != Some(v)) + .map(|(s, v)| (*s, v.clone())) + .collect(); + } + summary.returned = result.returned.unwrap_or_default(); + summary + } +} + +/// Reachable gaps and non-convergence, which make a result incomplete. +fn incompleteness( + program: &Program, + result: &RegionResult>, +) -> Vec<(TextRange, String)> { + let mut out: Vec<(TextRange, String)> = result + .gaps + .iter() + .map(|(gap, range)| (*range, format!("{gap:?} is not modeled"))) + .collect(); + if result.convergence != Convergence::Converged { + let range = program.region(result.region).range; + out.push((range, "the analysis did not converge".to_owned())); + } + out +} + +/// The reachable `raise` statements of a region. A failing `assert` is not reported, +/// like any other operation that may raise, and a re-raise out of a `try` is covered by +/// the gap that `try` already reports. +fn explicit_raises(result: &RegionResult>) -> Vec { + result + .raise_sites + .iter() + .filter(|site| site.cause == RaiseCause::Explicit) + .map(|site| site.range) + .collect() +} diff --git a/static_analysis/lifeguard/src/domain.rs b/static_analysis/lifeguard/src/domain.rs new file mode 100644 index 0000000000..295ae10987 --- /dev/null +++ b/static_analysis/lifeguard/src/domain.rs @@ -0,0 +1,1562 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! The lazy-import effect domain: a transfer function per shared operation that tracks +//! value origins and reports import-time effects. +//! +//! Mutation is judged by the owner of the mutated object, never by the syntax of the +//! target. Rebinding a name is a local write; storing into an attribute or item of an +//! object owned by another module, or calling a mutating method on it, is an effect. +//! Calls to project functions are applied through [`Analyzer::summarize`], which runs +//! the same engine over the callee's region with the caller's cells. + +use pyrefly_static_analysis::Access; +use pyrefly_static_analysis::Arg; +use pyrefly_static_analysis::BinaryOp; +use pyrefly_static_analysis::Binding; +use pyrefly_static_analysis::BuildKind; +use pyrefly_static_analysis::Cx; +use pyrefly_static_analysis::Domain; +use pyrefly_static_analysis::Element; +use pyrefly_static_analysis::ImportItem; +use pyrefly_static_analysis::Literal; +use pyrefly_static_analysis::ModuleName; +use pyrefly_static_analysis::Name; +use pyrefly_static_analysis::OpId; +use pyrefly_static_analysis::OpKind; +use pyrefly_static_analysis::Param; +use pyrefly_static_analysis::ParamKind; +use pyrefly_static_analysis::Presence; +use pyrefly_static_analysis::Program; +use pyrefly_static_analysis::Reg; +use pyrefly_static_analysis::ScopeId; +use pyrefly_static_analysis::SymbolId; +use pyrefly_static_analysis::SymbolOrigin; +use pyrefly_static_analysis::Trigger; + +use crate::Kind; +use crate::analyzer::Activation; +use crate::analyzer::Analyzer; +use crate::analyzer::Summary; +use crate::known; +use crate::known::Iteration; +use crate::known::Known; +use crate::value::Accessors; +use crate::value::ClassInfo; +use crate::value::FuncRef; +use crate::value::Origin; +use crate::value::Value; + +/// How calls are treated. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Mode { + /// Calls to project functions run their bodies through summaries. + Execute, + /// Calls produce unknown values without running anything. Used to scan function + /// bodies for effects that matter wherever they occur. + Scan, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum Event { + Report { + kind: Kind, + message: String, + }, + /// A parameter of this or an enclosing activation is mutated: the parameter's object + /// itself (`Origin::Param`) or an object reached from it (`Origin::Member`). + MutatesParam(Origin), + /// A parameter (as for `MutatesParam`) is iterated or awaited, which runs it if the + /// argument is a generator or coroutine. + ConsumesParam(Origin), +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct Finding { + pub start: u32, + pub end: u32, + pub event: Event, +} + +/// Who owns an object, relative to the module being imported. +enum Owner { + Local, + Param(Origin), + /// Another module; the string describes the object. + External(String), + Unknown, +} + +/// How an operation mutates objects. +#[derive(Clone, Copy)] +enum Mutation<'s> { + /// The operation mutates the object; the string describes the operation. + Definite(&'s str), + /// The operation mutates the object if its type is mutable. + Possible(&'s str), + /// A callee mutates the parameter this object is passed to. + Argument, +} + +/// The arguments of one call, as values. +#[derive(Clone, Default)] +struct CallArgs { + /// Positional arguments before any starred argument. + positional: Vec, + keywords: Vec<(Name, Value)>, + /// What `*` and `**` arguments, and positional arguments after them, may supply to + /// any parameter. + spread: Option, +} + +impl CallArgs { + fn new(cx: &Cx<'_, LifeguardDomain<'_>>, args: &[Arg]) -> Self { + let mut out = Self::default(); + for arg in args { + match arg { + Arg::Positional(r) if out.spread.is_none() => { + out.positional.push(cx.reg(*r).clone()) + } + Arg::Positional(r) => out + .spread + .get_or_insert_default() + .extend(cx.reg(*r).clone()), + Arg::Star(r) | Arg::DoubleStar(r) => out + .spread + .get_or_insert_default() + .extend(cx.reg(*r).flat_map(|o| Value::of(o.member(None)))), + Arg::Keyword(name, r) => out.keywords.push((name.clone(), cx.reg(*r).clone())), + } + } + out + } + + fn positional(positional: Vec) -> Self { + Self { + positional, + ..Self::default() + } + } + + fn with_receiver(&self, receiver: &Value) -> Self { + let mut out = self.clone(); + out.positional.insert(0, receiver.clone()); + out + } + + /// The argument at `index` or passed as keyword `name`, if it is known exactly. + fn get(&self, index: usize, name: &str) -> Option<&Value> { + self.positional.get(index).or_else(|| { + self.keywords + .iter() + .find(|(n, _)| n.as_str() == name) + .map(|(_, v)| v) + }) + } + + fn all(&self) -> impl Iterator { + self.positional + .iter() + .chain(self.keywords.iter().map(|(_, v)| v)) + .chain(&self.spread) + } +} + +pub struct LifeguardDomain<'a> { + analyzer: &'a Analyzer<'a>, + program: &'a Program, + /// The module whose import is executing this code. + importer: ModuleName, + mode: Mode, +} + +impl Domain for LifeguardDomain<'_> { + type Value = Value; + type State = (); + type Diagnostic = Finding; + + fn top(&self) -> Value { + Value::unknown() + } + + fn unbound(&self) -> Value { + Value::default() + } + + fn join(&self, a: &Value, b: &Value) -> Value { + a.join(b) + } + + fn join_state(&self, _: &(), _: &()) {} + + /// Testing an instance's truth runs its `__bool__` or `__len__`. + fn assume(&self, cond: Reg, _: bool, cx: &mut Cx<'_, Self>) -> bool { + let value = cx.reg(cond).clone(); + self.protocol(cx, &value, &["__bool__", "__len__"], &CallArgs::default()); + true + } + + fn transfer(&self, op: OpId, kind: &OpKind, cx: &mut Cx<'_, Self>) { + match kind { + OpKind::Name { + symbol, + access: Access::Store(_) | Access::Delete, + } => self.rebind(cx, *symbol), + OpKind::Name { .. } => {} + OpKind::Attribute { + object, + attr, + access, + } => { + let object = cx.reg(*object).clone(); + match access { + Access::Load(dst) => { + let value = self.load_attr(cx, &object, attr); + cx.set_reg(*dst, value); + } + Access::Store(src) => { + let value = cx.reg(*src).clone(); + self.escape(cx, &value); + self.store_attr(cx, &object, attr, Some(value)); + } + Access::Delete => self.store_attr(cx, &object, attr, None), + } + } + OpKind::Subscript { + object, + index, + access, + } => { + let object = cx.reg(*object).clone(); + let index = cx.reg(*index).clone(); + // `sys.modules` is the import system's state: any access other than a read + // that cannot fail depends on import order, and the module read is opaque. + let (sys_modules, object): (Vec, Vec) = + object.iter().cloned().partition(is_sys_modules); + let object: Value = object.into_iter().collect(); + let (hook, hook_args) = match access { + Access::Load(_) => ("__getitem__", vec![index.clone()]), + Access::Store(src) => { + ("__setitem__", vec![index.clone(), cx.reg(*src).clone()]) + } + Access::Delete => ("__delitem__", vec![index.clone()]), + }; + let hooked = self.protocol(cx, &object, &[hook], &CallArgs::positional(hook_args)); + let safe_read = matches!(access, Access::Load(_)) + && index.as_str().is_some_and(|k| self.loaded(k)); + if !sys_modules.is_empty() && !safe_read { + self.report( + cx, + Kind::SysModulesAccess, + "accesses `sys.modules` with a key that may not be imported yet", + ); + } + match access { + Access::Load(dst) => { + let mut value = object.flat_map(|o| Value::of(o.member(None))); + value.extend(hooked); + if !sys_modules.is_empty() { + value.extend(Value::unknown()); + } + cx.set_reg(*dst, value); + } + Access::Store(src) => { + let value = cx.reg(*src).clone(); + self.escape(cx, &value); + self.mutate(cx, &object, Mutation::Definite("stores item")); + } + Access::Delete => self.mutate(cx, &object, Mutation::Definite("deletes item")), + } + } + OpKind::Literal { value, dst } => { + let origin = match value { + Literal::Str(s) => Origin::Str(s.clone()), + _ => Origin::Const, + }; + cx.set_reg(*dst, Value::of(origin)); + } + OpKind::Build { + kind, + elements, + dst, + } => { + for element in elements { + self.escape_element(cx, element); + } + let origin = match kind { + BuildKind::String | BuildKind::Slice => Origin::Const, + _ => Origin::Object(self.importer), + }; + cx.set_reg(*dst, Value::of(origin)); + } + OpKind::Append { element, .. } => self.escape_element(cx, element), + OpKind::Call { func, args, dst } => { + let args = CallArgs::new(cx, args); + let func = cx.reg(*func).clone(); + let value = func.flat_map(|o| self.call(cx, o, &args)); + cx.set_reg(*dst, value); + } + OpKind::Unary { operand, dst, .. } => { + let operand = cx.reg(*operand).clone(); + let mut value = self.protocol(cx, &operand, UNARY_METHODS, &CallArgs::default()); + value.extend(Value::of(Origin::Const)); + cx.set_reg(*dst, value); + } + OpKind::Binary { + op, + left, + right, + dst, + } => { + let names = match op { + BinaryOp::Compare(_) => COMPARISON_METHODS, + BinaryOp::Arith(_) | BinaryOp::InPlace(_) => ARITHMETIC_METHODS, + }; + let (l, r) = (cx.reg(*left).clone(), cx.reg(*right).clone()); + let mut hooked = + self.protocol(cx, &l, names, &CallArgs::positional(vec![r.clone()])); + hooked.extend(self.protocol(cx, &r, names, &CallArgs::positional(vec![l]))); + let mut value = match op { + BinaryOp::Compare(_) => Value::of(Origin::Const), + BinaryOp::Arith(_) => Value::of(Origin::Object(self.importer)), + // In-place operators mutate mutable receivers and rebind immutable ones; + // without types, a non-local receiver may be mutated. + BinaryOp::InPlace(_) => { + let left = cx.reg(*left).clone(); + let mutation = Mutation::Possible("applies an in-place operator to"); + self.mutate(cx, &left, mutation); + left + } + }; + value.extend(hooked); + cx.set_reg(*dst, value); + } + // Iterating a generator, awaiting a coroutine, or delegating to either with + // `yield from` runs its body. + OpKind::Iter { + iterable: value, .. + } + | OpKind::Await { value, .. } + | OpKind::Yield { + value: Some(value), + from: true, + .. + } => { + let value = cx.reg(*value).clone(); + self.consume(cx, &value); + if matches!(kind, OpKind::Iter { .. }) { + let names = ["__iter__", "__next__"]; + let iterator = self.protocol(cx, &value, &names, &CallArgs::default()); + self.consume(cx, &iterator); + } + } + OpKind::Yield { + value: Some(value), .. + } => { + let value = cx.reg(*value).clone(); + self.escape(cx, &value); + } + OpKind::MakeFunction { region, dst, .. } => { + let func = FuncRef { + module: cx.program().module, + region: *region, + }; + cx.set_reg(*dst, Value::of(Origin::Function(func))); + } + OpKind::MakeClass { + name, + scope, + bases, + dst, + } => { + let class = self.make_class(cx, op, name, *scope, bases); + cx.set_reg(*dst, class); + } + OpKind::Import { module, item, dst } => { + let value = match item { + ImportItem::Root => { + self.module_value(ModuleName::from_name(&module.first_component())) + } + ImportItem::Module => self.module_value(*module), + ImportItem::Name(name) => self.analyzer.imported_name(*module, name), + }; + cx.set_reg(*dst, value); + } + OpKind::ContextEnter { + manager, is_async, .. + } => { + let manager = cx.reg(*manager).clone(); + let method = if *is_async { "__aenter__" } else { "__enter__" }; + self.context(cx, &manager, method); + } + OpKind::ContextExit { + manager, is_async, .. + } => { + let manager = cx.reg(*manager).clone(); + if manager.iter().all(|o| matches!(o, Origin::Instance(..))) { + let method = if *is_async { "__aexit__" } else { "__exit__" }; + self.context(cx, &manager, method); + } + } + // The remaining results (unpacked items, iterator items, parameters, sent + // values, gaps) take the engine's defaults, which are `top` when not tracked. + OpKind::Unpack { .. } + | OpKind::Next { .. } + | OpKind::Param { .. } + | OpKind::Yield { value: None, .. } + | OpKind::Gap { .. } => {} + } + } +} + +impl<'a> LifeguardDomain<'a> { + pub fn new( + analyzer: &'a Analyzer<'a>, + program: &'a Program, + importer: ModuleName, + mode: Mode, + ) -> Self { + Self { + analyzer, + program, + importer, + mode, + } + } + + fn report(&self, cx: &mut Cx<'_, Self>, kind: Kind, message: impl Into) { + self.event( + cx, + Event::Report { + kind, + message: message.into(), + }, + ); + } + + fn event(&self, cx: &mut Cx<'_, Self>, event: Event) { + let range = cx.range(); + cx.report(Finding { + start: range.start().into(), + end: range.end().into(), + event, + }); + } + + fn unknown_call(&self, cx: &mut Cx<'_, Self>, what: &str) -> Value { + let line = self.program.line_col(cx.range()).0; + let reason = format!("{}:{line}: {what}", self.program.module); + self.report(cx, Kind::UnknownCall, reason); + Value::unknown() + } + + /// Rebinding a local or closure cell is local. Rebinding a global from a function + /// changes the defining module's namespace, which is another module's state unless + /// that module is the one being imported. + fn rebind(&self, cx: &mut Cx<'_, Self>, symbol: SymbolId) { + if self.program.owner(symbol) == Some(Program::MODULE) && cx.region() != Program::MODULE { + let namespace = Value::of(Origin::Module(self.program.module)); + let action = format!("rebinds global `{}`", self.program.symbol(symbol).name); + self.mutate(cx, &namespace, Mutation::Definite(&action)); + } + } + + /// True if `sys.modules[key]` cannot fail because the module is always loaded or is + /// the reading module or one of its packages. + fn loaded(&self, key: &str) -> bool { + let module = self.program.module; + matches!(key, "__main__" | "sys" | "builtins") + || module.as_str() == key + || module + .as_str() + .strip_prefix(key) + .is_some_and(|rest| rest.starts_with('.')) + } + + fn module_value(&self, module: ModuleName) -> Value { + Value::of(if self.analyzer.contains(module) { + Origin::Module(module) + } else { + Origin::Imported(module) + }) + } + + fn owner(&self, origin: &Origin) -> Owner { + let module_owner = |m: ModuleName, what: &str| { + if m == self.importer { + Owner::Local + } else { + Owner::External(format!("{what} of module `{m}`")) + } + }; + match origin { + Origin::Const | Origin::Str(_) | Origin::Identity => Owner::Local, + Origin::Object(m) | Origin::Instance(_, m) => module_owner(*m, "an object"), + Origin::Function(f) + | Origin::Bound(f, _) + | Origin::StaticMethod(f) + | Origin::ClassMethod(f) + | Origin::Generator(f, _) => module_owner(f.module, "a function"), + Origin::Property(accessors) => { + let accessor = accessors.getter.or(accessors.setter).or(accessors.deleter); + let module = accessor.map_or(self.program.module, |f| f.module); + module_owner(module, "a property") + } + Origin::Class(c) => module_owner(c.module, &format!("class `{}`", c.name)), + Origin::Module(m) => module_owner(*m, "the namespace"), + Origin::Imported(path) => Owner::External(format!("imported `{path}`")), + Origin::Builtin(name) => Owner::External(format!("builtin `{name}`")), + // Attributes and items of objects are not tracked: an object reached from a + // parameter or an import keeps that owner, and one reached from anything + // else may be any object at all. + Origin::Member { root, .. } => match self.owner(root) { + Owner::Param(_) => Owner::Param(origin.clone()), + Owner::External(what) => Owner::External(format!("an object reached from {what}")), + Owner::Local | Owner::Unknown => Owner::Unknown, + }, + Origin::Param(..) => Owner::Param(origin.clone()), + Origin::Unknown => Owner::Unknown, + } + } + + /// Report the mutation of every object in `value`. + fn mutate(&self, cx: &mut Cx<'_, Self>, value: &Value, mutation: Mutation<'_>) { + for origin in value.iter() { + let (kind, message) = match (self.owner(origin), mutation) { + (Owner::Local, _) => continue, + (Owner::Param(param), _) => { + self.event(cx, Event::MutatesParam(param)); + continue; + } + (Owner::External(what), Mutation::Definite(action)) => { + (Kind::ImportedMutation, format!("{action} on {what}")) + } + (Owner::External(what), Mutation::Possible(action)) => ( + Kind::UnknownMutation, + format!("{action} {what}, which may mutate it"), + ), + (Owner::External(what), Mutation::Argument) => ( + Kind::ImportedArgument, + format!("passes {what} to a parameter that the callee mutates"), + ), + (Owner::Unknown, Mutation::Definite(action)) => ( + Kind::UnknownMutation, + format!("{action} on a value the analysis cannot resolve"), + ), + (Owner::Unknown, Mutation::Possible(action)) => ( + Kind::UnknownMutation, + format!("{action} a value the analysis cannot resolve"), + ), + (Owner::Unknown, Mutation::Argument) => ( + Kind::UnknownMutation, + "passes a value to a parameter whose objects the callee mutates, and the \ + analysis cannot resolve who owns them" + .to_owned(), + ), + }; + self.report(cx, kind, message); + } + } + + fn load_attr(&self, cx: &mut Cx<'_, Self>, object: &Value, attr: &Name) -> Value { + let name = CallArgs::positional(vec![Value::of(Origin::Str(attr.as_str().into()))]); + object.flat_map(|o| match o { + Origin::Module(m) => self.analyzer.module_attr(*m, attr), + Origin::Class(c) => match c.lookup(attr) { + Some(member) => self.bind_member(cx, o, c, &member, attr), + None => Value::of(o.member(Some(attr))), + }, + // `__getattribute__` runs on every read, and `__getattr__` when the class + // does not define the attribute. + Origin::Instance(c, _) => { + let me = Value::of(o.clone()); + let mut value = self.protocol(cx, &me, &["__getattribute__"], &name); + value.extend(match c.lookup(attr) { + Some(member) => self.bind_member(cx, o, c, &member, attr), + None => { + let mut v = self.protocol(cx, &me, &["__getattr__"], &name); + v.extend(Value::of(o.member(Some(attr)))); + v + } + }); + value + } + _ => Value::of(o.member(Some(attr))), + }) + } + + /// Read a class attribute through `owner` (the class or an instance of it), + /// applying the descriptor protocol for functions, `staticmethod`, `classmethod` and + /// `property`. + fn bind_member( + &self, + cx: &mut Cx<'_, Self>, + owner: &Origin, + class: &ClassInfo, + member: &Value, + attr: &Name, + ) -> Value { + let instance = matches!(owner, Origin::Instance(..)); + let me = Value::of(owner.clone()); + member.flat_map(|m| match m { + Origin::Function(f) if instance => Value::of(Origin::Bound(*f, Box::new(me.clone()))), + Origin::StaticMethod(f) => Value::of(Origin::Function(*f)), + Origin::ClassMethod(f) => { + let class = Value::of(Origin::Class(Box::new(class.clone()))); + Value::of(Origin::Bound(*f, Box::new(class))) + } + Origin::Property(accessors) if instance => match accessors.getter { + Some(getter) => self.invoke(cx, getter, &CallArgs::positional(vec![me.clone()])), + None => self.unknown_call(cx, &format!("reads property `{attr}` without a getter")), + }, + // The instance's own attributes may shadow class data. + _ if instance => Value::of(m.clone()).join(&Value::of(owner.member(Some(attr)))), + _ => Value::of(m.clone()), + }) + } + + /// The bound special method `name` that Python looks up on the type of `owner`. + /// `__new__` and `__init_subclass__` are implicitly static and class methods. + fn special( + &self, + cx: &mut Cx<'_, Self>, + owner: &Origin, + class: &ClassInfo, + name: &str, + ) -> Option { + let name = Name::new(name); + let member = class.lookup(&name)?; + Some(match name.as_str() { + "__new__" | "__init_subclass__" => member.flat_map(|m| match m { + Origin::Function(f) | Origin::StaticMethod(f) | Origin::ClassMethod(f) => { + Value::of(Origin::Bound(*f, Box::new(Value::of(owner.clone())))) + } + other => Value::of(other.clone()), + }), + _ => self.bind_member(cx, owner, class, &member, &name), + }) + } + + /// Run the special methods in `names` that the classes of the instances in `value` + /// define, as an operator, iteration or builtin does, and join their results. + fn protocol( + &self, + cx: &mut Cx<'_, Self>, + value: &Value, + names: &[&str], + args: &CallArgs, + ) -> Value { + let mut out = Value::default(); + for origin in value.iter() { + let Origin::Instance(c, _) = origin else { + continue; + }; + for name in names { + if let Some(method) = self.special(cx, origin, c, name) { + for f in method.iter() { + out.extend(self.call(cx, f, args)); + } + } + } + } + out + } + + /// Store (`value = Some`) or delete (`value = None`) an attribute. + fn store_attr(&self, cx: &mut Cx<'_, Self>, object: &Value, attr: &Name, value: Option) { + let action = match value { + Some(_) => format!("stores attribute `{attr}`"), + None => format!("deletes attribute `{attr}`"), + }; + for origin in object.iter() { + match origin { + Origin::Imported(p) + if p.as_str() == "builtins" && attr.as_str() == "__import__" => + { + self.report( + cx, + Kind::BuiltinsImportOverride, + "replaces or deletes `builtins.__import__`", + ); + } + Origin::Imported(p) if p.as_str() == "sys" && attr.as_str() == "modules" => { + self.report(cx, Kind::SysModulesAccess, "replaces `sys.modules`"); + } + Origin::Instance(c, _) => self.store_instance_attr(cx, origin, c, attr, &value), + // Class values and module globals are snapshots; a write through the class + // or module object would make later dispatch and reads stale. + Origin::Class(c) if c.module == self.importer => { + self.report( + cx, + Kind::UnknownMutation, + format!( + "{action} on class `{}` after creation, which is not tracked", + c.name + ), + ); + } + Origin::Module(m) if *m == self.importer => { + self.report( + cx, + Kind::UnknownMutation, + format!("{action} through the module object, which is not tracked"), + ); + } + _ => self.mutate(cx, &Value::of(origin.clone()), Mutation::Definite(&action)), + } + } + } + + fn store_instance_attr( + &self, + cx: &mut Cx<'_, Self>, + instance: &Origin, + class: &ClassInfo, + attr: &Name, + value: &Option, + ) { + let hook = if value.is_some() { + "__setattr__" + } else { + "__delattr__" + }; + if let Some(hook) = self.special(cx, instance, class, hook) { + let mut args = vec![Value::of(Origin::Str(attr.as_str().into()))]; + args.extend(value.clone()); + let args = CallArgs::positional(args); + for f in hook.iter() { + self.call(cx, f, &args); + } + return; + } + let me = Value::of(instance.clone()); + let action = format!("stores attribute `{attr}`"); + let Some(member) = class.lookup(attr) else { + return self.mutate(cx, &me, Mutation::Definite(&action)); + }; + for m in member.iter() { + match m { + Origin::Property(accessors) => { + let accessor = if value.is_some() { + accessors.setter + } else { + accessors.deleter + }; + match accessor { + Some(f) => { + let mut args = vec![me.clone()]; + args.extend(value.clone()); + self.invoke(cx, f, &CallArgs::positional(args)); + } + None => { + self.unknown_call(cx, &format!("writes property `{attr}`, which has no accessor for it")); + } + } + } + // Dispatch reads methods from the class snapshot, so an instance attribute + // that shadows one is not tracked. + Origin::Function(_) + | Origin::Bound(..) + | Origin::StaticMethod(_) + | Origin::ClassMethod(_) + | Origin::Class(_) + | Origin::Builtin(_) + | Origin::Identity => self.report( + cx, + Kind::UnknownMutation, + format!("shadows method `{attr}` of class `{}` on an instance, which is not tracked", class.name), + ), + _ => self.mutate(cx, &me, Mutation::Definite(&action)), + } + } + } + + /// Call one callee origin. + fn call(&self, cx: &mut Cx<'_, Self>, callee: &Origin, args: &CallArgs) -> Value { + match callee { + Origin::Function(f) | Origin::StaticMethod(f) => self.invoke(cx, *f, args), + Origin::Bound(f, receiver) => self.invoke(cx, *f, &args.with_receiver(receiver)), + Origin::Member { + root, + attr: Some(name), + .. + } => self.call_method(cx, callee, root, name, args), + Origin::Class(c) => self.construct(cx, c, args), + Origin::Builtin(name) => self.call_builtin(cx, name, args), + Origin::Imported(path) => self.call_external(cx, *path, args), + Origin::Identity => self.identity(args), + Origin::Instance(c, _) => match self.special(cx, callee, c, "__call__") { + Some(v) => v.flat_map(|f| self.call(cx, f, args)), + None => self.unknown_call(cx, &format!("calls an instance of `{}`", c.name)), + }, + Origin::Param(..) => self.unknown_call(cx, "calls a parameter"), + _ => self.unknown_call(cx, "calls a value the analysis cannot resolve"), + } + } + + /// Call `root.name(...)`, where `method` is the `Member` origin of the attribute. + fn call_method( + &self, + cx: &mut Cx<'_, Self>, + method: &Origin, + root: &Origin, + name: &Name, + args: &CallArgs, + ) -> Value { + // The object the method is called on: `root` itself, or an object reached from + // it when the attribute chain is longer (`root.a.name`). + let receiver = match method { + Origin::Member { deep: true, .. } => Value::of(root.member(None).member(None)), + _ => Value::of(root.clone()), + }; + let name = name.as_str(); + match root { + Origin::Property(accessors) if matches!(name, "setter" | "getter" | "deleter") => { + match args.positional.as_slice() { + [f] if f.iter().all(|o| matches!(o, Origin::Function(_))) => f.flat_map(|o| { + let Origin::Function(f) = o else { + unreachable!("checked above") + }; + let mut accessors = accessors.clone(); + match name { + "getter" => accessors.getter = Some(*f), + "setter" => accessors.setter = Some(*f), + _ => accessors.deleter = Some(*f), + } + Value::of(Origin::Property(accessors)) + }), + _ => self.unknown_call( + cx, + &format!("calls `property.{name}` with an unresolved function"), + ), + } + } + // Without types, builtin method names decide: a known mutator mutates its + // receiver, a known pure method does nothing, and anything else is unknown. + Origin::Object(_) + | Origin::Const + | Origin::Str(_) + | Origin::Param(..) + | Origin::Imported(_) => { + if known::is_mutating_method(name) || known::is_pure_method(name) { + for value in args.all() { + if known::is_storing_method(name) { + self.escape(cx, value); + } else { + self.consume(cx, value); + } + } + self.callback(cx, args.get(usize::MAX, "key")); + } + if known::is_mutating_method(name) { + self.mutate( + cx, + &receiver, + Mutation::Definite(&format!("calls `{name}`")), + ); + receiver.flat_map(|o| Value::of(o.member(None))) + } else if known::is_pure_method(name) && !matches!(root, Origin::Imported(_)) { + match name { + "copy" => Value::of(Origin::Object(self.importer)), + "get" | "keys" | "values" | "items" => { + receiver.flat_map(|o| Value::of(o.member(None))) + } + _ => Value::of(Origin::Const), + } + } else { + if matches!(root, Origin::Param(..)) { + self.mutate( + cx, + &receiver, + Mutation::Definite(&format!("calls `{name}`")), + ); + } + self.unknown_call(cx, &format!("calls unresolved method `{name}`")) + } + } + _ => self.unknown_call(cx, &format!("calls unresolved method `{name}`")), + } + } + + fn identity(&self, args: &CallArgs) -> Value { + match args.positional.first() { + Some(v) + if v.iter() + .all(|o| matches!(o, Origin::Function(_) | Origin::Class(_))) => + { + v.clone() + } + _ => Value::of(Origin::Identity), + } + } + + fn call_builtin(&self, cx: &mut Cx<'_, Self>, name: &Name, args: &CallArgs) -> Value { + match name.as_str() { + "exec" => { + self.report(cx, Kind::ExecCall, "calls `exec`"); + Value::unknown() + } + "eval" => { + if args.get(0, "source").is_some_and(|v| v.as_str().is_some()) { + self.unknown_call(cx, "evaluates a string literal, which is not analyzed") + } else { + self.report(cx, Kind::ExecCall, "calls `eval` on a computed string"); + Value::unknown() + } + } + "getattr" | "setattr" | "delattr" => { + let object = args.get(0, "").cloned().unwrap_or_else(Value::unknown); + let Some(attr) = args.get(1, "").and_then(|v| v.as_str().map(Name::new)) else { + let message = format!("calls `{name}` with a computed attribute name"); + self.report(cx, Kind::ProhibitedCall, message); + return Value::unknown(); + }; + match name.as_str() { + "getattr" => self.load_attr(cx, &object, &attr), + "setattr" => { + let value = args.get(2, "").cloned().unwrap_or_else(Value::unknown); + self.store_attr(cx, &object, &attr, Some(value)); + Value::of(Origin::Const) + } + _ => { + self.store_attr(cx, &object, &attr, None); + Value::of(Origin::Const) + } + } + } + "property" => wrap_functions(args.get(0, "fget"), property), + "staticmethod" => wrap_functions(args.get(0, ""), Origin::StaticMethod), + "classmethod" => wrap_functions(args.get(0, ""), Origin::ClassMethod), + // Dynamic namespaces, `super()` and iterator items are opaque. + "globals" | "locals" | "vars" | "super" | "type" => Value::unknown(), + "next" => { + if let Some(iterator) = args.get(0, "") { + self.consume(cx, &iterator.clone()); + } + Value::unknown() + } + _ => match known::builtin(name.as_str()) { + Some(known) => { + for value in args.all() { + self.protocol(cx, value, CONVERSION_METHODS, &CallArgs::default()); + } + if let Some((index, keyword, eager)) = known::builtin_callback(name.as_str()) { + let callback = args.get(index, keyword); + // `iter(callable, sentinel)` is the only form of `iter` with a callback. + let callback = match name.as_str() { + "iter" if args.positional.len() == 2 => args.positional.first(), + "iter" => None, + _ => callback, + }; + if eager { + self.callback(cx, callback); + } else if callback.is_some_and(|v| v.iter().any(|o| *o != Origin::Const)) { + // Lazy callbacks run during a later iteration that the analysis + // does not connect back to them, so any callback is unknown. + self.unknown_call( + cx, + &format!("passes a callback to lazy `{name}`, whose calls are not tracked"), + ); + } + } + let mut result = self.apply_known(cx, known, name.as_str(), args); + match known::builtin_iteration(name.as_str()) { + Iteration::Consumes => { + for value in args.all() { + self.consume(cx, &value.clone()); + } + } + // The lazy iterator runs the generators it wraps when it is iterated. + Iteration::Wraps => { + for value in args.all() { + result.extend( + value + .iter() + .filter(|o| matches!(o, Origin::Generator(..))) + .cloned() + .collect(), + ); + } + } + Iteration::Ignores => {} + } + result + } + None => self.unknown_call( + cx, + &format!("calls unmodeled builtin or undefined name `{name}`"), + ), + }, + } + } + + fn call_external(&self, cx: &mut Cx<'_, Self>, path: ModuleName, args: &CallArgs) -> Value { + // A dataclass's generated `__init__` calls default factories, which construction + // does not model. + if path.as_str() == "dataclasses.field" && args.get(usize::MAX, "default_factory").is_some() + { + return self.unknown_call( + cx, + "declares a dataclass default factory, which is not modeled", + ); + } + if let Some(known) = known::external(path.as_str()) { + if args + .all() + .any(|v| v.iter().any(|o| matches!(o, Origin::Generator(..)))) + { + let what = format!("passes a generator to `{path}`, which may iterate it"); + return self.unknown_call(cx, &what); + } + return self.apply_known(cx, known, path.as_str(), args); + } + let (Some(owner), Some(method)) = (path.parent(), path.components().pop()) else { + return self.unknown_call(cx, &format!("calls unresolved `{path}`")); + }; + if owner.as_str() == "sys.modules" { + self.report(cx, Kind::SysModulesAccess, format!("calls `{path}`")); + Value::unknown() + } else if known::is_mutating_method(method.as_str()) { + let receiver = Value::of(Origin::Imported(owner)); + self.mutate( + cx, + &receiver, + Mutation::Definite(&format!("calls `{method}`")), + ); + Value::unknown() + } else { + self.unknown_call(cx, &format!("calls unresolved `{path}`")) + } + } + + fn apply_known( + &self, + cx: &mut Cx<'_, Self>, + known: Known, + name: &str, + args: &CallArgs, + ) -> Value { + match known { + Known::Const => Value::of(Origin::Const), + Known::Fresh => Value::of(Origin::Object(self.importer)), + Known::Element => { + let mut items = Value::default(); + for v in args.positional.iter().chain(&args.spread) { + items.extend(v.flat_map(|o| Value::of(o.member(None)))); + } + items + } + Known::Returns(index, keyword) => args + .get(index, keyword) + .cloned() + .unwrap_or_else(Value::unknown), + Known::Identity => self.identity(args), + Known::Property => wrap_functions(args.get(0, "func"), property), + Known::IdentityFactory => Value::of(Origin::Identity), + Known::Prohibited => { + self.report(cx, Kind::ProhibitedCall, format!("calls `{name}`")); + Value::unknown() + } + } + } + + /// Iterating a project generator or awaiting a project coroutine runs its body with + /// the arguments it was created with. Iterating a parameter is recorded so that the + /// caller runs the generator it passes. + fn consume(&self, cx: &mut Cx<'_, Self>, value: &Value) { + for origin in value.iter() { + match origin { + Origin::Generator(f, bound) => { + self.run(cx, *f, bound.clone()); + } + Origin::Param(..) | Origin::Member { .. } => { + if let Owner::Param(param) = self.owner(origin) { + self.event(cx, Event::ConsumesParam(param)); + } + } + _ => {} + } + } + } + + /// A generator stored where the analysis does not follow it (a collection, an + /// attribute, a yielded value) may run later at import time or not at all. + fn escape(&self, cx: &mut Cx<'_, Self>, value: &Value) { + if value.iter().any(|o| matches!(o, Origin::Generator(..))) { + self.unknown_call( + cx, + "stores a generator or coroutine where its iteration is not tracked", + ); + } + } + + fn escape_element(&self, cx: &mut Cx<'_, Self>, element: &Element) { + let regs = match element { + Element::Value(r) | Element::DoubleStar(r) => vec![*r], + Element::Pair(k, v) => vec![*k, *v], + // Unpacking a generator into a display iterates it. + Element::Star(r) => { + let value = cx.reg(*r).clone(); + self.consume(cx, &value); + Vec::new() + } + Element::Missing => Vec::new(), + }; + for r in regs { + let value = cx.reg(r).clone(); + self.escape(cx, &value); + } + } + + /// Run a callback that a builtin calls with arguments the analysis does not track. + /// `None` is the usual "no callback" default. + fn callback(&self, cx: &mut Cx<'_, Self>, value: Option<&Value>) { + let args = CallArgs { + spread: Some(Value::unknown()), + ..CallArgs::default() + }; + for origin in value.into_iter().flat_map(|v| v.iter()) { + match origin { + Origin::Const => {} + // Calling a builtin on items the analysis does not track may run their + // special methods (`str` runs `__str__`), unless the builtin has effects + // of its own to report or calls none. + Origin::Builtin(name) + if name.as_str() != "id" + && !matches!( + known::builtin(name.as_str()), + Some(Known::Prohibited) | None + ) => + { + let what = format!( + "passes builtin `{name}` as a callback, which may run special methods of untracked items" + ); + self.unknown_call(cx, &what); + } + _ => { + self.call(cx, origin, &args); + } + } + } + } + + /// Run a context manager's `method`. The asynchronous methods are awaited, which + /// runs the coroutine they return. + fn context(&self, cx: &mut Cx<'_, Self>, manager: &Value, method: &str) { + for origin in manager.iter() { + let Origin::Instance(c, _) = origin else { + self.unknown_call(cx, "enters a context manager the analysis cannot resolve"); + continue; + }; + match self.special(cx, origin, c, method) { + Some(v) => { + for f in v.iter() { + let result = self.call(cx, f, &CallArgs::default()); + if method.starts_with("__a") { + self.consume(cx, &result); + } + } + } + None => { + self.unknown_call(cx, &format!("enters a context manager without `{method}`")); + } + } + } + } + + fn make_class( + &self, + cx: &mut Cx<'_, Self>, + def: OpId, + name: &Name, + scope: ScopeId, + bases: &[Arg], + ) -> Value { + // The namespace holds the class-body bindings themselves, not what a load would + // read: a deleted or never-bound name is absent, not its global fallback. + let members: Vec<(Name, Value)> = self + .program + .scope(scope) + .symbols + .iter() + .filter_map(|(n, s)| { + let binding = cx.frame().binding(self.program, *s)?; + let value = match binding.presence { + Presence::Bound => binding.value.clone(), + Presence::Maybe => binding.value.join(&Value::unknown()), + Presence::Unbound => return None, + }; + Some((n.clone(), value)) + }) + .collect(); + if members.iter().any(|(n, _)| n.as_str() == "__del__") { + let message = format!("class `{name}` defines `__del__`"); + self.report(cx, Kind::CustomFinalizer, message); + } + let mut base_values = Vec::new(); + for arg in bases { + match arg { + Arg::Positional(r) => base_values.push(cx.reg(*r).clone()), + Arg::Keyword(k, r) if k.as_str() == "metaclass" => { + let safe = cx.reg(*r).iter().all(|o| match o { + Origin::Imported(p) => p.as_str() == "abc.ABCMeta", + Origin::Builtin(b) => b.as_str() == "type", + _ => false, + }); + if !safe { + self.unknown_call( + cx, + &format!("creates class `{name}` with an unresolved metaclass"), + ); + } + } + _ => { + self.unknown_call( + cx, + &format!("creates class `{name}` with unmodeled class arguments"), + ); + } + } + } + let class = Origin::Class(Box::new(ClassInfo { + module: self.program.module, + def, + name: name.clone(), + members, + bases: base_values.clone(), + })); + // Creating a subclass runs the bases' metaclass and `__init_subclass__`. + for base in &base_values { + for origin in base.iter() { + match origin { + Origin::Class(c) => { + if let Some(hook) = self.special(cx, &class, c, "__init_subclass__") { + for f in hook.iter() { + self.call(cx, f, &CallArgs::default()); + } + } + } + Origin::Builtin(b) if known::is_safe_builtin_base(b.as_str()) => {} + Origin::Imported(p) if known::is_safe_external_base(p.as_str()) => {} + _ => { + self.unknown_call( + cx, + &format!("creates class `{name}` from an unresolved base"), + ); + } + } + } + } + Value::of(class) + } + + fn construct(&self, cx: &mut Cx<'_, Self>, class: &ClassInfo, args: &CallArgs) -> Value { + let class_origin = Origin::Class(Box::new(class.clone())); + let instance = Origin::Instance(Box::new(class.clone()), self.importer); + let mut found = false; + for (owner, method) in [(&class_origin, "__new__"), (&instance, "__init__")] { + if let Some(v) = self.special(cx, owner, class, method) { + found = true; + for f in v.iter() { + self.call(cx, f, args); + } + } + } + if !found && !self.bases_resolved(class) { + self.unknown_call( + cx, + &format!( + "constructs `{}`, whose constructor is inherited from an unresolved base", + class.name + ), + ); + } + Value::of(instance) + } + + fn bases_resolved(&self, class: &ClassInfo) -> bool { + class.bases.iter().all(|base| { + !base.is_empty() + && base.iter().all(|o| match o { + Origin::Class(c) => self.bases_resolved(c), + Origin::Builtin(b) => known::is_safe_builtin_base(b.as_str()), + Origin::Imported(p) => known::is_safe_external_base(p.as_str()), + _ => false, + }) + }) + } + + /// Call a project function: bind the arguments, then apply its summary. + fn invoke(&self, cx: &mut Cx<'_, Self>, func: FuncRef, args: &CallArgs) -> Value { + if self.mode == Mode::Scan { + return Value::unknown(); + } + let region = self.analyzer.program(func.module).region(func.region); + let bound = bind(®ion.params, args); + match region.trigger { + Trigger::OnCall => self.run(cx, func, bound), + // Calling a generator or coroutine function creates it without running it. + Trigger::OnIteration | Trigger::OnAwait => Value::of(Origin::Generator(func, bound)), + Trigger::OnImport => unreachable!("only module regions run on import"), + } + } + + /// Run a function body with `bound` arguments (missing ones are unknown) and apply + /// its summary to the caller. + fn run(&self, cx: &mut Cx<'_, Self>, func: FuncRef, mut bound: Vec) -> Value { + if self.mode == Mode::Scan { + return Value::unknown(); + } + let callee = self.analyzer.program(func.module); + bound.resize(callee.region(func.region).params.len(), Value::unknown()); + let same_program = func.module == self.program.module; + // Cells are read at invocation time: the callee's free symbols, plus every global + // that some function of its module reads, since the callee may call those + // functions. + let exit_globals = (!same_program) + .then(|| self.analyzer.exit_globals(func.module)) + .flatten(); + let mut symbols: Vec = callee + .free_symbols(func.region) + .iter() + .chain(self.analyzer.function_globals(func.module)) + .copied() + .collect(); + symbols.sort(); + symbols.dedup(); + let cells = symbols + .into_iter() + .filter_map(|s| { + let binding = if let Some(builtin) = builtin_binding(callee, s) { + builtin + } else if same_program { + cx.frame().binding(callee, s)?.clone() + } else { + // Another module's function sees that module's finished globals; + // closure cells of a function that escaped another module are unknown. + exit_globals.as_ref()?.get(&s)?.clone() + }; + Some((s, binding)) + }) + .collect(); + let summary = self.analyzer.summarize(Activation { + func, + importer: self.importer, + cells, + }); + self.apply(cx, func, &bound, &summary, same_program) + } + + fn apply( + &self, + cx: &mut Cx<'_, Self>, + func: FuncRef, + bound: &[Value], + summary: &Summary, + same_program: bool, + ) -> Value { + if let Some(reason) = summary.unsafe_reasons.first() { + self.report(cx, Kind::UnsafeCall, reason.clone()); + } + if let Some(reason) = summary.unknown_reasons.first() { + self.report(cx, Kind::UnknownCall, reason.clone()); + } + for param in &summary.consumed { + if is_own_param(param, func) { + let argument = Value::of(param.clone()).substitute(func, bound); + self.consume(cx, &argument); + } else { + self.event(cx, Event::ConsumesParam(param.clone())); + } + } + for param in &summary.mutated { + if is_own_param(param, func) { + let argument = Value::of(param.clone()).substitute(func, bound); + self.mutate(cx, &argument, Mutation::Argument); + } else { + self.event(cx, Event::MutatesParam(param.clone())); + } + } + if same_program { + for (symbol, binding) in &summary.writes { + let binding = Binding { + presence: binding.presence, + value: binding.value.substitute(func, bound), + }; + cx.set_binding(*symbol, binding); + } + } else if !summary.writes.is_empty() && func.module == self.importer { + let what = format!( + "rebinds globals of `{}` through another module", + func.module + ); + self.unknown_call(cx, &what); + } + summary.returned.substitute(func, bound) + } +} + +const UNARY_METHODS: &[&str] = &["__neg__", "__pos__", "__invert__"]; + +const COMPARISON_METHODS: &[&str] = &[ + "__eq__", + "__ne__", + "__lt__", + "__le__", + "__gt__", + "__ge__", + "__contains__", +]; + +const ARITHMETIC_METHODS: &[&str] = &[ + "__add__", + "__radd__", + "__iadd__", + "__sub__", + "__rsub__", + "__isub__", + "__mul__", + "__rmul__", + "__imul__", + "__matmul__", + "__rmatmul__", + "__imatmul__", + "__truediv__", + "__rtruediv__", + "__itruediv__", + "__floordiv__", + "__rfloordiv__", + "__ifloordiv__", + "__mod__", + "__rmod__", + "__imod__", + "__pow__", + "__rpow__", + "__ipow__", + "__lshift__", + "__rlshift__", + "__ilshift__", + "__rshift__", + "__rrshift__", + "__irshift__", + "__and__", + "__rand__", + "__iand__", + "__xor__", + "__rxor__", + "__ixor__", + "__or__", + "__ror__", + "__ior__", +]; + +/// Special methods that builtins may call on their arguments. +const CONVERSION_METHODS: &[&str] = &[ + "__len__", + "__iter__", + "__next__", + "__reversed__", + "__contains__", + "__str__", + "__repr__", + "__format__", + "__hash__", + "__bool__", + "__index__", + "__int__", + "__float__", + "__abs__", + "__round__", +]; + +/// The binding of a builtin symbol, which always holds the builtin itself: the program +/// has no way to rebind it except through the `builtins` module object. +pub fn builtin_binding(program: &Program, symbol: SymbolId) -> Option> { + let s = program.symbol(symbol); + (s.origin == SymbolOrigin::Builtin) + .then(|| Binding::bound(Value::of(Origin::Builtin(s.name.clone())))) +} + +/// A `property` with only a getter. +fn property(getter: FuncRef) -> Origin { + Origin::Property(Accessors { + getter: Some(getter), + setter: None, + deleter: None, + }) +} + +/// Wrap each function in `value` with `wrap`; anything else is unknown. +fn wrap_functions(value: Option<&Value>, wrap: impl Fn(FuncRef) -> Origin) -> Value { + match value { + Some(v) => v.flat_map(|o| match o { + Origin::Function(f) => Value::of(wrap(*f)), + _ => Value::unknown(), + }), + None => Value::unknown(), + } +} + +/// True if `param`, a parameter origin from a summary, is a parameter of `func` (or an +/// object reached from one) rather than of an enclosing activation. +fn is_own_param(param: &Origin, func: FuncRef) -> bool { + match param { + Origin::Param(f, _) => *f == func, + Origin::Member { root, .. } => matches!(&**root, Origin::Param(f, _) if *f == func), + _ => unreachable!("summaries record only parameter origins"), + } +} + +fn is_sys_modules(origin: &Origin) -> bool { + matches!(origin, Origin::Imported(p) if p.as_str() == "sys.modules") +} + +/// Bind call arguments to parameters following Python's rules. A value behind `*` or +/// `**` may reach any parameter. A parameter left unbound takes its default value, +/// which is unknown here because function objects do not carry their defaults. +fn bind(params: &[Param], args: &CallArgs) -> Vec { + let mut bound: Vec> = vec![None; params.len()]; + let mut put = |i: usize, v: &Value| match &mut bound[i] { + Some(old) => old.extend(v.clone()), + slot @ None => *slot = Some(v.clone()), + }; + let mut positional_slots = params + .iter() + .enumerate() + .filter(|(_, p)| matches!(p.kind, ParamKind::PositionalOnly | ParamKind::Positional)) + .map(|(i, _)| i); + let varargs = params.iter().position(|p| p.kind == ParamKind::VarArgs); + let kwargs = params.iter().position(|p| p.kind == ParamKind::KwArgs); + for value in &args.positional { + if let Some(i) = positional_slots.next().or(varargs) { + put(i, value); + } + } + for (name, value) in &args.keywords { + let target = params.iter().position(|p| { + p.name == *name && matches!(p.kind, ParamKind::Positional | ParamKind::KeywordOnly) + }); + if let Some(i) = target.or(kwargs) { + put(i, value); + } + } + bound + .into_iter() + .zip(params) + .map(|(b, param)| { + let mut value = match b { + Some(v) => v, + None if param.has_default => Value::unknown(), + None => Value::default(), + }; + if let Some(s) = &args.spread { + value.extend(s.clone()); + } + value + }) + .collect() +} diff --git a/static_analysis/lifeguard/src/known.rs b/static_analysis/lifeguard/src/known.rs new file mode 100644 index 0000000000..7297d16a68 --- /dev/null +++ b/static_analysis/lifeguard/src/known.rs @@ -0,0 +1,224 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Hand-written effect summaries for builtins, builtin methods and a few standard +//! library functions. Anything not listed here is unresolved, and the analysis reports +//! it as unknown rather than assuming it is safe. + +/// What calling a known builtin or external function does at import time. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Known { + /// No effect; returns an immutable value. + Const, + /// No effect; returns a new mutable object owned by the caller. + Fresh, + /// Returns an element of its positional arguments (`min`, `max`). + Element, + /// Returns the argument at this position or with this keyword (`typing.cast`). + Returns(usize, &'static str), + /// A decorator that returns the function or class it decorates (`functools.cache`). + Identity, + /// A decorator that turns a method into a property (`functools.cached_property`). + Property, + /// Returns a decorator that returns what it decorates (`functools.wraps(f)`). + IdentityFactory, + /// Has an effect that lazy imports can reorder or skip (I/O, process state). + Prohibited, +} + +/// Builtins by name. `exec`, `eval`, `getattr`, `setattr`, `delattr`, `next`, +/// `property`, `classmethod` and `staticmethod` are modeled separately by the domain. +pub fn builtin(name: &str) -> Option { + use Known::*; + Some(match name { + "abs" | "all" | "any" | "ascii" | "bin" | "bool" | "callable" | "chr" | "complex" + | "divmod" | "float" | "format" | "hasattr" | "hash" | "hex" | "id" | "int" + | "isinstance" | "issubclass" | "len" | "oct" | "ord" | "pow" | "print" | "repr" + | "round" | "str" | "sum" | "bytes" | "frozenset" | "tuple" | "range" | "slice" + | "compile" => Const, + "bytearray" | "dict" | "dir" | "enumerate" | "filter" | "iter" | "list" | "map" + | "memoryview" | "object" | "reversed" | "set" | "sorted" | "zip" => Fresh, + "max" | "min" => Element, + "input" | "open" | "breakpoint" | "exit" | "quit" | "help" | "__import__" => Prohibited, + _ if is_exception_class(name) => Fresh, + _ => return None, + }) +} + +/// How a builtin treats generator arguments. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Iteration { + /// Iterates its arguments now. + Consumes, + /// Returns a lazy iterator over its arguments. + Wraps, + /// Does not iterate its arguments. + Ignores, +} + +pub fn builtin_iteration(name: &str) -> Iteration { + match name { + "list" | "tuple" | "set" | "frozenset" | "dict" | "sorted" | "sum" | "min" | "max" + | "any" | "all" | "bytes" | "bytearray" | "next" => Iteration::Consumes, + "iter" | "enumerate" | "zip" | "map" | "filter" | "reversed" => Iteration::Wraps, + _ => Iteration::Ignores, + } +} + +/// Builtins that call a callback argument: `(position, keyword, eager)`. Eager callbacks +/// run during the call; lazy ones run when the result is iterated. +pub fn builtin_callback(name: &str) -> Option<(usize, &'static str, bool)> { + match name { + "sorted" | "min" | "max" => Some((usize::MAX, "key", true)), + "map" | "filter" => Some((0, "function", false)), + "iter" => Some((usize::MAX, "callable", false)), + _ => None, + } +} + +/// Methods of builtin containers that store their arguments rather than iterate them. +pub fn is_storing_method(name: &str) -> bool { + matches!( + name, + "append" | "insert" | "add" | "setdefault" | "__setitem__" + ) +} + +/// Builtin classes that are safe to subclass: creating the subclass runs no user code. +pub fn is_safe_builtin_base(name: &str) -> bool { + matches!( + name, + "object" | "int" | "str" | "bytes" | "float" | "dict" | "list" | "set" | "tuple" + ) || is_exception_class(name) +} + +fn is_exception_class(name: &str) -> bool { + name.ends_with("Error") + || name.ends_with("Exception") + || name.ends_with("Warning") + || matches!( + name, + "StopIteration" | "StopAsyncIteration" | "KeyboardInterrupt" | "SystemExit" + ) +} + +/// Functions of modules outside the project, by dotted path. +pub fn external(path: &str) -> Option { + use Known::*; + Some(match path { + "os.path.join" | "os.path.dirname" | "os.path.basename" | "os.path.abspath" + | "os.path.splitext" | "os.path.normpath" | "os.fspath" | "re.escape" + | "typing.TypeVar" | "typing.NewType" | "typing.ParamSpec" => Const, + "re.compile" + | "collections.namedtuple" + | "collections.defaultdict" + | "collections.OrderedDict" + | "collections.Counter" + | "collections.deque" + | "functools.partial" + | "dataclasses.field" + | "threading.Lock" + | "threading.RLock" + | "enum.auto" => Fresh, + "functools.cached_property" => Property, + "functools.cache" + | "functools.lru_cache" + | "functools.total_ordering" + | "typing.final" + | "typing.overload" + | "typing.runtime_checkable" + | "abc.abstractmethod" + | "dataclasses.dataclass" + | "contextlib.contextmanager" => Identity, + "typing.cast" => Returns(1, "val"), + "functools.wraps" => IdentityFactory, + "os.system" + | "os.chdir" + | "os.putenv" + | "subprocess.run" + | "subprocess.call" + | "subprocess.check_call" + | "subprocess.check_output" + | "subprocess.Popen" + | "sys.exit" + | "sys.setrecursionlimit" + | "signal.signal" + | "atexit.register" + | "warnings.filterwarnings" + | "warnings.simplefilter" + | "logging.basicConfig" + | "random.seed" + | "importlib.__import__" => Prohibited, + _ => return None, + }) +} + +/// External classes that are safe to subclass. +pub fn is_safe_external_base(path: &str) -> bool { + matches!( + path, + "abc.ABC" | "typing.Generic" | "typing.Protocol" | "typing.NamedTuple" | "typing.TypedDict" + ) +} + +/// Methods that mutate their receiver when it is a builtin container. +pub fn is_mutating_method(name: &str) -> bool { + matches!( + name, + "append" + | "extend" + | "insert" + | "pop" + | "popitem" + | "remove" + | "clear" + | "update" + | "add" + | "discard" + | "setdefault" + | "sort" + | "reverse" + | "__setitem__" + | "__delitem__" + | "__setattr__" + | "__delattr__" + ) +} + +/// Methods that builtin types define without side effects. +pub fn is_pure_method(name: &str) -> bool { + matches!( + name, + "copy" + | "get" + | "index" + | "count" + | "keys" + | "values" + | "items" + | "join" + | "split" + | "rsplit" + | "strip" + | "lstrip" + | "rstrip" + | "lower" + | "upper" + | "replace" + | "startswith" + | "endswith" + | "format" + | "encode" + | "decode" + | "union" + | "intersection" + | "difference" + | "issubset" + | "issuperset" + | "find" + ) +} diff --git a/static_analysis/lifeguard/src/lib.rs b/static_analysis/lifeguard/src/lib.rs new file mode 100644 index 0000000000..83add812dd --- /dev/null +++ b/static_analysis/lifeguard/src/lib.rs @@ -0,0 +1,191 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! An experimental lazy-import safety analysis in the style of +//! [Lifeguard](https://github.com/facebook/Lifeguard), built as a consumer of the neutral +//! `pyrefly_static_analysis` frontend and engine. +//! +//! A module is safe to import lazily when executing its body has no effect that another +//! module could observe: it does not mutate imported objects or other modules' state, +//! does not call functions with such effects, and does not depend on import order +//! through `sys.modules`, `exec` or `builtins.__import__`. +//! +//! The analysis never walks syntax. Each module is lowered once by the shared frontend, +//! and an abstract interpretation ([`domain`]) runs over the module region with the +//! shared [`Engine`](pyrefly_static_analysis::Engine). A call to a project function runs +//! the same engine over the callee's region, so function bodies count only when called, +//! while decorators and defaults count where the function is defined. Anything the +//! analysis cannot resolve is reported with an unknown kind instead of being treated as +//! safe. See `README.md` for the supported subset and its provenance. + +mod analyzer; +mod domain; +mod known; +mod value; + +use std::fmt; + +use pyrefly_static_analysis::ExportProvider; +use pyrefly_static_analysis::ModuleName; + +use crate::analyzer::Analyzer; + +/// What a diagnostic reports. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum Kind { + /// Import-time mutation of an object owned by another module (attribute, item, + /// deletion, a mutating method, or rebinding another module's global). + ImportedMutation, + /// An imported object passed to a parameter that the callee mutates. + ImportedArgument, + /// A call whose body has an import-time effect. + UnsafeCall, + /// A call to a builtin or library function with process-wide effects. + ProhibitedCall, + /// A `raise` that import-time execution reaches. + UnhandledRaise, + /// A class defining `__del__`, which runs at an unpredictable time. + CustomFinalizer, + /// `exec`, or `eval` of a computed string. + ExecCall, + /// An access to `sys.modules` that depends on which modules were already imported. + SysModulesAccess, + /// A store to or deletion of `builtins.__import__`. + BuiltinsImportOverride, + /// A call whose callee, or some operation in it, cannot be resolved. + UnknownCall, + /// A mutation of a value whose owner cannot be determined. + UnknownMutation, + /// A construct the frontend or the engine does not model, or a non-converged analysis. + Unsupported, +} + +impl Kind { + pub fn as_str(self) -> &'static str { + match self { + Kind::ImportedMutation => "imported-mutation", + Kind::ImportedArgument => "imported-argument", + Kind::UnsafeCall => "unsafe-call", + Kind::ProhibitedCall => "prohibited-call", + Kind::UnhandledRaise => "unhandled-raise", + Kind::CustomFinalizer => "custom-finalizer", + Kind::ExecCall => "exec-call", + Kind::SysModulesAccess => "sys-modules-access", + Kind::BuiltinsImportOverride => "builtins-import-override", + Kind::UnknownCall => "unknown-call", + Kind::UnknownMutation => "unknown-mutation", + Kind::Unsupported => "unsupported", + } + } + + /// True if the diagnostic is an absence of knowledge rather than a known effect. + pub fn is_unknown(self) -> bool { + matches!( + self, + Kind::UnknownCall | Kind::UnknownMutation | Kind::Unsupported + ) + } + + /// True if the module's own imports must run eagerly. These are reported wherever + /// they occur in the module, including in functions that import time never calls. + pub fn forces_eager_imports(self) -> bool { + matches!( + self, + Kind::CustomFinalizer + | Kind::ExecCall + | Kind::SysModulesAccess + | Kind::BuiltinsImportOverride + ) + } +} + +impl fmt::Display for Kind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct Diagnostic { + /// One-based line and column of the operation. + pub line: usize, + pub column: usize, + pub kind: Kind, + pub message: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Verdict { + /// No diagnostics: importing the module lazily is safe within the supported subset. + Safe, + /// Some known import-time effect. + Unsafe, + /// No known effect, but something could not be analyzed. + Unknown, +} + +impl Verdict { + pub fn as_str(self) -> &'static str { + match self { + Verdict::Safe => "safe", + Verdict::Unsafe => "unsafe", + Verdict::Unknown => "unknown", + } + } +} + +#[derive(Debug, Clone)] +pub struct ModuleReport { + pub name: ModuleName, + pub diagnostics: Vec, +} + +impl ModuleReport { + pub fn verdict(&self) -> Verdict { + if self.diagnostics.iter().any(|d| !d.kind.is_unknown()) { + Verdict::Unsafe + } else if self.diagnostics.is_empty() { + Verdict::Safe + } else { + Verdict::Unknown + } + } + + pub fn load_imports_eagerly(&self) -> bool { + self.diagnostics + .iter() + .any(|d| d.kind.forces_eager_imports()) + } +} + +/// A set of in-memory modules analyzed together. Imports of modules outside the set +/// are treated as external objects. +#[derive(Default)] +pub struct Project { + modules: Vec<(ModuleName, String, bool)>, +} + +impl Project { + pub fn new() -> Self { + Self::default() + } + + /// Add a module. `is_init` marks a package `__init__`, which affects relative imports. + pub fn add(&mut self, name: &str, source: &str, is_init: bool) -> &mut Self { + self.modules + .push((ModuleName::from_str(name), source.to_owned(), is_init)); + self + } + + /// Analyze every module. `exports` resolves `from m import n` across modules; with + /// `SyntacticExports`, re-exports are still followed through the analyzed modules' + /// own import statements. Reports are sorted by module name, and diagnostics by + /// position. + pub fn analyze(&self, exports: &dyn ExportProvider) -> Vec { + Analyzer::new(&self.modules, exports).run() + } +} diff --git a/static_analysis/lifeguard/src/main.rs b/static_analysis/lifeguard/src/main.rs new file mode 100644 index 0000000000..1984cccb01 --- /dev/null +++ b/static_analysis/lifeguard/src/main.rs @@ -0,0 +1,77 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! `lifeguard-spike NAME=PATH...`: analyze an explicit list of Python files as one +//! project and print lazy-import diagnostics and a verdict per module. +//! +//! Each argument names a module and its file. A path ending in `__init__.py` marks a +//! package `__init__`. Nothing is discovered: imports of modules not listed are treated +//! as external. +//! +//! Exit status: 0 when every module is safe, 1 when some module is unsafe, 2 when none +//! is unsafe but some module could not be fully analyzed, 64 for usage or I/O errors. + +use std::env; +use std::fs; +use std::path::Path; +use std::process::ExitCode; + +use pyrefly_static_analysis::SyntacticExports; +use pyrefly_static_analysis_lifeguard::Project; +use pyrefly_static_analysis_lifeguard::Verdict; + +const USAGE: &str = "usage: lifeguard-spike NAME=PATH [NAME=PATH ...]"; + +fn main() -> ExitCode { + let args: Vec = env::args().skip(1).collect(); + if args.is_empty() || args.iter().any(|a| a == "-h" || a == "--help") { + eprintln!("{USAGE}"); + return ExitCode::from(64); + } + let mut project = Project::new(); + for arg in &args { + let Some((name, path)) = arg.split_once('=') else { + eprintln!("expected NAME=PATH, got `{arg}`\n{USAGE}"); + return ExitCode::from(64); + }; + let source = match fs::read_to_string(path) { + Ok(source) => source, + Err(e) => { + eprintln!("cannot read `{path}`: {e}"); + return ExitCode::from(64); + } + }; + let is_init = Path::new(path).file_stem().is_some_and(|s| s == "__init__"); + project.add(name, &source, is_init); + } + + let reports = project.analyze(&SyntacticExports); + for report in &reports { + for d in &report.diagnostics { + println!( + "{}:{}:{}: {}: {}", + report.name, d.line, d.column, d.kind, d.message + ); + } + } + for report in &reports { + let eager = if report.load_imports_eagerly() { + " load-imports-eagerly" + } else { + "" + }; + println!("{}: {}{eager}", report.name, report.verdict().as_str()); + } + let verdicts: Vec = reports.iter().map(|r| r.verdict()).collect(); + if verdicts.contains(&Verdict::Unsafe) { + ExitCode::from(1) + } else if verdicts.contains(&Verdict::Unknown) { + ExitCode::from(2) + } else { + ExitCode::SUCCESS + } +} diff --git a/static_analysis/lifeguard/src/value.rs b/static_analysis/lifeguard/src/value.rs new file mode 100644 index 0000000000..1cac33f515 --- /dev/null +++ b/static_analysis/lifeguard/src/value.rs @@ -0,0 +1,229 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! The abstract values the lazy-import domain tracks for symbols and temporaries. +//! +//! A [`Value`] is the set of [`Origin`]s an object may come from. The origin decides +//! what mutating the object means at import time: an object created while importing +//! the module under analysis is local state, while an object reached through an import +//! belongs to another module. + +use std::collections::BTreeSet; + +use pyrefly_static_analysis::ModuleName; +use pyrefly_static_analysis::Name; +use pyrefly_static_analysis::OpId; +use pyrefly_static_analysis::RegionId; + +/// A function body: a deferred region of one project module. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct FuncRef { + pub module: ModuleName, + pub region: RegionId, +} + +/// A class created by a `class` statement in a project module, with the values its +/// body bound when the class was created. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct ClassInfo { + pub module: ModuleName, + pub def: OpId, + pub name: Name, + pub members: Vec<(Name, Value)>, + pub bases: Vec, +} + +impl ClassInfo { + /// Look up a class attribute through the class and its project bases. `None` + /// means the attribute is not defined by any class body the analysis can see. + pub fn lookup(&self, attr: &Name) -> Option { + if let Some((_, v)) = self.members.iter().find(|(n, _)| n == attr) { + return Some(v.clone()); + } + self.bases.iter().find_map(|base| { + base.iter().find_map(|o| match o { + Origin::Class(c) => c.lookup(attr), + _ => None, + }) + }) + } + + fn substitute(&self, func: FuncRef, args: &[Value]) -> ClassInfo { + ClassInfo { + members: self + .members + .iter() + .map(|(n, v)| (n.clone(), v.substitute(func, args))) + .collect(), + bases: self + .bases + .iter() + .map(|b| b.substitute(func, args)) + .collect(), + module: self.module, + def: self.def, + name: self.name.clone(), + } + } +} + +/// A `property` object and the accessor functions it holds. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub struct Accessors { + pub getter: Option, + pub setter: Option, + pub deleter: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +pub enum Origin { + /// An immutable constant other than a string. + Const, + /// A string constant, kept for literal attribute names, `sys.modules` keys and `eval`. + Str(Box), + /// A mutable object created while importing the given project module. + Object(ModuleName), + /// An object reached through a module outside the project, named by its dotted path. + Imported(ModuleName), + /// A project module object. + Module(ModuleName), + /// A builtin function or class. + Builtin(Name), + /// A function object for a project function body. + Function(FuncRef), + /// A function bound to a receiver: a method read from an instance, or a + /// `classmethod` read from its class or an instance. + Bound(FuncRef, Box), + /// A function wrapped by `staticmethod`. + StaticMethod(FuncRef), + /// A function wrapped by `classmethod`. + ClassMethod(FuncRef), + /// The generator or coroutine returned by calling a generator or `async` function + /// with the given arguments; its body has not run yet. + Generator(FuncRef, Vec), + Property(Accessors), + /// A decorator known to return the object it decorates. + Identity, + Class(Box), + /// An instance of a project class, created while importing `owner`. + Instance(Box, ModuleName), + /// An attribute (`attr = Some`) or item (`attr = None`) of an object reached from + /// `root`: of `root` itself, or of something reachable from it when `deep`. Mutating + /// it mutates the object graph of `root`. `root` is never itself a `Member`. + Member { + root: Box, + attr: Option, + deep: bool, + }, + /// A parameter of a function activation, as seen inside that function's summary. + /// Callers replace it with the argument (see [`Value::substitute`]), so it never + /// reaches a module body. + Param(FuncRef, usize), + /// Anything the analysis cannot attribute. + Unknown, +} + +impl Origin { + /// The origin of `self.attr` (or `self[...]` when `attr` is `None`) when nothing more + /// specific is known about the attribute. + pub fn member(&self, attr: Option<&Name>) -> Origin { + match (self, attr) { + (Origin::Imported(path), Some(attr)) => Origin::Imported(path.append(attr)), + (Origin::Unknown, _) => Origin::Unknown, + (Origin::Member { root, .. }, _) => Origin::Member { + root: root.clone(), + attr: attr.cloned(), + deep: true, + }, + _ => Origin::Member { + root: Box::new(self.clone()), + attr: attr.cloned(), + deep: false, + }, + } + } + + fn substitute(&self, func: FuncRef, args: &[Value]) -> Value { + let sub = |v: &Value| v.substitute(func, args); + Value::of(match self { + Origin::Param(f, i) if *f == func => return args[*i].clone(), + Origin::Member { root, attr, deep } => { + return root.substitute(func, args).flat_map(|r| { + let base = if *deep { r.member(None) } else { r.clone() }; + Value::of(base.member(attr.as_ref())) + }); + } + Origin::Bound(f, receiver) => Origin::Bound(*f, Box::new(sub(receiver))), + Origin::Generator(f, bound) => Origin::Generator(*f, bound.iter().map(sub).collect()), + Origin::Class(c) => Origin::Class(Box::new(c.substitute(func, args))), + Origin::Instance(c, owner) => { + Origin::Instance(Box::new(c.substitute(func, args)), *owner) + } + _ => self.clone(), + }) + } +} + +/// A set of origins. The empty set is the value of an unbound name: reading it raises, +/// so it contributes nothing to the normal continuation. +#[derive(Debug, Clone, Default, PartialEq, Eq, PartialOrd, Ord)] +pub struct Value(BTreeSet); + +impl Value { + pub fn unknown() -> Self { + Self::of(Origin::Unknown) + } + + pub fn of(origin: Origin) -> Self { + Self(BTreeSet::from([origin])) + } + + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + pub fn iter(&self) -> impl Iterator { + self.0.iter() + } + + pub fn join(&self, other: &Value) -> Value { + Value(self.0.union(&other.0).cloned().collect()) + } + + pub fn extend(&mut self, other: Value) { + self.0.extend(other.0); + } + + /// The single string constant this value holds, if it is exactly one. + pub fn as_str(&self) -> Option<&str> { + match self.0.first() { + Some(Origin::Str(s)) if self.0.len() == 1 => Some(s), + _ => None, + } + } + + /// Replace the parameters of `func` with the corresponding `args`, including inside + /// structured values (classes, instances, bound methods and generators). + pub fn substitute(&self, func: FuncRef, args: &[Value]) -> Value { + self.flat_map(|o| o.substitute(func, args)) + } + + /// Apply `f` to every origin and join the results. + pub fn flat_map(&self, mut f: impl FnMut(&Origin) -> Value) -> Value { + let mut out = Value::default(); + for o in &self.0 { + out.extend(f(o)); + } + out + } +} + +impl FromIterator for Value { + fn from_iter>(iter: I) -> Self { + Value(iter.into_iter().collect()) + } +} diff --git a/static_analysis/lifeguard/tests/common/mod.rs b/static_analysis/lifeguard/tests/common/mod.rs new file mode 100644 index 0000000000..bace88253a --- /dev/null +++ b/static_analysis/lifeguard/tests/common/mod.rs @@ -0,0 +1,66 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Fixture harness. Each module's source marks expected diagnostics with trailing +//! `# E: ` comments, one per diagnostic kind expected on that line. The harness +//! compares the set of `(line, kind)` pairs; messages are shown on mismatch. + +use pyrefly_static_analysis::SyntacticExports; +use pyrefly_static_analysis_lifeguard::ModuleReport; +use pyrefly_static_analysis_lifeguard::Project; + +fn expected(source: &str) -> Vec<(usize, String)> { + let mut out = Vec::new(); + for (i, line) in source.lines().enumerate() { + for part in line.split("# E:").skip(1) { + let kind = part.split_whitespace().next().expect("a kind after `# E:`"); + out.push((i + 1, kind.to_owned())); + } + } + out.sort(); + out.dedup(); + out +} + +/// Analyze `modules` together and check every module's expectations. +pub fn check_all(modules: &[(&str, &str)]) -> Vec { + let mut project = Project::new(); + for (name, source) in modules { + project.add(name, source, false); + } + let reports = project.analyze(&SyntacticExports); + for (name, source) in modules { + let report = reports + .iter() + .find(|r| r.name.as_str() == *name) + .expect("every module has a report"); + let mut actual: Vec<(usize, String)> = report + .diagnostics + .iter() + .map(|d| (d.line, d.kind.as_str().to_owned())) + .collect(); + actual.sort(); + actual.dedup(); + let details: Vec = report + .diagnostics + .iter() + .map(|d| format!(" {}:{}: {}: {}", d.line, d.column, d.kind, d.message)) + .collect(); + assert_eq!( + expected(source), + actual, + "module `{name}` diagnostics:\n{}\nsource:\n{source}", + details.join("\n"), + ); + } + reports +} + +/// Analyze a single module named `test`. +pub fn check(source: &str) -> ModuleReport { + check_all(&[("test", source)]).remove(0) +} diff --git a/static_analysis/lifeguard/tests/parity.rs b/static_analysis/lifeguard/tests/parity.rs new file mode 100644 index 0000000000..e4245e28ca --- /dev/null +++ b/static_analysis/lifeguard/tests/parity.rs @@ -0,0 +1,1545 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Fixtures adapted from the public Lifeguard test suite +//! (https://github.com/facebook/Lifeguard, commit +//! c04fde1358c467d7ec84244ddfb066186381f066). Each test names its upstream file and +//! test. Expectations use this crate's diagnostic kinds; where the result differs from +//! upstream, a comment says how and why. See `README.md` for the parity matrix. + +mod common; + +use common::check; +use common::check_all; + +// --------------------------------------------------------------------------- +// tests/assign.rs +// --------------------------------------------------------------------------- + +/// `test_assign_to_module_var`, `test_annotated_assign_to_module_var`. +#[test] +fn assign_to_module_var() { + check( + r#" +import foo +foo.bar = 1 # E: imported-mutation +foo.baz: int = 1 # E: imported-mutation +"#, + ); +} + +/// `test_augmented_assign_to_module_var`. The store is a definite mutation; the +/// in-place operator may additionally mutate `foo.bar` itself (upstream reports only +/// the store). +#[test] +fn augmented_assign_to_module_var() { + check( + r#" +import foo +foo.bar += 1 # E: imported-mutation # E: unknown-mutation +"#, + ); +} + +/// `test_update_to_import_array`. +#[test] +fn update_to_import_array() { + check( + r#" +from foo import bar +bar[0] = 1 # E: imported-mutation +"#, + ); +} + +/// `test_assign_to_import_var`: rebinding the imported name is local. +#[test] +fn assign_to_import_var() { + check( + r#" +from foo import bar +bar = 1 +"#, + ); +} + +/// `test_augmented_assign_to_import_var`. Upstream treats this as a rebinding and +/// reports nothing. Without types, `bar += 1` may call `bar.__iadd__`, which mutates a +/// list in place, so this analysis reports an unknown mutation instead of a safe verdict. +#[test] +fn augmented_assign_to_import_var() { + check( + r#" +from foo import bar +bar += 1 # E: unknown-mutation +"#, + ); +} + +/// `test_class_attribute_shadowing_import_is_not_a_mutation`. +#[test] +fn class_attribute_shadowing_import() { + check( + r#" +from foo import bar + +class C: + bar = 1 +"#, + ); +} + +/// `test_subscript_assign_to_alias`: the alias is followed through another project module. +#[test] +fn subscript_assign_to_alias() { + check_all(&[ + ("mod1", "A = []\n"), + ( + "mod2", + r#" +import mod1 +x = mod1.A +x[1] = 2 # E: imported-mutation +"#, + ), + ]); +} + +/// `test_call_with_import_var_alias`. Upstream reports `unsafe-function-call` for the +/// unresolved `x.a()`; this analysis reports it as unknown. +#[test] +fn call_with_import_var_alias() { + check( + r#" +import foo + +baz = foo.bar + +def f(x): + x.a() + +f(baz) # E: imported-argument # E: unknown-call +"#, + ); +} + +/// `test_assign_to_builtins_import`, `..._in_function`, `..._via_alias`, +/// `test_setattr_builtins_import`, `test_delete_builtins_import`. The override is +/// reported wherever it occurs, including in functions import time never calls. +#[test] +fn builtins_import_override() { + check( + r#" +import builtins +import builtins as b + +def hook(name, *args, **kwargs): + return name + +builtins.__import__ = hook # E: builtins-import-override + +def install(hook): + b.__import__ = hook # E: builtins-import-override + setattr(builtins, "__import__", hook) # E: builtins-import-override + +def uninstall(): + del builtins.__import__ # E: builtins-import-override +"#, + ); +} + +/// `test_assign_to_other_builtins_attr`, `test_read_builtins_import`, and the +/// destructuring variants. +#[test] +fn other_builtins_attributes() { + check( + r#" +import builtins +builtins.my_helper = 1 # E: imported-mutation +original = builtins.__import__ +builtins.other, x = 1, 2 # E: imported-mutation +[builtins.third, y] = 1, 2 # E: imported-mutation +a, (builtins.fourth, c) = 1, (2, 3) # E: imported-mutation +"#, + ); +} + +// --------------------------------------------------------------------------- +// tests/basic.rs +// --------------------------------------------------------------------------- + +/// `test_getattr_literal`, `test_setattr_literal`: attributes of a function this module +/// defines are local state. +#[test] +fn literal_getattr_setattr() { + check( + r#" +def f(): ... +getattr(f, "__module__") +setattr(f, "__module__", "foo") +"#, + ); +} + +/// `test_getattr_nonliteral`, `test_setattr_nonliteral`. The upstream fixtures bind the +/// name to a string literal first; flow-sensitive constants resolve that, so a truly +/// computed name is used here and the literal-variable form is safe. +#[test] +fn computed_getattr_setattr() { + check( + r#" +def f(): ... +x = "foo" +getattr(f, x) +name = x + "bar" +getattr(f, name) # E: prohibited-call +setattr(f, name, 1) # E: prohibited-call +"#, + ); +} + +/// `test_setattr_global_assign`, `test_global_var_assign`: mutating or rebinding a +/// module's own state is safe while that module is importing, and an effect when +/// another module's import runs it. +#[test] +fn own_state_is_safe_only_for_its_own_import() { + check_all(&[ + ( + "a", + r#" +counter = 0 +def f(): ... +def g(): + setattr(f, "__module__", "foo") +def h(): + global counter + counter = 1 +g() +h() +"#, + ), + ( + "b", + r#" +from a import g, h +g() # E: unsafe-call +h() # E: unsafe-call +"#, + ), + ]); +} + +/// `test_imported_subscript_access`, `test_imported_subscript_assignment`. +#[test] +fn imported_subscript() { + check( + r#" +from foo import bar +bar['hello'] +bar['hello'] = 10 # E: imported-mutation +"#, + ); +} + +/// `test_unsafe_subexpression`: a raise in the callee makes the call unsafe. +#[test] +fn unsafe_subexpression() { + check( + r#" +def f(x, y): + raise(ValueError()) +def g(x, y): return x + y +a = g(1, 2) + g(4, f(4, 5)) # E: unsafe-call +"#, + ); +} + +/// `test_nested_class`. Reported at the class statement rather than at `__del__`. +#[test] +fn custom_finalizer() { + check( + r#" +class A: + class B: # E: custom-finalizer + def __del__(self): ... +"#, + ); +} + +// --------------------------------------------------------------------------- +// tests/calls.rs +// --------------------------------------------------------------------------- + +/// `test_top_level_function_call`, `test_top_level_function_call_from_import`. +#[test] +fn computed_getattr_through_import() { + check_all(&[ + ( + "a", + r#" +def f(x, y): + getattr(x, y) +"#, + ), + ( + "b", + r#" +from a import f +getattr(x, y) # E: prohibited-call +f(1, 2) # E: unsafe-call +"#, + ), + ]); +} + +/// `test_top_level_call_through_package_re_export`. +#[test] +fn call_through_package_re_export() { + check_all(&[ + ( + "pkg.impl", + r#" +REGISTRY = [] +def register(x): + REGISTRY.append(x) +class Registered: + def __init__(self): + REGISTRY.append(self) +"#, + ), + ("pkg", "from pkg.impl import register, Registered\n"), + ( + "consumer", + r#" +import pkg +from pkg import register +pkg.register(1) # E: unsafe-call +register(1) # E: unsafe-call +pkg.Registered() # E: unsafe-call +"#, + ), + ]); +} + +/// `test_safe_call_through_package_re_export`. +#[test] +fn safe_call_through_package_re_export() { + check_all(&[ + ( + "pkg.impl", + r#" +def pure(x): + return x + 1 +class Plain: + def __init__(self): + self.x = 1 +"#, + ), + ("pkg", "from pkg.impl import pure, Plain\n"), + ( + "consumer", + r#" +import pkg +from pkg import pure +pkg.pure(1) +pure(1) +pkg.Plain() +"#, + ), + ]); +} + +/// `test_parameterized_decorator_through_package_re_export`, +/// `test_helper_calling_package_re_export`. +#[test] +fn decorator_and_helper_through_re_export() { + check_all(&[ + ( + "pkg.impl", + r#" +REGISTRY = [] +def register(name): + def wrap(f): + REGISTRY.append(f) + return f + return wrap +def add(x): + REGISTRY.append(x) +"#, + ), + ("pkg", "from pkg.impl import register, add\n"), + ( + "helper", + r#" +import pkg +def add_later(x): + pkg.add(x) +"#, + ), + ( + "consumer", + r#" +from pkg import register +from helper import add_later +@register("f") # E: unsafe-call +def f(): + pass +add_later(1) # E: unsafe-call +"#, + ), + ]); +} + +/// `test_definition_with_side_effect_call`, `test_local_function_call`. +#[test] +fn deferred_and_safe_local_calls() { + check( + r#" +import os + +def foo(): + os.path.join("a", "b") + +def f(x, y): ... +f("a", "b") +a = f("a", "b") +"#, + ); +} + +/// `test_unsafe_call_returned_from_a_function`. `f` comes from a module outside the +/// project, so the call is unknown here (upstream: `unsafe-function-call`). +#[test] +fn unresolved_call_returned_from_a_function() { + check( + r#" +from foo import f +def g(): + return f() + +a = g() # E: unknown-call +"#, + ); +} + +/// `test_unsafe_local_function_call`: only the import-time call site is reported. +#[test] +fn unsafe_local_function_chain() { + check( + r#" +from foo import f +def g(): + f.x = 10 + +def h(): + g() + +a = h() # E: unsafe-call +"#, + ); +} + +/// `test_unsafe_recursive_function_call`. Recursion is solved to a fixpoint and its +/// termination is reported as unknown (upstream: `unsafe-function-call`). +#[test] +fn recursive_call() { + check( + r#" +def f(): + g() + +def g(): + f() + +a = f() # E: unknown-call +"#, + ); +} + +/// `test_exec`, `test_exec_in_nested_function`. +#[test] +fn exec_in_nested_function() { + check( + r#" +def f(x): + def g(x): + exec("import foo") # E: exec-call +"#, + ); +} + +/// `test_method_call`, `test_safe_method_call`, `test_method_call_on_constructor_result`. +#[test] +fn method_calls() { + check( + r#" +class C: + def m(self): + raise Exception() + def ok(self): + pass +a = C() +a.ok() +a.m() # E: unsafe-call +C().ok() +C().m() # E: unsafe-call +"#, + ); +} + +/// `test_method_call_on_literal_safe`. +#[test] +fn method_call_on_literal() { + check( + r#" +a = ":".join(["x", "y"]) +b = "hello".upper() +c = [1, 2].index(1) +d = {"k": 1}.get("k") +e = (1, 2).count(1) +f = {1, 2}.union({3}) +g = b"x".decode() +h = f"v={a}".strip() +i = [x for x in range(3)].pop() +"#, + ); +} + +/// `test_imported_method_call`, `test_method_call_on_reexported_class_instance`, +/// `test_imported_inherited_unsafe_method_call`. +#[test] +fn imported_method_calls() { + check_all(&[ + ( + "m1", + r#" +class Pure: + def m(self): + pass + +class C: + def m(self): + raise Exception() + +class Derived(C): + pass +"#, + ), + ("m2", "from m1 import Pure, C, Derived\n"), + ( + "m3", + r#" +from m2 import Pure, C, Derived +Pure().m() +pure = Pure() +pure.m() +C().m() # E: unsafe-call +c = C() +c.m() # E: unsafe-call +Derived().m() # E: unsafe-call +"#, + ), + ]); +} + +/// `test_imported_class_alias_preserves_qualified_call`. +#[test] +fn imported_class_alias() { + check_all(&[ + ( + "m1", + r#" +class C: + @classmethod + def f(cls) -> None: + input() +"#, + ), + ( + "m2", + r#" +from m1 import C +Alias = C +Alias.f() # E: unsafe-call +x = C.f +x() # E: unsafe-call +"#, + ), + ]); +} + +/// `test_builtins`, `test_dynamic_imports_dunder_import`, +/// `test_dynamic_imports_in_func_are_safe`. +#[test] +fn prohibited_builtins() { + check( + r#" +a = input() # E: prohibited-call +print(a) +c = __import__("sys") # E: prohibited-call + +def test_func(): + c = __import__("sys") +"#, + ); +} + +/// `test_eval_*`. Evaluating a computed string is an exec hazard. Upstream parses +/// literal strings; this analysis never reparses source, so a literal is unknown. +#[test] +fn eval() { + check( + r#" +a = eval("1 + 2") # E: unknown-call +x = input() # E: prohibited-call +eval(x) # E: exec-call + +def f(): + eval("input()") +"#, + ); +} + +/// `test_lambda_body_not_analyzed`, `test_unsafe_lambda_call`, `test_lambda_default_analyzed`, +/// `test_lambda_call`, `test_aliased_function_call`. Lambdas and aliases of functions +/// resolve through value flow, so their calls run the body (upstream reports +/// `unknown-function-call` for both). +#[test] +fn lambdas_and_aliases() { + check( + r#" +import os +f = lambda: os.getcwd() +f() # E: unknown-call +g = lambda x=os.system("true"): x # E: prohibited-call +h = lambda x: x + 1 +h(10) +def k(x): pass +alias = k +alias(10) +"#, + ); +} + +/// `test_function_from_subscript`. +#[test] +fn function_from_subscript() { + check( + r#" +import foo +a = foo.funcs[1]() # E: unknown-call +"#, + ); +} + +/// `test_local_list_append_is_safe`. +#[test] +fn local_list_append() { + check( + r#" +x = [] +x.append(10) +"#, + ); +} + +/// `test_dynamic_imports_importlib`. Upstream treats `import_module` as safe; it is not +/// modeled here, so it is unknown. +#[test] +fn importlib() { + check( + r#" +import importlib +a = importlib.import_module("sys") # E: unknown-call +b = importlib.__import__("math") # E: prohibited-call +"#, + ); +} + +// --------------------------------------------------------------------------- +// tests/sys_modules.rs +// --------------------------------------------------------------------------- + +/// `test_reading_a_module_the_interpreter_always_has`, `test_reading_itself_or_an_ancestor`. +#[test] +fn sys_modules_reads_that_cannot_fail() { + let reports = check_all(&[ + ("pkg", ""), + ( + "pkg.child", + r#" +import sys +main_file = sys.modules["__main__"].__file__ +path = sys.modules["pkg"].__path__ +me = sys.modules["pkg.child"] +"#, + ), + ]); + assert!(reports.iter().all(|r| !r.load_imports_eagerly())); +} + +/// `test_reading_another_module_stays_eager`, `test_literal_key_write_stays_eager`, +/// `test_method_call_stays_eager`, `test_del_call_still_triggers_eager`, +/// `test_computed_key_stays_eager`. +#[test] +fn sys_modules_accesses_that_force_eager_imports() { + let report = check( + r#" +import sys +import loud +x = sys.modules["loud"] # E: sys-modules-access +sys.modules["sys"] = loud # E: sys-modules-access +sys.modules.setdefault("sys", None) # E: sys-modules-access +del sys.modules["builtins"] # E: sys-modules-access +name = input() # E: prohibited-call +y = sys.modules[name] # E: sys-modules-access +"#, + ); + assert!(report.load_imports_eagerly()); +} + +/// `test_membership_test_makes_the_read_a_probe`. Upstream recognizes the membership +/// guard and keeps the module lazy. Guards are not modeled here, so the read stays eager. +#[test] +fn sys_modules_membership_probe_is_not_modeled() { + check( + r#" +import sys + +def numpy_array(values): + if "loud" in sys.modules: + loud = sys.modules["loud"] # E: sys-modules-access + return loud.array(values) + return None +"#, + ); +} + +// --------------------------------------------------------------------------- +// tests/nested_functions.rs +// --------------------------------------------------------------------------- + +/// `test_nested_function_returned_not_called`, `test_closure_factory`, +/// `test_nested_function_stored_not_called`, `test_deeply_nested_uncalled`, +/// `test_multiple_nested_uncalled`. +#[test] +fn uncalled_nested_functions() { + check( + r#" +from foo import obj + +def f(): + def g(): + obj.x = 10 + def h(): + def k(): + obj.y = 20 + return k + alias = g + return (alias, h) +f() +"#, + ); +} + +/// `test_nested_function_called_in_parent`, `test_deeply_nested_called`, +/// `test_nested_one_called_one_not`. +#[test] +fn called_nested_functions() { + check( + r#" +from foo import obj + +def f(): + def g(): + def h(): + obj.x = 10 + h() + def unused(): + obj.y = 20 + g() + return unused +f() # E: unsafe-call +"#, + ); +} + +/// `test_decorator_factory_no_call`, `test_decorator_with_side_effect_in_body`, +/// `test_decorator_calls_nested_function`. +#[test] +fn decorators_run_at_definition() { + check_all(&[ + ( + "foo", + r#" +from bar import obj + +def returns_wrapper(f): + def wrapper(*args): + obj.x = 10 + return wrapper + +def mutates(f): + obj.x = 10 + return f + +def calls_nested(f): + def setup(): + obj.x = 10 + setup() + return f +"#, + ), + ( + "main", + r#" +from foo import returns_wrapper, mutates, calls_nested + +@returns_wrapper +def a(): + pass + +@mutates # E: unsafe-call +def b(): + pass + +@calls_nested # E: unsafe-call +def c(): + pass +"#, + ), + ]); +} + +/// `test_parameterized_decorator_factory`, `..._imported_var`, `..._transitive_call`. +#[test] +fn parameterized_decorators() { + check_all(&[ + ( + "registry", + r#" +from bar import obj +REGISTRY = {} + +def register(name): + def decorator(cls): + REGISTRY[name] = cls + return cls + return decorator + +def configure(key): + def decorator(cls): + obj[key] = cls + return cls + return decorator + +def helper(): + obj.x = 10 + +def indirect(name): + def decorator(cls): + helper() + return cls + return decorator +"#, + ), + ( + "main", + r#" +from registry import register, configure, indirect + +@register("FiLMLayer") # E: unsafe-call +class A: + pass + +@configure("test") # E: unsafe-call +class B: + pass + +@indirect("name") # E: unsafe-call +class C: + pass +"#, + ), + ]); +} + +/// `test_class_body_in_function`, `test_nested_class_body_in_function`, +/// `test_class_body_in_nested_uncalled_function`, `test_method_in_class_in_function_not_called`, +/// `test_class_with_side_effect_and_method`. +#[test] +fn class_bodies_inside_functions() { + check( + r#" +from foo import obj + +def runs_body(): + class A: + class B: + obj.x = 10 + +def defers_method(): + class C: + def m(self): + obj.x = 10 + +def defers_nested(): + def g(): + class D: + obj.x = 10 + return g + +runs_body() # E: unsafe-call +defers_method() +defers_nested() +"#, + ); +} + +/// `test_imported_closure_factory`, `test_imported_function_that_calls_nested`. +#[test] +fn imported_closures() { + check_all(&[ + ( + "foo", + r#" +from bar import obj + +def make_callback(): + def callback(): + obj.x = 10 + return callback + +def do_setup(): + def setup_impl(): + obj.x = 10 + setup_impl() +"#, + ), + ( + "main", + r#" +from foo import make_callback, do_setup +cb = make_callback() +do_setup() # E: unsafe-call +"#, + ), + ]); +} + +/// `test_param_mutation_in_called_nested`, `test_param_mutation_in_uncalled_nested`. +/// Upstream also flags the uncalled closure because it could mutate the argument later; +/// that happens only if something calls it, so it is not an import-time effect here. +#[test] +fn param_mutation_in_nested_functions() { + check( + r#" +from foo import obj + +def called(x): + def inner(): + x.enabled = True + inner() + +def uncalled(x): + def inner(): + x.enabled = True + return inner + +called(obj) # E: imported-argument +uncalled(obj) +"#, + ); +} + +/// `test_parameterized_decorator_pure_wraps_wrapper`, `..._pure_wraps_from_import`, +/// `..._returned_wrapper_side_effect_is_safe`, `..._wraps_plus_mutation_still_unsafe`. +#[test] +fn functools_wraps_decorators() { + check_all(&[ + ( + "lib", + r#" +import functools +from functools import wraps + +REGISTRY = {} + +def trace(name): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + return wrapper + return decorator + +def validate(name): + def decorator(func): + @wraps(func) + def wrapper(*args, **kwargs): + raise ValueError("only at call time") + return wrapper + return decorator + +def register(name): + def decorator(func): + REGISTRY[name] = func + @functools.wraps(func) + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + return wrapper + return decorator +"#, + ), + ( + "main", + r#" +from lib import trace, validate, register + +@trace("op") +def a(): + pass + +@validate("op") +def b(): + pass + +@register("op") # E: unsafe-call +def c(): + pass +"#, + ), + ]); +} + +// --------------------------------------------------------------------------- +// tests/decorators.rs +// --------------------------------------------------------------------------- + +/// `test_local_decorator`, `test_imported_function_decorator`, `test_imported_class_decorator`, +/// `test_imported_method_decorator`, `test_unknown_decorator`. +#[test] +fn local_and_unresolved_decorators() { + check( + r#" +from foo import dec + +def local(f): + return f + +@local +def a(x): + ... + +@dec # E: unknown-call +def b(x): + ... + +@dec # E: unknown-call +class B: + @dec # E: unknown-call + def m(self): + ... + +@undefined # E: unknown-call +def c(): + ... +"#, + ); +} + +/// `test_safe_imported_decorator`, `test_unsafe_imported_decorator`. +#[test] +fn imported_decorators() { + check_all(&[ + ( + "foo", + r#" +def dec(f): + raise() + +def safe(f): + return f +"#, + ), + ( + "main", + r#" +from foo import dec, safe +import foo + +class A: + @dec # E: unsafe-call + def f(x): + ... + @safe + def g(x): + ... + +@dec(args) # E: unsafe-call +def f(x): + ... + +@foo.dec # E: unsafe-call +def g(x): + ... +"#, + ), + ]); +} + +/// `test_decorator_arguments_on_definitions`. +#[test] +fn decorator_arguments() { + check( + r#" +def safe_factory(*args, **kwargs): + return lambda f: f + +def unsafe(): + raise() + +@safe_factory(unsafe()) # E: unsafe-call +def function(): + ... + +@safe_factory(value=unsafe()) # E: unsafe-call +class Class: + ... + +@safe_factory(*[unsafe()]) # E: unsafe-call +async def async_function(): + ... + +class Container: + @safe_factory(**{"value": unsafe()}) # E: unsafe-call + def method(self): + ... +"#, + ); +} + +/// `test_property`, `test_property_setter_is_safe`, `test_property_deleter_is_safe`. +#[test] +fn properties() { + check( + r#" +class Foo: + @property + def bar(self): + return self._bar + + @bar.setter + def bar(self, value): + self._bar = value + + @bar.deleter + def bar(self): + del self._bar +"#, + ); +} + +/// `test_subscript_decorator`. The decorated name is then unknown, so calling it is too. +#[test] +fn subscript_decorator() { + check( + r#" +decorators = [ + lambda fn: fn, +] + +@decorators[0] # E: unknown-call +def foo(value): + print(value) + +foo(37) # E: unknown-call +"#, + ); +} + +// --------------------------------------------------------------------------- +// tests/param_mutation.rs +// --------------------------------------------------------------------------- + +/// `test_param_subscript_mutation_*`, `test_param_attr_mutation_*`, +/// `test_multiple_param_attr_mutations`, `test_param_del_*`. +#[test] +fn param_mutations() { + check( + r#" +from foo import d, obj + +def set_item(x): + x["key"] = "value" + +def set_attr(x): + x.enabled = True + x.verbose = False + +def del_item(x): + del x["key"] + +def del_attr(x): + del x.enabled + +set_item(d) # E: imported-argument +set_item({}) +set_attr(obj) # E: imported-argument +set_attr(10) +del_item(d) # E: imported-argument +del_attr(obj) # E: imported-argument +"#, + ); +} + +/// `test_param_method_call_with_imported_arg`, `test_param_method_call_unsafe_without_imported_arg`. +/// Unresolved methods on parameters are unknown (upstream: `unsafe-function-call`) and +/// may mutate the argument. +#[test] +fn unresolved_param_method() { + check( + r#" +from foo import A + +def f(x): + x.bar() + +f(A) # E: imported-argument # E: unknown-call +f(10) # E: unknown-call +"#, + ); +} + +/// `test_imported_function_mutates_param`, `test_registry_pattern_global_mutation`. +#[test] +fn cross_module_param_and_global_mutation() { + check_all(&[ + ( + "setup", + r#" +registry = [] + +def configure(x): + x.enabled = True + +def register(item): + registry.append(item) +"#, + ), + ( + "main", + r#" +from setup import configure, register +from config import settings + +configure(settings) # E: imported-argument +register("my_item") # E: unsafe-call +"#, + ), + ]); +} + +/// `test_precise_match_*`, `test_keyword_arg_precise_matching*`, +/// `test_mixed_positional_and_keyword_only_keyword_mutated`. +#[test] +fn precise_argument_matching() { + check( + r#" +from foo import A + +def second(a, b): + b.x = 1 + +def kwonly(a, *, target): + target["k"] = a + +second(A, 1) +second(1, A) # E: imported-argument +second(1, b=A) # E: imported-argument +kwonly(A, target={}) +kwonly(1, target=A) # E: imported-argument +"#, + ); +} + +/// `test_safe_function_no_mutations_with_imported_arg`, `test_safe_subscript_read_with_imported_arg`, +/// `test_safe_builtin_calls_with_imported_arg`, `test_mutation_in_function_scope_only`, +/// `test_star_args_with_imported_var`, `test_kwargs_with_imported_var`. +#[test] +fn safe_uses_of_imported_arguments() { + check( + r#" +from foo import A + +def reads(x): + return x[0] + +def ignores(*args, **kwargs): + pass + +def modify(x): + x.append(1) + +def caller(): + modify(A) + +reads(A) +ignores(A, x=A) +x = len(A) +y = str(A) +z = list(A) +"#, + ); +} + +/// `test_imported_var_alias_passed_to_mutating_function`, `test_module_attr_passed_to_mutating_function`, +/// `test_mutating_builtin_method_on_param_is_unsafe`. +#[test] +fn imported_values_passed_to_mutators() { + check( + r#" +import foo +from foo import bar + +baz = bar + +def modify(x): + x.append(1) + +modify(baz) # E: imported-argument +modify(foo.bar) # E: imported-argument +"#, + ); +} + +/// `test_non_mutating_method_on_param_is_safe`, `test_dict_get_on_param_is_safe`, +/// `test_list_index_on_param_is_safe`, `test_param_copied_then_mutated_is_not_an_imported_var_argument`. +/// The copy is a fresh local object, so mutating it is safe (upstream still reports +/// `unsafe-function-call` for the unresolved `y.append`). +#[test] +fn non_mutating_methods_on_params() { + check( + r#" +from foo import A + +def f(x): + y = x.copy() + y.append(1) + return x.get("key"), x.index(42) + +f(A) +"#, + ); +} + +/// `test_transitive_param_mutation`, `test_three_level_transitive_param_forwarding`, +/// `test_diamond_param_forwarding`. +#[test] +fn forwarded_param_mutation() { + check( + r#" +from foo import A + +def h(z): + z.append(1) + +def g(y): + h(y) + +def i(y): + h(y) + +def f(x): + g(x) + i(x) + +f(A) # E: imported-argument +"#, + ); +} + +/// `test_cycle_param_forwarding`: the cycle converges with the mutation propagated, +/// and its termination is reported as unknown. +#[test] +fn cyclic_param_forwarding() { + check( + r#" +from foo import A + +def g(y): + y.append(1) + f(y) + +def f(x): + g(x) + +f(A) # E: imported-argument # E: unknown-call +"#, + ); +} + +/// `test_reassigned_param_not_forwarded`, `test_param_reassigned_then_aliased_is_safe`: +/// documented upstream false positives. Reaching definitions after the rebinding see +/// only the fresh list, so neither is reported here. +#[test] +fn rebound_params_are_not_forwarded() { + check( + r#" +from foo import A + +def g(y): + y.append(1) + +def f(x): + x = [] + g(x) + +def k(x): + x = [] + y = x + y.append(1) + +f(A) +k(A) +"#, + ); +} + +/// `test_param_aliased_then_mutated_is_unsafe`, `test_param_aliased_two_hops_then_mutated_is_unsafe`, +/// `test_param_alias_then_reassigned_is_safe`, `test_closure_capture_param_alias_mutation`. +#[test] +fn param_aliases() { + check( + r#" +from foo import A + +def one_hop(x): + y = x + y.append(1) + +def two_hops(x): + y = x + z = y + z.append(1) + +def rebound(x): + y = x + y = {} + y["k"] = 1 + +def closure(x): + def g(): + y = x + y.append(1) + g() + +one_hop(A) # E: imported-argument +two_hops(A) # E: imported-argument +rebound(A) +closure(A) # E: imported-argument +"#, + ); +} + +/// `test_aliased_param_forwarded_to_mutating_call`, `test_aliased_param_forwarded_two_hops`. +#[test] +fn aliased_param_forwarded_across_modules() { + check_all(&[ + ( + "a", + r#" +def g(item): + item["k"] = 1 +"#, + ), + ( + "main", + r#" +from a import g +from foo import A + +def f(x): + y = x + z = y + g(z) + +f(A) # E: imported-argument +"#, + ), + ]); +} + +/// `test_tuple_unpack_alias_not_tracked`: the unpacked value is unknown, so the method +/// call on it is unknown. +#[test] +fn tuple_unpack_alias() { + check( + r#" +from foo import A + +def f(x): + y, _ = x, 1 + y.append(1) + +f(A) # E: unknown-call +"#, + ); +} + +/// `test_nested_function_outer_param_is_not_inner_forwarded_param`. The inner call +/// forwards the outer parameter, which is bound to a fresh list (upstream additionally +/// reports the unresolved `v.append`). +#[test] +fn outer_param_is_not_inner_param() { + check( + r#" +from foo import A + +def mutate(v): + v.append(1) + +def outer(x): + def inner(**kwargs): + mutate(x) + inner(**A) + +outer([]) +"#, + ); +} + +// --------------------------------------------------------------------------- +// tests/blocks.rs +// --------------------------------------------------------------------------- + +/// `test_parameter_default_effects`, `test_class_base_effects`: defaults and bases run +/// where the function or class is defined. +#[test] +fn defaults_and_bases_run_at_definition() { + check( + r#" +from foo import f, g +def h(a, b=f(), *args, c=f(), **kwargs): # E: unknown-call + pass +class C(f(), metaclass=g()): # E: unknown-call + pass +"#, + ); +} + +/// `test_for_target`. +#[test] +fn for_target() { + check( + r#" +import foo +for foo.x in [1, 2, 3]: # E: imported-mutation + ... +"#, + ); +} diff --git a/static_analysis/lifeguard/tests/regressions.rs b/static_analysis/lifeguard/tests/regressions.rs new file mode 100644 index 0000000000..610e3527c4 --- /dev/null +++ b/static_analysis/lifeguard/tests/regressions.rs @@ -0,0 +1,433 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Regressions for cases where an earlier version reported a module as safe (or +//! panicked) although it has an import-time effect or cannot be analyzed. + +mod common; + +use common::check; +use common::check_all; + +/// A class created inside a function captures the function's parameter; the class +/// escapes through the return value, so the parameter is substituted inside it. +#[test] +fn returned_class_member_is_substituted() { + check( + r#" +import foreign +def make(x): + class C: + y = x + return C +make(foreign.items).y.append(1) # E: imported-mutation +"#, + ); +} + +/// `staticmethod` and `classmethod` bind as descriptors: no receiver, or the class. +#[test] +fn static_and_class_methods_bind_correctly() { + check( + r#" +import foreign + +class C: + @staticmethod + def f(x): + x.append(1) + + @classmethod + def g(cls, x): + x.append(1) + +C().f(foreign.items) # E: imported-argument +C.f(foreign.items) # E: imported-argument +C.g(foreign.items) # E: imported-argument +C().g(foreign.items) # E: imported-argument +C.f([]) +C().g([]) +"#, + ); +} + +/// Writing or deleting a property runs its setter or deleter. +#[test] +fn property_setters_and_deleters_run() { + check( + r#" +import foreign + +class P: + @property + def v(self): + return 1 + + @v.setter + def v(self, value): + foreign.items.append(value) + + @v.deleter + def v(self): + foreign.items.clear() + +class ReadOnly: + @property + def v(self): + return 1 + +p = P() +p.v = 1 # E: unsafe-call +del p.v # E: unsafe-call +ReadOnly().v = 1 # E: unknown-call +"#, + ); +} + +/// A class's `__setattr__` and `__delattr__` run on instance attribute writes. +#[test] +fn custom_setattr_runs() { + check( + r#" +import foreign + +class S: + def __setattr__(self, name, value): + foreign.items.append(value) + + def __delattr__(self, name): + foreign.items.clear() + +s = S() +s.x = 1 # E: unsafe-call +del s.x # E: unsafe-call +setattr(s, "y", 2) # E: unsafe-call +"#, + ); +} + +/// Class values are snapshots of the class body, so later writes that change method +/// dispatch are reported as unknown rather than silently ignored. +#[test] +fn dynamic_class_writes_are_unknown() { + check( + r#" +import foreign + +class K: + def m(self): + pass + +K.m = lambda self: foreign.items.append(1) # E: unknown-mutation +k = K() +k.m = print # E: unknown-mutation +k.data = 1 +"#, + ); +} + +/// `typing.cast` returns its second argument, and `min`/`max` return an element. +/// Container contents are not tracked, so an element of a copied container is unknown. +#[test] +fn returned_arguments_keep_their_origin() { + check( + r#" +import foreign +import typing +from typing import cast + +x = cast("list", foreign.items) +x[0] = 1 # E: imported-mutation +y = typing.cast(typ="list", val=foreign.other) +y.append(1) # E: imported-mutation +z = max(foreign.lists) +z.append(1) # E: imported-mutation +t = tuple(foreign.lists) +t[0].append(1) # E: unknown-mutation +"#, + ); +} + +/// `next` runs a generator, and a generator keeps the arguments it was created with. +#[test] +fn generators_run_with_their_arguments() { + check( + r#" +import foreign + +def gen(): + foreign.items.append(1) + yield 1 + +def mutator(x): + x.append(1) + yield x + +first = next(gen()) # E: unsafe-call +list(mutator(foreign.items)) # E: imported-argument +list(mutator([])) +"#, + ); +} + +/// Special methods that a project class defines run when operators, iteration, +/// subscripts, truth tests, attribute fallbacks or builtins use its instances. +#[test] +fn protocol_methods_run() { + check( + r#" +import foreign + +class Loud: + def __iter__(self): + foreign.items.append(1) + yield 1 + def __getitem__(self, key): + foreign.items.append(key) + def __add__(self, other): + foreign.items.append(other) + def __bool__(self): + foreign.items.clear() + return True + def __getattr__(self, name): + foreign.items.append(name) + def __len__(self): + foreign.items.append(0) + return 0 + +class Quiet: + pass + +loud = Loud() +for _ in loud: # E: unsafe-call + pass +loud[0] # E: unsafe-call +loud + 1 # E: unsafe-call +if loud: # E: unsafe-call + pass +loud.missing # E: unsafe-call +len(loud) # E: unsafe-call +quiet = Quiet() +if quiet: + pass +len(quiet) +"#, + ); +} + +/// `functools.cached_property` runs its function on read, a dataclass default factory +/// runs on construction (not modeled), and `async with` uses `__aenter__`. +#[test] +fn library_descriptors_and_async_context_managers() { + check( + r#" +import dataclasses +import foreign +import functools + +class C: + @functools.cached_property + def value(self): + foreign.items.append(1) + return 1 + +@dataclasses.dataclass +class D: + items: list = dataclasses.field(default_factory=list) # E: unknown-call + +class Manager: + async def __aenter__(self): + foreign.items.append(1) + async def __aexit__(self, *args): + pass + +async def use(): + async with Manager(): + pass + +C().value # E: unsafe-call +for _ in use(): # E: unsafe-call + pass +"#, + ); +} + +/// Builtins run eager callbacks (`sorted`, `min`, `max` with `key=`, `list.sort`). Any +/// callback given to a lazy builtin (`map`, `filter`, `iter(callable, sentinel)`) is +/// unknown, and so is a builtin callback that may run special methods of the items. +#[test] +fn builtin_callbacks() { + check( + r#" +import foreign + +def key(x): + foreign.registry.append(x) + return x + +def pure_key(x): + return x + +sorted([1], key=key) # E: unsafe-call +max([1], key=key) # E: unsafe-call +items = [2, 1] +items.sort(key=key) # E: unsafe-call +sorted([1], key=pure_key) +sorted([1], key=None) +list(map(key, [1])) # E: unknown-call +list(map(str, [1])) # E: unknown-call +list(map(exec, ["x = 1"])) # E: unknown-call +list(map(open, ["/tmp/not-opened"])) # E: unknown-call +list(filter(input, ["prompt"])) # E: unknown-call +next(iter(input, "stop")) # E: unknown-call +sorted(["/tmp/x"], key=open) # E: prohibited-call +sorted([1], key=id) +filter(None, [1]) +list(key(x) for x in [1]) # E: unsafe-call + +class Loud: + def __str__(self): + foreign.registry.append(self) + return "" + +list(map(str, [Loud()])) # E: unknown-call +sorted([Loud()], key=str) # E: unknown-call +"#, + ); +} + +/// A generator passed to a project function runs if the function iterates it. +#[test] +fn generators_passed_to_functions_run_where_iterated() { + check( + r#" +import foreign + +def gen(): + foreign.registry.append(1) + yield 1 + +def drain(xs): + for _ in xs: + pass + +def keep(xs): + return xs + +drain(gen()) # E: unsafe-call +held = keep(gen()) +for _ in held: # E: unsafe-call + pass +for _ in enumerate(gen()): # E: unsafe-call + pass +"#, + ); +} + +/// The class namespace holds the class body's own bindings: a deleted class attribute +/// is absent, not the global of the same name. +#[test] +fn deleted_class_attribute_is_absent() { + check( + r#" +import foreign + +def helper(): + foreign.registry.append(1) + +class C: + helper = 1 + del helper + +C.helper() # E: unknown-call +"#, + ); +} + +/// A module that may raise still exposes the globals bound on the raising path to an +/// importer that catches the exception. +#[test] +fn globals_join_normal_and_raising_exits() { + check_all(&[ + ( + "a", + r#" +import foreign +items = [] +if foreign.flag: + items = foreign.items + raise ValueError() # E: unhandled-raise +"#, + ), + ( + "b", + r#" +from a import items +items.append(1) # E: imported-mutation +"#, + ), + ]); +} + +/// Many functions and globals: call summaries carry only the cells a call can observe. +#[test] +fn many_decorated_functions() { + let mut source = String::from( + "def deco(f): + return f +", + ); + for i in 0..1000 { + source.push_str(&format!( + "@deco +def f{i}(): + pass +" + )); + } + check(&source); +} + +/// Deleting a global exposes the builtin of the same name, both after a call that +/// deletes it and inside a function called after the module deletes it. +#[test] +fn deleted_globals_fall_back_to_builtins() { + check( + r#" +open = lambda *args: None + +def remove(): + global open + del open + +def use(): + open("x") + +open("fine") +use() +remove() +open("oops") # E: prohibited-call +use() # E: unsafe-call +"#, + ); +} + +#[test] +fn module_level_delete_falls_back_to_builtin() { + check( + r#" +open = lambda *args: None + +def use(): + open("x") + +use() +del open +use() # E: unsafe-call +"#, + ); +} diff --git a/static_analysis/lifeguard/tests/semantics.rs b/static_analysis/lifeguard/tests/semantics.rs new file mode 100644 index 0000000000..ea526ef61a --- /dev/null +++ b/static_analysis/lifeguard/tests/semantics.rs @@ -0,0 +1,336 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Behavior that the shared frontend and engine make possible beyond the upstream +//! fixtures: flow-sensitive rebinding, invocation-time closure cells, deferred +//! generator bodies, value-flow aliases of hazards, and explicit unknown results. + +mod common; + +use std::fs; +use std::process::Command; + +use common::check; +use common::check_all; +use pyrefly_static_analysis_lifeguard::Verdict; + +#[test] +fn local_rebinding_versus_external_mutation() { + check( + r#" +import foo +x = foo.A +x = [] +x.append(1) +y = foo.B +y.append(1) # E: imported-mutation +if foo.flag: + z = foo.C +else: + z = [] +z.append(1) # E: imported-mutation +"#, + ); +} + +/// A called function's `global` rebinding flows back into the module frame. +#[test] +fn global_rebinding_by_a_call_updates_the_module() { + check( + r#" +import foo +x = [] +def f(): + global x + x = foo.A +x.append(1) +f() +x.append(2) # E: imported-mutation +"#, + ); +} + +/// Closure cells are read when the closure runs, not when it is defined. +#[test] +fn closure_cells_are_read_at_invocation() { + check( + r#" +from foo import obj + +def safe(): + target = [] + def g(): + target.append(1) + g() + +def unsafe(): + target = [] + def g(): + target.append(1) + g() + target = obj + g() + +safe() +unsafe() # E: unsafe-call +"#, + ); +} + +/// A closure that escapes its defining activation has unknown cells. +#[test] +fn escaped_closure_cells_are_unknown() { + check( + r#" +from foo import obj + +def outer(x): + def inner(): + x.enabled = True + return inner + +h = outer(obj) +h() # E: unknown-call +"#, + ); +} + +/// Calling a generator or coroutine function does not run its body; iterating does. +/// A generator stored in a collection is unknown, because iteration through the +/// collection is not followed, and passing one to a builtin that does not iterate it +/// runs nothing. +#[test] +fn generator_bodies_run_when_iterated() { + check( + r#" +import foo + +def gen(): + foo.x = 1 + yield 1 + +async def coro(): + foo.y = 1 + +g = gen() +c = coro() +for _ in gen(): # E: unsafe-call + pass +items = list(gen()) # E: unsafe-call + +async def runner(): + await coro() + +r = runner() +for _ in [runner()]: # E: unknown-call + pass +print(gen()) +"#, + ); +} + +/// Awaiting a coroutine inside a running coroutine runs the awaited body. +#[test] +fn awaited_coroutines_run() { + check( + r#" +import foo + +async def coro(): + foo.y = 1 + +def drive(): + async def runner(): + await coro() + for _ in runner(): + pass + +drive() # E: unsafe-call +"#, + ); +} + +#[test] +fn property_reads_run_the_getter() { + check( + r#" +import foo + +class C: + @property + def p(self): + foo.x = 1 + return 1 + +c = C() +v = c.p # E: unsafe-call +"#, + ); +} + +/// The constructor's own mutation of the new instance is local, while mutating an +/// object stored in one of its attributes is unknown: attributes are not tracked. +#[test] +fn constructor_arguments_reach_init() { + check( + r#" +import foo + +class C: + def __init__(self, target): + self.items = [] + target.x = 1 + +class D: + def __init__(self): + self.items = [] + self.items.append(1) + +C([]) +C(foo.obj) # E: imported-argument +D() # E: unknown-mutation +"#, + ); +} + +/// Hazards are found through value flow, not by spelling. +#[test] +fn hazards_through_aliases() { + let report = check( + r#" +import sys +import builtins +e = exec +e("x = 1") # E: exec-call +m = sys.modules +m["k"] = None # E: sys-modules-access +b = builtins +b.__import__ = None # E: builtins-import-override +"#, + ); + assert!(report.load_imports_eagerly()); +} + +#[test] +fn class_creation_with_unresolved_bases_is_unknown() { + let report = check( + r#" +import abc +from foo import Base + +class A(Exception): + pass + +class B(abc.ABC): + pass + +class C(Base): # E: unknown-call + pass +"#, + ); + assert_eq!(report.verdict(), Verdict::Unknown); +} + +/// Reachable frontend gaps are reported instead of a safety claim. A failing `assert` +/// is not a raise the analysis reports, and the re-raise path out of a `try` is part of +/// the `try` gap. +#[test] +fn unsupported_constructs_are_not_safe() { + let report = check( + r#" +from foo import * # E: unsupported +try: # E: unsupported + import bar +except ImportError: + bar = None +def g(x): + assert x + return x +g(1) +"#, + ); + assert_eq!(report.verdict(), Verdict::Unknown); +} + +/// Code after an unconditional raise is unreachable and reports nothing, and a raise +/// that a handler catches does not leave the module. +#[test] +fn unreachable_code_reports_nothing() { + check( + r#" +import foo +try: # E: unsupported + raise ValueError() +except ValueError: + pass +raise SystemExit # E: unhandled-raise +foo.x = 1 +"#, + ); +} + +/// A class-body name read before the class binds it falls back to the module global. +#[test] +fn class_body_name_fallback() { + check( + r#" +import foo +x = foo.A +class C: + x.append(1) # E: imported-mutation + x = [] + x.append(2) +"#, + ); +} + +/// In an import cycle, the partially initialized module's names are unknown. +#[test] +fn import_cycle_values_are_unknown() { + let reports = check_all(&[ + ("a", "from b import Y\nX = []\n"), + ( + "b", + "from a import X\nY = 1\nX.append(1) # E: unknown-call\n", + ), + ]); + let verdicts: Vec = reports.iter().map(|r| r.verdict()).collect(); + assert_eq!(verdicts, [Verdict::Safe, Verdict::Unknown]); +} + +#[test] +fn cli_reports_and_exit_status() { + let dir = std::env::temp_dir().join(format!("lifeguard-spike-{}", std::process::id())); + fs::create_dir_all(&dir).unwrap(); + let a = dir.join("a.py"); + let b = dir.join("b.py"); + fs::write( + &a, + "REGISTRY = []\ndef register(x):\n REGISTRY.append(x)\n", + ) + .unwrap(); + fs::write(&b, "from a import register\nregister(1)\n").unwrap(); + let run = |args: &[String]| { + Command::new(env!("CARGO_BIN_EXE_lifeguard-spike")) + .args(args) + .output() + .unwrap() + }; + let arg = |name: &str, path: &std::path::Path| format!("{name}={}", path.display()); + + let out = run(&[arg("a", &a), arg("b", &b)]); + let stdout = String::from_utf8(out.stdout).unwrap(); + assert_eq!(out.status.code(), Some(1), "{stdout}"); + assert!(stdout.contains("b:2:1: unsafe-call: a:3: calls `append` on an object of module `a`")); + assert!(stdout.ends_with("a: safe\nb: unsafe\n"), "{stdout}"); + + assert_eq!(run(&[arg("a", &a)]).status.code(), Some(0)); + fs::write(&b, "import foo\nfoo.f()\n").unwrap(); + assert_eq!(run(&[arg("b", &b)]).status.code(), Some(2)); + assert_eq!(run(&["no-equals-sign".to_owned()]).status.code(), Some(64)); + fs::remove_dir_all(&dir).unwrap(); +} diff --git a/static_analysis/pyrefly_exports/Cargo.toml b/static_analysis/pyrefly_exports/Cargo.toml new file mode 100644 index 0000000000..e0182a298a --- /dev/null +++ b/static_analysis/pyrefly_exports/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "pyrefly_static_analysis_exports" +version = "0.0.0" +edition = "2024" +license.workspace = true +publish = false +repository.workspace = true +description = "Experimental export-provenance bridge for a shared Python analysis frontend" + +[dependencies] +pyrefly = { path = "../../pyrefly", default-features = false } +pyrefly_python = { path = "../../crates/pyrefly_python" } +pyrefly_static_analysis = { path = ".." } +pyrefly_types = { path = "../../crates/pyrefly_types" } +ruff_python_ast = "0.0.14" +ruff_text_size = "0.0.14" +starlark_map = "0.14.2" + +[lints] +workspace = true diff --git a/static_analysis/pyrefly_exports/README.md b/static_analysis/pyrefly_exports/README.md new file mode 100644 index 0000000000..56e986d91c --- /dev/null +++ b/static_analysis/pyrefly_exports/README.md @@ -0,0 +1,46 @@ +# Export-provenance bridge (experimental) + +This optional package uses Pyrefly's real `Exports` calculation without constructing +`Bindings` or requesting solved `Answers`. It implements the generic core's +`ExportProvider`; the core and primary consumers do not depend on the checker crate. + +Inputs are an explicit immutable set of source/stub modules and configured `SysInfo`. +There is no filesystem discovery or incremental mutation. Export dependencies are +recorded for inspection; changing the universe requires rebuilding the index. + +## Resolution and boundaries + +- Named and renamed re-exports follow declaration provenance through modules. +- `__all__` controls wildcard enumeration. A dynamic/unresolvable `__all__`, missing + wildcard dependency or calculation cycle is incomplete, never an exhaustive empty + surface. Failure evidence is cached per calculation and propagated to its dependents; + it does not contaminate unrelated queries. +- Module-valued imports and simple assignments are classified through the shared + frontend/engine, not a separate import AST walker. Rebinding does not redirect an + existing alias. Ambiguous or effectful module candidates remain unknown. +- Supplied package submodules are distinguishable from named declarations. Dynamic + module attributes (`__getattr__`) remain unknown. +- The direct API retains `Found`, `Module`, `Missing` and `Unknown` outcomes. The generic + provider maps missing/unknown to `ExportTarget::Unknown` for consumer safety. + +**Declaration provenance is not runtime purity or identity proof.** Missing runtime +packages, source/stub differences, dynamic namespace mutation and unsupported syntax +still require separate assumptions or validation. The bridge uses existing Pyrefly +export policies; it is not a lossless runtime module model. + +The experimental integration exposes Pyrefly's `export` module with `#[doc(hidden)]`. +That is a deliberately unstable visibility change on this branch, not a supported +upstream plugin API. + +The adapter currently parses for Pyrefly Exports and again for the shared module-value +classifier. A parse-reuse entry point is part of the proposed frontend evolution; this +bridge is not evidence of acceptable replacement-performance overhead. + +```sh +cargo test -p pyrefly_static_analysis_exports +cargo clippy -p pyrefly_static_analysis_exports --all-targets --no-deps -- -D warnings +``` + +Tests cover named/wildcard/relative re-exports, module aliases, rebinding, submodules, +dynamic attributes/`__all__`, cached failures and cycles, independent queries after +failure, and provenance flowing through the shared name/frame bookkeeping. diff --git a/static_analysis/pyrefly_exports/src/lib.rs b/static_analysis/pyrefly_exports/src/lib.rs new file mode 100644 index 0000000000..8084edf568 --- /dev/null +++ b/static_analysis/pyrefly_exports/src/lib.rs @@ -0,0 +1,634 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Export-only provenance adapter. The generic analysis frontend does not depend +//! on Pyrefly's binding graph or solved Answers. This adapter owns an immutable +//! source universe, so recorded dependencies need no incremental invalidation. + +use std::cell::RefCell; +use std::collections::BTreeSet; +use std::collections::HashMap; +use std::collections::HashSet; +use std::path::PathBuf; +use std::sync::Arc; + +use pyrefly::export::exports::ExportLocation; +use pyrefly::export::exports::ExportOrigin; +use pyrefly::export::exports::Exports; +use pyrefly::export::exports::LookupExport; +use pyrefly::export::special::SpecialExport; +use pyrefly::state::loader::FindError; +use pyrefly::state::loader::FindingOrError; +use pyrefly_python::ast::Ast; +use pyrefly_python::module::Module; +use pyrefly_python::module_name::ModuleName; +use pyrefly_python::module_path::ModulePath; +use pyrefly_python::sys_info::SysInfo; +use pyrefly_static_analysis::ExportProvider; +use pyrefly_static_analysis::ExportTarget; +use pyrefly_types::function::Deprecation; +use ruff_python_ast::PySourceType; +use ruff_python_ast::name::Name; +use ruff_text_size::TextRange; +use starlark_map::small_map::SmallMap; +use starlark_map::small_set::SmallSet; + +mod modules; + +type ExportTable = Arc>; + +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq)] +enum Calculation { + Exports, + Wildcard, +} + +type CalculationKey = (ModuleName, Calculation); + +/// A source or stub supplied by the caller; no filesystem discovery is performed. +pub struct Source { + pub module: String, + pub path: PathBuf, + pub contents: String, +} + +/// Static declaration provenance is not evidence of runtime purity or identity. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Resolution { + Found { module: String, name: String }, + Module(String), + Missing, + Unknown(String), +} + +/// Pyrefly's real export calculation over an explicitly supplied source universe. +pub struct ExportIndex { + modules: HashMap, + module_values: HashMap<(ModuleName, Name), Option>, + dependencies: RefCell)>>, + active: RefCell>, + incomplete: RefCell>>, +} + +impl ExportIndex { + /// Parse modules and compute declaration metadata without building Bindings. + pub fn new(sources: Vec, sys_info: SysInfo) -> Result { + let mut modules = HashMap::new(); + let mut module_values = HashMap::new(); + for source in sources { + let name = ModuleName::from_str(&source.module); + let (parsed, errors, unsupported) = + Ast::parse_with_version(&source.contents, sys_info.version(), PySourceType::Python); + if !errors.is_empty() || !unsupported.is_empty() { + return Err(format!( + "cannot parse {}: {errors:?} {unsupported:?}", + source.module + )); + } + let path = ModulePath::memory(source.path); + module_values.extend( + modules::classify(&source.contents, name, path.is_init()) + .into_iter() + .map(|(symbol, value)| ((name, symbol), value)), + ); + let info = Module::new(name, path, Arc::new(source.contents)); + let exports = Exports::new(&parsed.into_syntax().body, &info, sys_info, false); + if modules.insert(name, exports).is_some() { + return Err(format!("duplicate source module {}", source.module)); + } + } + Ok(Self { + modules, + module_values, + dependencies: RefCell::new(BTreeSet::new()), + active: RefCell::new(Vec::new()), + incomplete: RefCell::new(HashMap::new()), + }) + } + + /// Resolve a named declaration through renamed and wildcard re-exports. + /// Missing/cyclic/synthesized declarations are never reported as found. + pub fn resolve(&self, module: &str, name: &str) -> Resolution { + let mut module = ModuleName::from_str(module); + let mut name = Name::new(name); + let mut seen = HashSet::new(); + loop { + self.record(module, Some(&name)); + if !seen.insert((module, name.clone())) { + return Resolution::Unknown("cyclic re-export".to_owned()); + } + if !self.modules.contains_key(&module) { + return Resolution::Missing; + } + let Some(table) = self.table(module) else { + return Resolution::Unknown(self.errors((module, Calculation::Exports)).join("; ")); + }; + if let Some(value) = self.module_values.get(&(module, name.clone())) { + return match value { + Some(module) => Resolution::Module(module.to_string()), + None => Resolution::Unknown("ambiguous module-valued export".to_owned()), + }; + } + match table.get(&name) { + Some(ExportLocation::ThisModule(export)) if export.symbol_kind.is_some() => { + return Resolution::Found { + module: module.to_string(), + name: name.to_string(), + }; + } + Some(ExportLocation::ThisModule(_)) => { + return Resolution::Unknown("synthesized or invalid declaration".to_owned()); + } + Some(ExportLocation::OtherModule(next, alias)) => { + module = *next; + if let Some(alias) = alias { + name = alias.clone(); + } + } + None if table.contains_key(&Name::new("__getattr__")) => { + return Resolution::Unknown("dynamic module attribute".to_owned()); + } + None if self.modules.contains_key(&module.append(&name)) => { + return Resolution::Module(module.append(&name).to_string()); + } + None => return Resolution::Missing, + } + } + } + + /// Enumerate `import *` names. An incomplete calculation is distinct from empty. + pub fn wildcard(&self, module: &str) -> Result, Vec> { + let id = ModuleName::from_str(module); + let names = self.get_wildcard(id); + let issues = self.errors((id, Calculation::Wildcard)); + if !issues.is_empty() { + return Err(issues); + } + match names { + Some(names) => { + let mut names: Vec<_> = names.iter().map(ToString::to_string).collect(); + names.sort(); + Ok(names) + } + None => Err(vec![format!("missing module {module}")]), + } + } + + /// Dependencies requested by this immutable export calculation. + pub fn dependencies(&self) -> BTreeSet<(String, Option)> { + self.dependencies.borrow().clone() + } + + fn errors(&self, key: CalculationKey) -> Vec { + self.incomplete + .borrow() + .get(&key) + .map_or_else(Vec::new, |errors| errors.iter().cloned().collect()) + } + + fn issue(&self, key: CalculationKey, error: String) { + let mut issues = self.incomplete.borrow_mut(); + issues.entry(key).or_default().insert(error.clone()); + for caller in self.active.borrow().iter() { + issues.entry(*caller).or_default().insert(error.clone()); + } + } + + /// Cache failure evidence alongside the calculation it belongs to. Cached partial + /// Pyrefly tables never lose their incompleteness, and unrelated queries remain usable. + fn calculate(&self, key: CalculationKey, run: impl FnOnce() -> T) -> Option { + let errors = self.errors(key); + if !errors.is_empty() { + for error in errors { + self.issue(key, error); + } + return None; + } + if self.active.borrow().contains(&key) { + self.issue(key, format!("cyclic export calculation at {}", key.0)); + return None; + } + self.active.borrow_mut().push(key); + let result = run(); + assert_eq!( + self.active.borrow_mut().pop(), + Some(key), + "nested calculations unwind in order" + ); + let errors = self.errors(key); + if errors.is_empty() { + Some(result) + } else { + for error in errors { + self.issue(key, error); + } + None + } + } + + fn table(&self, module: ModuleName) -> Option { + let exports = self.modules.get(&module)?; + self.calculate((module, Calculation::Exports), || exports.exports(self)) + } + + fn record(&self, module: ModuleName, name: Option<&Name>) { + self.dependencies + .borrow_mut() + .insert((module.to_string(), name.map(ToString::to_string))); + } +} + +impl ExportProvider for ExportIndex { + fn resolve(&self, module: ModuleName, name: &Name) -> ExportTarget { + match ExportIndex::resolve(self, &module.to_string(), name.as_str()) { + Resolution::Found { module, name } => { + ExportTarget::Definition(ModuleName::from_str(&module), Name::new(&name)) + } + Resolution::Module(module) => ExportTarget::Module(ModuleName::from_str(&module)), + Resolution::Missing | Resolution::Unknown(_) => ExportTarget::Unknown, + } + } +} + +impl LookupExport for ExportIndex { + fn export_exists(&self, module: ModuleName, name: &Name) -> bool { + self.record(module, Some(name)); + self.table(module) + .is_some_and(|table| table.contains_key(name)) + } + + fn module_exists(&self, module: ModuleName) -> FindingOrError<()> { + if self.modules.contains_key(&module) { + FindingOrError::new_finding(()) + } else { + // This immutable adapter has no search path. A missing universe entry + // is not an ignored runtime import; callers receive Missing above. + FindingOrError::Error(FindError::Ignored) + } + } + + fn get_wildcard(&self, module: ModuleName) -> Option>> { + self.record(module, None); + let key = (module, Calculation::Wildcard); + let Some(exports) = self.modules.get(&module) else { + self.issue(key, format!("missing wildcard module {module}")); + return None; + }; + if exports.unresolvable_dunder_all_range().is_some() { + self.issue(key, format!("dynamic __all__ in {module}")); + return None; + } + self.calculate(key, || exports.wildcard(self)) + } + + fn get_every_export_untracked(&self, module: ModuleName) -> Option> { + self.table(module) + .map(|table| table.keys().cloned().collect()) + } + + fn is_submodule_imported_implicitly(&self, module: ModuleName, name: &Name) -> bool { + self.record(module, Some(name)); + self.modules + .get(&module) + .is_some_and(|exports| exports.is_submodule_imported_implicitly(name)) + } + + fn get_deprecated(&self, module: ModuleName, name: &Name) -> Option { + self.record(module, Some(name)); + None + } + + fn reexport_source(&self, module: ModuleName, name: &Name) -> Option { + self.record(module, Some(name)); + let table = self.table(module)?; + match table.get(name) { + Some(ExportLocation::OtherModule(module, _)) => Some(*module), + _ => None, + } + } + + fn is_implicit_reexport(&self, module: ModuleName, name: &Name) -> bool { + self.record(module, Some(name)); + self.modules + .get(&module) + .is_some_and(|exports| exports.is_implicit_reexport(name)) + } + + fn is_special_export(&self, module: ModuleName, name: &Name) -> Option { + self.record(module, Some(name)); + None + } + + fn docstring_range(&self, module: ModuleName, name: &Name) -> Option { + self.record(module, Some(name)); + let table = self.table(module)?; + match table.get(name) { + Some(ExportLocation::ThisModule(export)) => export.docstring_range, + _ => None, + } + } + + fn export_origin(&self, module: ModuleName, name: &Name) -> ExportOrigin { + let origin = match self.resolve(&module.to_string(), name.as_str()) { + Resolution::Found { module, name } => (ModuleName::from_str(&module), Name::new(&name)), + Resolution::Module(_) | Resolution::Missing | Resolution::Unknown(_) => { + (module, name.clone()) + } + }; + ExportOrigin { + origin, + is_final: false, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pyrefly_python::sys_info::PythonPlatform; + use pyrefly_python::sys_info::PythonVersion; + + fn index(sources: &[(&str, &str)]) -> ExportIndex { + ExportIndex::new( + sources + .iter() + .map(|(module, contents)| Source { + module: (*module).to_owned(), + path: PathBuf::from(format!("{module}.py")), + contents: (*contents).to_owned(), + }) + .collect(), + SysInfo::new_without_type_checking(PythonVersion::default(), PythonPlatform::default()), + ) + .unwrap() + } + + #[derive(Clone, Debug, PartialEq)] + enum Provenance { + Target(ExportTarget), + Unknown, + } + + struct ProvenanceDomain<'a>(&'a ExportIndex); + + impl pyrefly_static_analysis::Domain for ProvenanceDomain<'_> { + type Value = Provenance; + type State = (); + type Diagnostic = (); + + fn top(&self) -> Provenance { + Provenance::Unknown + } + + fn join(&self, a: &Provenance, b: &Provenance) -> Provenance { + if a == b { + a.clone() + } else { + Provenance::Unknown + } + } + + fn join_state(&self, _: &(), _: &()) {} + + fn transfer( + &self, + _: pyrefly_static_analysis::OpId, + kind: &pyrefly_static_analysis::OpKind, + cx: &mut pyrefly_static_analysis::Cx<'_, Self>, + ) { + if let pyrefly_static_analysis::OpKind::Import { + module, + item: pyrefly_static_analysis::ImportItem::Name(name), + dst, + } = kind + { + cx.set_reg( + *dst, + Provenance::Target(ExportProvider::resolve(self.0, *module, name)), + ); + } + } + } + + #[test] + fn export_provenance_uses_shared_name_bookkeeping() { + let exports = index(&[ + ("original", "def work(): pass"), + ("facade", "from original import work as run"), + ]); + let program = pyrefly_static_analysis::analyze_python( + "from facade import run as f\nalias = f\nf = 0\n", + ModuleName::from_str("consumer"), + false, + ); + let domain = ProvenanceDomain(&exports); + let result = pyrefly_static_analysis::Engine::new(&program, &domain).analyze( + pyrefly_static_analysis::Program::MODULE, + pyrefly_static_analysis::Entry::new(()), + ); + assert!(!result.is_incomplete()); + let exit = result.exit.unwrap(); + assert_eq!( + exit.symbol(&program, program.global("alias").unwrap()), + Some(&Provenance::Target(ExportTarget::Definition( + ModuleName::from_str("original"), + Name::new("work") + ))) + ); + assert_eq!( + exit.symbol(&program, program.global("f").unwrap()), + Some(&Provenance::Unknown) + ); + } + + #[test] + fn named_and_wildcard_reexports_use_actual_pyrefly_exports() { + let exports = index(&[ + ("original", "def work(): pass\n__all__ = ['work']"), + ( + "alias", + "from original import work as renamed\n__all__ = ['renamed']", + ), + ("facade", "from alias import *"), + ]); + assert_eq!( + exports.resolve("facade", "renamed"), + Resolution::Found { + module: "original".into(), + name: "work".into() + } + ); + assert_eq!(exports.wildcard("facade"), Ok(vec!["renamed".into()])); + assert_eq!(exports.resolve("facade", "missing"), Resolution::Missing); + assert!( + exports + .dependencies() + .contains(&("original".into(), Some("work".into()))) + ); + } + + #[test] + fn relative_reexport_uses_package_identity() { + let sources = vec![ + Source { + module: "pkg".into(), + path: PathBuf::from("pkg/__init__.py"), + contents: "from .worker import work as run\n__all__ = ['run']".into(), + }, + Source { + module: "pkg.worker".into(), + path: PathBuf::from("pkg/worker.py"), + contents: "def work(): pass".into(), + }, + ]; + let exports = ExportIndex::new( + sources, + SysInfo::new_without_type_checking(PythonVersion::default(), PythonPlatform::default()), + ) + .unwrap(); + assert_eq!( + exports.resolve("pkg", "run"), + Resolution::Found { + module: "pkg.worker".into(), + name: "work".into() + } + ); + } + + #[test] + fn explicit_all_filters_wildcard_but_not_named_imports() { + let exports = index(&[("m", "public = 1\nhidden = 2\n__all__ = ['public']")]); + assert_eq!(exports.wildcard("m"), Ok(vec!["public".into()])); + assert!(matches!( + exports.resolve("m", "hidden"), + Resolution::Found { .. } + )); + } + + #[test] + fn missing_wildcard_and_invalid_all_are_incomplete() { + let exports = index(&[("facade", "from missing import *")]); + assert!(exports.wildcard("facade").is_err()); + let exports = index(&[("m", "__all__ = ['not_defined']")]); + assert!(matches!( + exports.resolve("m", "not_defined"), + Resolution::Unknown(_) + )); + } + + #[test] + fn dynamic_wildcards_do_not_poison_other_queries_or_become_complete() { + let exports = index(&[ + ("original", "def work(): pass"), + ("dynamic", "__all__ = compute()\ndef work(): pass"), + ("bad", "from missing import *\nfrom original import work"), + ]); + assert!(exports.wildcard("dynamic").is_err()); + assert!(exports.wildcard("dynamic").is_err()); + assert!(matches!( + exports.resolve("dynamic", "work"), + Resolution::Found { .. } + )); + assert!(exports.wildcard("bad").is_err()); + assert!(matches!( + exports.resolve("bad", "work"), + Resolution::Unknown(_) + )); + assert!(matches!( + exports.resolve("original", "work"), + Resolution::Found { .. } + )); + assert!(matches!( + exports.resolve("bad", "work"), + Resolution::Unknown(_) + )); + assert!(matches!( + exports.resolve("original", "work"), + Resolution::Found { .. } + )); + } + + #[test] + fn module_aliases_keep_value_identity_through_rebinding_and_reexports() { + let exports = index(&[ + ("original", "def work(): pass"), + ("facade", "import original as x\ny = x\nx = 0"), + ("reexport", "from facade import y as z"), + ]); + assert_eq!( + exports.resolve("facade", "y"), + Resolution::Module("original".into()) + ); + assert_eq!( + ExportProvider::resolve(&exports, ModuleName::from_str("reexport"), &Name::new("z")), + ExportTarget::Module(ModuleName::from_str("original")) + ); + assert!(matches!( + exports.resolve("facade", "x"), + Resolution::Found { .. } + )); + let exports = index(&[("original", ""), ("facade", "import original as x")]); + assert_eq!( + exports.resolve("facade", "x"), + Resolution::Module("original".into()) + ); + let exports = index(&[ + ("original", ""), + ("other", ""), + ( + "facade", + "if flag:\n import original as x\nelse:\n import other as x", + ), + ]); + assert!(matches!( + exports.resolve("facade", "x"), + Resolution::Unknown(_) + )); + } + + #[test] + fn submodules_and_dynamic_attributes_are_distinct() { + let sources = vec![ + Source { + module: "pkg".into(), + path: PathBuf::from("pkg/__init__.py"), + contents: "".into(), + }, + Source { + module: "pkg.child".into(), + path: PathBuf::from("pkg/child.py"), + contents: "".into(), + }, + ]; + let exports = ExportIndex::new( + sources, + SysInfo::new_without_type_checking(PythonVersion::default(), PythonPlatform::default()), + ) + .unwrap(); + assert_eq!( + exports.resolve("pkg", "child"), + Resolution::Module("pkg.child".into()) + ); + let exports = index(&[("m", "def __getattr__(name): return object()")]); + assert!(matches!( + exports.resolve("m", "anything"), + Resolution::Unknown(_) + )); + } + + #[test] + fn missing_and_cyclic_exports_are_not_success() { + let exports = index(&[("a", "from b import thing"), ("b", "from a import thing")]); + assert!(matches!( + exports.resolve("a", "thing"), + Resolution::Unknown(_) + )); + assert_eq!(exports.resolve("missing", "thing"), Resolution::Missing); + let exports = index(&[("a", "from b import *"), ("b", "from a import *")]); + assert!(exports.wildcard("a").is_err()); + } +} diff --git a/static_analysis/pyrefly_exports/src/modules.rs b/static_analysis/pyrefly_exports/src/modules.rs new file mode 100644 index 0000000000..1d71e50263 --- /dev/null +++ b/static_analysis/pyrefly_exports/src/modules.rs @@ -0,0 +1,128 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Classify module-valued exports using the shared frontend and frame semantics. +//! This is declaration provenance, not a claim that arbitrary module effects are pure. + +use std::collections::BTreeSet; +use std::collections::HashMap; + +use pyrefly_static_analysis::Access; +use pyrefly_static_analysis::Cx; +use pyrefly_static_analysis::Domain; +use pyrefly_static_analysis::Engine; +use pyrefly_static_analysis::Entry; +use pyrefly_static_analysis::ImportItem; +use pyrefly_static_analysis::ModuleName; +use pyrefly_static_analysis::Name; +use pyrefly_static_analysis::OpId; +use pyrefly_static_analysis::OpKind; +use pyrefly_static_analysis::Program; +use pyrefly_static_analysis::SymbolId; +use pyrefly_static_analysis::analyze_python; + +#[derive(Clone, Debug, PartialEq)] +enum Value { + Module(ModuleName), + Other, + Unknown, +} + +#[derive(Clone, Debug, Default, PartialEq)] +struct State { + candidates: BTreeSet, + effectful: bool, +} + +struct Modules; + +impl Domain for Modules { + type Value = Value; + type State = State; + type Diagnostic = (); + + fn top(&self) -> Value { + Value::Unknown + } + + fn join(&self, a: &Value, b: &Value) -> Value { + if a == b { a.clone() } else { Value::Unknown } + } + + fn join_state(&self, a: &State, b: &State) -> State { + State { + candidates: a.candidates.union(&b.candidates).copied().collect(), + effectful: a.effectful || b.effectful, + } + } + + fn transfer(&self, _: OpId, kind: &OpKind, cx: &mut Cx<'_, Self>) { + match kind { + OpKind::Import { module, item, dst } => { + let value = match item { + ImportItem::Root => { + Value::Module(ModuleName::from_name(&module.first_component())) + } + ImportItem::Module => Value::Module(*module), + ImportItem::Name(_) => Value::Unknown, + }; + cx.set_reg(*dst, value); + } + OpKind::Literal { dst, .. } + | OpKind::Build { dst, .. } + | OpKind::MakeFunction { dst, .. } => cx.set_reg(*dst, Value::Other), + OpKind::Name { + symbol, + access: Access::Store(reg), + } => { + let is_module = matches!(cx.reg(*reg), Value::Module(_)); + let module_scope = cx.program().region(Program::MODULE).scope; + if is_module && cx.program().symbol(*symbol).scope == module_scope { + cx.state_mut().candidates.insert(*symbol); + } + } + OpKind::Name { .. } | OpKind::Param { .. } => {} + // Unknown execution can alter the namespace. Do not turn its surviving + // import-shaped values into a proof of module-valued identity. + _ => cx.state_mut().effectful = true, + } + } +} + +/// Known modules, or an explicitly ambiguous module candidate. Other declaration +/// kinds are absent and are resolved by Pyrefly's export table instead. +pub(crate) fn classify( + source: &str, + module: ModuleName, + is_init: bool, +) -> HashMap> { + let program = analyze_python(source, module, is_init); + let result = + Engine::new(&program, &Modules).analyze(Program::MODULE, Entry::new(State::default())); + let Some(frame) = result.exit else { + return HashMap::new(); + }; + let incomplete = result.convergence != pyrefly_static_analysis::Convergence::Converged + || !result.gaps.is_empty() + || frame.state.effectful; + program + .module_symbols() + .filter_map(|symbol| { + if !frame.state.candidates.contains(&symbol) { + return None; + } + let value = frame.symbol(&program, symbol); + match value { + Some(Value::Other) => None, + Some(Value::Module(module)) if !incomplete => { + Some((program.symbol(symbol).name.clone(), Some(*module))) + } + _ => Some((program.symbol(symbol).name.clone(), None)), + } + }) + .collect() +} diff --git a/static_analysis/spmd/Cargo.toml b/static_analysis/spmd/Cargo.toml new file mode 100644 index 0000000000..b3ab28254e --- /dev/null +++ b/static_analysis/spmd/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "pyrefly_static_analysis_spmd" +version = "0.0.0" +edition = "2024" +license.workspace = true +repository.workspace = true +publish = false +description = "Experimental SPMD contract consumer of the neutral Python analysis engine" + +[dependencies] +pyrefly_static_analysis = { path = ".." } +pyrefly_python = { path = "../../crates/pyrefly_python" } +serde_json = "1" + +[lints] +workspace = true diff --git a/static_analysis/spmd/README.md b/static_analysis/spmd/README.md new file mode 100644 index 0000000000..c5320047b9 --- /dev/null +++ b/static_analysis/spmd/README.md @@ -0,0 +1,184 @@ +# SPMD contract consumer (experimental) + +`pyrefly_static_analysis_spmd` is a separate Cargo package using the shared +`pyrefly_static_analysis` frontend and worklist engine. It supplies an arbitrary +stateful domain: imported identities and tensor descriptors flow through the +core's frame, while active mesh context and unknown-effect invalidation flow +through the domain's state. It does not parse Python, traverse its AST, execute +Torch, insert communication, or depend on Pyrefly's type inference solver. + +## Run a complete tensor-parallel MLP + +From the repository root: + +```bash +cargo run -p pyrefly_static_analysis_spmd -- \ + static_analysis/spmd/examples/tp_mlp.py \ + static_analysis/spmd/examples/tp_mlp.json +``` + +The example has this type cycle on `tp`: + +```text +x:I -> invariant_to_replicate -> R -> column matmul -> S(1) + -> ReLU -> S(1) -> row matmul [explicit Partial intent] -> P + -> all_reduce(dst=I) -> I +``` + +The sidecar is a **trusted assumption**, not a claim inferred from the Python +body. It describes real tensors' entry contracts, the ambient named compute +mesh, and the required return contract. On a sharded contraction it supplies an +explicit per-call Partial output contract rather than inventing a keyword that +real `torch.matmul` does not accept. + +Try deleting the `invariant_to_replicate` entry (retain the source line or update +the sidecar line), changing the exit to `dst=s.R`, or applying `t.relu(p)` before +the reduction. All are rejected: the entry and wrong exit can have identical +forward values while differing in the required backward communication. The +`p.stride()` metadata query is accepted without discharging Partial. + +The CLI prints deterministic located diagnostics and `contract-validated` or +`not-validated`, plus a separate incompleteness flag. Exit codes are 0 for a +validated supported contract, 1 for violations/incompleteness, and 2 for file, +argument or sidecar errors. Use explicit source-file paths; no checkout scan, +project discovery, Python import execution, or tensor execution is performed. + +## Sidecar schema + +The example JSON is complete and executable with the command above. Additional +input forms are `{"mesh": ["dp", "tp"]}` for a trusted mesh context argument +and `{"predicate": true}` for a trusted Boolean of unknown truth value with +no user-defined truthiness effects. A tensor has: + +- `rank` (0 through 32), `grad` (`yes`, `no`, or `unknown`), and `axes` mapping + names to `R`, `I`, `V`, `P`, or `S(i)`; +- optional `partition`, one entry per dimension: `null`, an axis string, or an + ordered array of axes (outermost first). Those axes must have local type V, + and an axis may occur at most once. Use V plus `partition` for several axes + sharding one dimension; several `S(i)` entries cannot express their ordering. + +`partial_outputs` entries identify a source `line` and its `axes`. Exactly one +call/binary operation must start on that line in the target region; multiple +calls on one line are rejected rather than guessed. The Rust API accepts stable +`OpId`s directly. Sidecars reject unknown fields, invalid placements, and +ambiguous target functions. Function default expressions are analyzed by the +shared frontend, but parameter defaults are not filled into entry contracts: +provide every used parameter explicitly. + +```python +import torch +import spmd_types as spmd + + +def nested(x, dp, inner_mesh): + with spmd.set_current_mesh(inner_mesh): + dp = torch.relu(dp) + return torch.relu(x) +``` + +Give `inner_mesh` the mesh descriptor `{"mesh": ["dp"]}`, `dp` a dp tensor, +and `x` a tp tensor with ambient mesh `["tp"]`. The inner context overrides +and then restores the outer mesh; the engine also emits restoration on normal +early returns. Named axes are logical identities supplied by the integration: +this checker does **not** establish actual process-group orthogonality, size, +ranks, or cross-mesh equivalence. + +## Supported rules + +- Per-axis R/I/V/P gradient duality: R <-> P, I <-> I, V <-> V. Unknown + gradient information is conservative, never treated as `requires_grad=False`. +- Ordinary **local** matmul/linear: R,V -> V and V,V -> V. P is not inferred + simply because both inputs vary. Explicit Partial intent permits V -> P. +- Restricted global rank-two matmul (`mk,kn->mn`) and linear (`mk,nk->mn`): + consistent shard dimensions/order, free-label shard propagation and explicit + Partial intent for sharded contraction. Pointwise shard propagation is + checked for ReLU, add and multiply; coincident output shards must have a + consistent ordering supplied by their inputs, not an invented order. Every + produced tensor is checked for representation invariants independently of the + optional return contract; global V axes must all occur in its PartitionSpec. + No tensor broadcasting is modeled. +- Addition P,P -> P; P,R and P,constant are rejected (an affine term would be + summed once per rank). Multiply/matmul P,R -> P; P,P is rejected. ReLU(P) + is rejected. Known no-grad R/I may adopt the other replicated flavor when + needed; gradient-bearing I cannot mix with R/V/P. +- `invariant_to_replicate`, a conservative subset of `convert` (I->R, + I/R->S(i), R->P and identity), and `reinterpret(V,P)`. Unbind-form + convert, mixed V/S all-to-all signatures, expert-mode, + unusual conversions and additional call options are unsupported. +- `all_reduce`, `all_gather`, `reduce_scatter`, `all_to_all`: validated source, + destination, axis and shard dimension. Type-only `spmd_types.rules` reduction + calls require explicit `src=V` when reducing V. Runtime reduction calls permit + the reference API's convenience V reinterpret. Missing input facts are errors, + never established by `src`. +- Negative shard dimensions normalize by rank. Local V all-gather adds a + leading dimension; local V reduce-scatter unbinds dimension 0. S(i) forms + preserve rank. Global collective variants require explicit S(i) whenever an + endpoint is V. A single-axis collective may change only an innermost shard. +- Metadata attributes `shape`, `ndim`, `dtype`, `device`; methods `size`, + `stride`, `dim`, `ndimension`, `storage_offset`; builtin `len` on nonscalars. + They return metadata, not a newly replicated tensor. Unsupported methods + cannot masquerade as metadata by a similar name. +- Renamed imports, imported function aliases and lexical shadowing use shared + symbol/frame bookkeeping. `ExportProvider` can resolve re-exported functions; + the CLI uses the face-value `SyntacticExports` provider and assumes stable + Torch/SPMD library namespaces, not arbitrary monkey-patching. +- Named-mesh ambient context override/restoration and completeness checks on + compute, not metadata. Branch joins preserve ambiguity; worklist convergence + limits and reachable frontend gaps propagate to incompleteness. + +## Honest boundaries + +Validation means only that supported rules hold under the supplied entry/call +assumptions. It is **not** full Python safety, global-SPMD soundness, numerical +correctness, shape/dtype correctness, proof of equal R/I forward data, or exact +behavioral parity with a runtime checker. No backward graph is executed or +inferred; backward contracts follow from declared forward types. + +Unknown calls/protocols, object or tensor mutation, unsupported contexts, +unsupported signatures (including Torch `out=`/inplace arguments), suspensions, +and reachable frontend gaps make the result incomplete. Unknown effects +permanently invalidate tensor knowledge for the activation, including aliases +and outer bindings; later annotations or metadata cannot turn it back into a +validated result. `assert_type` is intentionally unsupported as a fact setter: +use an explicitly trusted entry and a checked return contract instead. + +Only direct module-level plain-function activations without variadic parameters +are checked. Calls to +user-defined helpers, custom autograd hooks, deferred/generator/async execution, +annotation overlays, tensor attributes on user objects, raw distributed APIs, +mesh folding/reinterpretation, reshape, batched matmul and general einsum are +not summarized. Module setup is analyzed separately; unsupported module effects +preclude function validation. Sidecar arguments override invocation inputs, not +lexical bindings; the Rust `outer` map is also an explicit trusted assumption. + +## Public reference and validation + +Reference: [meta-pytorch/spmd_types](https://github.com/meta-pytorch/spmd_types) +at commit `e5ebc6d840fa2399bcd10306664faf4a2b5feaf6`. The rules source is +`spmd_types/rules.py` (there is no `rules/source` directory at this revision). +Semantics were selected from `docs/{local_spmd_types,design,key_concepts,rules}.md`, +`spmd_types/rules.py`, the local inference source, and public +`tests/{checker,rules,global_spmd,collectives,types}_test.py` scenarios. + +There is a deliberate scope distinction: public `docs/design.md` requests +explicit `out_partial_axes` intent for ordinary global operations, whereas +`tests/global_spmd_test.py::test_mm_contracted_sharded_dim` currently infers P +for `torch.mm`. Composable `rules.einsum` also derives P from its explicit +contraction equation. This spike chooses explicit Partial intent for ordinary +calls, while validating the rank-two descriptor rule. It also deliberately does +not copy the runtime rule helper's ability to establish missing input types +from a collective `src` assertion. + +```bash +cargo test -p pyrefly_static_analysis_spmd +cargo clippy -p pyrefly_static_analysis_spmd --all-targets -- -D warnings +cargo fmt --check -p pyrefly_static_analysis_spmd +``` + +Tests include the TP MLP and both backward-only failures, mesh restoration, +renamed imports/shadowing, unknown-effect anti-fabrication, convergence/gaps, +collective contracts, JSON validation and the CLI. Pure Rust numerical +counterexamples distinguish partial addition from partial multiplication and +ReLU, without a Torch dependency. These are descriptor tests, not numerical +verification of arbitrary user kernels. Validation on other operating systems +must be reported separately; source fixtures and CLI paths use Rust path APIs. diff --git a/static_analysis/spmd/examples/tp_mlp.json b/static_analysis/spmd/examples/tp_mlp.json new file mode 100644 index 0000000000..f100205595 --- /dev/null +++ b/static_analysis/spmd/examples/tp_mlp.json @@ -0,0 +1,12 @@ +{ + "function": "mlp", + "global": true, + "mesh": ["tp"], + "inputs": { + "x": {"rank": 2, "grad": "yes", "axes": {"tp": "I"}}, + "w1": {"rank": 2, "grad": "yes", "axes": {"tp": "S(1)"}}, + "w2": {"rank": 2, "grad": "yes", "axes": {"tp": "S(0)"}} + }, + "return": {"rank": 2, "grad": "yes", "axes": {"tp": "I"}}, + "partial_outputs": [{"line": 14, "axes": ["tp"]}] +} diff --git a/static_analysis/spmd/examples/tp_mlp.py b/static_analysis/spmd/examples/tp_mlp.py new file mode 100644 index 0000000000..1e43b5870d --- /dev/null +++ b/static_analysis/spmd/examples/tp_mlp.py @@ -0,0 +1,16 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import spmd_types as s +import torch as t + + +def mlp(x, w1, w2): + x = s.invariant_to_replicate(x, "tp") + h = t.matmul(x, w1) + h = t.relu(h) + p = t.matmul(h, w2) + p.stride() + return s.all_reduce(p, "tp", src=s.P, dst=s.I) diff --git a/static_analysis/spmd/src/domain.rs b/static_analysis/spmd/src/domain.rs new file mode 100644 index 0000000000..b04e1c7286 --- /dev/null +++ b/static_analysis/spmd/src/domain.rs @@ -0,0 +1,876 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! A stateful consumer of the shared operation CFG. No Python syntax is walked here. + +use std::collections::BTreeMap; +use std::collections::BTreeSet; + +use pyrefly_static_analysis::Access; +use pyrefly_static_analysis::Arg; +use pyrefly_static_analysis::BinaryOp; +use pyrefly_static_analysis::Binding; +use pyrefly_static_analysis::BuildKind; +use pyrefly_static_analysis::Cx; +use pyrefly_static_analysis::Domain; +use pyrefly_static_analysis::Element; +use pyrefly_static_analysis::Engine; +use pyrefly_static_analysis::Entry; +use pyrefly_static_analysis::ExportProvider; +use pyrefly_static_analysis::ExportTarget; +use pyrefly_static_analysis::ImportItem; +use pyrefly_static_analysis::Limits; +use pyrefly_static_analysis::Literal; +use pyrefly_static_analysis::OpId; +use pyrefly_static_analysis::OpKind; +use pyrefly_static_analysis::Operator; +use pyrefly_static_analysis::ParamKind; +use pyrefly_static_analysis::Program; +use pyrefly_static_analysis::Reg; +use pyrefly_static_analysis::RegionId; +use pyrefly_static_analysis::SymbolId; +use pyrefly_static_analysis::SymbolOrigin; +use pyrefly_static_analysis::Trigger; +use pyrefly_static_analysis::UnaryOp; + +use crate::types::Collective; +use crate::types::Compute; +use crate::types::Grad; +use crate::types::Local; +use crate::types::Mesh; +use crate::types::Placement; +use crate::types::Tensor; +use crate::types::compute; +use crate::types::transition; + +/// Abstract values stay finite: unequal path values join to Ambiguous. Imported +/// identity flows through the shared frame, so aliases and shadowing are semantic. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Value { + Unknown, + Unbound, + Ambiguous, + None, + Bool(bool), + /// A trusted Boolean input of unknown truth value, with no truthiness effects. + Predicate, + Int(i64), + Str(String), + Sequence(Vec), + Placement(Placement), + Tensor(Tensor), + Mesh(Mesh), + MeshContext(Mesh), + Module(String), + Qualified(String), + Function(RegionId), + Metadata, + MetadataMethod { + name: String, + rank: usize, + }, +} + +impl Value { + fn validate_tensors(&self, global: bool) -> Result<(), String> { + match self { + Self::Tensor(tensor) => tensor.validate(global), + Self::Sequence(values) => values + .iter() + .try_for_each(|value| value.validate_tensors(global)), + _ => Ok(()), + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Ambient { + Known(Option), + Ambiguous, +} + +/// Unknown effects permanently invalidate tensor facts for this activation, +/// including aliases and outer bindings. A context never clears this flag. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct State { + pub ambient: Ambient, + pub contexts: Option>, + pub invalidated: bool, +} + +impl State { + pub fn new(mesh: Option) -> Self { + Self { + ambient: Ambient::Known(mesh), + contexts: Some(Vec::new()), + invalidated: false, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub struct Diagnostic { + pub op: Option, + pub code: &'static str, + pub message: String, +} + +/// Entry values and per-operation partial intent are trusted assumptions, not +/// facts inferred from an annotation call. The expected return is checked exactly. +#[derive(Clone, Debug, Default)] +pub struct Contract { + pub arguments: BTreeMap, + pub outer: BTreeMap, + pub mesh: Option, + pub returns: Option, + pub partial_outputs: BTreeMap>, + pub global: bool, + pub limits: Limits, +} + +#[derive(Clone, Debug)] +pub struct Report { + pub returned: Option, + pub diagnostics: Vec, + pub incomplete: bool, +} + +impl Report { + /// This certifies only the supported contract rules under the supplied + /// assumptions; it is not a Python or distributed-program soundness claim. + pub fn validated(&self) -> bool { + !self.incomplete && self.diagnostics.is_empty() + } +} + +pub struct Spmd<'a> { + pub exports: &'a dyn ExportProvider, + pub partial_outputs: &'a BTreeMap>, + pub global: bool, +} + +impl Spmd<'_> { + fn fail( + &self, + op: OpId, + cx: &mut Cx<'_, Self>, + code: &'static str, + message: impl Into, + ) { + cx.report(Diagnostic { + op: Some(op), + code, + message: message.into(), + }); + } + + fn unsupported(&self, op: OpId, cx: &mut Cx<'_, Self>, message: impl Into) { + cx.state_mut().invalidated = true; + self.fail(op, cx, "unsupported", message); + } + + fn ambient(&self, cx: &Cx<'_, Self>) -> Result, String> { + if cx.state().invalidated { + return Err("tensor facts were invalidated by an unsupported effect".into()); + } + match &cx.state().ambient { + Ambient::Known(mesh) => Ok(mesh.clone()), + Ambient::Ambiguous => Err("active mesh is ambiguous across control-flow paths".into()), + } + } + + fn compute( + &self, + op: OpId, + kind: Compute, + values: &[Value], + cx: &Cx<'_, Self>, + ) -> Result { + let ambient = self.ambient(cx)?; + let mut tensors = Vec::new(); + let template = values.iter().find_map(|v| { + if let Value::Tensor(t) = v { + Some(t) + } else { + None + } + }); + for value in values { + match value { + Value::Tensor(tensor) => tensors.push(tensor.clone()), + Value::Int(_) if matches!(kind, Compute::Add | Compute::Multiply) => { + let template = template.ok_or("scalar-only operations are unsupported")?; + tensors.push(Tensor::new( + template.rank(), + Grad::No, + template + .local() + .keys() + .map(|axis| (axis.clone(), Placement::Local(Local::R))), + )?); + } + Value::Ambiguous => { + return Err("tensor contract is ambiguous across branches".into()); + } + _ => return Err("operand has no verified tensor contract".into()), + } + } + let empty = BTreeSet::new(); + let partial = self.partial_outputs.get(&op).unwrap_or(&empty); + if !partial.is_empty() && !matches!(kind, Compute::Matmul | Compute::Linear) { + return Err( + "explicit Partial output intent is supported only for matmul/linear".into(), + ); + } + compute(kind, &tensors, ambient.as_ref(), partial, self.global).map(Value::Tensor) + } + + fn call( + &self, + op: OpId, + function: Value, + args: &[Arg], + cx: &mut Cx<'_, Self>, + ) -> Result { + let mut positional = Vec::new(); + let mut keywords = BTreeMap::new(); + for arg in args { + match arg { + Arg::Positional(reg) => positional.push(cx.reg(*reg).clone()), + Arg::Keyword(name, reg) => { + if keywords + .insert(name.as_str(), cx.reg(*reg).clone()) + .is_some() + { + return Err("duplicate call keyword".into()); + } + } + _ => { + self.unsupported(op, cx, "starred call arguments are unsupported"); + return Ok(Value::Unknown); + } + } + } + if self.partial_outputs.contains_key(&op) + && !matches!(&function, Value::Qualified(name) if matches!(name.as_str(), + "torch.matmul" | "torch.mm" | "torch.nn.functional.linear")) + { + return Err( + "Partial output contract does not identify a supported matmul/linear".into(), + ); + } + if let Value::MetadataMethod { name, rank } = &function { + let valid = keywords.is_empty() + && match name.as_str() { + "size" | "stride" => { + positional.is_empty() + || (positional.len() == 1 + && matches!(&positional[0], Value::Int(dim) + if *dim >= -(i64::try_from(*rank).unwrap_or(i64::MAX)) && *dim < *rank as i64)) + } + _ => positional.is_empty(), + }; + if !valid { + return Err("unsupported tensor metadata query signature".into()); + } + return Ok(if matches!(name.as_str(), "dim" | "ndimension") { + Value::Int(*rank as i64) + } else { + Value::Metadata + }); + } + let Value::Qualified(name) = function else { + self.unsupported(op, cx, "callee has no supported qualified identity"); + return Ok(Value::Unknown); + }; + let compute = match name.as_str() { + "torch.matmul" | "torch.mm" => Some(Compute::Matmul), + "torch.nn.functional.linear" => Some(Compute::Linear), + "torch.relu" | "torch.nn.functional.relu" => Some(Compute::Relu), + "torch.add" => Some(Compute::Add), + "torch.mul" => Some(Compute::Multiply), + _ => None, + }; + if let Some(kind) = compute { + // Only these exact signatures are summarized. In particular, `out=`, + // inplace=True, and nontrivial linear bias must never look pure. + let count = if kind == Compute::Relu { 1 } else { 2 }; + let bias_none = kind == Compute::Linear + && positional.len() == 3 + && positional.last() == Some(&Value::None); + if !keywords.is_empty() || !(positional.len() == count || bias_none) { + self.unsupported(op, cx, format!("unsupported {name} call signature")); + return Ok(Value::Unknown); + } + return self.compute(op, kind, &positional[..count], cx); + } + if matches!(name.as_str(), "spmd_types.S" | "spmd_types.Shard") { + if keywords.is_empty() + && positional.len() == 1 + && let Value::Int(dim) = positional[0] + { + return Ok(Value::Placement(Placement::Shard(dim))); + } + return Err("S(i) needs a literal integer dimension".into()); + } + if name == "spmd_types.set_current_mesh" { + if keywords.is_empty() + && positional.len() == 1 + && let Value::Mesh(mesh) = &positional[0] + { + return Ok(Value::MeshContext(mesh.clone())); + } + self.unsupported(op, cx, "set_current_mesh requires a trusted Mesh argument"); + return Ok(Value::Unknown); + } + if name == "spmd_types.assert_type" { + self.unsupported(op, cx, "assert_type cannot establish static facts; supply a trusted entry contract and use a checked exit contract"); + return Ok(Value::Unknown); + } + let (kind, rules) = match name.as_str() { + "spmd_types.invariant_to_replicate" => (Collective::InvariantToReplicate, false), + "spmd_types.convert" | "spmd_types.rules.convert" => { + (Collective::Convert, name.contains(".rules.")) + } + "spmd_types.reinterpret" | "spmd_types.rules.reinterpret" => { + (Collective::Reinterpret, name.contains(".rules.")) + } + "spmd_types.all_reduce" | "spmd_types.rules.all_reduce" => { + (Collective::AllReduce, name.contains(".rules.")) + } + "spmd_types.all_gather" | "spmd_types.rules.all_gather" => { + (Collective::AllGather, name.contains(".rules.")) + } + "spmd_types.reduce_scatter" | "spmd_types.rules.reduce_scatter" => { + (Collective::ReduceScatter, name.contains(".rules.")) + } + "spmd_types.all_to_all" | "spmd_types.rules.all_to_all" => { + (Collective::AllToAll, name.contains(".rules.")) + } + _ => { + self.unsupported(op, cx, format!("unsupported call {name}")); + return Ok(Value::Unknown); + } + }; + if positional.len() != 2 || keywords.keys().any(|key| !matches!(*key, "src" | "dst")) { + self.unsupported(op, cx, format!("unsupported {name} call signature")); + return Ok(Value::Unknown); + } + let ambient = self.ambient(cx)?; + let Value::Tensor(input) = &positional[0] else { + return Err("collective input has no verified tensor contract".into()); + }; + let Value::Str(axis) = &positional[1] else { + return Err("collective axis must be an explicit named axis token".into()); + }; + if let Some(mesh) = &ambient + && (!mesh.axes().contains(axis) + || input.local().keys().any(|a| !mesh.axes().contains(a)) + || mesh.axes().iter().any(|a| !input.local().contains_key(a))) + { + return Err("collective operand and active mesh are incompatible or incomplete".into()); + } + let default_src = match kind { + Collective::InvariantToReplicate => Some(Local::I), + Collective::AllReduce | Collective::ReduceScatter => Some(Local::P), + Collective::AllGather | Collective::AllToAll => Some(Local::V), + _ => None, + }; + let default_dst = match kind { + Collective::InvariantToReplicate => Some(Local::R), + Collective::ReduceScatter | Collective::AllToAll => Some(Local::V), + _ => None, + }; + let placement = |key: &str, default: Option| -> Result { + match keywords.get(key) { + Some(Value::Placement(p)) => Ok(*p), + Some(_) => Err(format!("{key} is not a known SPMD placement")), + None => default + .map(Placement::Local) + .ok_or_else(|| format!("missing collective {key}")), + } + }; + let mut src = placement("src", default_src)?; + let dst = placement("dst", default_dst)?; + if kind == Collective::InvariantToReplicate && !keywords.is_empty() { + return Err("invariant_to_replicate does not accept src/dst overrides".into()); + } + // The runtime reduction API interprets V as P even when src defaults to P; + // the type-only rules API requires the explicit source claim instead. + if !rules + && matches!(kind, Collective::AllReduce | Collective::ReduceScatter) + && src == Placement::Local(Local::P) + && input.local().get(axis) == Some(&Local::V) + { + src = Placement::Local(Local::V); + } + transition(input, kind, axis, src, dst, self.global).map(Value::Tensor) + } +} + +impl Domain for Spmd<'_> { + type Value = Value; + type State = State; + type Diagnostic = Diagnostic; + + fn top(&self) -> Value { + Value::Unknown + } + fn unbound(&self) -> Value { + Value::Unbound + } + + fn join(&self, a: &Value, b: &Value) -> Value { + if a == b { + a.clone() + } else if matches!(a, Value::Unknown) || matches!(b, Value::Unknown) { + Value::Unknown + } else { + Value::Ambiguous + } + } + + fn join_state(&self, a: &State, b: &State) -> State { + State { + ambient: if a.ambient == b.ambient { + a.ambient.clone() + } else { + Ambient::Ambiguous + }, + contexts: if a.contexts == b.contexts { + a.contexts.clone() + } else { + None + }, + invalidated: a.invalidated || b.invalidated, + } + } + + fn transfer(&self, op: OpId, kind: &OpKind, cx: &mut Cx<'_, Self>) { + match kind { + OpKind::Name { + symbol, + access: Access::Load(dst), + } => { + if cx.program().symbol(*symbol).origin == SymbolOrigin::Builtin + && cx.program().symbol(*symbol).name.as_str() == "len" + { + // Lexical shadowing changes the symbol, not just its spelling. + cx.set_reg(*dst, Value::Qualified("builtins.len".into())); + } else if matches!(cx.load(*symbol), Value::Unknown | Value::Unbound) { + self.unsupported(op, cx, "name has no verified reaching value"); + } + } + OpKind::Name { .. } | OpKind::Param { .. } => {} + OpKind::Literal { value, dst } => { + let value = match value { + Literal::None => Value::None, + Literal::Bool(v) => Value::Bool(*v), + Literal::Int(Some(v)) => Value::Int(*v), + Literal::Str(v) => Value::Str(v.to_string()), + _ => Value::Unknown, + }; + cx.set_reg(*dst, value); + } + OpKind::Import { module, item, dst } => { + let module_text = module.to_string(); + let value = match item { + ImportItem::Root => Value::Module( + module_text + .split('.') + .next() + .expect("module name has a root") + .into(), + ), + ImportItem::Module => Value::Module(module_text), + ImportItem::Name(name) => match self.exports.resolve(*module, name) { + ExportTarget::Definition(module, name) => { + Value::Qualified(format!("{module}.{name}")) + } + ExportTarget::Module(module) => Value::Module(module.to_string()), + ExportTarget::Unknown => Value::Unknown, + }, + }; + cx.set_reg(*dst, normalize_qualified(value)); + } + OpKind::Attribute { + object, + attr, + access: Access::Load(dst), + } => { + let value = match cx.reg(*object).clone() { + Value::Module(module) | Value::Qualified(module) + if trusted_namespace(&module) => + { + normalize_qualified(Value::Qualified(format!("{module}.{attr}"))) + } + Value::Tensor(tensor) => match attr.as_str() { + "ndim" => Value::Int(tensor.rank() as i64), + "shape" | "dtype" | "device" => Value::Metadata, + "size" | "stride" | "dim" | "ndimension" | "storage_offset" => { + Value::MetadataMethod { + name: attr.to_string(), + rank: tensor.rank(), + } + } + _ => { + self.unsupported( + op, + cx, + format!("unsupported tensor attribute {attr}"), + ); + Value::Unknown + } + }, + _ => { + self.unsupported( + op, + cx, + "attribute access may invoke an unsupported protocol", + ); + Value::Unknown + } + }; + cx.set_reg(*dst, value); + } + OpKind::Call { func, args, dst } => { + let function = cx.reg(*func).clone(); + let value = if function == Value::Qualified("builtins.len".into()) { + if args.len() == 1 + && matches!(args[0], Arg::Positional(r) if matches!(cx.reg(r), Value::Tensor(t) if t.rank() > 0)) + { + Ok(Value::Metadata) + } else { + self.unsupported( + op, + cx, + "len is supported only as a non-scalar tensor metadata query", + ); + Ok(Value::Unknown) + } + } else { + self.call(op, function, args, cx) + }; + match value { + Ok(value) => cx.set_reg(*dst, value), + Err(message) => { + self.fail(op, cx, "contract", message); + cx.set_reg(*dst, Value::Unknown); + } + } + } + OpKind::Binary { + op: binary, + left, + right, + dst, + } => { + let compute = match binary { + BinaryOp::Arith(Operator::Add) => Some(Compute::Add), + BinaryOp::Arith(Operator::Mult) => Some(Compute::Multiply), + BinaryOp::Arith(Operator::MatMult) => Some(Compute::Matmul), + _ => None, + }; + if let Some(kind) = compute { + let values = [cx.reg(*left).clone(), cx.reg(*right).clone()]; + match self.compute(op, kind, &values, cx) { + Ok(value) => cx.set_reg(*dst, value), + Err(message) => { + self.fail(op, cx, "contract", message); + cx.set_reg(*dst, Value::Unknown); + } + } + } else { + self.unsupported(op, cx, "unsupported binary operator or in-place effect"); + } + } + OpKind::Unary { + op: UnaryOp::USub, + operand, + dst, + } => match cx.reg(*operand) { + Value::Int(value) => match value.checked_neg() { + Some(value) => cx.set_reg(*dst, Value::Int(value)), + None => self.unsupported(op, cx, "integer negation overflows the descriptor"), + }, + _ => self.unsupported(op, cx, "unsupported unary protocol"), + }, + OpKind::Build { + kind: BuildKind::Tuple | BuildKind::List | BuildKind::Set, + elements, + dst, + } => { + if elements.len() <= 32 && elements.iter().all(|e| matches!(e, Element::Value(_))) { + let values = elements + .iter() + .map(|element| match element { + Element::Value(reg) => cx.reg(*reg).clone(), + _ => unreachable!("collection elements were checked above"), + }) + .collect(); + cx.set_reg(*dst, Value::Sequence(values)); + } else { + self.unsupported(op, cx, "unsupported collection construction"); + } + } + OpKind::MakeFunction { region, dst, .. } => { + cx.set_reg(*dst, Value::Function(*region)); + } + OpKind::ContextEnter { + manager, + dst, + context, + is_async: false, + } => { + if let Value::MeshContext(mesh) = cx.reg(*manager).clone() { + let previous = cx.state().ambient.clone(); + match &mut cx.state_mut().contexts { + Some(contexts) if contexts.len() < 32 => { + contexts.push((*context, previous)) + } + _ => { + self.unsupported( + op, + cx, + "mesh context stack is ambiguous or exceeds its bound", + ); + } + } + cx.state_mut().ambient = Ambient::Known(Some(mesh)); + cx.set_reg(*dst, Value::None); + } else { + cx.state_mut().ambient = Ambient::Ambiguous; + self.unsupported( + op, + cx, + "unsupported context manager can mutate tensor contracts", + ); + } + } + OpKind::ContextExit { + context, + is_async: false, + .. + } => { + let restored = cx.state_mut().contexts.as_mut().and_then(Vec::pop); + match restored { + Some((expected, ambient)) if expected == *context => { + cx.state_mut().ambient = ambient; + } + _ => { + cx.state_mut().ambient = Ambient::Ambiguous; + self.unsupported(op, cx, "mesh context restoration is ambiguous"); + } + } + } + OpKind::Gap { .. } => { + cx.state_mut().invalidated = true; + } + _ => self.unsupported(op, cx, "operation has no supported SPMD effect summary"), + } + } + + fn assume(&self, cond: Reg, truth: bool, cx: &mut Cx<'_, Self>) -> bool { + match cx.reg(cond) { + Value::Bool(value) => *value == truth, + Value::None => !truth, + Value::Int(value) => (*value != 0) == truth, + Value::Str(value) => !value.is_empty() == truth, + Value::Predicate | Value::Metadata => true, + _ => { + // An unknown object's truthiness can invoke arbitrary Python code. + cx.state_mut().invalidated = true; + true + } + } + } +} + +fn trusted_namespace(name: &str) -> bool { + ["torch", "spmd_types"] + .iter() + .any(|root| name == *root || name.starts_with(&format!("{root}."))) +} + +fn normalize_qualified(value: Value) -> Value { + if let Value::Qualified(name) = &value { + let local = match name.as_str() { + "spmd_types.R" => Some(Local::R), + "spmd_types.I" => Some(Local::I), + "spmd_types.V" => Some(Local::V), + "spmd_types.P" => Some(Local::P), + _ => None, + }; + if let Some(local) = local { + return Value::Placement(Placement::Local(local)); + } + } + value +} + +/// Analyze a selected function under trusted entry/call contracts. Module execution +/// is analyzed first to obtain reaching imported identities, not scanned for imports. +/// Nested/deferred helpers are intentionally unsupported at calls in this subset. +pub fn check( + program: &Program, + region: RegionId, + contract: &Contract, + exports: &dyn ExportProvider, +) -> Report { + let domain = Spmd { + exports, + partial_outputs: &contract.partial_outputs, + global: contract.global, + }; + let engine = Engine::new(program, &domain).with_limits(contract.limits); + let module = engine.analyze(Program::MODULE, Entry::new(State::new(None))); + let mut entry: Entry> = Entry::new(State::new(contract.mesh.clone())); + let mut diagnostics = module.diagnostics.clone(); + let mut incomplete = + module.is_incomplete() || module.exit.as_ref().is_none_or(|f| f.state.invalidated); + if region == Program::MODULE || region.index() >= program.regions.len() { + diagnostics.push(Diagnostic { + op: None, + code: "entry", + message: "select a valid function region".into(), + }); + return Report { + returned: None, + diagnostics, + incomplete: true, + }; + } + let function = program.region(region); + if function.trigger != Trigger::OnCall + || function + .params + .iter() + .any(|param| matches!(param.kind, ParamKind::VarArgs | ParamKind::KwArgs)) + || function + .parent + .is_none_or(|(parent, _)| parent != Program::MODULE) + { + diagnostics.push(Diagnostic { + op: None, + code: "unsupported", + message: "only plain module-level function activations are supported".into(), + }); + incomplete = true; + } + for param in &function.params { + entry.args.push( + contract + .arguments + .get(param.name.as_str()) + .cloned() + .unwrap_or(Value::Unknown), + ); + } + if contract + .arguments + .keys() + .any(|name| !function.params.iter().any(|p| p.name.as_str() == name)) + { + diagnostics.push(Diagnostic { + op: None, + code: "entry", + message: "entry contract names a nonexistent parameter".into(), + }); + } + if let Some(frame) = &module.exit { + for (index, _) in program.symbols.iter().enumerate() { + let symbol = SymbolId(index as u32); + if program.owner(symbol) == Some(Program::MODULE) + && let Some(binding) = frame.binding(program, symbol) + { + entry.outer.insert(symbol, binding.clone()); + } + } + } + for op in contract.partial_outputs.keys() { + if program.ops.get(op.index()).is_none_or(|operation| { + operation.region != region + || !matches!(operation.kind, OpKind::Call { .. } | OpKind::Binary { .. }) + }) { + diagnostics.push(Diagnostic { + op: None, + code: "entry", + message: + "partial output contract must identify a call/binary op in the selected region" + .into(), + }); + } + } + for (symbol, value) in &contract.outer { + if symbol.index() >= program.symbols.len() { + diagnostics.push(Diagnostic { + op: None, + code: "entry", + message: "outer entry contract names a nonexistent symbol".into(), + }); + } else { + entry.outer.insert(*symbol, Binding::bound(value.clone())); + } + } + for value in contract.arguments.values().chain(contract.outer.values()) { + if let Err(message) = value.validate_tensors(contract.global) { + diagnostics.push(Diagnostic { + op: None, + code: "entry", + message, + }); + incomplete = true; + } + } + entry.state.invalidated = incomplete; + let result = engine.analyze(region, entry); + incomplete |= result.is_incomplete() + || result.exit.as_ref().is_none_or(|frame| { + frame.state.invalidated + || frame.state.ambient == Ambient::Ambiguous + || frame.state.contexts.is_none() + }) + || matches!( + result.returned, + Some(Value::Unknown | Value::Ambiguous | Value::Unbound) + ); + diagnostics.extend(result.diagnostics); + if let Some(value) = &result.returned + && let Err(message) = value.validate_tensors(contract.global) + { + diagnostics.push(Diagnostic { + op: None, + code: "exit", + message, + }); + incomplete = true; + } + if incomplete { + diagnostics.push(Diagnostic { + op: None, + code: "incomplete", + message: "reachable gaps, unknown effects/values, or an unconverged/absent normal exit prevent contract validation".into(), + }); + } + if let Some(expected) = &contract.returns { + match &result.returned { + Some(Value::Tensor(actual)) if actual.matches(expected) => {} + value => diagnostics.push(Diagnostic { + op: None, + code: "exit", + message: format!("return contract {value:?} does not match expected {expected:?}"), + }), + } + } + diagnostics.sort(); + diagnostics.dedup(); + Report { + returned: result.returned, + diagnostics, + incomplete, + } +} diff --git a/static_analysis/spmd/src/lib.rs b/static_analysis/spmd/src/lib.rs new file mode 100644 index 0000000000..68b5863716 --- /dev/null +++ b/static_analysis/spmd/src/lib.rs @@ -0,0 +1,32 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Experimental local gradient contracts and restricted rank-two sharding rules. +//! The Python frontend, lexical binding and fixpoint engine belong to the core; +//! this package supplies only a stateful domain and trusted contract descriptors. + +mod domain; +pub mod sidecar; +mod types; + +pub use domain::Ambient; +pub use domain::Contract; +pub use domain::Diagnostic; +pub use domain::Report; +pub use domain::Spmd; +pub use domain::State; +pub use domain::Value; +pub use domain::check; +pub use types::Collective; +pub use types::Compute; +pub use types::Grad; +pub use types::Local; +pub use types::Mesh; +pub use types::Placement; +pub use types::Tensor; +pub use types::compute; +pub use types::transition; diff --git a/static_analysis/spmd/src/main.rs b/static_analysis/spmd/src/main.rs new file mode 100644 index 0000000000..1c05ca261c --- /dev/null +++ b/static_analysis/spmd/src/main.rs @@ -0,0 +1,67 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use std::env; +use std::fs; +use std::process::ExitCode; + +use pyrefly_python::module_name::ModuleName; +use pyrefly_static_analysis::analyze_python; +use pyrefly_static_analysis::exports::SyntacticExports; +use pyrefly_static_analysis_spmd::check; +use pyrefly_static_analysis_spmd::sidecar; + +fn run() -> Result { + let args: Vec<_> = env::args_os().skip(1).collect(); + if args.len() != 2 { + return Err("usage: pyrefly_static_analysis_spmd ".into()); + } + let source = fs::read_to_string(&args[0]).map_err(|error| error.to_string())?; + let contracts = fs::read_to_string(&args[1]).map_err(|error| error.to_string())?; + let program = analyze_python(&source, ModuleName::from_str("spmd_input"), false); + let (region, contract) = sidecar::parse(&program, &contracts)?; + let report = check(&program, region, &contract, &SyntacticExports); + for diagnostic in &report.diagnostics { + let range = diagnostic + .op + .map(|op| program.op(op).range) + .unwrap_or(program.region(region).range); + let (line, column) = program.line_col(range); + println!( + "{}:{}:{}: {}: {}", + args[0].to_string_lossy(), + line, + column, + diagnostic.code, + diagnostic.message + ); + } + println!( + "status: {}; incomplete: {}", + if report.validated() { + "contract-validated" + } else { + "not-validated" + }, + report.incomplete + ); + println!( + "Scope: supported SPMD rules under trusted sidecar assumptions; not full Python/global-SPMD soundness." + ); + Ok(report.validated()) +} + +fn main() -> ExitCode { + match run() { + Ok(true) => ExitCode::SUCCESS, + Ok(false) => ExitCode::from(1), + Err(error) => { + eprintln!("{error}"); + ExitCode::from(2) + } + } +} diff --git a/static_analysis/spmd/src/sidecar.rs b/static_analysis/spmd/src/sidecar.rs new file mode 100644 index 0000000000..3c13a5fec0 --- /dev/null +++ b/static_analysis/spmd/src/sidecar.rs @@ -0,0 +1,223 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Strict JSON sidecars are trusted integration inputs, never executable code. + +use std::collections::BTreeSet; + +use pyrefly_static_analysis::OpId; +use pyrefly_static_analysis::OpKind; +use pyrefly_static_analysis::Program; +use pyrefly_static_analysis::RegionId; +use serde_json::Map; +use serde_json::Value as Json; + +use crate::Contract; +use crate::Grad; +use crate::Local; +use crate::Mesh; +use crate::Placement; +use crate::Tensor; +use crate::Value; + +fn object(value: &Json) -> Result<&Map, String> { + value + .as_object() + .ok_or_else(|| "expected JSON object".into()) +} + +fn fields(value: &Map, allowed: &[&str]) -> Result<(), String> { + if let Some(key) = value.keys().find(|key| !allowed.contains(&key.as_str())) { + return Err(format!("unknown sidecar field {key}")); + } + Ok(()) +} + +fn strings(value: &Json) -> Result, String> { + value + .as_array() + .ok_or("expected axis name array")? + .iter() + .map(|value| { + value + .as_str() + .map(String::from) + .ok_or_else(|| "expected string axis name".into()) + }) + .collect() +} + +fn placement(value: &str) -> Result { + let local = match value { + "R" => Some(Local::R), + "I" => Some(Local::I), + "V" => Some(Local::V), + "P" => Some(Local::P), + _ => None, + }; + if let Some(local) = local { + return Ok(Placement::Local(local)); + } + if let Some(dim) = value + .strip_prefix("S(") + .and_then(|value| value.strip_suffix(')')) + { + return dim + .parse::() + .map(Placement::Shard) + .map_err(|_| "invalid S(i) dimension".into()); + } + Err(format!("unknown placement {value}")) +} + +fn tensor(value: &Json) -> Result { + let value = object(value)?; + fields(value, &["rank", "grad", "axes", "partition"])?; + let rank = value + .get("rank") + .and_then(Json::as_u64) + .ok_or("tensor needs unsigned rank")?; + let rank = usize::try_from(rank).map_err(|_| "rank does not fit this platform")?; + if rank > 32 { + return Err("descriptor rank exceeds 32".into()); + } + let grad = match value.get("grad").and_then(Json::as_str) { + Some("yes") => Grad::Yes, + Some("no") => Grad::No, + Some("unknown") => Grad::Unknown, + _ => return Err("tensor grad must be yes, no or unknown".into()), + }; + let axes = object(value.get("axes").ok_or("tensor needs axes")?)?; + let mut placements = Vec::new(); + for (axis, value) in axes { + placements.push(( + axis.clone(), + placement(value.as_str().ok_or("axis placement must be a string")?)?, + )); + } + let mut result = Tensor::new(rank, grad, placements)?; + if let Some(partition) = value.get("partition") { + let mut dims = Vec::new(); + for dim in partition.as_array().ok_or("partition must be an array")? { + dims.push(if dim.is_null() { + Vec::new() + } else if let Some(axis) = dim.as_str() { + vec![axis.into()] + } else { + strings(dim)? + }); + } + result = result.with_partition(dims)?; + } + Ok(result) +} + +fn input(value: &Json) -> Result { + let value_object = object(value)?; + if let Some(mesh) = value_object.get("mesh") { + fields(value_object, &["mesh"])?; + Ok(Value::Mesh(Mesh::new(strings(mesh)?)?)) + } else if value_object.get("predicate") == Some(&Json::Bool(true)) { + fields(value_object, &["predicate"])?; + Ok(Value::Predicate) + } else { + Ok(Value::Tensor(tensor(value)?)) + } +} + +/// Resolve a partial-output contract only when exactly one compute operation +/// starts at the specified line in the selected region. Ambiguity is an error; +/// source text is never reparsed or pattern-matched for operation semantics. +pub fn parse(program: &Program, text: &str) -> Result<(RegionId, Contract), String> { + let value: Json = serde_json::from_str(text).map_err(|error| error.to_string())?; + let value = object(&value)?; + fields( + value, + &[ + "function", + "global", + "mesh", + "inputs", + "return", + "partial_outputs", + ], + )?; + let name = value + .get("function") + .and_then(Json::as_str) + .ok_or("sidecar needs function name")?; + let candidates: Vec<_> = program + .regions + .iter() + .enumerate() + .filter(|(_, region)| { + region.name.as_str() == name + && region + .parent + .is_some_and(|(parent, _)| parent == Program::MODULE) + }) + .collect(); + if candidates.len() != 1 { + return Err("function must name exactly one module-level region".into()); + } + let region = RegionId(candidates[0].0 as u32); + let mut contract = Contract::default(); + if let Some(global) = value.get("global") { + contract.global = global.as_bool().ok_or("global must be a boolean")?; + } + if let Some(mesh) = value.get("mesh") { + contract.mesh = Some(Mesh::new(strings(mesh)?)?); + } + for (name, value) in object(value.get("inputs").ok_or("sidecar needs inputs")?)? { + contract.arguments.insert(name.clone(), input(value)?); + } + if let Some(value) = value.get("return") { + contract.returns = Some(tensor(value)?); + } + if let Some(partial) = value.get("partial_outputs") { + for value in partial + .as_array() + .ok_or("partial_outputs must be an array")? + { + let value = object(value)?; + fields(value, &["line", "axes"])?; + let line = value + .get("line") + .and_then(Json::as_u64) + .ok_or("partial output needs unsigned line")?; + let axes: BTreeSet<_> = strings(value.get("axes").ok_or("partial output needs axes")?)? + .into_iter() + .collect(); + if axes.is_empty() { + return Err("partial output must name at least one axis".into()); + } + let ops: Vec<_> = program + .ops + .iter() + .enumerate() + .filter(|(_, op)| { + op.region == region + && program.line_col(op.range).0.get() as u64 == line + && matches!(op.kind, OpKind::Call { .. } | OpKind::Binary { .. }) + }) + .collect(); + if ops.len() != 1 { + return Err(format!( + "line {line} must identify exactly one call or binary operation" + )); + } + if contract + .partial_outputs + .insert(OpId(ops[0].0 as u32), axes) + .is_some() + { + return Err(format!("duplicate partial output line {line}")); + } + } + } + Ok((region, contract)) +} diff --git a/static_analysis/spmd/src/types.rs b/static_analysis/spmd/src/types.rs new file mode 100644 index 0000000000..89491149e8 --- /dev/null +++ b/static_analysis/spmd/src/types.rs @@ -0,0 +1,555 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Pure, finite SPMD descriptors. They describe contracts, not executing tensors. + +use std::collections::BTreeMap; +use std::collections::BTreeSet; + +/// The backward contract is determined by the forward local contract. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Local { + R, + I, + V, + P, +} + +impl Local { + pub fn gradient(self) -> Self { + match self { + Self::R => Self::P, + Self::P => Self::R, + Self::I => Self::I, + Self::V => Self::V, + } + } +} + +/// Unknown gradient information never enables a no-gradient relaxation. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Grad { + Yes, + No, + Unknown, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Placement { + Local(Local), + Shard(i64), +} + +impl Placement { + pub fn local(self) -> Local { + match self { + Self::Local(local) => local, + Self::Shard(_) => Local::V, + } + } +} + +/// Named, independent axes supplied by a trusted integration. Names do not prove +/// the physical rank layout or orthogonality of actual process groups. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Mesh { + axes: BTreeSet, +} + +impl Mesh { + pub fn new(axes: impl IntoIterator>) -> Result { + let mut result = BTreeSet::new(); + for axis in axes { + let axis = axis.into(); + if axis.is_empty() || !result.insert(axis) { + return Err("mesh axes must be nonempty and distinct".into()); + } + } + Ok(Self { axes: result }) + } + + pub fn axes(&self) -> &BTreeSet { + &self.axes + } +} + +/// An ordered list of axes per dimension, outermost first. Construction validates +/// that only V axes occur and that an axis occurs in at most one dimension. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Tensor { + rank: usize, + grad: Grad, + local: BTreeMap, + partition: Vec>, +} + +impl Tensor { + pub fn new( + rank: usize, + grad: Grad, + axes: impl IntoIterator, Placement)>, + ) -> Result { + if rank > 32 { + return Err("descriptor rank exceeds 32".into()); + } + let mut result = Self { + rank, + grad, + local: BTreeMap::new(), + partition: vec![Vec::new(); rank], + }; + for (axis, placement) in axes { + let axis = axis.into(); + if axis.is_empty() + || result + .local + .insert(axis.clone(), placement.local()) + .is_some() + { + return Err("tensor axes must be nonempty and distinct".into()); + } + if let Placement::Shard(dim) = placement { + result.partition[dimension(dim, rank)?].push(axis); + } + } + if result.partition.iter().any(|axes| axes.len() > 1) { + return Err("multiple axes on one dimension require an ordered PartitionSpec".into()); + } + Ok(result) + } + + pub fn with_partition(mut self, partition: Vec>) -> Result { + self.partition = partition; + self.validate(false)?; + Ok(self) + } + + /// Check representation invariants independently of an expected return contract. + /// Local V may omit a shard dimension; global V must occur exactly once in the + /// partition. Every producer checks this before publishing its result. + pub fn validate(&self, global: bool) -> Result<(), String> { + if self.rank > 32 || self.partition.len() != self.rank { + return Err("PartitionSpec must match a tensor rank of at most 32".into()); + } + if self.local.keys().any(String::is_empty) { + return Err("tensor axes must be nonempty".into()); + } + let mut seen = BTreeSet::new(); + for axis in self.partition.iter().flatten() { + if self.local.get(axis) != Some(&Local::V) || !seen.insert(axis) { + return Err("PartitionSpec axes must be V and occur at most once".into()); + } + } + if global { + for (axis, local) in &self.local { + if *local == Local::V && !seen.contains(axis) { + return Err(format!( + "global V axis {axis} is missing from PartitionSpec" + )); + } + } + } + Ok(()) + } + + pub fn rank(&self) -> usize { + self.rank + } + + pub fn grad(&self) -> Grad { + self.grad + } + + pub fn local(&self) -> &BTreeMap { + &self.local + } + + pub fn partition(&self) -> &[Vec] { + &self.partition + } + + /// Exact forward and gradient contract, including explicit shard ordering. + pub fn matches(&self, expected: &Self) -> bool { + self == expected + } + + fn shard(&self, axis: &str) -> Option { + self.partition + .iter() + .position(|axes| axes.iter().any(|a| a == axis)) + } + + fn complete(&self, axes: &BTreeSet) -> Result<(), String> { + for axis in axes { + if !self.local.contains_key(axis) { + return Err(format!("tensor is missing active mesh axis {axis}")); + } + } + Ok(()) + } +} + +fn dimension(dim: i64, rank: usize) -> Result { + let rank = i64::try_from(rank).map_err(|_| "tensor rank is too large")?; + let canonical = if dim < 0 { + rank.checked_add(dim) + } else { + Some(dim) + }; + match canonical { + Some(dim) if dim >= 0 && dim < rank => Ok(dim as usize), + _ => Err(format!("shard dimension {dim} is invalid for rank {rank}")), + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Compute { + Add, + Multiply, + Relu, + Matmul, + Linear, +} + +/// Check local propagation and, optionally, the restricted rank-two sharding rule. +/// A sharded contraction requires explicit `partial_axes` intent; this never +/// inserts a collective or guesses whether a local operation meant a global one. +pub fn compute( + op: Compute, + inputs: &[Tensor], + ambient: Option<&Mesh>, + partial_axes: &BTreeSet, + global: bool, +) -> Result { + if !partial_axes.is_empty() && !matches!(op, Compute::Matmul | Compute::Linear) { + return Err("explicit Partial output intent is supported only for matmul/linear".into()); + } + let count = if op == Compute::Relu { 1 } else { 2 }; + if inputs.len() != count { + return Err(format!("{op:?} requires {count} tensor operands")); + } + let mut axes = partial_axes.clone(); + for input in inputs { + axes.extend(input.local.keys().cloned()); + } + if let Some(mesh) = ambient { + axes.extend(mesh.axes.iter().cloned()); + if inputs + .iter() + .any(|t| t.local.keys().any(|a| !mesh.axes.contains(a))) + { + return Err("foreign mesh presentation is unsupported".into()); + } + } + for input in inputs { + input.complete(&axes)?; + } + let grad = if inputs.iter().any(|t| t.grad == Grad::Yes) { + Grad::Yes + } else if inputs.iter().all(|t| t.grad == Grad::No) { + Grad::No + } else { + Grad::Unknown + }; + let rank = inputs[0].rank; + if inputs.iter().any(|t| t.rank != rank) { + return Err("broadcasting and unequal tensor ranks are unsupported".into()); + } + if matches!(op, Compute::Matmul | Compute::Linear) && rank != 2 { + return Err("only rank-two matmul and linear are supported".into()); + } + let mut output = Tensor { + rank, + grad, + local: BTreeMap::new(), + partition: vec![Vec::new(); rank], + }; + for axis in &axes { + let mut locals: Vec<_> = inputs.iter().map(|t| t.local[axis]).collect(); + // Preserve homogeneous I. Otherwise a known no-grad R/I adopts the + // gradient-bearing flavor (or R when no operand has a gradient edge). + if locals.contains(&Local::I) && locals.iter().any(|l| *l != Local::I) { + let flavor = inputs + .iter() + .zip(&locals) + .find_map(|(t, l)| { + (t.grad != Grad::No && matches!(l, Local::I | Local::R)).then_some(*l) + }) + .unwrap_or(Local::R); + for (input, local) in inputs.iter().zip(&mut locals) { + if input.grad == Grad::No && matches!(local, Local::I | Local::R) { + *local = flavor; + } + } + } + let mut local = if locals.iter().all(|l| *l == Local::I) { + Local::I + } else if locals.contains(&Local::I) { + return Err(format!( + "gradient-bearing I cannot mix with other contracts on {axis}; an I->R entry may be missing" + )); + } else if locals.contains(&Local::P) { + let valid = match op { + Compute::Add => locals.iter().all(|l| *l == Local::P), + Compute::Multiply | Compute::Matmul | Compute::Linear => { + locals.iter().filter(|l| **l == Local::P).count() == 1 + && locals.iter().all(|l| matches!(l, Local::P | Local::R)) + } + Compute::Relu => false, + }; + if !valid { + return Err(format!( + "{op:?} cannot consume these Partial contracts on {axis}" + )); + } + Local::P + } else if locals.contains(&Local::V) { + Local::V + } else { + Local::R + }; + if partial_axes.contains(axis) { + if !matches!(local, Local::V | Local::P) { + return Err(format!("explicit partial output requires V or P on {axis}")); + } + local = Local::P; + } + output.local.insert(axis.clone(), local); + } + if !global { + output.validate(false)?; + return Ok(output); + } + if matches!(op, Compute::Matmul | Compute::Linear) { + let right_labels = if op == Compute::Linear { + [1, 2] + } else { + [2, 1] + }; + let labels = [[0, 2], right_labels]; // output labels are [0, 1]; 2 is K. + for axis in &axes { + let mut label = None; + let mut order = None; + for (input, labels) in inputs.iter().zip(labels) { + if let Some(dim) = input.shard(axis) { + let next = labels[dim]; + if label.is_some_and(|previous| previous != next) { + return Err(format!("mesh axis {axis} shards different matmul labels")); + } + label = Some(next); + let next_order = &input.partition[dim]; + if order.is_some_and(|previous| previous != next_order) { + return Err(format!("shard order differs on matmul label for {axis}")); + } + order = Some(next_order); + } else if input.local[axis] == Local::V { + return Err(format!( + "global matmul needs a shard dimension for V on {axis}" + )); + } + } + if let Some(label) = label { + for (input, labels) in inputs.iter().zip(labels) { + if labels.contains(&label) && input.shard(axis).is_none() { + return Err(format!("matmul label sharding is inconsistent on {axis}")); + } + } + if label == 2 { + if !partial_axes.contains(axis) { + return Err(format!( + "sharded contraction on {axis} requires an explicit Partial output contract" + )); + } + } else { + if partial_axes.contains(axis) { + return Err(format!( + "free matmul label on {axis} cannot be declared Partial" + )); + } + let order = order.expect("a sharded label records its axis order"); + if output.partition[label].is_empty() { + output.partition[label] = order.clone(); + } else if &output.partition[label] != order { + return Err("output shard order is inconsistent".into()); + } + } + } else if partial_axes.contains(axis) + && inputs.iter().all(|t| t.local[axis] != Local::P) + { + return Err(format!( + "no sharded contraction justifies Partial on {axis}" + )); + } + } + } else { + for axis in &axes { + let shards: Vec<_> = inputs + .iter() + .filter(|t| t.local[axis] == Local::V) + .map(|t| t.shard(axis)) + .collect(); + if shards.iter().any(|dim| dim.is_none()) { + return Err(format!( + "global pointwise compute needs a shard dimension on {axis}" + )); + } + if let Some(Some(dim)) = shards.first() { + if shards.iter().any(|d| d != &Some(*dim)) { + return Err(format!("pointwise shard dimensions disagree on {axis}")); + } + let orders: Vec<_> = inputs + .iter() + .filter(|t| t.local[axis] == Local::V) + .map(|t| &t.partition[*dim]) + .collect(); + if orders.iter().any(|order| *order != orders[0]) { + return Err(format!("pointwise shard ordering disagrees on {axis}")); + } + if output.partition[*dim].is_empty() { + output.partition[*dim] = orders[0].clone(); + } else if &output.partition[*dim] != orders[0] { + return Err(format!( + "coincident output shards have incompatible ordering on dimension {dim}" + )); + } + } + } + } + output.validate(true)?; + Ok(output) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Collective { + Convert, + InvariantToReplicate, + Reinterpret, + AllReduce, + AllGather, + ReduceScatter, + AllToAll, +} + +/// Validate a transition before changing any descriptor. Unlike a runtime +/// annotation setter, a missing source axis is an error, never an assumption. +pub fn transition( + input: &Tensor, + kind: Collective, + axis: &str, + src: Placement, + dst: Placement, + global: bool, +) -> Result { + let source = src.local(); + let destination = dst.local(); + let allowed = match kind { + Collective::InvariantToReplicate => source == Local::I && destination == Local::R, + Collective::Convert => { + (matches!( + (source, destination), + (Local::I, Local::R) | (Local::R, Local::P) + ) || (matches!(source, Local::I | Local::R) && matches!(dst, Placement::Shard(_)))) + || source == destination + } + Collective::Reinterpret => { + source == Local::V + && destination == Local::P + && matches!(src, Placement::Local(_)) + && matches!(dst, Placement::Local(_)) + } + Collective::AllReduce => { + matches!(source, Local::P | Local::V) + && matches!(destination, Local::R | Local::I) + && matches!(src, Placement::Local(_)) + } + Collective::AllGather => source == Local::V && matches!(destination, Local::R | Local::I), + Collective::ReduceScatter => { + matches!(source, Local::P | Local::V) + && destination == Local::V + && matches!(src, Placement::Local(_)) + } + Collective::AllToAll => { + source == Local::V + && destination == Local::V + && (matches!(src, Placement::Shard(_)) == matches!(dst, Placement::Shard(_))) + } + }; + if !allowed { + return Err(format!("{kind:?} does not admit {src:?} -> {dst:?}")); + } + let actual = input + .local + .get(axis) + .ok_or_else(|| format!("tensor has no source contract on axis {axis}"))?; + if *actual != source + && !(input.grad == Grad::No + && matches!( + (*actual, source), + (Local::I, Local::R) | (Local::R, Local::I) + )) + { + return Err(format!( + "collective src {source:?} does not match {actual:?} on {axis}" + )); + } + let source_dim = match src { + Placement::Shard(dim) => Some(dimension(dim, input.rank)?), + Placement::Local(_) => None, + }; + let destination_dim = match dst { + Placement::Shard(dim) => Some(dimension(dim, input.rank)?), + Placement::Local(_) => None, + }; + if let Some(dim) = source_dim { + if input.shard(axis) != Some(dim) { + return Err(format!( + "collective source shard dimension does not match on {axis}" + )); + } + } else if global && source == Local::V && input.shard(axis).is_some() { + return Err("global collective requires an explicit S(i) source".into()); + } + if global + && ((source == Local::V && source_dim.is_none()) + || (destination == Local::V && destination_dim.is_none())) + { + return Err("global collectives require S(i), not unqualified V".into()); + } + if kind == Collective::Convert + && source_dim != destination_dim + && source == Local::V + && destination == Local::V + { + return Err("convert cannot change a shard dimension without communication".into()); + } + if let Some(dim) = input.shard(axis) + && input.partition[dim].last().map(String::as_str) != Some(axis) + { + return Err("a single-axis collective cannot change an outer shard axis".into()); + } + let mut result = input.clone(); + for axes in &mut result.partition { + axes.retain(|a| a != axis); + } + result.local.insert(axis.into(), destination); + if let Some(dim) = destination_dim { + result.partition[dim].push(axis.into()); + } + if kind == Collective::AllGather && src == Placement::Local(Local::V) { + result.rank += 1; + result.partition.insert(0, Vec::new()); + } + if kind == Collective::ReduceScatter && dst == Placement::Local(Local::V) { + if result.rank == 0 || !result.partition[0].is_empty() { + return Err("unbind reduce_scatter requires an unsharded leading dimension".into()); + } + result.rank -= 1; + result.partition.remove(0); + } + result.validate(global)?; + Ok(result) +} diff --git a/static_analysis/spmd/tests/consumer.rs b/static_analysis/spmd/tests/consumer.rs new file mode 100644 index 0000000000..d1e49edba1 --- /dev/null +++ b/static_analysis/spmd/tests/consumer.rs @@ -0,0 +1,472 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use std::collections::BTreeMap; +use std::path::Path; +use std::process::Command; + +use pyrefly_python::module_name::ModuleName; +use pyrefly_static_analysis::Domain; +use pyrefly_static_analysis::Engine; +use pyrefly_static_analysis::Entry; +use pyrefly_static_analysis::ExportProvider; +use pyrefly_static_analysis::ExportTarget; +use pyrefly_static_analysis::Limits; +use pyrefly_static_analysis::Program; +use pyrefly_static_analysis::RegionId; +use pyrefly_static_analysis::analyze_python; +use pyrefly_static_analysis::exports::SyntacticExports; +use pyrefly_static_analysis_spmd::Ambient; +use pyrefly_static_analysis_spmd::Contract; +use pyrefly_static_analysis_spmd::Grad; +use pyrefly_static_analysis_spmd::Local; +use pyrefly_static_analysis_spmd::Mesh; +use pyrefly_static_analysis_spmd::Placement; +use pyrefly_static_analysis_spmd::Report; +use pyrefly_static_analysis_spmd::Spmd; +use pyrefly_static_analysis_spmd::State; +use pyrefly_static_analysis_spmd::Tensor; +use pyrefly_static_analysis_spmd::Value; +use pyrefly_static_analysis_spmd::check; +use pyrefly_static_analysis_spmd::sidecar; + +const MLP: &str = include_str!("../examples/tp_mlp.py"); +const MLP_CONTRACT: &str = include_str!("../examples/tp_mlp.json"); + +fn program(source: &str) -> Program { + analyze_python(source, ModuleName::from_str("fixture"), false) +} + +fn region(program: &Program, name: &str) -> RegionId { + RegionId( + program + .regions + .iter() + .position(|r| r.name.as_str() == name) + .unwrap() as u32, + ) +} + +fn tensor(local: Local) -> Tensor { + Tensor::new(2, Grad::Yes, [("tp", Placement::Local(local))]).unwrap() +} + +fn contract() -> Contract { + Contract { + mesh: Some(Mesh::new(["tp"]).unwrap()), + ..Contract::default() + } +} + +fn report(source: &str, contract: &Contract) -> Report { + let program = program(source); + check(&program, region(&program, "f"), contract, &SyntacticExports) +} + +fn mlp(source: &str) -> Report { + let program = program(source); + let (region, contract) = sidecar::parse(&program, MLP_CONTRACT).unwrap(); + check(&program, region, &contract, &SyntacticExports) +} + +#[test] +fn tp_mlp_checks_forward_and_backward_contracts() { + let result = mlp(MLP); + assert!(result.validated(), "{result:?}"); + assert_eq!(result.returned, Some(Value::Tensor(tensor(Local::I)))); + let result = mlp(&MLP.replace("x = s.invariant_to_replicate(x, \"tp\")", "x = x")); + assert!(!result.validated()); + assert!( + result + .diagnostics + .iter() + .any(|d| d.message.contains("I->R entry")), + "{result:?}" + ); + let result = mlp(&MLP.replace("dst=s.I", "dst=s.R")); + assert!(!result.validated()); + assert!( + result.diagnostics.iter().any(|d| d.code == "exit"), + "{result:?}" + ); + // R and I have equal forward denotation, but the exit's gradient contract differs. + assert_eq!(result.returned, Some(Value::Tensor(tensor(Local::R)))); + let result = mlp(&MLP.replace( + "return s.all_reduce(p, \"tp\", src=s.P, dst=s.I)", + "return p", + )); + assert!(!result.validated()); + assert_eq!(result.returned, Some(Value::Tensor(tensor(Local::P)))); +} + +#[test] +fn metadata_is_transparent_but_nonlinear_partial_reads_are_not() { + let result = mlp(&MLP.replace("p.stride()", "t.relu(p)")); + assert!( + result + .diagnostics + .iter() + .any(|d| d.message.contains("Partial")), + "{result:?}" + ); + let mut contract = contract(); + contract + .arguments + .insert("p".into(), Value::Tensor(tensor(Local::P))); + contract.returns = Some(tensor(Local::P)); + let result = report( + "def f(p):\n p.shape\n p.dtype\n p.device\n p.ndim\n p.size()\n p.stride(0)\n p.dim()\n p.ndimension()\n p.storage_offset()\n len(p)\n return p\n", + &contract, + ); + assert!(result.validated(), "{result:?}"); + let result = report( + "import torch\ndef f(p):\n p.stride()\n return torch.relu(p)\n", + &contract, + ); + assert!(!result.validated()); + assert!( + result + .diagnostics + .iter() + .any(|d| d.message.contains("Partial")) + ); + let result = report("def f(p):\n p.values()\n return p\n", &contract); + assert!(result.incomplete && !result.validated()); +} + +#[test] +fn renamed_imports_aliases_and_shadowing_use_reaching_identity() { + let mut contract = contract(); + contract + .arguments + .insert("x".into(), Value::Tensor(tensor(Local::I))); + contract.returns = Some(tensor(Local::R)); + let result = report( + "from spmd_types import invariant_to_replicate as enter\ndef f(x):\n renamed = enter\n return renamed(x, 'tp')\n", + &contract, + ); + assert!(result.validated(), "{result:?}"); + let result = report( + "from spmd_types import invariant_to_replicate as enter\ndef f(x):\n enter = x\n return enter(x, 'tp')\n", + &contract, + ); + assert!(result.incomplete && !result.validated()); + let result = report( + "import torch as t\ndef f(x):\n if False:\n t = x\n return t.relu(x)\n", + &contract, + ); + assert!(result.incomplete && !result.validated(), "{result:?}"); + let result = report( + "from spmd_types import invariant_to_replicate as enter\ndef f(x, enter):\n return enter(x, 'tp')\n", + &contract, + ); + assert!(result.incomplete && !result.validated()); +} + +#[test] +fn no_grad_entry_relaxation_is_not_guessed() { + let source = "import torch\ndef f(x, w):\n return torch.matmul(x, w)\n"; + let mut contract = contract(); + contract + .arguments + .insert("w".into(), Value::Tensor(tensor(Local::V))); + contract.returns = Some(tensor(Local::V)); + for (grad, valid) in [(Grad::No, true), (Grad::Yes, false), (Grad::Unknown, false)] { + contract.arguments.insert( + "x".into(), + Value::Tensor(Tensor::new(2, grad, [("tp", Placement::Local(Local::I))]).unwrap()), + ); + let result = report(source, &contract); + assert_eq!(result.validated(), valid, "{result:?}"); + } +} + +#[test] +fn ambient_mesh_restores_on_normal_and_early_exits() { + let mut contract = contract(); + contract + .arguments + .insert("x".into(), Value::Tensor(tensor(Local::I))); + contract + .arguments + .insert("inner".into(), Value::Mesh(Mesh::new(["dp"]).unwrap())); + contract.arguments.insert( + "dp".into(), + Value::Tensor(Tensor::new(2, Grad::Yes, [("dp", Placement::Local(Local::I))]).unwrap()), + ); + contract.returns = Some(tensor(Local::I)); + let source = "import torch\nimport spmd_types as s\ndef f(x, dp, inner):\n with s.set_current_mesh(inner):\n dp = torch.relu(dp)\n return torch.relu(x)\n"; + let result = report(source, &contract); + assert!(result.validated(), "{result:?}"); + let result = report( + &source.replace("dp = torch.relu(dp)", "torch.relu(x)"), + &contract, + ); + assert!(!result.validated(), "{result:?}"); + // Exercise the public full-state API directly and verify unwind restoration. + let source = "import spmd_types as s\ndef f(x, inner):\n with s.set_current_mesh(inner):\n return x\n"; + let program = program(source); + let empty = BTreeMap::new(); + let domain = Spmd { + exports: &SyntacticExports, + partial_outputs: &empty, + global: false, + }; + let engine = Engine::new(&program, &domain); + let module = engine.analyze(Program::MODULE, Entry::new(State::new(None))); + let mut entry = Entry::new(State::new(contract.mesh.clone())); + entry.args = vec![ + Value::Tensor(tensor(Local::I)), + contract.arguments["inner"].clone(), + ]; + let s = program.global("s").unwrap(); + entry.outer.insert( + s, + module.exit.unwrap().binding(&program, s).unwrap().clone(), + ); + let result = engine.analyze(region(&program, "f"), entry); + let state = result.exit.unwrap().state; + assert_eq!(state.ambient, Ambient::Known(contract.mesh)); + assert_eq!(state.contexts, Some(Vec::new())); +} + +#[test] +fn active_mesh_completeness_is_not_inferred_from_operand_union() { + let mut contract = contract(); + contract.mesh = Some(Mesh::new(["tp", "dp"]).unwrap()); + contract + .arguments + .insert("x".into(), Value::Tensor(tensor(Local::R))); + let result = report( + "import torch\ndef f(x):\n return torch.relu(x)\n", + &contract, + ); + assert!(!result.validated()); + assert!( + result + .diagnostics + .iter() + .any(|d| d.message.contains("missing active mesh axis dp")), + "{result:?}" + ); + let result = report("def f(x):\n x.stride()\n return x\n", &contract); + assert!( + result.validated(), + "metadata does not compute on tensor values: {result:?}" + ); +} + +#[test] +fn branch_ambiguity_does_not_become_a_safe_contract() { + let mut contract = contract(); + contract + .arguments + .insert("x".into(), Value::Tensor(tensor(Local::I))); + contract.arguments.insert("flag".into(), Value::Predicate); + let source = "import spmd_types as s\nimport torch\ndef f(x, flag):\n if flag:\n x = s.invariant_to_replicate(x, 'tp')\n return torch.relu(x)\n"; + let result = report(source, &contract); + assert!(!result.validated()); + assert!( + result + .diagnostics + .iter() + .any(|d| d.message.contains("ambiguous across branches")), + "{result:?}" + ); + let result = report( + "def f(x, flag):\n if flag:\n y = x\n return y\n", + &contract, + ); + assert!(result.incomplete && !result.validated(), "{result:?}"); + let empty = BTreeMap::new(); + let domain = Spmd { + exports: &SyntacticExports, + partial_outputs: &empty, + global: false, + }; + let left = State::new(Some(Mesh::new(["tp"]).unwrap())); + let right = State::new(Some(Mesh::new(["dp"]).unwrap())); + assert_eq!(domain.join_state(&left, &right).ambient, Ambient::Ambiguous); + assert_eq!( + domain.join( + &Value::Tensor(tensor(Local::I)), + &Value::Tensor(tensor(Local::R)) + ), + Value::Ambiguous + ); +} + +#[test] +fn unknown_effects_cannot_be_repaired_by_metadata_or_annotations() { + let mut contract = contract(); + contract + .arguments + .insert("x".into(), Value::Tensor(tensor(Local::I))); + contract.returns = Some(tensor(Local::I)); + for source in [ + "def f(x):\n opaque(x)\n return x\n", + "def f(x):\n x.foo = x\n return x\n", + "import torch\ndef f(x):\n torch.relu(x, inplace=True)\n return x\n", + "def f(x):\n with unknown_context():\n x.stride()\n return x\n", + "import spmd_types as s\ndef f(x):\n x = opaque(x)\n s.assert_type(x, s.I)\n return x\n", + "def f(x):\n try:\n x.stride()\n except Exception:\n pass\n return x\n", + "def f(x):\n missing\n return x\n", + "def f(x):\n if x:\n pass\n return x\n", + ] { + let result = report(source, &contract); + assert!( + result.incomplete && !result.validated(), + "{source}\n{result:?}" + ); + } + let result = report( + "import torch\ndef f(x):\n return torch.relu(*[x])\n", + &contract, + ); + assert!(result.incomplete && !result.validated()); +} + +#[test] +fn collective_src_dst_axis_and_type_only_strictness() { + let mut contract = contract(); + contract + .arguments + .insert("x".into(), Value::Tensor(tensor(Local::P))); + contract.returns = Some(tensor(Local::I)); + let source = + "import spmd_types as s\ndef f(x):\n return s.all_reduce(x, 'tp', src=s.P, dst=s.I)\n"; + assert!(report(source, &contract).validated()); + for wrong in [ + source.replace("src=s.P", "src=s.R"), + source.replace("dst=s.I", "dst=s.V"), + source.replace("'tp'", "'wrong'"), + ] { + assert!(!report(&wrong, &contract).validated()); + } + contract + .arguments + .insert("x".into(), Value::Tensor(tensor(Local::V))); + assert!( + report(source, &contract).validated(), + "runtime reduction permits V reinterpret" + ); + let strict = source.replace("s.all_reduce", "s.rules.all_reduce"); + assert!(!report(&strict, &contract).validated()); + assert!(report(&strict.replace("src=s.P", "src=s.V"), &contract).validated()); + contract.arguments.insert("x".into(), Value::Unknown); + assert!( + !report(source, &contract).validated(), + "src cannot invent an input contract" + ); +} + +#[test] +fn unreachable_effects_are_not_reported_and_loops_terminate() { + let mut contract = contract(); + contract + .arguments + .insert("x".into(), Value::Tensor(tensor(Local::R))); + contract.arguments.insert("flag".into(), Value::Predicate); + contract.returns = Some(tensor(Local::R)); + let result = report( + "import torch\ndef f(x):\n if False:\n opaque(x)\n return torch.relu(x)\n", + &contract, + ); + // This source has no flag parameter; remove the extra entry contract. + assert!(!result.validated()); + contract.arguments.remove("flag"); + assert!( + report( + "import torch\ndef f(x):\n if False:\n opaque(x)\n return torch.relu(x)\n", + &contract + ) + .validated() + ); + contract.arguments.insert("flag".into(), Value::Predicate); + let source = + "import torch\ndef f(x, flag):\n while flag:\n x = torch.relu(x)\n return x\n"; + assert!(report(source, &contract).validated()); + contract.limits = Limits { + max_block_visits: 0, + widen_after: 0, + }; + assert!(report(source, &contract).incomplete); +} + +#[test] +fn sidecars_reject_ambiguous_lines_and_unknown_fields() { + let parsed = program(MLP); + assert!( + sidecar::parse( + &parsed, + &MLP_CONTRACT.replace("\"global\": true", "\"globla\": true") + ) + .is_err() + ); + let mut invalid_line: serde_json::Value = serde_json::from_str(MLP_CONTRACT).unwrap(); + invalid_line["partial_outputs"][0]["line"] = serde_json::json!(1); + assert!(sidecar::parse(&parsed, &invalid_line.to_string()).is_err()); + assert!(sidecar::parse(&parsed, &MLP_CONTRACT.replace("\"S(1)\"", "\"S(100)\"")).is_err()); + let source = MLP.replace("p = t.matmul(h, w2)", "p = t.matmul(t.relu(h), w2)"); + assert!(sidecar::parse(&program(&source), MLP_CONTRACT).is_err()); +} + +#[test] +fn runnable_cli_has_explicit_contract_status_and_nonzero_errors() { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("examples"); + let output = Command::new(env!("CARGO_BIN_EXE_pyrefly_static_analysis_spmd")) + .arg(path.join("tp_mlp.py")) + .arg(path.join("tp_mlp.json")) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + assert!(String::from_utf8_lossy(&output.stdout).contains("contract-validated")); + let output = Command::new(env!("CARGO_BIN_EXE_pyrefly_static_analysis_spmd")) + .output() + .unwrap(); + assert_eq!(output.status.code(), Some(2)); +} + +// Verify the consumer can use a genuine re-export resolver rather than only +// matching the syntactic import spelling. Name is inferred from the trait API. +struct Reexports; +impl ExportProvider for Reexports { + fn resolve(&self, module: ModuleName, name: &pyrefly_static_analysis::Name) -> ExportTarget { + if module.to_string() == "my_ops" && name.as_str() == "entry" { + ExportTarget::Definition( + ModuleName::from_str("spmd_types"), + pyrefly_static_analysis::Name::new("invariant_to_replicate"), + ) + } else { + ExportTarget::Unknown + } + } +} + +#[test] +fn generic_export_provider_controls_function_identity() { + let program = program("from my_ops import entry\ndef f(x):\n return entry(x, 'tp')\n"); + let mut contract = contract(); + contract + .arguments + .insert("x".into(), Value::Tensor(tensor(Local::I))); + contract.returns = Some(tensor(Local::R)); + assert!(check(&program, region(&program, "f"), &contract, &Reexports).validated()); + assert!( + !check( + &program, + region(&program, "f"), + &contract, + &SyntacticExports + ) + .validated() + ); +} diff --git a/static_analysis/spmd/tests/denotations.rs b/static_analysis/spmd/tests/denotations.rs new file mode 100644 index 0000000000..508431e4d5 --- /dev/null +++ b/static_analysis/spmd/tests/denotations.rs @@ -0,0 +1,39 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Pure numerical counterexamples for the supported rank-two contract rule. + +use std::collections::BTreeSet; + +use pyrefly_static_analysis_spmd::Compute; +use pyrefly_static_analysis_spmd::Grad; +use pyrefly_static_analysis_spmd::Local; +use pyrefly_static_analysis_spmd::Placement; +use pyrefly_static_analysis_spmd::Tensor; +use pyrefly_static_analysis_spmd::compute; + +#[test] +fn row_sharded_matmul_partial_sum_reconstructs_full_matmul() { + // x=[2,3], w=[5,7]^T, with one K coordinate on each rank. + let x = [2_i64, 3]; + let w = [5_i64, 7]; + let partials = [x[0] * w[0], x[1] * w[1]]; + let full = x.iter().zip(w).map(|(x, w)| x * w).sum::(); + assert_eq!(partials.iter().sum::(), full); + let x = Tensor::new(2, Grad::Yes, [("tp", Placement::Shard(1))]).unwrap(); + let w = Tensor::new(2, Grad::Yes, [("tp", Placement::Shard(0))]).unwrap(); + let axes = BTreeSet::from([String::from("tp")]); + let descriptor = compute(Compute::Matmul, &[x, w], None, &axes, true).unwrap(); + assert_eq!(descriptor.local()["tp"], Local::P); +} + +#[test] +fn unsupported_output_partial_intent_never_leaves_invalid_partition_metadata() { + let x = Tensor::new(2, Grad::Yes, [("tp", Placement::Shard(1))]).unwrap(); + let axes = BTreeSet::from([String::from("tp")]); + assert!(compute(Compute::Relu, &[x], None, &axes, true).is_err()); +} diff --git a/static_analysis/spmd/tests/descriptors.rs b/static_analysis/spmd/tests/descriptors.rs new file mode 100644 index 0000000000..18f3e22a2a --- /dev/null +++ b/static_analysis/spmd/tests/descriptors.rs @@ -0,0 +1,350 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use std::collections::BTreeSet; + +use pyrefly_static_analysis_spmd::Collective; +use pyrefly_static_analysis_spmd::Compute; +use pyrefly_static_analysis_spmd::Grad; +use pyrefly_static_analysis_spmd::Local; +use pyrefly_static_analysis_spmd::Mesh; +use pyrefly_static_analysis_spmd::Placement; +use pyrefly_static_analysis_spmd::Tensor; +use pyrefly_static_analysis_spmd::compute; +use pyrefly_static_analysis_spmd::transition; + +fn tensor(placement: Placement) -> Tensor { + Tensor::new(2, Grad::Yes, [("tp", placement)]).unwrap() +} + +fn local(local: Local) -> Tensor { + tensor(Placement::Local(local)) +} + +fn apply(op: Compute, values: &[Tensor]) -> Result { + compute(op, values, None, &BTreeSet::new(), false) +} + +#[test] +fn gradient_contracts_are_dual_but_forward_r_and_i_agree() { + for (forward, backward) in [ + (Local::R, Local::P), + (Local::P, Local::R), + (Local::I, Local::I), + (Local::V, Local::V), + ] { + assert_eq!(forward.gradient(), backward); + assert_eq!(backward.gradient(), forward); + } + let replicated = [3.0_f64, 3.0]; + let invariant = [3.0_f64, 3.0]; + assert_eq!(replicated, invariant); + assert_ne!(Local::R.gradient(), Local::I.gradient()); +} + +#[test] +fn local_rules_do_not_invent_partial_contractions() { + assert_eq!( + apply(Compute::Matmul, &[local(Local::V), local(Local::V)]) + .unwrap() + .local()["tp"], + Local::V + ); + assert_eq!( + apply(Compute::Matmul, &[local(Local::R), local(Local::V)]) + .unwrap() + .local()["tp"], + Local::V + ); + assert!( + apply(Compute::Matmul, &[local(Local::I), local(Local::V)]) + .unwrap_err() + .contains("I->R") + ); + assert_eq!( + apply(Compute::Relu, &[local(Local::I)]).unwrap().local()["tp"], + Local::I + ); +} + +#[test] +fn linearity_rejects_affine_and_multilinear_partial_misuse() { + assert_eq!( + apply(Compute::Add, &[local(Local::P), local(Local::P)]) + .unwrap() + .local()["tp"], + Local::P + ); + for op in [Compute::Multiply, Compute::Matmul] { + assert_eq!( + apply(op, &[local(Local::P), local(Local::R)]) + .unwrap() + .local()["tp"], + Local::P + ); + assert!(apply(op, &[local(Local::P), local(Local::P)]).is_err()); + assert!(apply(op, &[local(Local::P), local(Local::V)]).is_err()); + } + assert!(apply(Compute::Add, &[local(Local::P), local(Local::R)]).is_err()); + assert!(apply(Compute::Relu, &[local(Local::P)]).is_err()); + // Concrete denotations expose the missing cross terms and nonlinear error. + let a = [1.0_f64, 1.0]; + let b = [1.0_f64, 1.0]; + assert_eq!( + a.iter().zip(b).map(|(a, b)| a + b).sum::(), + a.iter().sum::() + b.iter().sum::() + ); + assert_ne!( + a.iter().zip(b).map(|(a, b)| a * b).sum::(), + a.iter().sum::() * b.iter().sum::() + ); + let p = [-2.0_f64, 3.0]; + assert_ne!( + p.iter().map(|x| x.max(0.0)).sum::(), + p.iter().sum::().max(0.0) + ); +} + +#[test] +fn no_grad_relaxation_is_explicit_and_does_not_fabricate_it() { + let invariant = Tensor::new(2, Grad::No, [("tp", Placement::Local(Local::I))]).unwrap(); + assert_eq!( + apply(Compute::Matmul, &[invariant, local(Local::V)]) + .unwrap() + .local()["tp"], + Local::V + ); + let invariant = Tensor::new(2, Grad::Unknown, [("tp", Placement::Local(Local::I))]).unwrap(); + assert!(apply(Compute::Matmul, &[invariant, local(Local::V)]).is_err()); + let replicate = Tensor::new(2, Grad::No, [("tp", Placement::Local(Local::R))]).unwrap(); + assert_eq!( + apply(Compute::Multiply, &[local(Local::I), replicate]) + .unwrap() + .local()["tp"], + Local::I + ); +} + +#[test] +fn global_matmul_preserves_free_shards_and_requires_partial_intent() { + let empty = BTreeSet::new(); + let output = compute( + Compute::Matmul, + &[local(Local::R), tensor(Placement::Shard(1))], + None, + &empty, + true, + ) + .unwrap(); + assert_eq!(output.partition(), &[vec![], vec![String::from("tp")]]); + let inputs = [tensor(Placement::Shard(1)), tensor(Placement::Shard(0))]; + assert!( + compute(Compute::Matmul, &inputs, None, &empty, true) + .unwrap_err() + .contains("explicit Partial") + ); + let partial = BTreeSet::from([String::from("tp")]); + let output = compute(Compute::Matmul, &inputs, None, &partial, true).unwrap(); + assert_eq!(output.local()["tp"], Local::P); + assert!(output.partition().iter().all(Vec::is_empty)); + assert!( + compute( + Compute::Matmul, + &[tensor(Placement::Shard(0)), tensor(Placement::Shard(1))], + None, + &empty, + true + ) + .is_err() + ); + assert!( + compute( + Compute::Matmul, + &[local(Local::R), tensor(Placement::Shard(1))], + None, + &partial, + true + ) + .is_err() + ); +} + +#[test] +fn partition_and_dimension_invariants_are_checked() { + assert!(Tensor::new(0, Grad::No, [("tp", Placement::Shard(0))]).is_err()); + assert!(Tensor::new(2, Grad::Yes, [("tp", Placement::Shard(-3))]).is_err()); + assert_eq!(tensor(Placement::Shard(-1)), tensor(Placement::Shard(1))); + assert!( + local(Local::R) + .with_partition(vec![vec!["tp".into()], vec![]]) + .is_err() + ); + assert!( + tensor(Placement::Shard(0)) + .with_partition(vec![vec!["tp".into()], vec!["tp".into()]]) + .is_err() + ); + assert!(Mesh::new(["tp", "tp"]).is_err()); +} + +#[test] +fn collective_source_and_destination_are_not_assumptions() { + let src = Placement::Local(Local::P); + let dst = Placement::Local(Local::I); + assert_eq!( + transition( + &local(Local::P), + Collective::AllReduce, + "tp", + src, + dst, + false + ) + .unwrap() + .local()["tp"], + Local::I + ); + assert!( + transition( + &local(Local::R), + Collective::AllReduce, + "tp", + src, + dst, + false + ) + .is_err() + ); + assert!( + transition( + &local(Local::P), + Collective::AllReduce, + "wrong", + src, + dst, + false + ) + .is_err() + ); + assert!( + transition( + &local(Local::P), + Collective::AllReduce, + "tp", + src, + Placement::Local(Local::V), + false + ) + .is_err() + ); + assert!( + transition( + &tensor(Placement::Shard(0)), + Collective::AllGather, + "tp", + Placement::Shard(1), + Placement::Local(Local::R), + true + ) + .is_err() + ); +} + +#[test] +fn stack_unbind_and_innermost_shard_collectives() { + let output = transition( + &local(Local::V), + Collective::AllGather, + "tp", + Placement::Local(Local::V), + Placement::Local(Local::R), + false, + ) + .unwrap(); + assert_eq!(output.rank(), 3); + let output = transition( + &local(Local::P), + Collective::ReduceScatter, + "tp", + Placement::Local(Local::P), + Placement::Local(Local::V), + false, + ) + .unwrap(); + assert_eq!(output.rank(), 1); + let scalar = Tensor::new(0, Grad::Yes, [("tp", Placement::Local(Local::P))]).unwrap(); + assert!( + transition( + &scalar, + Collective::ReduceScatter, + "tp", + Placement::Local(Local::P), + Placement::Local(Local::V), + false + ) + .is_err() + ); + let input = Tensor::new( + 2, + Grad::Yes, + [ + ("dp", Placement::Local(Local::V)), + ("tp", Placement::Local(Local::V)), + ], + ) + .unwrap() + .with_partition(vec![vec!["dp".into(), "tp".into()], vec![]]) + .unwrap(); + assert!( + transition( + &input, + Collective::AllGather, + "dp", + Placement::Shard(0), + Placement::Local(Local::R), + true + ) + .is_err() + ); + let output = transition( + &input, + Collective::AllGather, + "tp", + Placement::Shard(0), + Placement::Local(Local::R), + true, + ) + .unwrap(); + assert_eq!(output.partition()[0], vec![String::from("dp")]); + assert!( + transition( + &tensor(Placement::Shard(0)), + Collective::Convert, + "tp", + Placement::Shard(0), + Placement::Shard(1), + true + ) + .is_err() + ); +} + +#[test] +fn active_mesh_requires_every_operand_axis() { + let mesh = Mesh::new(["dp", "tp"]).unwrap(); + assert!( + compute( + Compute::Relu, + &[local(Local::P)], + Some(&mesh), + &BTreeSet::new(), + false + ) + .unwrap_err() + .contains("missing active") + ); +} diff --git a/static_analysis/spmd/tests/neutral_contracts.rs b/static_analysis/spmd/tests/neutral_contracts.rs new file mode 100644 index 0000000000..90885b2f6e --- /dev/null +++ b/static_analysis/spmd/tests/neutral_contracts.rs @@ -0,0 +1,72 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use pyrefly_static_analysis::ModuleName; +use pyrefly_static_analysis::RegionId; +use pyrefly_static_analysis::SyntacticExports; +use pyrefly_static_analysis::analyze_python; +use pyrefly_static_analysis_spmd::Contract; +use pyrefly_static_analysis_spmd::Grad; +use pyrefly_static_analysis_spmd::Local; +use pyrefly_static_analysis_spmd::Mesh; +use pyrefly_static_analysis_spmd::Placement; +use pyrefly_static_analysis_spmd::Tensor; +use pyrefly_static_analysis_spmd::Value; +use pyrefly_static_analysis_spmd::check; + +#[test] +fn qualified_shard_constructor_and_negative_dimension_use_neutral_operations() { + let source = "from spmd_types import S as shard, all_gather, R\ndef f(x):\n return all_gather(x, 'tp', src=shard(-1), dst=R)\n"; + let mut contract = Contract { + mesh: Some(Mesh::new(["tp"]).unwrap()), + global: true, + ..Contract::default() + }; + contract.arguments.insert( + "x".into(), + Value::Tensor(Tensor::new(2, Grad::Yes, [("tp", Placement::Shard(1))]).unwrap()), + ); + contract.returns = + Some(Tensor::new(2, Grad::Yes, [("tp", Placement::Local(Local::R))]).unwrap()); + for (source, valid) in [ + (source.into(), true), + (source.replace("shard(-1)", "shard(0)"), false), + ] { + let program = analyze_python(&source, ModuleName::from_str("fixture"), false); + let region = RegionId( + program + .regions + .iter() + .position(|region| region.name.as_str() == "f") + .unwrap() as u32, + ); + let result = check(&program, region, &contract, &SyntacticExports); + assert_eq!(result.validated(), valid, "{result:?}"); + } +} + +#[test] +fn variadic_entry_cannot_fabricate_tensor_parameter_shapes() { + let mut contract = Contract::default(); + let tensor = Tensor::new(2, Grad::Yes, [("tp", Placement::Local(Local::I))]).unwrap(); + contract + .arguments + .insert("x".into(), Value::Tensor(tensor.clone())); + contract.returns = Some(tensor); + for source in ["def f(*x):\n return x\n", "def f(**x):\n return x\n"] { + let program = analyze_python(source, ModuleName::from_str("fixture"), false); + let region = RegionId( + program + .regions + .iter() + .position(|region| region.name.as_str() == "f") + .unwrap() as u32, + ); + let result = check(&program, region, &contract, &SyntacticExports); + assert!(result.incomplete && !result.validated(), "{result:?}"); + } +} diff --git a/static_analysis/spmd/tests/output_invariants.rs b/static_analysis/spmd/tests/output_invariants.rs new file mode 100644 index 0000000000..77efb49ceb --- /dev/null +++ b/static_analysis/spmd/tests/output_invariants.rs @@ -0,0 +1,253 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use std::collections::BTreeSet; + +use pyrefly_static_analysis::ModuleName; +use pyrefly_static_analysis::RegionId; +use pyrefly_static_analysis::SyntacticExports; +use pyrefly_static_analysis::analyze_python; +use pyrefly_static_analysis_spmd::Collective; +use pyrefly_static_analysis_spmd::Compute; +use pyrefly_static_analysis_spmd::Contract; +use pyrefly_static_analysis_spmd::Grad; +use pyrefly_static_analysis_spmd::Local; +use pyrefly_static_analysis_spmd::Mesh; +use pyrefly_static_analysis_spmd::Placement; +use pyrefly_static_analysis_spmd::Tensor; +use pyrefly_static_analysis_spmd::Value; +use pyrefly_static_analysis_spmd::check; +use pyrefly_static_analysis_spmd::compute; +use pyrefly_static_analysis_spmd::transition; + +fn pointwise(inputs: &[Tensor]) -> Result { + compute( + Compute::Add, + inputs, + Some(&Mesh::new(["dp", "tp"]).unwrap()), + &BTreeSet::new(), + true, + ) +} + +fn ordered(order: &[&str]) -> Tensor { + Tensor::new( + 2, + Grad::Yes, + [ + ("dp", Placement::Local(Local::V)), + ("tp", Placement::Local(Local::V)), + ], + ) + .unwrap() + .with_partition(vec![ + order.iter().map(|axis| String::from(*axis)).collect(), + vec![], + ]) + .unwrap() +} + +fn collisions() -> [Tensor; 2] { + [ + Tensor::new( + 2, + Grad::Yes, + [ + ("dp", Placement::Local(Local::R)), + ("tp", Placement::Shard(0)), + ], + ) + .unwrap(), + Tensor::new( + 2, + Grad::Yes, + [ + ("dp", Placement::Shard(0)), + ("tp", Placement::Local(Local::R)), + ], + ) + .unwrap(), + ] +} + +fn contract(inputs: [Tensor; 2]) -> Contract { + let mut contract = Contract { + mesh: Some(Mesh::new(["dp", "tp"]).unwrap()), + global: true, + // No expected return contract: representation validity is mandatory itself. + ..Contract::default() + }; + let [x, y] = inputs; + contract.arguments.insert("x".into(), Value::Tensor(x)); + contract.arguments.insert("y".into(), Value::Tensor(y)); + contract +} + +#[test] +fn coincident_pointwise_shards_need_a_shared_order() { + for op in [Compute::Add, Compute::Multiply] { + let result = compute(op, &collisions(), None, &BTreeSet::new(), true); + assert!(result.unwrap_err().contains("coincident output shards")); + } + assert!(pointwise(&[ordered(&["dp", "tp"]), ordered(&["tp", "dp"])]).is_err()); +} + +#[test] +fn consistent_ordered_pointwise_shards_are_preserved() { + let input = ordered(&["dp", "tp"]); + let replicated = Tensor::new( + 2, + Grad::Yes, + [ + ("dp", Placement::Local(Local::R)), + ("tp", Placement::Local(Local::R)), + ], + ) + .unwrap(); + for second in [input.clone(), replicated] { + let output = pointwise(&[input.clone(), second]).unwrap(); + output.validate(true).unwrap(); + assert_eq!(output.partition(), input.partition()); + assert_eq!(output.local()["dp"], Local::V); + assert_eq!(output.local()["tp"], Local::V); + } + let distinct_dims = Tensor::new( + 2, + Grad::Yes, + [("dp", Placement::Shard(0)), ("tp", Placement::Shard(1))], + ) + .unwrap(); + assert_eq!( + pointwise(&[distinct_dims.clone(), distinct_dims.clone()]).unwrap(), + distinct_dims + ); +} + +#[test] +fn consumer_rejects_output_collision_without_expected_return() { + let program = analyze_python( + "import torch\ndef f(x, y):\n return torch.add(x, y)\n", + ModuleName::from_str("fixture"), + false, + ); + let region = RegionId( + program + .regions + .iter() + .position(|region| region.name.as_str() == "f") + .unwrap() as u32, + ); + let invalid = contract(collisions()); + assert!(invalid.returns.is_none()); + let result = check(&program, region, &invalid, &SyntacticExports); + assert!(!result.validated(), "{result:?}"); + assert!( + result + .diagnostics + .iter() + .any(|diagnostic| diagnostic.message.contains("coincident output shards")), + "{result:?}" + ); + let valid = contract([ordered(&["dp", "tp"]), ordered(&["dp", "tp"])]); + assert!(valid.returns.is_none()); + let result = check(&program, region, &valid, &SyntacticExports); + assert!(result.validated(), "{result:?}"); + let Some(Value::Tensor(output)) = result.returned else { + panic!("expected a tensor output") + }; + output.validate(true).unwrap(); + assert_eq!( + output.partition()[0], + vec![String::from("dp"), String::from("tp")] + ); +} + +#[test] +fn discarded_invalid_tensor_result_is_still_rejected() { + let program = analyze_python( + "import torch\ndef f(x, y):\n torch.add(x, y)\n return x\n", + ModuleName::from_str("fixture"), + false, + ); + let region = RegionId( + program + .regions + .iter() + .position(|region| region.name.as_str() == "f") + .unwrap() as u32, + ); + let contract = contract(collisions()); + let result = check(&program, region, &contract, &SyntacticExports); + assert_eq!(result.returned, Some(contract.arguments["x"].clone())); + assert!(!result.validated(), "{result:?}"); + assert!( + result + .diagnostics + .iter() + .any(|diagnostic| diagnostic.message.contains("coincident output shards")), + "{result:?}" + ); +} + +#[test] +fn output_tensor_invariants_are_checked_without_an_expected_contract() { + let varying = Tensor::new( + 2, + Grad::Yes, + [ + ("dp", Placement::Local(Local::V)), + ("tp", Placement::Local(Local::R)), + ], + ) + .unwrap(); + assert!(varying.validate(false).is_ok()); + assert!( + varying + .validate(true) + .unwrap_err() + .contains("missing from PartitionSpec") + ); + let input = Tensor::new(32, Grad::Yes, [("tp", Placement::Local(Local::V))]).unwrap(); + assert!( + transition( + &input, + Collective::AllGather, + "tp", + Placement::Local(Local::V), + Placement::Local(Local::R), + false + ) + .is_err() + ); + // Returning a trusted local-V descriptor does not certify global reconstruction, + // including when that descriptor is nested inside a returned collection. + for returned in ["x", "[x]"] { + let source = format!("def f(x, y):\n return {returned}\n"); + let program = analyze_python(&source, ModuleName::from_str("fixture"), false); + let region = RegionId( + program + .regions + .iter() + .position(|region| region.name.as_str() == "f") + .unwrap() as u32, + ); + let result = check( + &program, + region, + &contract([varying.clone(), varying.clone()]), + &SyntacticExports, + ); + assert!(!result.validated(), "{result:?}"); + assert!( + result + .diagnostics + .iter() + .any(|diagnostic| diagnostic.message.contains("missing from PartitionSpec")), + "{result:?}" + ); + } +} diff --git a/static_analysis/src/declarations.rs b/static_analysis/src/declarations.rs new file mode 100644 index 0000000000..153a851075 --- /dev/null +++ b/static_analysis/src/declarations.rs @@ -0,0 +1,597 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! The declarations of one lexical scope, as a stream of events. +//! +//! [`walk_suite`] and [`walk_child`] decide which syntax belongs to the scope being walked +//! and report every name the scope binds or declares, every name expression it contains, +//! and every nested scope at the point where it is created. A [`DeclarationSink`] interprets the +//! events; it never walks statements or expressions itself. +//! +//! Statements are reported in source order. Within a statement, the walker owns the +//! order and the structural rules that do not depend on the consumer: +//! +//! - the suites of `if`, `while`, `for`, `with`, `try` and `match` belong to the scope, +//! while function, lambda, class and comprehension bodies are nested scopes; +//! - the names of every `except` clause are declared before the `try` body, and the +//! captures of every `case` before the case bodies; +//! - an assigned value is visited before its targets; the names an assignment target +//! binds are declared before the expressions inside it are visited; a deleted target is +//! visited before its names are declared; a walrus target is declared before its value +//! is visited; +//! - every name expression is reported with its [`NameSite`], in any context. Names that +//! are identifiers rather than expressions (`def`, `class`, import, `except`, capture, +//! `global`, `nonlocal` and parameter names) are reported only as declarations. Name +//! occurrences are syntactic: they are not the runtime's reads, which lowering models; +//! - a walrus binds in the nearest enclosing scope that is not a comprehension (PEP 572), +//! but not past a lambda body; +//! - decorators, parameter defaults, class arguments and a comprehension's first iterable +//! are evaluated in the enclosing scope; +//! - annotations, type parameters and `type` alias values are never visited: they are +//! evaluated lazily in annotation scopes, which the walker does not model. +//! +//! The consumer owns policy: [`Position`] lets it leave out expression positions, and +//! [`DeclarationSink::static_truth`] lets it prune `if` arms and conditional expressions +//! (a statically false arm is skipped; a statically true arm or an `else` ends the chain). +//! A sink that keeps every role and answers `None` sees what the runtime binds. + +use pyrefly_python::ast::Ast; +use pyrefly_util::visit::Visit; +use ruff_python_ast::Alias; +use ruff_python_ast::Comprehension; +use ruff_python_ast::ExceptHandler; +use ruff_python_ast::Expr; +use ruff_python_ast::ExprIf; +use ruff_python_ast::ExprLambda; +use ruff_python_ast::ExprName; +use ruff_python_ast::Identifier; +use ruff_python_ast::Parameters; +use ruff_python_ast::Stmt; +use ruff_python_ast::StmtAnnAssign; +use ruff_python_ast::StmtAssign; +use ruff_python_ast::StmtAugAssign; +use ruff_python_ast::StmtClassDef; +use ruff_python_ast::StmtFunctionDef; +use ruff_python_ast::StmtImportFrom; +use ruff_text_size::TextRange; + +/// A site that binds or declares a name in the scope being walked. +#[derive(Debug, Clone, Copy)] +pub enum Declaration<'a> { + /// `import a.b` binds `a`; `import a.b as c` binds `c`. + Import(&'a Alias), + /// `from m import x, y as z`, or `from m import *`, which binds names the walker + /// cannot know. + ImportFrom(&'a StmtImportFrom), + Function(&'a StmtFunctionDef), + Class(&'a StmtClassDef), + /// A name that is an entire target of an assignment, `x = value`. + Assign(&'a ExprName, &'a StmtAssign), + /// The name target of an augmented assignment, `x += value`. + AugAssign(&'a ExprName, &'a StmtAugAssign), + /// The name target of an annotated assignment, with or without a value. + Annotated(&'a ExprName, &'a StmtAnnAssign), + /// Any other stored name: part of an assignment target, a `for`, `with` or + /// comprehension target, or a `type` alias. + Store(&'a ExprName), + Delete(&'a ExprName), + /// A walrus target, reported to the scope it binds in. + Walrus(&'a ExprName), + /// The name of an `except` clause. + Handler(&'a Identifier), + /// A capture in a `case` pattern. + Capture(&'a Identifier), + Global(&'a Identifier), + Nonlocal(&'a Identifier), + Parameter(&'a Identifier), +} + +/// An expression position whose inclusion is the sink's choice. Leaving a position out +/// skips everything inside it: its reads, its walrus targets and its nested scopes. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Position { + /// The default values of `def` or `lambda` parameters. + ParameterDefault, + /// The bases and keywords of a `class` statement. + ClassArguments, + /// The test of an `elif` clause. The first test of an `if` is always visited. + ElifTest, + /// The guard of a `case` clause. + MatchGuard, + /// The exception type of an `except` clause. + ExceptType, + /// A target of an augmented assignment that is not a name, such as `a[i] += 1`. + AugmentedTarget, +} + +/// A region of the walk that a sink may want to track. Contexts nest, and every +/// `enter` is followed by an `exit` of the same context. +#[derive(Debug, Clone, Copy)] +pub enum Context<'a> { + /// The part of a comprehension that the enclosing scope evaluates: the first iterable, + /// and the walrus targets the comprehension binds in the enclosing scope. + Comprehension, + /// A live arm of an `if` statement: its test, and its static truth. An `else` arm has + /// no test and is statically true. + IfArm { + test: Option<&'a Expr>, + truth: Option, + }, +} + +/// Where a name expression occurs. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NameSite { + /// An expression the walk visits, including assignment, `for`, `with`, `del` and + /// comprehension targets and the name of a `type` alias. + Expression, + /// The name target of an annotated assignment. + AnnAssignTarget, + /// The name target of an augmented assignment. + AugAssignTarget, + /// A walrus target, reported to the scope it binds in. + WalrusTarget, + /// A name in a value or class pattern of a `case`, such as `Color` in `Color.RED`. + Pattern, +} + +/// Where a test whose static truth is asked about appears. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TestSite { + If, + /// The test of a conditional expression, `a if test else b`. + Conditional, +} + +/// A scope nested inside the scope being walked. +#[derive(Debug, Clone, Copy)] +pub enum ChildScope<'a> { + Function(&'a StmtFunctionDef), + Lambda(&'a ExprLambda), + Class(&'a StmtClassDef), + Comprehension { + range: TextRange, + generators: &'a [Comprehension], + /// The key of a dict comprehension. + key: Option<&'a Expr>, + /// The element, or the value of a dict comprehension. + elt: &'a Expr, + /// Generator expressions run deferred, other comprehensions inline. + is_generator: bool, + }, +} + +/// The consumer of a scope's events. Only `declare` is required. +pub trait DeclarationSink<'a> { + fn declare(&mut self, declaration: Declaration<'a>); + + /// A name expression of the scope. + fn name(&mut self, _name: &'a ExprName, _site: NameSite) {} + + /// Whether to visit an expression in this position. + fn keeps(&mut self, _position: Position) -> bool { + true + } + + fn enter(&mut self, _context: Context<'a>) {} + + fn exit(&mut self, _context: Context<'a>) {} + + fn static_truth(&mut self, _test: &'a Expr, _site: TestSite) -> Option { + None + } + + /// A statement of the scope, after all of its events, including its nested suites. + fn statement(&mut self, _stmt: &'a Stmt) {} + + /// A nested scope, at the point where the enclosing scope creates it. The sink walks + /// it with [`walk_child`] if it wants its contents. + fn child(&mut self, _child: ChildScope<'a>) {} +} + +/// Walk a module, class or function body. +pub fn walk_suite<'a>(body: &'a [Stmt], sink: &mut impl DeclarationSink<'a>) { + Walker { + sink, + in_comprehension: false, + } + .suite(body) +} + +/// Walk the scope a [`ChildScope`] introduces: parameters and body, or the parts of a +/// comprehension that its own scope evaluates. A walrus inside a comprehension is not +/// reported here: the enclosing walk reported it to the scope it binds in. +pub fn walk_child<'a>(child: ChildScope<'a>, sink: &mut impl DeclarationSink<'a>) { + let mut walker = Walker { + sink, + in_comprehension: false, + }; + match child { + ChildScope::Function(x) => { + walker.parameters(&x.parameters); + walker.suite(&x.body); + } + ChildScope::Lambda(x) => { + if let Some(parameters) = &x.parameters { + walker.parameters(parameters); + } + walker.expr(&x.body); + } + ChildScope::Class(x) => walker.suite(&x.body), + ChildScope::Comprehension { + generators, + key, + elt, + .. + } => { + walker.in_comprehension = true; + for (i, g) in generators.iter().enumerate() { + if i > 0 { + walker.expr(&g.iter); + } + walker.target(&g.target); + for cond in &g.ifs { + walker.expr(cond); + } + } + if let Some(key) = key { + walker.expr(key); + } + walker.expr(elt); + } + } +} + +struct Walker<'s, S> { + sink: &'s mut S, + /// Whether the walked scope is a comprehension, whose walrus targets bind elsewhere. + in_comprehension: bool, +} + +impl<'a, S: DeclarationSink<'a>> Walker<'_, S> { + fn suite(&mut self, body: &'a [Stmt]) { + for x in body { + self.stmt(x); + } + } + + fn declare(&mut self, declaration: Declaration<'a>) { + self.sink.declare(declaration) + } + + /// Run `f` if the sink keeps `position`. + fn at(&mut self, position: Position, f: impl FnOnce(&mut Self)) { + if self.sink.keeps(position) { + f(self); + } + } + + fn within(&mut self, context: Context<'a>, f: impl FnOnce(&mut Self)) { + self.sink.enter(context); + f(self); + self.sink.exit(context); + } + + /// Declare the names an assignment target binds, then visit it. + fn target(&mut self, x: &'a Expr) { + Ast::expr_lvalue(x, &mut |name| self.declare(Declaration::Store(name))); + self.expr(x); + } + + fn parameters(&mut self, x: &'a Parameters) { + for p in x.iter() { + self.declare(Declaration::Parameter(p.name())); + } + } + + fn defaults(&mut self, x: &'a Parameters, mut visit: impl FnMut(&mut Self, &'a Expr)) { + self.at(Position::ParameterDefault, |w| { + for p in x.iter_non_variadic_params() { + if let Some(default) = &p.default { + visit(w, default); + } + } + }); + } + + fn stmt(&mut self, x: &'a Stmt) { + match x { + Stmt::FunctionDef(f) => { + for d in &f.decorator_list { + self.expr(&d.expression); + } + self.defaults(&f.parameters, Self::expr); + self.declare(Declaration::Function(f)); + self.sink.child(ChildScope::Function(f)); + } + Stmt::ClassDef(c) => { + for d in &c.decorator_list { + self.expr(&d.expression); + } + if let Some(arguments) = &c.arguments { + self.at(Position::ClassArguments, |w| { + arguments.recurse(&mut |e| w.expr(e)) + }); + } + self.declare(Declaration::Class(c)); + self.sink.child(ChildScope::Class(c)); + } + Stmt::Import(x) => { + for a in &x.names { + self.declare(Declaration::Import(a)); + } + } + Stmt::ImportFrom(x) => self.declare(Declaration::ImportFrom(x)), + Stmt::Global(x) => { + for n in &x.names { + self.declare(Declaration::Global(n)); + } + } + Stmt::Nonlocal(x) => { + for n in &x.names { + self.declare(Declaration::Nonlocal(n)); + } + } + Stmt::Assign(assign) => { + self.expr(&assign.value); + for t in &assign.targets { + match t { + Expr::Name(name) => { + self.declare(Declaration::Assign(name, assign)); + self.sink.name(name, NameSite::Expression); + } + t => self.target(t), + } + } + } + Stmt::AnnAssign(x) => { + if let Some(value) = &x.value { + self.expr(value); + } + match &*x.target { + Expr::Name(name) => { + self.declare(Declaration::Annotated(name, x)); + self.sink.name(name, NameSite::AnnAssignTarget); + } + target => self.target(target), + } + } + Stmt::AugAssign(x) => { + self.expr(&x.value); + match &*x.target { + Expr::Name(name) => { + self.declare(Declaration::AugAssign(name, x)); + self.sink.name(name, NameSite::AugAssignTarget); + } + target => self.at(Position::AugmentedTarget, |w| w.expr(target)), + } + } + Stmt::Delete(x) => { + for t in &x.targets { + self.expr(t); + Ast::expr_lvalue(t, &mut |name| self.declare(Declaration::Delete(name))); + } + } + Stmt::TypeAlias(x) => { + if let Expr::Name(name) = &*x.name { + self.declare(Declaration::Store(name)); + self.sink.name(name, NameSite::Expression); + } + } + Stmt::Expr(x) => self.expr(&x.value), + Stmt::Return(x) => { + if let Some(value) = &x.value { + self.expr(value); + } + } + Stmt::Raise(x) => { + if let Some(exc) = &x.exc { + self.expr(exc); + } + if let Some(cause) = &x.cause { + self.expr(cause); + } + } + Stmt::Assert(x) => { + self.expr(&x.test); + if let Some(msg) = &x.msg { + self.expr(msg); + } + } + Stmt::For(x) => { + self.expr(&x.iter); + self.target(&x.target); + self.suite(&x.body); + self.suite(&x.orelse); + } + Stmt::While(x) => { + self.expr(&x.test); + self.suite(&x.body); + self.suite(&x.orelse); + } + Stmt::With(x) => { + for item in &x.items { + self.expr(&item.context_expr); + if let Some(target) = &item.optional_vars { + self.target(target); + } + } + self.suite(&x.body); + } + Stmt::Match(x) => { + self.expr(&x.subject); + for case in &x.cases { + Ast::pattern_lvalue(&case.pattern, &mut |id| { + self.declare(Declaration::Capture(id)) + }); + } + for case in &x.cases { + case.pattern.recurse(&mut |e| self.pattern_names(e)); + if let Some(guard) = &case.guard { + self.at(Position::MatchGuard, |w| w.expr(guard)); + } + self.suite(&case.body); + } + } + Stmt::Try(x) => { + for ExceptHandler::ExceptHandler(h) in &x.handlers { + if let Some(name) = &h.name { + self.declare(Declaration::Handler(name)); + } + } + self.suite(&x.body); + for ExceptHandler::ExceptHandler(h) in &x.handlers { + if let Some(type_) = &h.type_ { + self.at(Position::ExceptType, |w| w.expr(type_)); + } + self.suite(&h.body); + } + self.suite(&x.orelse); + self.suite(&x.finalbody); + } + Stmt::If(x) => { + self.expr(&x.test); + for (i, (test, body)) in Ast::if_branches(x).enumerate() { + let truth = match test { + None => Some(true), + Some(test) => { + if i > 0 { + self.at(Position::ElifTest, |w| w.expr(test)); + } + self.sink.static_truth(test, TestSite::If) + } + }; + if truth == Some(false) { + continue; + } + self.within(Context::IfArm { test, truth }, |w| w.suite(body)); + if truth == Some(true) { + break; + } + } + } + Stmt::Pass(_) | Stmt::Break(_) | Stmt::Continue(_) | Stmt::IpyEscapeCommand(_) => {} + } + self.sink.statement(x); + } + + /// Visit an expression the walked scope evaluates. + fn expr(&mut self, x: &'a Expr) { + match x { + Expr::Name(name) => self.sink.name(name, NameSite::Expression), + Expr::Named(x) => { + self.walrus(&x.target); + self.expr(&x.value); + } + Expr::Lambda(lambda) => { + if let Some(parameters) = &lambda.parameters { + self.defaults(parameters, Self::expr); + } + self.sink.child(ChildScope::Lambda(lambda)); + } + Expr::If(x) => { + self.expr(&x.test); + self.conditional(x, Self::expr); + } + Expr::ListComp(c) => self.comprehension(c.range, &c.generators, None, &c.elt, false), + Expr::SetComp(c) => self.comprehension(c.range, &c.generators, None, &c.elt, false), + Expr::Generator(c) => self.comprehension(c.range, &c.generators, None, &c.elt, true), + Expr::DictComp(c) => { + self.comprehension(c.range, &c.generators, c.key.as_deref(), &c.value, false) + } + _ => x.recurse(&mut |x| self.expr(x)), + } + } + + /// Declare a walrus target, unless the walked scope is a comprehension, where the + /// enclosing walk declared it. + fn walrus(&mut self, target: &'a Expr) { + if !self.in_comprehension { + Ast::expr_lvalue(target, &mut |name| { + self.declare(Declaration::Walrus(name)); + self.sink.name(name, NameSite::WalrusTarget); + }); + } + } + + /// Report the names in an expression of a `case` pattern, which can only be a dotted + /// name, a literal or a mapping key. + fn pattern_names(&mut self, x: &'a Expr) { + match x { + Expr::Name(name) => self.sink.name(name, NameSite::Pattern), + _ => x.recurse(&mut |x| self.pattern_names(x)), + } + } + + /// Visit the branches of `a if test else b` that the sink does not rule out. + fn conditional(&mut self, x: &'a ExprIf, mut visit: impl FnMut(&mut Self, &'a Expr)) { + let truth = self.sink.static_truth(&x.test, TestSite::Conditional); + if truth != Some(false) { + visit(self, &x.body); + } + if truth != Some(true) { + visit(self, &x.orelse); + } + } + + /// The enclosing scope's share of a comprehension: the walrus targets in it and the + /// first iterable, in source order, then the comprehension's own scope. + fn comprehension( + &mut self, + range: TextRange, + generators: &'a [Comprehension], + key: Option<&'a Expr>, + elt: &'a Expr, + is_generator: bool, + ) { + self.within(Context::Comprehension, |w| { + if let Some(key) = key { + w.hoist(key); + } + w.hoist(elt); + for (i, g) in generators.iter().enumerate() { + if i == 0 { + w.expr(&g.iter); + } else { + w.hoist(&g.iter); + } + for cond in &g.ifs { + w.hoist(cond); + } + } + }); + self.sink.child(ChildScope::Comprehension { + range, + generators, + key, + elt, + is_generator, + }); + } + + /// Find the walrus targets in an expression a nested comprehension evaluates, which + /// bind in the walked scope. Everything else there belongs to the comprehension. + fn hoist(&mut self, x: &'a Expr) { + match x { + Expr::Named(x) => { + self.walrus(&x.target); + self.hoist(&x.value); + } + // A lambda's defaults run in the comprehension; its body is another scope. + Expr::Lambda(lambda) => { + if let Some(parameters) = &lambda.parameters { + self.defaults(parameters, Self::hoist); + } + } + Expr::If(x) => { + self.hoist(&x.test); + self.conditional(x, Self::hoist); + } + _ => x.recurse(&mut |x| self.hoist(x)), + } + } +} diff --git a/static_analysis/src/engine.rs b/static_analysis/src/engine.rs new file mode 100644 index 0000000000..dc527b89d6 --- /dev/null +++ b/static_analysis/src/engine.rs @@ -0,0 +1,810 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! A generic forward dataflow engine over one region of a [`Program`]. +//! +//! The engine owns the shared bookkeeping: a [`Frame`] holding the value of every symbol +//! the region owns, a cell for every outer symbol (closure variable, global or builtin) +//! the region has seen, the value of every temporary, and an arbitrary domain-owned state +//! (ambient context, typestate, heap summaries, ...). A [`Domain`] supplies the lattice +//! and a transfer function per operation; it never walks syntax. + +use std::collections::BTreeMap; +use std::collections::BTreeSet; +use std::fmt; +use std::fmt::Debug; + +use ruff_text_size::TextRange; +use ruff_text_size::TextSize; +use starlark_map::small_map::SmallMap; + +use crate::ir::Access; +use crate::ir::BlockId; +use crate::ir::GapKind; +use crate::ir::OpId; +use crate::ir::OpKind; +use crate::ir::Program; +use crate::ir::RaiseCause; +use crate::ir::Reg; +use crate::ir::RegionId; +use crate::ir::SymbolId; +use crate::ir::Terminator; + +/// An abstract interpretation of Python operations. +/// +/// The engine calls [`Domain::transfer`] for every reachable operation with the frame as +/// it was before the operation. Afterwards, any result temporary the domain did not set +/// receives a default value, and a name `Store`/`Delete` the domain did not override +/// updates the frame: +/// +/// - `Name` load: see [`load`]: the symbol's slot (owned symbols, falling back as +/// CPython does while unbound) or cell (other symbols, `top` if absent). +/// - `Param`: the value in [`Entry::args`], else `top`. +/// - `Name` store: the stored temporary's value. `Name` delete: `unbound`. +/// - Every other result: `top`. +/// +/// The defaults never claim anything about effects: an unhandled `Call`, `Attribute` or +/// `Subscript` simply produces `top`, and a domain that tracks effects must treat +/// unknown callees and protocol methods conservatively itself. +pub trait Domain { + type Value: Clone + PartialEq + Debug; + /// Domain-owned state carried along every path and joined at merges. + type State: Clone + PartialEq + Debug; + /// Diagnostics are collected only from reachable operations after the fixpoint, + /// then sorted and deduplicated. + type Diagnostic: Clone + Ord + Debug; + + /// The value of anything the domain knows nothing about. + fn top(&self) -> Self::Value; + + /// The value of a symbol before any binding (and after `del`). + fn unbound(&self) -> Self::Value { + self.top() + } + + fn join(&self, a: &Self::Value, b: &Self::Value) -> Self::Value; + + fn join_state(&self, a: &Self::State, b: &Self::State) -> Self::State; + + /// Applied at loop heads, once `Limits::widen_after` visits have happened, to the + /// old entry value and the joined new one. + fn widen(&self, old: &Self::Value, new: &Self::Value) -> Self::Value { + self.join(old, new) + } + + fn widen_state(&self, old: &Self::State, new: &Self::State) -> Self::State { + self.join_state(old, new) + } + + /// Interpret one operation. The default does nothing beyond the shared bookkeeping. + fn transfer(&self, op: OpId, kind: &OpKind, cx: &mut Cx<'_, Self>) { + let _ = (op, kind, cx); + } + + /// Refine the frame on a `Branch` edge where `cond` has truthiness `truth`. Return + /// `false` if the edge is infeasible. + fn assume(&self, cond: Reg, truth: bool, cx: &mut Cx<'_, Self>) -> bool { + let _ = (cond, truth, cx); + true + } +} + +/// Whether a symbol holds a binding, tracked independently of the domain's values so +/// that "unknown value" and "no binding" stay distinct. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Presence { + Bound, + Unbound, + /// Bound on some paths only, or not known. + Maybe, +} + +impl Presence { + fn join(self, other: Self) -> Self { + if self == other { self } else { Presence::Maybe } + } +} + +/// The binding state of one symbol: an owned slot or an outer cell. +#[derive(Debug, Clone, PartialEq)] +pub struct Binding { + pub presence: Presence, + /// The bound value; meaningful when `presence` is not `Unbound`. + pub value: V, +} + +impl Binding { + pub fn bound(value: V) -> Self { + Self { + presence: Presence::Bound, + value, + } + } +} + +/// The full abstract state at a program point. +pub struct Frame { + region: RegionId, + slots: Vec>, + outer: BTreeMap>, + /// Only temporaries that may still be read: block entry frames keep just the + /// temporaries live into the block. + regs: BTreeMap, + pub state: D::State, +} + +impl Clone for Frame { + fn clone(&self) -> Self { + Self { + region: self.region, + slots: self.slots.clone(), + outer: self.outer.clone(), + regs: self.regs.clone(), + state: self.state.clone(), + } + } +} + +impl PartialEq for Frame { + fn eq(&self, other: &Self) -> bool { + self.slots == other.slots + && self.outer == other.outer + && self.regs == other.regs + && self.state == other.state + } +} + +impl Debug for Frame { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Frame") + .field("slots", &self.slots) + .field("outer", &self.outer) + .field("state", &self.state) + .finish_non_exhaustive() + } +} + +impl Frame { + /// The value of a temporary; panics if the temporary is not set on this path. + pub fn reg(&self, reg: Reg) -> &D::Value { + self.regs + .get(®) + .expect("lowering defines every temporary before use on each path") + } + + /// The binding of a symbol: its slot if the region owns it, else its cell if this + /// activation has one. A missing cell means nothing is known about the symbol. + pub fn binding(&self, program: &Program, symbol: SymbolId) -> Option<&Binding> { + match self.slot(program, symbol) { + Some(slot) => Some(&self.slots[slot]), + None => self.outer.get(&symbol), + } + } + + pub fn presence(&self, program: &Program, symbol: SymbolId) -> Option { + self.binding(program, symbol).map(|b| b.presence) + } + + /// The raw value of a symbol's binding. See [`load`] for what a load reads, + /// including fallbacks. + pub fn symbol(&self, program: &Program, symbol: SymbolId) -> Option<&D::Value> { + self.binding(program, symbol).map(|b| &b.value) + } + + /// The cells of symbols owned by other regions: the entry's [`Entry::outer`] cells + /// plus every store or delete this activation performed on such symbols. Comparing + /// an exit frame's cells with the entry's gives a callee's effect on its caller. + pub fn outer(&self) -> &BTreeMap> { + &self.outer + } + + fn slot(&self, program: &Program, symbol: SymbolId) -> Option { + let s = program.symbol(symbol); + match s.slot { + Some(slot) if program.scope(s.scope).region == self.region => Some(slot as usize), + _ => None, + } + } + + fn write(&mut self, program: &Program, symbol: SymbolId, binding: Binding) { + match self.slot(program, symbol) { + Some(slot) => self.slots[slot] = binding, + None => { + self.outer.insert(symbol, binding); + } + } + } + + /// Join (or widen) `other` into `self`. + fn merge(&mut self, other: &Self, domain: &D, widen: bool) { + let value = |a: &D::Value, b: &D::Value| { + let joined = domain.join(a, b); + if widen { + domain.widen(a, &joined) + } else { + joined + } + }; + let binding = |a: &Binding, b: &Binding| Binding { + presence: a.presence.join(b.presence), + value: value(&a.value, &b.value), + }; + for (a, b) in self.slots.iter_mut().zip(&other.slots) { + *a = binding(a, b); + } + // A temporary set on only one incoming path is dead here. + self.regs.retain(|r, _| other.regs.contains_key(r)); + for (r, a) in self.regs.iter_mut() { + *a = value(a, &other.regs[r]); + } + // A cell missing on one side is unknown there. + let unknown = Binding { + presence: Presence::Maybe, + value: domain.top(), + }; + for (symbol, b) in &other.outer { + let joined = binding(self.outer.get(symbol).unwrap_or(&unknown), b); + self.outer.insert(*symbol, joined); + } + for (symbol, a) in self.outer.iter_mut() { + if !other.outer.contains_key(symbol) { + *a = binding(a, &unknown); + } + } + let joined = domain.join_state(&self.state, &other.state); + self.state = if widen { + domain.widen_state(&self.state, &joined) + } else { + joined + }; + } +} + +/// The context handed to [`Domain::transfer`] and [`Domain::assume`]. +pub struct Cx<'a, D: Domain + ?Sized> { + program: &'a Program, + domain: &'a D, + range: TextRange, + op: Option, + frame: &'a mut Frame, + diagnostics: Option<&'a mut BTreeSet>, + set_regs: Vec, + set_symbol: bool, +} + +impl<'a, D: Domain + ?Sized> Cx<'a, D> { + pub fn program(&self) -> &'a Program { + self.program + } + + pub fn region(&self) -> RegionId { + self.frame.region + } + + /// The range of the current operation (or branch condition). + pub fn range(&self) -> TextRange { + self.range + } + + pub fn reg(&self, reg: Reg) -> &D::Value { + self.frame.reg(reg) + } + + /// Set a result temporary, suppressing its default value. + pub fn set_reg(&mut self, reg: Reg, value: D::Value) { + self.frame.regs.insert(reg, value); + self.set_regs.push(reg); + } + + /// The value a load of `symbol` produces by default: see [`load`]. + pub fn load(&self, symbol: SymbolId) -> D::Value { + load(self.program, self.domain, self.frame, symbol) + } + + /// Set a symbol's slot or cell. While transferring a `Name` store or delete of the + /// same symbol, this replaces the default update. + pub fn set_binding(&mut self, symbol: SymbolId, binding: Binding) { + if let Some(op) = self.op + && let OpKind::Name { + symbol: s, + access: Access::Store(_) | Access::Delete, + } = self.program.op(op).kind + && s == symbol + { + self.set_symbol = true; + } + self.frame.write(self.program, symbol, binding); + } + + /// Bind a symbol to `value`, as [`Cx::set_binding`] does. + pub fn set_symbol(&mut self, symbol: SymbolId, value: D::Value) { + self.set_binding(symbol, Binding::bound(value)); + } + + pub fn state(&self) -> &D::State { + &self.frame.state + } + + pub fn state_mut(&mut self) -> &mut D::State { + &mut self.frame.state + } + + pub fn frame(&self) -> &Frame { + self.frame + } + + /// Report a diagnostic. Ignored until the final pass over the fixpoint states. + pub fn report(&mut self, diagnostic: D::Diagnostic) { + if let Some(d) = &mut self.diagnostics { + d.insert(diagnostic); + } + } +} + +/// The starting point of one region analysis. +pub struct Entry { + /// Argument values by parameter index; missing entries are `top`. + pub args: Vec, + /// Initial cells for symbols owned by other regions (closure cells, globals and + /// builtins) as this activation sees them. A missing cell is unknown: loads of it + /// read `top`. Seed builtin cells to give builtins values. + pub outer: SmallMap>, + pub state: D::State, +} + +impl Entry { + pub fn new(state: D::State) -> Self { + Self { + args: Vec::new(), + outer: SmallMap::new(), + state, + } + } +} + +#[derive(Debug, Clone, Copy)] +pub struct Limits { + /// Visits of a loop head before joins there switch to widening. + pub widen_after: u32, + /// Visits of any single block before the analysis gives up. + pub max_block_visits: u32, +} + +impl Default for Limits { + fn default() -> Self { + Self { + widen_after: 3, + max_block_visits: 64, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Convergence { + Converged, + /// The visit limit was hit; states, exits and diagnostics are incomplete. + LimitReached, +} + +/// A reachable exit by raising, with the frame there. +pub struct RaiseSite { + pub cause: RaiseCause, + pub range: TextRange, + pub frame: Frame, +} + +impl Debug for RaiseSite { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RaiseSite") + .field("cause", &self.cause) + .field("range", &self.range) + .field("frame", &self.frame) + .finish() + } +} + +/// The outcome of analyzing one region. +pub struct RegionResult { + pub region: RegionId, + pub convergence: Convergence, + /// The join of frames at normal `Return`s; `None` if no normal exit is reachable. + pub exit: Option>, + /// The join of returned values (function regions only). + pub returned: Option, + /// The join of frames at reachable `Raise` terminators that leave the region. + pub raised: Option>, + /// Each reachable `Raise` terminator, in reverse postorder of its block. + pub raise_sites: Vec>, + pub diagnostics: Vec, + /// Gaps on reachable operations, deduplicated and in source order. + pub gaps: Vec<(GapKind, TextRange)>, + args: Vec, + block_entries: Vec>>, +} + +impl Debug for RegionResult { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RegionResult") + .field("region", &self.region) + .field("convergence", &self.convergence) + .field("exit", &self.exit) + .field("returned", &self.returned) + .field("diagnostics", &self.diagnostics) + .field("gaps", &self.gaps) + .finish_non_exhaustive() + } +} + +impl RegionResult { + /// True if the result is not a complete claim: unconverged, or a gap is reachable. + pub fn is_incomplete(&self) -> bool { + self.convergence != Convergence::Converged || !self.gaps.is_empty() + } + + pub fn is_reachable(&self, op: OpId, program: &Program) -> bool { + let op = program.op(op); + op.region == self.region && self.block_entries[op.block.index()].is_some() + } +} + +pub struct Engine<'a, D: Domain + ?Sized> { + program: &'a Program, + domain: &'a D, + limits: Limits, +} + +/// A callback that sees each operation with the frame before it. +type Visit<'b, D> = &'b mut dyn FnMut(OpId, &Frame); + +/// Where a block run reports what it sees. +struct Observer<'b, D: Domain + ?Sized> { + diagnostics: Option<&'b mut BTreeSet>, + gaps: Option<&'b mut BTreeSet<(TextSize, TextSize, GapKind)>>, + visit: Option>, +} + +impl<'a, D: Domain + ?Sized> Engine<'a, D> { + pub fn new(program: &'a Program, domain: &'a D) -> Self { + Self { + program, + domain, + limits: Limits::default(), + } + } + + pub fn with_limits(mut self, limits: Limits) -> Self { + self.limits = limits; + self + } + + /// Run the region to a fixpoint, then replay reachable blocks once to collect + /// exits, diagnostics and gaps. + pub fn analyze(&self, region: RegionId, entry: Entry) -> RegionResult { + let r = self.program.region(region); + let order = self.program.region(region).reverse_postorder(); + let mut rank = vec![usize::MAX; r.blocks.len()]; + for (i, b) in order.iter().enumerate() { + rank[b.index()] = i; + } + let initial = Frame { + region, + slots: vec![ + Binding { + presence: Presence::Unbound, + value: self.domain.unbound(), + }; + r.num_slots as usize + ], + outer: entry.outer.into_iter().collect(), + regs: BTreeMap::new(), + state: entry.state, + }; + let args = entry.args; + let live = live_temporaries(self.program, region); + let mut entries: Vec>> = vec![None; r.blocks.len()]; + entries[r.entry.index()] = Some(initial); + let mut visits = vec![0u32; r.blocks.len()]; + let mut worklist = BTreeSet::from([rank[r.entry.index()]]); + let mut convergence = Convergence::Converged; + while let Some(i) = worklist.pop_first() { + let block = order[i]; + visits[block.index()] += 1; + if visits[block.index()] > self.limits.max_block_visits { + convergence = Convergence::LimitReached; + break; + } + let mut frame = entries[block.index()] + .clone() + .expect("only blocks with an entry frame are queued"); + let mut observer = Observer { + diagnostics: None, + gaps: None, + visit: None, + }; + for (target, mut out) in self.run_block(block, &mut frame, &args, &mut observer) { + let t = target.index(); + out.regs.retain(|r, _| live[t].contains(r)); + let changed = match &mut entries[t] { + None => { + entries[t] = Some(out); + true + } + Some(old) => { + // A loop head is a block reached by an edge that does not go + // forward in reverse postorder. + let widen = rank[t] <= i && visits[t] >= self.limits.widen_after; + let before = old.clone(); + old.merge(&out, self.domain, widen); + *old != before + } + }; + if changed { + worklist.insert(rank[t]); + } + } + } + + let mut result = RegionResult { + region, + convergence, + exit: None, + returned: None, + raised: None, + raise_sites: Vec::new(), + diagnostics: Vec::new(), + gaps: Vec::new(), + args, + block_entries: entries, + }; + let mut diagnostics = BTreeSet::new(); + let mut gaps = BTreeSet::new(); + for &block in &order { + let Some(mut frame) = result.block_entries[block.index()].clone() else { + continue; + }; + let mut observer = Observer { + diagnostics: Some(&mut diagnostics), + gaps: Some(&mut gaps), + visit: None, + }; + self.run_block(block, &mut frame, &result.args, &mut observer); + match &r.blocks[block.index()].terminator { + Terminator::Return(value) => { + if let Some(value) = value { + let v = frame.reg(*value).clone(); + result.returned = Some(match &result.returned { + Some(old) => self.domain.join(old, &v), + None => v, + }); + } + frame.regs.clear(); + join_into(&mut result.exit, frame, self.domain); + } + Terminator::Raise { cause, range, .. } => { + frame.regs.clear(); + result.raise_sites.push(RaiseSite { + cause: *cause, + range: *range, + frame: frame.clone(), + }); + join_into(&mut result.raised, frame, self.domain) + } + Terminator::Jump(_) | Terminator::Branch { .. } | Terminator::Choice(_) => {} + } + } + result.diagnostics = diagnostics.into_iter().collect(); + result.gaps = gaps + .into_iter() + .map(|(start, end, kind)| (kind, TextRange::new(start, end))) + .collect(); + result + } + + /// Visit every reachable operation of a finished analysis with the frame before it, + /// in a deterministic order (blocks in reverse postorder, operations in order). + pub fn replay(&self, result: &RegionResult, f: &mut dyn FnMut(OpId, &Frame)) { + for block in self.program.region(result.region).reverse_postorder() { + let Some(mut frame) = result.block_entries[block.index()].clone() else { + continue; + }; + let mut observer = Observer { + diagnostics: None, + gaps: None, + visit: Some(&mut *f), + }; + self.run_block(block, &mut frame, &result.args, &mut observer); + } + } + + /// Run a block's operations on `frame` and return the feasible successor frames. + fn run_block( + &self, + block: BlockId, + frame: &mut Frame, + args: &[D::Value], + observer: &mut Observer<'_, D>, + ) -> Vec<(BlockId, Frame)> { + let b = &self.program.region(frame.region).blocks[block.index()]; + for &id in &b.ops { + if let Some(visit) = &mut observer.visit { + visit(id, frame); + } + let op = self.program.op(id); + if let (OpKind::Gap { kind, .. }, Some(gaps)) = (&op.kind, &mut observer.gaps) { + gaps.insert((op.range.start(), op.range.end(), *kind)); + } + let mut cx = Cx { + program: self.program, + domain: self.domain, + range: op.range, + op: Some(id), + frame, + diagnostics: observer.diagnostics.as_deref_mut(), + set_regs: Vec::new(), + set_symbol: false, + }; + self.domain.transfer(id, &op.kind, &mut cx); + let Cx { + set_regs, + set_symbol, + .. + } = cx; + for reg in op.kind.writes() { + if set_regs.contains(®) { + continue; + } + let value = match &op.kind { + OpKind::Name { symbol, .. } => load(self.program, self.domain, frame, *symbol), + OpKind::Param { index, .. } => args + .get(*index) + .cloned() + .unwrap_or_else(|| self.domain.top()), + _ => self.domain.top(), + }; + frame.regs.insert(reg, value); + } + if !set_symbol && let OpKind::Name { symbol, access } = &op.kind { + match access { + Access::Store(src) => { + let v = frame.reg(*src).clone(); + frame.write(self.program, *symbol, Binding::bound(v)); + } + Access::Delete => frame.write( + self.program, + *symbol, + Binding { + presence: Presence::Unbound, + value: self.domain.unbound(), + }, + ), + Access::Load(_) => {} + } + } + } + match &b.terminator { + Terminator::Jump(t) => vec![(*t, frame.clone())], + Terminator::Branch { cond, then, orelse } => [(*then, true), (*orelse, false)] + .into_iter() + .filter_map(|(target, truth)| { + let mut out = frame.clone(); + let mut cx = Cx { + program: self.program, + domain: self.domain, + range: self.cond_range(frame.region, block), + op: None, + frame: &mut out, + diagnostics: observer.diagnostics.as_deref_mut(), + set_regs: Vec::new(), + set_symbol: false, + }; + self.domain + .assume(*cond, truth, &mut cx) + .then_some((target, out)) + }) + .collect(), + Terminator::Choice(targets) => targets.iter().map(|t| (*t, frame.clone())).collect(), + Terminator::Return(_) | Terminator::Raise { .. } => Vec::new(), + } + } + + /// The range of the operation that computed a branch condition: the block's last op. + fn cond_range(&self, region: RegionId, block: BlockId) -> TextRange { + self.program.region(region).blocks[block.index()] + .ops + .last() + .map_or(self.program.region(region).range, |op| { + self.program.op(*op).range + }) + } +} + +/// What a load of `symbol` reads in `frame`. A bound symbol reads its value; an unbound +/// one reads its [`crate::ir::Symbol::fallback`] (or `unbound` if it has none); a +/// possibly bound one reads the join of both. This applies equally to owned slots and +/// outer cells. A symbol with no slot or cell in the frame is unknown and reads `top`. +pub fn load( + program: &Program, + domain: &D, + frame: &Frame, + symbol: SymbolId, +) -> D::Value { + let Some(binding) = frame.binding(program, symbol) else { + return domain.top(); + }; + let fallback = || match program.symbol(symbol).fallback { + Some(f) => load(program, domain, frame, f), + None => domain.unbound(), + }; + match binding.presence { + Presence::Bound => binding.value.clone(), + Presence::Unbound => fallback(), + Presence::Maybe => domain.join(&binding.value, &fallback()), + } +} + +/// For each block, the temporaries that may be read before being written on some path +/// from its entry (classic backward liveness). +fn live_temporaries(program: &Program, region: RegionId) -> Vec> { + let r = program.region(region); + // Per block: temporaries read before any write in the block, and those written. + let local: Vec<(BTreeSet, BTreeSet)> = r + .blocks + .iter() + .map(|b| { + let mut used = BTreeSet::new(); + let mut defined = BTreeSet::new(); + let terminator_reads = match &b.terminator { + Terminator::Branch { cond, .. } => vec![*cond], + Terminator::Return(r) | Terminator::Raise { exc: r, .. } => { + r.iter().copied().collect() + } + Terminator::Jump(_) | Terminator::Choice(_) => Vec::new(), + }; + let reads = b + .ops + .iter() + .map(|op| (program.op(*op).kind.reads(), program.op(*op).kind.writes())) + .chain([(terminator_reads, Vec::new())]); + for (reads, writes) in reads { + for reg in reads { + if !defined.contains(®) { + used.insert(reg); + } + } + defined.extend(writes); + } + (used, defined) + }) + .collect(); + let mut live: Vec> = local.iter().map(|(used, _)| used.clone()).collect(); + let mut changed = true; + while changed { + changed = false; + for (i, b) in r.blocks.iter().enumerate().rev() { + let mut next = live[i].clone(); + for succ in b.terminator.successors() { + for reg in &live[succ.index()] { + if !local[i].1.contains(reg) { + next.insert(*reg); + } + } + } + if next != live[i] { + live[i] = next; + changed = true; + } + } + } + live +} + +fn join_into(acc: &mut Option>, frame: Frame, domain: &D) { + match acc { + Some(a) => a.merge(&frame, domain, false), + None => *acc = Some(frame), + } +} diff --git a/static_analysis/src/exports.rs b/static_analysis/src/exports.rs new file mode 100644 index 0000000000..c31b25d971 --- /dev/null +++ b/static_analysis/src/exports.rs @@ -0,0 +1,36 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! The interface through which domains resolve imported names across modules. + +use pyrefly_python::module_name::ModuleName; +use ruff_python_ast::name::Name; + +/// Where an imported name ultimately comes from. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum ExportTarget { + /// A definition `name` in module `module`, after following re-exports. + Definition(ModuleName, Name), + /// The name is a module (a submodule import, or a re-exported module alias). + Module(ModuleName), + /// The provider cannot resolve the name (missing module, dynamic `__getattr__`, cycle). + Unknown, +} + +/// Resolves `from module import name` across modules. +pub trait ExportProvider { + fn resolve(&self, module: ModuleName, name: &Name) -> ExportTarget; +} + +/// A provider with no cross-module knowledge: `module.name` is taken at face value. +pub struct SyntacticExports; + +impl ExportProvider for SyntacticExports { + fn resolve(&self, module: ModuleName, name: &Name) -> ExportTarget { + ExportTarget::Definition(module, name.clone()) + } +} diff --git a/static_analysis/src/facts.rs b/static_analysis/src/facts.rs new file mode 100644 index 0000000000..14c179c6d1 --- /dev/null +++ b/static_analysis/src/facts.rs @@ -0,0 +1,65 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Optional facts attached by an external pre-check to structured source points. +//! Facts carry explicit typing assumptions and coverage; they are not unconditional +//! runtime proofs, and the frontend supplies no type-analysis or merge policy. + +use crate::flow_point::FlowPointId; + +/// Whether a source arm can complete normally, under the provider's assumptions. +/// This says nothing about divergence versus exceptions, nor about branch truth. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum NormalCompletion { + /// No proof is available. Missing facts must not prune a path. + Unknown, + /// The provider's types/rules establish no normal completion of this arm, but + /// only if declared types and the provider's typing assumptions hold at runtime. + NoNormalCompletionIfTypesHold, +} + +/// These facts are conditional on the provider's typing model, not unconditional +/// execution proofs. Untyped calls, casts and incorrect annotations can violate it. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum FactAssumptions { + DeclaredTypesAndTypingRulesHold, +} + +/// The provider's status for pre-check diagnostics. Facts alone do not establish +/// that a program passed a diagnostic check; the consumer must account for this. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DiagnosticCheck { + NotProvided, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ObservationCoverage { + Unobserved, + /// Counts describe observed arm continuations, not the entire source CFG. + Observed { + /// Arm-exit mappings the provider observed, including repeated occurrences. + mapped_arms: usize, + /// Mappings the provider considers eligible to carry a completion fact. + eligible_arms: usize, + /// Source points carrying a conditional no-normal-completion fact. + conditional_facts: usize, + }, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct FeedbackMetadata { + pub assumptions: FactAssumptions, + pub diagnostics: DiagnosticCheck, + pub coverage: ObservationCoverage, +} + +/// An optional, source-bound facts view. Providers must reject stale source before +/// handing out the view. A downstream domain decides whether and how to use facts. +pub trait FlowFacts { + fn metadata(&self) -> FeedbackMetadata; + fn normal_completion(&self, point: FlowPointId) -> NormalCompletion; +} diff --git a/static_analysis/src/flow_point.rs b/static_analysis/src/flow_point.rs new file mode 100644 index 0000000000..8813857354 --- /dev/null +++ b/static_analysis/src/flow_point.rs @@ -0,0 +1,78 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Source-occurrence identities for the shared structured statement driver. +//! +//! An overlay must qualify these IDs with its module and exact source snapshot. They +//! are stable across parses and backend pruning of the same source, not across edits. +//! Repeated lowering of a `finally` body repeats its IDs intentionally: these identify +//! source occurrences, not runtime instances or allocated control-flow blocks. +//! +//! Points describe structural boundaries, not feasibility, normal return, narrowing, +//! or merge policy. The statement driver does not identify expression-level control +//! flow, lambda scopes, or comprehension scopes. + +use ruff_text_size::TextRange; + +use crate::ir::Program; + +/// A boundary's role in the source structure. None of these roles implies reachability. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum FlowPointKind { + SuiteEntry, + SuiteExit, + /// An envelope around traversal of a function/class definition, including headers. + ScopeEntry, + ScopeExit, + IfFork, + IfJoin, + ArmTest, + ArmEntry, + ArmExit, + LoopHeader, + LoopBody, + LoopBackedge, + /// Normal exhaustion of the loop, before its `else` suite. + LoopExit, + LoopElse, + /// The boundary after the loop and its `else`, including `break` paths. + LoopJoin, + Break, + Continue, +} + +/// A point within one source module/snapshot. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct FlowPointId { + /// Lexical function/class/module suite span, independent of backend region allocation. + pub region: TextRange, + /// The source construct owning this boundary, not a visit ordinal. + pub range: TextRange, + pub kind: FlowPointKind, +} + +/// A lowered program with an opt-in trace of source boundaries visited by its backend. +/// +/// The trace is not a block-location map: a structural join callback can precede the +/// backend's construction of its join block. Consumers may attach source-qualified +/// overlays here, but applying them to the engine's blocks requires a further mapping. +pub struct ProgramWithFlowPoints { + pub program: Program, + pub flow_points: Vec, +} + +/// An opt-in observer's ordered trace. An unobserved backend needs no collector. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct FlowPointCollector { + pub points: Vec, +} + +impl FlowPointCollector { + pub fn observe(&mut self, point: FlowPointId) { + self.points.push(point); + } +} diff --git a/static_analysis/src/history.rs b/static_analysis/src/history.rs new file mode 100644 index 0000000000..e72d214b50 --- /dev/null +++ b/static_analysis/src/history.rs @@ -0,0 +1,190 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Domain-free queries over a [`Program`]: reaching definitions, definition/use chains, +//! and per-symbol access history. These run a fixed set-based dataflow over operation +//! ids, without a user-supplied `Domain` and without per-operation abstract states. + +use std::collections::BTreeSet; + +use starlark_map::small_map::SmallMap; + +use crate::ir::Access; +use crate::ir::OpId; +use crate::ir::OpKind; +use crate::ir::Program; +use crate::ir::RegionId; +use crate::ir::SymbolId; + +/// A definition reaching a program point. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum Def { + /// The region entry: the symbol has had no binding on some path. A load of a + /// symbol with a [`crate::ir::Symbol::fallback`] then reads the fallback. + Entry, + /// A `Name` store or delete operation. + Op(OpId), + /// Code in another region (a `global` or `nonlocal` writer, run by a call) also + /// stores to or deletes the symbol, so any binding may have been replaced. + External, +} + +pub struct History { + /// Reaching definitions for every reachable `Name` load of a region-owned symbol. + reaching: SmallMap>, + /// The loads each store or delete reaches, in program order. + uses: SmallMap>, + /// Every `Name` operation on each symbol, in program order. + accesses: SmallMap>, +} + +impl History { + pub fn new(program: &Program) -> Self { + let externally_written: BTreeSet = program + .ops + .iter() + .filter_map(|op| match op.kind { + OpKind::Name { + symbol, + access: Access::Store(_) | Access::Delete, + } if program.owner(symbol).is_some_and(|o| o != op.region) => Some(symbol), + _ => None, + }) + .collect(); + let mut reaching = SmallMap::new(); + for region in 0..program.regions.len() { + reaching_definitions( + program, + RegionId(region as u32), + &externally_written, + &mut reaching, + ); + } + let mut uses: SmallMap> = SmallMap::new(); + let mut loads: Vec = reaching.keys().copied().collect(); + loads.sort(); + for load in loads { + for def in reaching.get(&load).expect("collected from the map's keys") { + if let Def::Op(def) = *def { + uses.entry(def).or_default().push(load); + } + } + } + let mut accesses: SmallMap> = SmallMap::new(); + for (i, op) in program.ops.iter().enumerate() { + if let OpKind::Name { symbol, .. } = op.kind { + accesses.entry(symbol).or_default().push(OpId(i as u32)); + } + } + Self { + reaching, + uses, + accesses, + } + } + + /// The definitions reaching a `Name` load. `None` if `load` is not a reachable load + /// of a symbol owned by its region: a load of a closure cell, global or builtin from + /// another region depends on when that region runs. Includes [`Def::External`] when + /// another region writes the symbol. + pub fn reaching(&self, load: OpId) -> Option<&BTreeSet> { + self.reaching.get(&load) + } + + /// The `Name` loads that a store or delete may reach. + pub fn uses(&self, def: OpId) -> &[OpId] { + self.uses.get(&def).map_or(&[], |v| v.as_slice()) + } + + /// Every `Name` load, store and delete of `symbol` across all regions. + pub fn accesses(&self, symbol: SymbolId) -> &[OpId] { + self.accesses.get(&symbol).map_or(&[], |v| v.as_slice()) + } +} + +/// Classic reaching definitions over the slots of one region. +fn reaching_definitions( + program: &Program, + region: RegionId, + externally_written: &BTreeSet, + out: &mut SmallMap>, +) { + let r = program.region(region); + let slot = |symbol: SymbolId| { + let s = program.symbol(symbol); + s.slot + .filter(|_| program.scope(s.scope).region == region) + .map(|slot| slot as usize) + }; + let order = r.reverse_postorder(); + let mut rank = vec![usize::MAX; r.blocks.len()]; + for (i, b) in order.iter().enumerate() { + rank[b.index()] = i; + } + let mut entries: Vec>>> = vec![None; r.blocks.len()]; + entries[r.entry.index()] = Some(vec![BTreeSet::from([Def::Entry]); r.num_slots as usize]); + // Visiting blocks in reverse postorder makes each block see all of its forward + // predecessors first, so only loops cause revisits. + let mut worklist = BTreeSet::from([0]); + let run = |block: usize, + state: &mut Vec>, + out: Option<&mut SmallMap>>| { + let mut out = out; + for &id in &r.blocks[block].ops { + if let OpKind::Name { symbol, access } = program.op(id).kind + && let Some(slot) = slot(symbol) + { + match access { + Access::Load(_) => { + if let Some(out) = &mut out { + let mut defs = state[slot].clone(); + if externally_written.contains(&symbol) { + defs.insert(Def::External); + } + out.insert(id, defs); + } + } + Access::Store(_) | Access::Delete => { + state[slot] = BTreeSet::from([Def::Op(id)]) + } + } + } + } + }; + while let Some(i) = worklist.pop_first() { + let block = order[i]; + let mut state = entries[block.index()] + .clone() + .expect("queued blocks have an entry"); + run(block.index(), &mut state, None); + for t in r.blocks[block.index()].terminator.successors() { + let changed = match &mut entries[t.index()] { + None => { + entries[t.index()] = Some(state.clone()); + true + } + Some(old) => { + let mut changed = false; + for (a, b) in old.iter_mut().zip(&state) { + for d in b { + changed |= a.insert(*d); + } + } + changed + } + }; + if changed { + worklist.insert(rank[t.index()]); + } + } + } + for (block, entry) in entries.into_iter().enumerate() { + if let Some(mut state) = entry { + run(block, &mut state, Some(out)); + } + } +} diff --git a/static_analysis/src/ir.rs b/static_analysis/src/ir.rs new file mode 100644 index 0000000000..af12e96c01 --- /dev/null +++ b/static_analysis/src/ir.rs @@ -0,0 +1,898 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! The neutral intermediate representation produced by lowering. +//! +//! A [`Program`] is one Python module. Its code is split into [`Region`]s: the module +//! body, plus one region per function, lambda and generator expression. A region is a +//! control-flow graph of [`Block`]s; each block holds [`Op`]s in Python's runtime +//! evaluation order and ends in a [`Terminator`]. Operations communicate through +//! region-local temporaries ([`Reg`]) and through symbols ([`SymbolId`]) that the +//! scope prepass resolved with Python's lexical rules. +//! +//! Class bodies and list/set/dict comprehensions execute immediately, so they are +//! lowered inline into the enclosing region and only contribute their own scope. + +use std::collections::BTreeSet; +use std::fmt; +use std::sync::OnceLock; + +use pyrefly_python::module_name::ModuleName; +use ruff_python_ast::CmpOp; +use ruff_python_ast::Operator; +use ruff_python_ast::UnaryOp; +use ruff_python_ast::name::Name; +use ruff_source_file::LineIndex; +use ruff_source_file::OneIndexed; +use ruff_text_size::TextRange; +use starlark_map::small_map::SmallMap; + +use crate::scope::Lexical; + +macro_rules! index_type { + ($(#[$attr:meta])* $name:ident) => { + $(#[$attr])* + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] + pub struct $name(pub u32); + + impl $name { + pub fn index(self) -> usize { + self.0 as usize + } + } + }; +} + +index_type!( + /// A lexical scope. Stable for a given source text. + ScopeId +); +index_type!( + /// A symbol: one name owned by one scope. Stable for a given source text. + SymbolId +); +index_type!( + /// A code region with its own control-flow graph. + RegionId +); +index_type!( + /// An operation, unique across the whole program. Stable for a given source text. + OpId +); +index_type!( + /// A basic block, local to its region. + BlockId +); +index_type!( + /// A temporary value, local to its region. A temporary may be assigned on several + /// branches that later join (for example both arms of `a if c else b`). + Reg +); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ScopeKind { + Module, + Class, + Function, + Lambda, + /// A list/set/dict comprehension (inline) or a generator expression (its own region). + Comprehension, + /// The implicit outermost scope that holds names no enclosing scope defines. + Builtins, +} + +#[derive(Debug, Clone)] +pub struct Scope { + pub kind: ScopeKind, + pub parent: Option, + /// The region whose frame stores this scope's symbols. Class and inline + /// comprehension scopes share the frame of the region they execute in. The + /// builtins scope is attributed to the module region but owns no frame slots. + pub region: RegionId, + pub range: TextRange, + pub symbols: SmallMap, +} + +/// How a scope came to own a symbol. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum SymbolOrigin { + /// Some statement in the scope binds the name (assignment, import, `def`, `del`, ...), + /// whether or not that statement is reachable. + Bound, + /// A parameter of a function or lambda scope. + Parameter, + /// A module symbol created only because a nested scope declared it `global`. + GlobalDeclaration, + /// A builtin, or a name that no scope defines (a `NameError` at runtime unless a + /// dynamic definition such as a star import provides it). + Builtin, +} + +#[derive(Debug, Clone)] +pub struct Symbol { + pub name: Name, + pub scope: ScopeId, + pub origin: SymbolOrigin, + /// The slot in the owning region's frame; `None` for builtins. + pub slot: Option, + /// True if a different region reads or writes the symbol (a closure cell, or a + /// module global used from a function). + pub captured: bool, + /// What a load reads while this symbol is unbound. A class-body name falls back to + /// the module symbol of the same name (or the builtin), and a module name falls back + /// to the builtin, following CPython's `LOAD_NAME`/`LOAD_GLOBAL`. Function locals + /// have none: reading them unbound raises `UnboundLocalError`. + pub fallback: Option, +} + +/// What a region's code runs in response to. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Trigger { + /// The module body runs when the module is imported. + OnImport, + /// A plain function or lambda body runs when the function object is called. + OnCall, + /// A generator (function containing `yield`, or a generator expression) runs + /// piecewise each time the generator is advanced, not when it is called. + OnIteration, + /// A coroutine (`async def`) runs when it is awaited or scheduled, not when called. + OnAwait, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ParamKind { + PositionalOnly, + Positional, + VarArgs, + KeywordOnly, + KwArgs, +} + +/// A neutral, unevaluated description of an annotation or other static expression. +/// Names are resolved in the scope that would evaluate the expression. +#[derive(Debug, Clone, PartialEq)] +pub enum StaticExpr { + Name(Name, SymbolId), + Attribute(Box, Name), + /// `base[a, b]`; a tuple index is flattened into its elements. + Subscript(Box, Vec), + Literal(Literal), + /// `[a, b]` or `(a, b)` outside a subscript. + Sequence(Vec), + /// Anything else (calls, operators, ...), identified by its source range. + Other(TextRange), +} + +#[derive(Debug, Clone)] +pub struct Param { + pub name: Name, + pub symbol: SymbolId, + pub kind: ParamKind, + pub annotation: Option, + pub has_default: bool, + pub range: TextRange, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum Literal { + None, + Bool(bool), + /// `None` when the integer does not fit in an `i64`. + Int(Option), + Float(f64), + Complex, + Str(Box), + Bytes, + Ellipsis, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum BuildKind { + Tuple, + List, + Set, + Dict, + Slice, + /// An f-string or t-string; the elements are the interpolated values in order. + String, +} + +/// One element of a built collection or comprehension accumulation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Element { + Value(Reg), + /// `*x` inside a tuple, list or set display. + Star(Reg), + /// `key: value` inside a dict display. + Pair(Reg, Reg), + /// `**m` inside a dict display. + DoubleStar(Reg), + /// An omitted slice bound, as in `a[:n]`. + Missing, +} + +/// One argument of a call or class definition. Arguments are listed in evaluation +/// order: positional and starred arguments, then keyword and `**` arguments. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Arg { + Positional(Reg), + Star(Reg), + Keyword(Name, Reg), + DoubleStar(Reg), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum BinaryOp { + Arith(Operator), + /// The in-place form, as performed by augmented assignment. + InPlace(Operator), + Compare(CmpOp), +} + +/// What an operation does to a name, attribute or subscript location. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Access { + /// Read the location into a temporary. + Load(Reg), + /// Write a temporary into the location. + Store(Reg), + Delete, +} + +/// What an import statement binds. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ImportItem { + /// `import a.b.c` loads `a.b.c` and binds the top-level package `a`. + Root, + /// `import a.b.c as x` binds the module `a.b.c` itself. + Module, + /// `from m import name [as x]` binds `m.name` (an attribute or submodule). + Name(Name), +} + +/// A construct whose semantics the frontend approximates or does not model. A +/// reachable gap means results for that region are not a complete safety claim. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum GapKind { + SyntaxError, + /// `try`: exceptions are modeled only at statement boundaries inside the body, + /// and `finally` on the exceptional path is approximated. + Exceptions, + /// `match`: case selection is a nondeterministic choice; captures are unknown. + Match, + /// `from m import *` defines names that the scope prepass cannot see. + StarImport, + /// A relative import that goes above the top-level package. + InvalidRelativeImport, + /// PEP 695 type parameters and `type` statements. + TypeParams, + /// An annotation with possible side effects; annotations are never evaluated. + AnnotationEvaluation, + /// `nonlocal` with no enclosing function binding, or a similar scope error. + ScopeError, + /// IPython escape commands. + IpyEscape, + /// A region needed more copies of `finally` bodies than the lowering budget allows, + /// and this path skips the `finally` body. Every duplicated exit counts toward the + /// budget: the normal path, the exception path, and each `return`, `break` and + /// `continue` leaving the `try`. Because copies multiply with nesting, moderate + /// nesting (about ten levels) can reach the budget. + FinallyLimit, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum OpKind { + /// Access a variable. Loads of a symbol owned by another region read that + /// region's current binding (closure cell or module global). + Name { + symbol: SymbolId, + access: Access, + }, + Attribute { + object: Reg, + attr: Name, + access: Access, + }, + Subscript { + object: Reg, + index: Reg, + access: Access, + }, + Literal { + value: Literal, + dst: Reg, + }, + Build { + kind: BuildKind, + elements: Vec, + dst: Reg, + }, + /// Append to a collection under construction by a comprehension. + Append { + collection: Reg, + element: Element, + }, + Call { + func: Reg, + args: Vec, + dst: Reg, + }, + Unary { + op: UnaryOp, + operand: Reg, + dst: Reg, + }, + Binary { + op: BinaryOp, + left: Reg, + right: Reg, + dst: Reg, + }, + /// Destructure an iterable into `dsts`; `star` is the index of a starred target. + Unpack { + value: Reg, + dsts: Vec, + star: Option, + }, + /// `iter(iterable)` (or `aiter` when `is_async`). + Iter { + iterable: Reg, + dst: Reg, + is_async: bool, + }, + /// The next item of an iterator. Exhaustion is a separate `Choice` edge. + Next { + iterator: Reg, + dst: Reg, + is_async: bool, + }, + /// The value of a parameter at region entry. + Param { + index: usize, + dst: Reg, + }, + /// Create a function object for a deferred region. `defaults` are already evaluated + /// in the enclosing region, in parameter order. Decorators are applied by later + /// `Call` operations. + MakeFunction { + region: RegionId, + defaults: Vec, + dst: Reg, + }, + /// Create a class after its body (lowered inline, before this op) has run. + MakeClass { + name: Name, + scope: ScopeId, + bases: Vec, + dst: Reg, + }, + /// Execute an import. `module` is absolute. + Import { + module: ModuleName, + item: ImportItem, + dst: Reg, + }, + /// `__enter__` (or `__aenter__`). `context` identifies the `with` item; the + /// matching `ContextExit` on every normal exit path carries the same identity. + ContextEnter { + manager: Reg, + dst: Reg, + context: OpId, + is_async: bool, + }, + ContextExit { + manager: Reg, + context: OpId, + is_async: bool, + }, + /// A suspension point. The resumed value is unknown to the frontend. + Await { + value: Reg, + dst: Reg, + }, + Yield { + value: Option, + dst: Reg, + from: bool, + }, + /// An approximated or unmodeled construct. `dst`, if any, receives the unknown value + /// the construct produces (a match capture, an exception bound by `except ... as`). + Gap { + kind: GapKind, + dst: Option, + }, +} + +impl OpKind { + /// The temporaries this operation reads, in evaluation order. + pub fn reads(&self) -> Vec { + fn element(e: &Element, out: &mut Vec) { + match *e { + Element::Value(r) | Element::Star(r) | Element::DoubleStar(r) => out.push(r), + Element::Pair(k, v) => out.extend([k, v]), + Element::Missing => {} + } + } + fn arg(a: &Arg) -> Reg { + match a { + Arg::Positional(r) | Arg::Star(r) | Arg::Keyword(_, r) | Arg::DoubleStar(r) => *r, + } + } + let stored = |access: &Access| match access { + Access::Store(r) => Some(*r), + Access::Load(_) | Access::Delete => None, + }; + let mut out = Vec::new(); + match self { + OpKind::Name { access, .. } => out.extend(stored(access)), + OpKind::Attribute { object, access, .. } => { + out.push(*object); + out.extend(stored(access)); + } + OpKind::Subscript { + object, + index, + access, + } => { + out.extend([*object, *index]); + out.extend(stored(access)); + } + OpKind::Build { elements, .. } => elements.iter().for_each(|e| element(e, &mut out)), + OpKind::Append { + collection, + element: e, + } => { + out.push(*collection); + element(e, &mut out); + } + OpKind::Call { func, args, .. } => { + out.push(*func); + out.extend(args.iter().map(arg)); + } + OpKind::Unary { operand, .. } => out.push(*operand), + OpKind::Binary { left, right, .. } => out.extend([*left, *right]), + OpKind::Unpack { value, .. } => out.push(*value), + OpKind::Iter { iterable, .. } => out.push(*iterable), + OpKind::Next { iterator, .. } => out.push(*iterator), + OpKind::MakeFunction { defaults, .. } => out.extend(defaults), + OpKind::MakeClass { bases, .. } => out.extend(bases.iter().map(arg)), + OpKind::ContextEnter { manager, .. } | OpKind::ContextExit { manager, .. } => { + out.push(*manager) + } + OpKind::Await { value, .. } => out.push(*value), + OpKind::Yield { value, .. } => out.extend(value), + OpKind::Literal { .. } + | OpKind::Param { .. } + | OpKind::Import { .. } + | OpKind::Gap { .. } => {} + } + out + } + + /// The temporaries this operation defines. + pub fn writes(&self) -> Vec { + match self { + OpKind::Name { access, .. } + | OpKind::Attribute { access, .. } + | OpKind::Subscript { access, .. } => match access { + Access::Load(r) => vec![*r], + Access::Store(_) | Access::Delete => Vec::new(), + }, + OpKind::Unpack { dsts, .. } => dsts.clone(), + OpKind::Literal { dst, .. } + | OpKind::Build { dst, .. } + | OpKind::Call { dst, .. } + | OpKind::Unary { dst, .. } + | OpKind::Binary { dst, .. } + | OpKind::Iter { dst, .. } + | OpKind::Next { dst, .. } + | OpKind::Param { dst, .. } + | OpKind::MakeFunction { dst, .. } + | OpKind::MakeClass { dst, .. } + | OpKind::Import { dst, .. } + | OpKind::ContextEnter { dst, .. } + | OpKind::Await { dst, .. } + | OpKind::Yield { dst, .. } => vec![*dst], + OpKind::Gap { dst, .. } => dst.iter().copied().collect(), + OpKind::Append { .. } | OpKind::ContextExit { .. } => Vec::new(), + } + } +} + +#[derive(Debug, Clone)] +pub struct Op { + pub kind: OpKind, + pub region: RegionId, + /// The lexical scope the operation is evaluated in (an inline class or + /// comprehension scope, or the region's own scope). + pub scope: ScopeId, + pub block: BlockId, + /// The source range of the expression or statement fragment that performs the op. + pub range: TextRange, + /// The range of the innermost enclosing statement: a statement-level view of the CFG. + pub stmt: TextRange, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum Terminator { + Jump(BlockId), + /// Branch on the truthiness of `cond`. + Branch { + cond: Reg, + then: BlockId, + orelse: BlockId, + }, + /// A choice the frontend cannot decide (loop exhaustion, exception edges, match + /// cases). Every target is feasible. + Choice(Vec), + /// Leave the region normally. Function returns always carry a value (a bare + /// `return` loads `None`); the end of the module carries none. + Return(Option), + /// Leave the region by raising. + Raise { + /// The exception value, if the source names one. + exc: Option, + cause: RaiseCause, + /// The `raise` or `assert` statement, or the `try` statement for a re-raise. + range: TextRange, + }, +} + +impl Terminator { + pub fn successors(&self) -> Vec { + match self { + Terminator::Jump(t) => vec![*t], + Terminator::Branch { then, orelse, .. } => vec![*then, *orelse], + Terminator::Choice(ts) => ts.clone(), + Terminator::Return(_) | Terminator::Raise { .. } => Vec::new(), + } + } +} + +/// Why a region is left by raising. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub enum RaiseCause { + /// A `raise` statement. + Explicit, + /// A failing `assert`. + AssertFailure, + /// An exception that reached a `try` and propagates onward: no handler matched, + /// or it is leaving through `finally`. + Reraise, +} + +#[derive(Debug, Clone)] +pub struct Block { + pub ops: Vec, + pub terminator: Terminator, +} + +#[derive(Debug, Clone)] +pub struct Region { + pub name: Name, + pub trigger: Trigger, + pub scope: ScopeId, + /// The region that creates this one, and the first `MakeFunction` op that does so. + /// A definition inside a `finally` body is lowered once per exit path, so several + /// `MakeFunction` ops (each with its own defaults) may create the same region. + pub parent: Option<(RegionId, OpId)>, + pub params: Vec, + pub returns: Option, + pub blocks: Vec, + pub entry: BlockId, + pub num_regs: u32, + pub num_slots: u32, + pub range: TextRange, +} + +impl Region { + /// Blocks reachable from the entry, in reverse postorder: every block precedes its + /// successors except along loop back edges. + pub fn reverse_postorder(&self) -> Vec { + let successors = |b: BlockId| self.blocks[b.index()].terminator.successors(); + let mut seen = vec![false; self.blocks.len()]; + let mut post = Vec::new(); + // Iterative depth-first search; each entry is a block and its next successor. + let mut stack = vec![(self.entry, successors(self.entry), 0)]; + seen[self.entry.index()] = true; + while let Some((block, succs, i)) = stack.last_mut() { + if let Some(&next) = succs.get(*i) { + *i += 1; + if !seen[next.index()] { + seen[next.index()] = true; + let s = successors(next); + stack.push((next, s, 0)); + } + } else { + post.push(*block); + stack.pop(); + } + } + post.reverse(); + post + } +} + +/// A lowered module. +pub struct Program { + pub module: ModuleName, + pub scopes: Vec, + pub symbols: Vec, + pub regions: Vec, + pub ops: Vec, + pub syntax_errors: Vec<(String, TextRange)>, + pub(crate) lexical: Lexical, + /// Computed on the first [`Program::free_symbols`] query. + pub(crate) free_symbols: OnceLock>>, + pub(crate) line_index: LineIndex, + pub(crate) source: String, +} + +impl fmt::Debug for Program { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Program") + .field("module", &self.module) + .field("regions", &self.regions.len()) + .field("ops", &self.ops.len()) + .finish() + } +} + +impl Program { + pub const MODULE: RegionId = RegionId(0); + + pub fn scope(&self, id: ScopeId) -> &Scope { + &self.scopes[id.index()] + } + + pub fn symbol(&self, id: SymbolId) -> &Symbol { + &self.symbols[id.index()] + } + + pub fn region(&self, id: RegionId) -> &Region { + &self.regions[id.index()] + } + + pub fn op(&self, id: OpId) -> &Op { + &self.ops[id.index()] + } + + /// The region whose frame stores `symbol`, or `None` for builtins. + pub fn owner(&self, symbol: SymbolId) -> Option { + let s = self.symbol(symbol); + s.slot.map(|_| self.scope(s.scope).region) + } + + /// Look up a name as code in `scope` would see it at runtime. `None` for a builtin + /// that no code in the module references. + pub fn resolve(&self, scope: ScopeId, name: &Name) -> Option { + self.lexical.resolve(&self.scopes, scope, name) + } + + /// The symbols of the module scope (not of classes lowered inline into the module). + pub fn module_symbols(&self) -> impl Iterator + '_ { + self.scope(self.region(Self::MODULE).scope) + .symbols + .values() + .copied() + } + + /// The symbols that `region` or a region nested in it accesses (or may read as a + /// [`Symbol::fallback`]) but that no region in that subtree owns: enclosing + /// function cells, module globals seen from functions, and builtins. Sorted. + pub fn free_symbols(&self, region: RegionId) -> &[SymbolId] { + &self + .free_symbols + .get_or_init(|| self.compute_free_symbols())[region.index()] + } + + fn compute_free_symbols(&self) -> Vec> { + // `ancestors[r]` is `r` followed by the regions that enclose it. + let ancestors: Vec> = (0..self.regions.len()) + .map(|r| { + let mut chain = vec![RegionId(r as u32)]; + while let Some((parent, _)) = self.region(*chain.last().unwrap()).parent { + chain.push(parent); + } + chain + }) + .collect(); + let mut free: Vec> = vec![BTreeSet::new(); self.regions.len()]; + for op in &self.ops { + let OpKind::Name { symbol, access } = op.kind else { + continue; + }; + // Only a load can read a fallback. + let is_load = matches!(access, Access::Load(_)); + let mut next = Some(symbol); + while let Some(s) = next { + // A symbol is free in every region on the path from the op's region up + // to, but excluding, the first region whose subtree contains the owner. + let owner_chain = self.owner(s).map(|o| &ancestors[o.index()]); + for r in &ancestors[op.region.index()] { + if owner_chain.is_some_and(|chain| chain.contains(r)) { + break; + } + free[r.index()].insert(s); + } + next = self.symbol(s).fallback.filter(|_| is_load); + } + } + free.into_iter().map(|s| s.into_iter().collect()).collect() + } + + /// The module-level symbol for `name`, if the module binds or declares it. + pub fn global(&self, name: &str) -> Option { + self.scope(self.region(Self::MODULE).scope) + .symbols + .get(&Name::new(name)) + .copied() + } + + /// One-based line and column of the start of `range`. + pub fn line_col(&self, range: TextRange) -> (OneIndexed, OneIndexed) { + let loc = self.line_index.line_column(range.start(), &self.source); + (loc.line, loc.column) + } + + pub fn text(&self, range: TextRange) -> &str { + &self.source[range] + } + + /// A readable listing of a region, one block per paragraph. Symbols owned by + /// another region are suffixed with `@` and the owning region's name. + pub fn dump(&self, region: RegionId) -> String { + let r = self.region(region); + let reg = |r: &Reg| format!("r{}", r.0); + let regs = |rs: &[Reg]| rs.iter().map(reg).collect::>().join(", "); + let sym = |s: &SymbolId| { + let symbol = self.symbol(*s); + match self.owner(*s) { + Some(owner) if owner == region => symbol.name.to_string(), + Some(owner) => format!("{}@{}", symbol.name, self.region(owner).name), + None => format!("{}@builtins", symbol.name), + } + }; + let element = |e: &Element| match e { + Element::Value(r) => reg(r), + Element::Star(r) => format!("*{}", reg(r)), + Element::Pair(k, v) => format!("{}: {}", reg(k), reg(v)), + Element::DoubleStar(r) => format!("**{}", reg(r)), + Element::Missing => "_".to_owned(), + }; + let args = |args: &[Arg]| { + args.iter() + .map(|a| match a { + Arg::Positional(r) => reg(r), + Arg::Star(r) => format!("*{}", reg(r)), + Arg::Keyword(k, r) => format!("{k}={}", reg(r)), + Arg::DoubleStar(r) => format!("**{}", reg(r)), + }) + .collect::>() + .join(", ") + }; + let access = |target: String, a: &Access| match a { + Access::Load(r) => format!("{} = {target}", reg(r)), + Access::Store(r) => format!("{target} = {}", reg(r)), + Access::Delete => format!("del {target}"), + }; + let mut out = String::new(); + for (i, block) in r.blocks.iter().enumerate() { + out.push_str(&format!("b{i}:\n")); + for op in &block.ops { + let line = match &self.op(*op).kind { + OpKind::Name { symbol, access: a } => access(sym(symbol), a), + OpKind::Attribute { + object, + attr, + access: a, + } => access(format!("{}.{attr}", reg(object)), a), + OpKind::Subscript { + object, + index, + access: a, + } => access(format!("{}[{}]", reg(object), reg(index)), a), + OpKind::Literal { value, dst } => format!("{} = {value:?}", reg(dst)), + OpKind::Build { + kind, + elements, + dst, + } => format!( + "{} = {kind:?}[{}]", + reg(dst), + elements.iter().map(element).collect::>().join(", ") + ), + OpKind::Append { + collection, + element: e, + } => { + format!("append {} <- {}", reg(collection), element(e)) + } + OpKind::Call { func, args: a, dst } => { + format!("{} = {}({})", reg(dst), reg(func), args(a)) + } + OpKind::Unary { op, operand, dst } => { + format!("{} = {op:?} {}", reg(dst), reg(operand)) + } + OpKind::Binary { + op, + left, + right, + dst, + } => { + format!("{} = {} {op:?} {}", reg(dst), reg(left), reg(right)) + } + OpKind::Unpack { value, dsts, star } => { + format!("{} = unpack {} star={star:?}", regs(dsts), reg(value)) + } + OpKind::Iter { iterable, dst, .. } => { + format!("{} = iter {}", reg(dst), reg(iterable)) + } + OpKind::Next { iterator, dst, .. } => { + format!("{} = next {}", reg(dst), reg(iterator)) + } + OpKind::Param { index, dst } => format!("{} = param {index}", reg(dst)), + OpKind::MakeFunction { + region, + defaults, + dst, + } => format!( + "{} = function {}({})", + reg(dst), + self.region(*region).name, + regs(defaults) + ), + OpKind::MakeClass { + name, bases, dst, .. + } => { + format!("{} = class {name}({})", reg(dst), args(bases)) + } + OpKind::Import { module, item, dst } => { + format!("{} = import {module} {item:?}", reg(dst)) + } + OpKind::ContextEnter { manager, dst, .. } => { + format!("{} = enter {}", reg(dst), reg(manager)) + } + OpKind::ContextExit { manager, .. } => format!("exit {}", reg(manager)), + OpKind::Await { value, dst } => format!("{} = await {}", reg(dst), reg(value)), + OpKind::Yield { value, dst, from } => format!( + "{} = yield{} {}", + reg(dst), + if *from { " from" } else { "" }, + value.as_ref().map_or(String::new(), reg) + ), + OpKind::Gap { kind, dst } => match dst { + Some(d) => format!("{} = gap {kind:?}", reg(d)), + None => format!("gap {kind:?}"), + }, + }; + out.push_str(&format!(" {line}\n")); + } + let t = match &block.terminator { + Terminator::Jump(b) => format!("jump b{}", b.0), + Terminator::Branch { cond, then, orelse } => { + format!("branch {} ? b{} : b{}", reg(cond), then.0, orelse.0) + } + Terminator::Choice(bs) => format!( + "choice [{}]", + bs.iter() + .map(|b| format!("b{}", b.0)) + .collect::>() + .join(", ") + ), + Terminator::Return(Some(r)) => format!("return {}", reg(r)), + Terminator::Return(None) => "return".to_owned(), + Terminator::Raise { exc, cause, .. } => format!( + "raise {} {cause:?}", + exc.as_ref().map_or("_".to_owned(), reg) + ), + }; + out.push_str(&format!(" {t}\n")); + } + out + } +} diff --git a/static_analysis/src/lexical.rs b/static_analysis/src/lexical.rs new file mode 100644 index 0000000000..140ffc9dc2 --- /dev/null +++ b/static_analysis/src/lexical.rs @@ -0,0 +1,210 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Python's lexical name resolution over any tree of scopes. +//! +//! A backend describes its scopes through [`ScopeChain`] and answers, per scope, what that +//! scope's tables say about a name ([`Probe`]). This module owns the navigation rules +//! (): +//! +//! - lookup proceeds from the innermost scope outward; +//! - a class body is visible only to its own code, and to an annotation scope or `type` +//! alias scope directly inside it, not to nested functions or comprehensions; +//! - `global` resolves in the root scope; +//! - `nonlocal` resolves in the nearest enclosing scope that is neither a class nor the +//! root, following an enclosing `nonlocal`; an enclosing `global` is an error. +//! +//! What a scope binds, whether a binding is initialized, and what a name means once found +//! stay with the backend. + +/// What kind of scope a backend scope is, as far as navigation is concerned. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Category { + Module, + Class, + /// A `def` or `lambda` body. + Function, + Comprehension, + /// The scope of a definition's type parameters (and, for a class, its bases). + Annotation { + class_scope: bool, + }, + /// The scope of a `type` alias value. + TypeAlias, +} + +/// A tree of scopes, navigated from a scope to its parent. Implementations are cheap +/// handles, such as a reference to the backend's scope table. +pub trait ScopeChain: Copy { + type Id: Copy; + /// What a scope puts between the scopes it encloses and the scopes enclosing it, in + /// the backend's terms. `Default` is no barrier. + type Barrier: Copy + Default; + + fn parent(&self, id: Self::Id) -> Option; + fn category(&self, id: Self::Id) -> Category; + fn barrier(&self, id: Self::Id) -> Self::Barrier; + /// The barrier of crossing both `a` and `b`. + fn join(a: Self::Barrier, b: Self::Barrier) -> Self::Barrier; +} + +/// A scope visible from the lookup origin. +pub struct View { + pub id: C::Id, + /// How many scopes, visible or not, lie between the origin and this one. + pub depth: usize, + /// The joined barriers of the visible scopes from the origin up to, but excluding, + /// this one. A class body hidden from the origin contributes nothing. + pub barrier: C::Barrier, +} + +/// The scopes visible from `origin`, innermost first. +pub fn visible(chain: C, origin: C::Id) -> Visible { + Visible { + chain, + next: Some(origin), + depth: 0, + barrier: C::Barrier::default(), + from_annotation_like: matches!( + chain.category(origin), + Category::Annotation { .. } | Category::TypeAlias + ), + } +} + +pub struct Visible { + chain: C, + next: Option, + depth: usize, + barrier: C::Barrier, + from_annotation_like: bool, +} + +impl Iterator for Visible { + type Item = View; + + fn next(&mut self) -> Option> { + loop { + let id = self.next?; + self.next = self.chain.parent(id); + let depth = self.depth; + self.depth += 1; + let category = self.chain.category(id); + if category == Category::Class + && !(depth == 0 || (self.from_annotation_like && depth == 1)) + { + continue; + } + let view = View { + id, + depth, + barrier: self.barrier, + }; + self.barrier = C::join(self.barrier, self.chain.barrier(id)); + return Some(view); + } + } +} + +/// What one scope's tables say about a name. +pub enum Probe { + /// The scope binds the name. + Owned(T), + Absent, + /// The scope declares the name `global`. + Global, + /// The scope declares the name `nonlocal`. + Nonlocal, +} + +/// Why a `nonlocal` declaration has no binding to refer to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum NonlocalError { + /// The name is global: only the root scope binds it, or an enclosing scope declares + /// it `global`. + Global, + NotFound, +} + +pub enum Resolution { + Found(T), + /// No scope binds the name, so it resolves to a builtin, if any. + Unbound, + /// The name is declared `nonlocal` and refers to nothing. + Nonlocal(NonlocalError), +} + +/// Resolve a name as code in `origin` sees it. +pub fn resolve( + chain: C, + origin: C::Id, + mut probe: impl FnMut(C::Id) -> Probe, +) -> Resolution { + for view in visible(chain, origin) { + match probe(view.id) { + Probe::Owned(found) => return Resolution::Found(found), + Probe::Absent => {} + // A backend's root probe reports what the root owns before its declarations: + // `global` at the root is a no-op and `nonlocal` there is a syntax error. + Probe::Global => { + return match probe(root(chain, view.id)) { + Probe::Owned(found) => Resolution::Found(found), + Probe::Absent | Probe::Global | Probe::Nonlocal => Resolution::Unbound, + }; + } + Probe::Nonlocal => { + return match resolve_nonlocal(chain, chain.parent(view.id), probe) { + Ok(found) => Resolution::Found(found), + Err(e) => Resolution::Nonlocal(e), + }; + } + } + } + Resolution::Unbound +} + +/// Find the binding a `nonlocal` declaration refers to, searching outward from `from` +/// (the declaring scope's parent). +pub fn resolve_nonlocal( + chain: C, + from: Option, + mut probe: impl FnMut(C::Id) -> Probe, +) -> Result { + let mut next = from; + while let Some(id) = next { + next = chain.parent(id); + match chain.category(id) { + Category::Class => {} + // A `nonlocal` declaration at the root is itself a syntax error, which refers to + // nothing. + Category::Module => { + return Err(match probe(id) { + Probe::Absent | Probe::Nonlocal => NonlocalError::NotFound, + Probe::Owned(_) | Probe::Global => NonlocalError::Global, + }); + } + Category::Function + | Category::Comprehension + | Category::Annotation { .. } + | Category::TypeAlias => match probe(id) { + Probe::Owned(found) => return Ok(found), + // An enclosing `nonlocal` refers further out. + Probe::Absent | Probe::Nonlocal => {} + Probe::Global => return Err(NonlocalError::Global), + }, + } + } + Err(NonlocalError::NotFound) +} + +/// The root scope enclosing `id`, where `global` names live. +pub fn root(chain: C, mut id: C::Id) -> C::Id { + while let Some(parent) = chain.parent(id) { + id = parent; + } + id +} diff --git a/static_analysis/src/lib.rs b/static_analysis/src/lib.rs new file mode 100644 index 0000000000..550a8b7f84 --- /dev/null +++ b/static_analysis/src/lib.rs @@ -0,0 +1,117 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! An experimental, domain-neutral Python static-analysis frontend and engine. +//! +//! [`analyze_python`] parses a module, resolves lexical scopes, and lowers the code into +//! a [`Program`] of regions whose control-flow graphs contain operations in runtime +//! evaluation order. An [`Engine`] runs any [`Domain`] over a region; [`History`] +//! answers reaching-definition queries without a domain. +//! +//! Lowering walks statement structure (suites, `if`, loops, `break`/`continue`, function +//! bodies) through the [`structured`] driver. Another backend, such as a type checker's +//! binder, can drive its own values over the same parsed AST with that driver, without +//! lowering. +//! +//! # Supported semantics +//! +//! - Scopes follow CPython's syntactic rules: any binding (including in dead code) +//! makes a name local; `global`/`nonlocal` redirect to the owning scope; class scopes +//! are invisible to nested functions and comprehensions; comprehension targets are +//! comprehension-local while a walrus binds in the enclosing non-comprehension scope; +//! unbound class-body names fall back to the module and unbound module names to +//! builtins ([`Symbol::fallback`]). Declarations come from the [`declarations`] walker +//! and lookup from the [`lexical`] engine, which a type checker's binder can consume +//! with its own policies. +//! - Evaluation order: assignment values before targets (targets left to right, each +//! evaluating its object and index before storing); augmented assignment reads the +//! target, then the value, then stores; the callee, then positional and starred +//! arguments, then keyword and `**` arguments (also for class bases); +//! decorators, then defaults, then function creation, then decorator application +//! bottom-up; class bases before the inline class body; the first iterable of a +//! comprehension in the enclosing scope. +//! - Control flow: `if`/`elif`/`else`, `while`/`for` with `else`, `break` and +//! `continue`, `and`/`or`, conditional expressions and chained comparisons as explicit +//! branches; `return`; `raise` and `assert`; `with` with `ContextExit` on fallthrough +//! and on every `return`/`break`/`continue`/`raise` leaving it; `finally` on every +//! normal exit. +//! - Regions: functions, lambdas and generator expressions are deferred regions with a +//! [`Trigger`]; class bodies and list/set/dict comprehensions run inline. +//! +//! # Approximations, reported as [`GapKind`] operations +//! +//! - `try`/`except` (including `except*`): an exception may arise at any statement +//! boundary inside the body, not mid-statement; handler matching is a +//! nondeterministic choice; the bound exception value is unknown. +//! - `match`: case selection is a nondeterministic choice tried in order; captures are +//! unknown values; value patterns are not evaluated. +//! - Star imports, invalid relative imports, PEP 695 type parameters and `type` +//! statements, annotations with possible side effects (annotations are never +//! evaluated), unresolvable `nonlocal`, syntax errors and IPython escapes. +//! +//! # Not modeled +//! +//! - Exceptions raised outside `try` simply leave the region; the engine reports only +//! explicit `raise`/`assert` exits in [`RegionResult::raised`]. +//! - A context manager that suppresses exceptions; `assert` removal under `-O`. +//! - Dynamic name definitions (`globals()`, `exec`, `setattr` on modules) and +//! interleaving at suspension points: these are domain concerns. + +pub mod declarations; +pub mod engine; +pub mod exports; +pub mod facts; +pub mod flow_point; +pub mod history; +pub mod ir; +pub mod lexical; +mod lower; +mod scope; +pub mod structured; + +pub use engine::Binding; +pub use engine::Convergence; +pub use engine::Cx; +pub use engine::Domain; +pub use engine::Engine; +pub use engine::Entry; +pub use engine::Frame; +pub use engine::Limits; +pub use engine::Presence; +pub use engine::RaiseSite; +pub use engine::RegionResult; +pub use engine::load; +pub use exports::ExportProvider; +pub use exports::ExportTarget; +pub use exports::SyntacticExports; +pub use flow_point::ProgramWithFlowPoints; +pub use history::Def; +pub use history::History; +pub use ir::*; +pub use pyrefly_python::module_name::ModuleName; +pub use ruff_python_ast::CmpOp; +pub use ruff_python_ast::Operator; +pub use ruff_python_ast::UnaryOp; +pub use ruff_python_ast::name::Name; +pub use ruff_text_size::TextRange; + +/// Parse and lower one module. `is_init` says whether the module is a package +/// `__init__`, which affects relative imports. Syntax errors are recorded in +/// [`Program::syntax_errors`] and as a [`GapKind::SyntaxError`] at module entry. +pub fn analyze_python(source: &str, module: ModuleName, is_init: bool) -> Program { + lower::lower(source, module, is_init) +} + +/// Parse and lower with opt-in source flow-point observation. The points identify +/// structured source boundaries, not allocated CFG blocks or branch feasibility. +pub fn analyze_python_with_flow_points( + source: &str, + module: ModuleName, + is_init: bool, +) -> ProgramWithFlowPoints { + lower::lower_with_flow_points(source, module, is_init) +} diff --git a/static_analysis/src/lower.rs b/static_analysis/src/lower.rs new file mode 100644 index 0000000000..aa9eda08e8 --- /dev/null +++ b/static_analysis/src/lower.rs @@ -0,0 +1,1902 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Lowering from the Ruff AST to the operation CFG. Statement structure (suites, +//! `if`, loops, `break`/`continue`, function bodies) is walked by the shared +//! [`structured`](crate::structured) driver; this module lowers what the driver hands it. +//! Every other component consumes [`Program`]. +//! +//! Operations are emitted in CPython's evaluation order: an assignment evaluates its +//! value before any target subexpression, a call evaluates the callee before its +//! arguments, decorators are evaluated before defaults, and `and`/`or`, conditional +//! expressions and chained comparisons branch explicitly. + +use std::sync::OnceLock; + +use pyrefly_python::ast::Ast; +use pyrefly_python::module_name::ModuleName; +use ruff_python_ast::Arguments; +use ruff_python_ast::BoolOp; +use ruff_python_ast::Comprehension; +use ruff_python_ast::ExceptHandler; +use ruff_python_ast::Expr; +use ruff_python_ast::Identifier; +use ruff_python_ast::InterpolatedStringElement; +use ruff_python_ast::Number; +use ruff_python_ast::Parameters; +use ruff_python_ast::PySourceType; +use ruff_python_ast::Stmt; +use ruff_python_ast::StmtFunctionDef; +use ruff_python_ast::StmtTry; +use ruff_python_ast::name::Name; +use ruff_python_ast::visitor::Visitor; +use ruff_python_ast::visitor::walk_expr; +use ruff_source_file::LineIndex; +use ruff_text_size::Ranged; +use ruff_text_size::TextRange; + +use crate::flow_point::FlowPointCollector; +use crate::flow_point::FlowPointId; +use crate::flow_point::ProgramWithFlowPoints; +use crate::ir::Access; +use crate::ir::Arg; +use crate::ir::BinaryOp; +use crate::ir::Block; +use crate::ir::BlockId; +use crate::ir::BuildKind; +use crate::ir::Element; +use crate::ir::GapKind; +use crate::ir::ImportItem; +use crate::ir::Literal; +use crate::ir::Op; +use crate::ir::OpId; +use crate::ir::OpKind; +use crate::ir::Param; +use crate::ir::ParamKind; +use crate::ir::Program; +use crate::ir::RaiseCause; +use crate::ir::Reg; +use crate::ir::Region; +use crate::ir::RegionId; +use crate::ir::ScopeId; +use crate::ir::ScopeKind; +use crate::ir::StaticExpr; +use crate::ir::Terminator; +use crate::ir::Trigger; +use crate::scope::GENERATOR_ARG; +use crate::scope::ScopeTable; +use crate::scope::annotation_may_have_effects; +use crate::structured; +use crate::structured::Arm; +use crate::structured::Borrowed; +use crate::structured::Driver; +use crate::structured::Jump; +use crate::structured::LoopHeader; +use crate::structured::Structured; + +pub fn lower(source: &str, module: ModuleName, is_init: bool) -> Program { + lower_impl(source, module, is_init, None).program +} + +/// Lower with an observer attached to the actual borrowed CFG backend. +pub fn lower_with_flow_points( + source: &str, + module: ModuleName, + is_init: bool, +) -> ProgramWithFlowPoints { + lower_impl(source, module, is_init, Some(FlowPointCollector::default())) +} + +fn lower_impl( + source: &str, + module: ModuleName, + is_init: bool, + flow_points: Option, +) -> ProgramWithFlowPoints { + let (ast, parse_errors, unsupported) = Ast::parse(source, PySourceType::Python); + let syntax_errors: Vec<(String, TextRange)> = parse_errors + .iter() + .map(|e| (e.error.to_string(), e.location)) + .chain(unsupported.iter().map(|e| (e.to_string(), e.range))) + .collect(); + let table = ScopeTable::new(&ast.body, ast.range); + let mut lowerer = Lowerer { + module, + is_init, + regions: (0..table.num_regions).map(|_| None).collect(), + table, + ops: Vec::new(), + r: RegionState::new(ScopeId(0)), + driver: Driver::default(), + flow_points, + }; + let module_scope = lowerer.table.module(); + lowerer.r = RegionState::new(module_scope); + if let Some((_, range)) = syntax_errors.first() { + lowerer.gap(GapKind::SyntaxError, None, *range); + } + structured::suite(&mut lowerer, &ast.body); + lowerer.terminate(Terminator::Return(None)); + let state = std::mem::replace(&mut lowerer.r, RegionState::new(module_scope)); + let (blocks, num_regs) = state.into_blocks(); + lowerer.regions[0] = Some(Region { + name: Name::new_static(""), + trigger: Trigger::OnImport, + scope: module_scope, + parent: None, + params: Vec::new(), + returns: None, + blocks, + entry: BlockId(0), + num_regs, + num_slots: 0, + range: ast.range, + }); + let Lowerer { + module, + regions, + table, + ops, + flow_points, + .. + } = lowerer; + let parts = table.into_parts(); + let mut program = Program { + module, + scopes: parts.scopes, + symbols: parts.symbols, + regions: regions + .into_iter() + .map(|r| r.expect("every region in the scope table is lowered")) + .collect(), + ops, + syntax_errors, + lexical: parts.lexical, + free_symbols: OnceLock::new(), + line_index: LineIndex::from_source_text(source), + source: source.to_owned(), + }; + for symbol in &program.symbols { + if let Some(slot) = symbol.slot { + let region = &mut program.regions[program.scopes[symbol.scope.index()].region.index()]; + region.num_slots = region.num_slots.max(slot + 1); + } + } + // A symbol is captured if code in another region accesses it. + for i in 0..program.ops.len() { + if let OpKind::Name { symbol, .. } = program.ops[i].kind + && program + .owner(symbol) + .is_some_and(|owner| owner != program.ops[i].region) + { + program.symbols[symbol.index()].captured = true; + } + } + ProgramWithFlowPoints { + program, + flow_points: flow_points.map_or_else(Vec::new, |collector| collector.points), + } +} + +/// The number of `finally` bodies a region may copy onto exit paths before further +/// copies are replaced by a [`GapKind::FinallyLimit`]. Nested `finally` blocks multiply +/// copies, so this bounds the size of the lowered graph. +const MAX_FINALLY_COPIES: usize = 256; + +/// A pending action on the way out of a construct. +enum Unwind<'a> { + /// A `with` item whose `ContextExit` must run. + With { + manager: Reg, + context: OpId, + is_async: bool, + }, + /// A `finally` body that must run. + Finally(&'a [Stmt]), + /// Where an exception raised at this depth is dispatched. + Handler(BlockId), + /// The name bound by `except ... as name`, deleted however the handler is left. + DeleteName(&'a Identifier), +} + +/// Where `break` and `continue` go, and the unwind depth at the loop body. +#[derive(Clone, Copy)] +struct LoopTarget { + break_to: BlockId, + continue_to: BlockId, + depth: usize, +} + +/// The blocks of a loop being lowered. +struct LoopBlocks { + head: BlockId, + orelse: BlockId, + after: BlockId, +} + +/// The blocks of an `if` being lowered: the join, and the block that the next arm's test +/// starts in. +struct IfBlocks { + after: BlockId, + next: Option, +} + +/// The region currently being lowered. +struct RegionState<'a> { + scope: ScopeId, + blocks: Vec<(Vec, Option)>, + /// `None` after a terminator: following code is unreachable. + current: Option, + num_regs: u32, + unwind: Vec>, + stmt: TextRange, + /// `finally` bodies lowered so far by unwinding. + finally_copies: usize, +} + +impl<'a> RegionState<'a> { + fn new(scope: ScopeId) -> Self { + Self { + scope, + blocks: vec![(Vec::new(), None)], + current: Some(BlockId(0)), + num_regs: 0, + unwind: Vec::new(), + stmt: TextRange::default(), + finally_copies: 0, + } + } + + /// The finished blocks and temporary count. + fn into_blocks(self) -> (Vec, u32) { + let blocks = self + .blocks + .into_iter() + .map(|(ops, terminator)| Block { + ops, + terminator: terminator.expect("every block is terminated"), + }) + .collect(); + (blocks, self.num_regs) + } +} + +struct Lowerer<'a> { + module: ModuleName, + is_init: bool, + table: ScopeTable, + regions: Vec>, + ops: Vec, + r: RegionState<'a>, + driver: Driver, + flow_points: Option, +} + +/// The source of a deferred region's body. +enum Body<'a> { + Function(&'a StmtFunctionDef), + Lambda(&'a Expr), + Generator(&'a [Comprehension], &'a Expr), +} + +impl<'a> Lowerer<'a> { + // Building blocks. + + fn new_block(&mut self) -> BlockId { + self.r.blocks.push((Vec::new(), None)); + BlockId(self.r.blocks.len() as u32 - 1) + } + + /// The current block, starting a fresh unreachable one after a terminator. + fn current(&mut self) -> BlockId { + match self.r.current { + Some(b) => b, + None => { + let b = self.new_block(); + self.r.current = Some(b); + b + } + } + } + + fn start(&mut self, block: BlockId) { + self.r.current = Some(block); + } + + fn terminate(&mut self, terminator: Terminator) { + let b = self.current(); + self.r.blocks[b.index()].1 = Some(terminator); + self.r.current = None; + } + + fn jump(&mut self, to: BlockId) { + if self.r.current.is_some() { + self.terminate(Terminator::Jump(to)); + } + } + + fn reg(&mut self) -> Reg { + self.r.num_regs += 1; + Reg(self.r.num_regs - 1) + } + + fn next_op(&self) -> OpId { + OpId(self.ops.len() as u32) + } + + fn emit(&mut self, kind: OpKind, range: TextRange) -> OpId { + let block = self.current(); + let id = self.next_op(); + self.ops.push(Op { + kind, + region: RegionId(self.regions_lowering_index()), + scope: self.r.scope, + block, + range, + stmt: self.r.stmt, + }); + self.r.blocks[block.index()].0.push(id); + id + } + + fn regions_lowering_index(&self) -> u32 { + self.table.scopes[self.r.scope.index()].region.0 + } + + fn gap(&mut self, kind: GapKind, dst: Option, range: TextRange) { + self.emit(OpKind::Gap { kind, dst }, range); + } + + fn name(&mut self, name: &Name, access: Access, range: TextRange) { + let (symbol, error) = self.table.resolve(self.r.scope, name); + if error { + self.gap(GapKind::ScopeError, None, range); + } + self.emit(OpKind::Name { symbol, access }, range); + } + + // Unwinding. + + /// Emit the exits of every pending `with` and `finally` above `depth`, innermost first. + fn unwind_to(&mut self, depth: usize) { + for i in (depth..self.r.unwind.len()).rev() { + if self.r.current.is_none() { + return; + } + match self.r.unwind[i] { + Unwind::With { + manager, + context, + is_async, + } => { + let range = self.op_range(context); + self.emit( + OpKind::ContextExit { + manager, + context, + is_async, + }, + range, + ); + } + Unwind::Finally(body) => { + // The finally body runs with only the entries outside it pending. + let saved = self.r.unwind.split_off(i); + self.finally_body(body); + self.r.unwind.extend(saved); + } + Unwind::DeleteName(name) => self.name(&name.id, Access::Delete, name.range), + Unwind::Handler(_) => {} + } + } + } + + fn op_range(&self, op: OpId) -> TextRange { + self.ops[op.index()].range + } + + fn innermost_handler(&self) -> Option<(usize, BlockId)> { + self.r + .unwind + .iter() + .enumerate() + .rev() + .find_map(|(i, u)| match u { + Unwind::Handler(b) => Some((i, *b)), + _ => None, + }) + } + + /// Raise from the current point: to the innermost handler, or out of the region. + fn raise(&mut self, exc: Option, cause: RaiseCause, range: TextRange) { + match self.innermost_handler() { + Some((depth, handler)) => { + self.unwind_to(depth + 1); + self.jump(handler); + } + None => { + self.unwind_to(0); + if self.r.current.is_some() { + self.terminate(Terminator::Raise { exc, cause, range }); + } + } + } + } + + /// Inside a `try`, the current point may raise: add an edge to the handler. + fn exception_edge(&mut self) { + if self.r.current.is_none() { + return; + } + if let Some((depth, handler)) = self.innermost_handler() { + let next = self.new_block(); + let edge = self.new_block(); + self.terminate(Terminator::Choice(vec![next, edge])); + self.start(edge); + self.unwind_to(depth + 1); + self.jump(handler); + self.start(next); + } + } + + // Statements. + + /// A statement that the structured driver does not decompose. + fn stmt(&mut self, x: &'a Stmt) { + match x { + Stmt::FunctionDef(f) => { + let decorators = self.decorators(&f.decorator_list); + if let Some(t) = &f.type_params { + self.gap(GapKind::TypeParams, None, t.range); + } + let annotations = f + .parameters + .iter() + .filter_map(|p| p.annotation()) + .chain(f.returns.as_deref()); + if let Some(a) = annotations + .into_iter() + .find(|a| annotation_may_have_effects(a)) + { + self.gap(GapKind::AnnotationEvaluation, None, a.range()); + } + let function = self.reg(); + self.function(f.range, Some(&f.parameters), Body::Function(f), function); + let value = self.apply_decorators(decorators, function, f.range); + self.name(&f.name.id, Access::Store(value), f.name.range); + } + Stmt::ClassDef(c) => { + let decorators = self.decorators(&c.decorator_list); + if let Some(t) = &c.type_params { + self.gap(GapKind::TypeParams, None, t.range); + } + let bases = match &c.arguments { + Some(arguments) => self.arguments(arguments), + None => Vec::new(), + }; + let scope = self.table.scope_of_node(c.range); + let outer = std::mem::replace(&mut self.r.scope, scope); + structured::suite(self, &c.body); + self.r.scope = outer; + let dst = self.reg(); + self.emit( + OpKind::MakeClass { + name: c.name.id.clone(), + scope, + bases, + dst, + }, + c.range, + ); + let value = self.apply_decorators(decorators, dst, c.range); + self.name(&c.name.id, Access::Store(value), c.name.range); + } + Stmt::Return(x) => { + let value = match &x.value { + Some(v) => self.expr(v), + None => self.literal(Literal::None, x.range), + }; + self.unwind_to(0); + if self.r.current.is_some() { + self.terminate(Terminator::Return(Some(value))); + } + } + Stmt::Delete(x) => { + for t in &x.targets { + self.delete(t); + } + } + Stmt::Assign(x) => { + let value = self.expr(&x.value); + for t in &x.targets { + self.assign(t, value); + } + } + Stmt::AugAssign(x) => { + let op = BinaryOp::InPlace(x.op); + let dst = self.reg(); + match &*x.target { + Expr::Name(n) => { + let current = self.reg(); + self.name(&n.id, Access::Load(current), n.range); + let right = self.expr(&x.value); + self.binary(op, current, right, dst, x.range); + self.name(&n.id, Access::Store(dst), n.range); + } + Expr::Attribute(a) => { + let object = self.expr(&a.value); + let current = self.reg(); + let attr = a.attr.id.clone(); + self.emit( + OpKind::Attribute { + object, + attr: attr.clone(), + access: Access::Load(current), + }, + a.range, + ); + let right = self.expr(&x.value); + self.binary(op, current, right, dst, x.range); + self.emit( + OpKind::Attribute { + object, + attr, + access: Access::Store(dst), + }, + a.range, + ); + } + Expr::Subscript(s) => { + let object = self.expr(&s.value); + let index = self.expr(&s.slice); + let current = self.reg(); + self.emit( + OpKind::Subscript { + object, + index, + access: Access::Load(current), + }, + s.range, + ); + let right = self.expr(&x.value); + self.binary(op, current, right, dst, x.range); + self.emit( + OpKind::Subscript { + object, + index, + access: Access::Store(dst), + }, + s.range, + ); + } + // Any other target is a syntax error that the parser reports. + _ => {} + } + } + Stmt::AnnAssign(x) => { + // Only module and class annotations would ever be evaluated at runtime. + let kind = self.table.scopes[self.r.scope.index()].kind; + if matches!(kind, ScopeKind::Module | ScopeKind::Class) + && annotation_may_have_effects(&x.annotation) + { + self.gap(GapKind::AnnotationEvaluation, None, x.annotation.range()); + } + match (&x.value, &*x.target) { + (Some(v), _) => { + let value = self.expr(v); + self.assign(&x.target, value); + } + // Without a value, CPython still evaluates an attribute target's object + // and a subscript target's object and index, but stores nothing. + (None, Expr::Attribute(a)) => { + self.expr(&a.value); + } + (None, Expr::Subscript(s)) => { + self.expr(&s.value); + self.expr(&s.slice); + } + (None, _) => {} + } + } + Stmt::TypeAlias(x) => { + let dst = self.reg(); + self.gap(GapKind::TypeParams, Some(dst), x.range); + self.assign(&x.name, dst); + } + Stmt::With(x) => { + let depth = self.r.unwind.len(); + for item in &x.items { + let manager = self.expr(&item.context_expr); + let dst = self.reg(); + let context = self.next_op(); + self.emit( + OpKind::ContextEnter { + manager, + dst, + context, + is_async: x.is_async, + }, + item.range, + ); + if let Some(target) = &item.optional_vars { + self.assign(target, dst); + } + self.r.unwind.push(Unwind::With { + manager, + context, + is_async: x.is_async, + }); + } + structured::suite(self, &x.body); + self.unwind_to(depth); + self.r.unwind.truncate(depth); + } + Stmt::Match(x) => { + self.expr(&x.subject); + self.gap(GapKind::Match, None, x.range); + let after = self.new_block(); + for case in &x.cases { + // Try this case or fall through to the next one. + let this = self.new_block(); + let next = self.new_block(); + self.terminate(Terminator::Choice(vec![this, next])); + self.start(this); + let mut captures = Vec::new(); + Ast::pattern_lvalue(&case.pattern, &mut |id| captures.push(id)); + for id in captures { + let dst = self.reg(); + self.gap(GapKind::Match, Some(dst), id.range); + self.name(&id.id, Access::Store(dst), id.range); + } + if let Some(guard) = &case.guard { + let cond = self.expr(guard); + let body = self.new_block(); + self.terminate(Terminator::Branch { + cond, + then: body, + orelse: next, + }); + self.start(body); + } + structured::suite(self, &case.body); + self.jump(after); + self.start(next); + if case.guard.is_none() && case.pattern.is_irrefutable() { + // No later case and no fall-through is reachable. + self.terminate(Terminator::Choice(Vec::new())); + } + } + self.jump(after); + self.start(after); + } + Stmt::Raise(x) => { + let exc = x.exc.as_ref().map(|e| self.expr(e)); + if let Some(cause) = &x.cause { + self.expr(cause); + } + self.raise(exc, RaiseCause::Explicit, x.range); + } + Stmt::Try(x) => self.try_(x), + Stmt::Assert(x) => { + let cond = self.expr(&x.test); + let ok = self.new_block(); + let fail = self.new_block(); + self.terminate(Terminator::Branch { + cond, + then: ok, + orelse: fail, + }); + self.start(fail); + // The message is evaluated only on failure; the raised exception is an + // `AssertionError` that wraps it, so no temporary holds the exception. + if let Some(m) = &x.msg { + self.expr(m); + } + self.raise(None, RaiseCause::AssertFailure, x.range); + self.start(ok); + } + Stmt::Import(x) => { + for a in &x.names { + let module = ModuleName::from_name(&a.name.id); + let dst = self.reg(); + match &a.asname { + Some(alias) => { + self.emit( + OpKind::Import { + module, + item: ImportItem::Module, + dst, + }, + a.range, + ); + self.name(&alias.id, Access::Store(dst), alias.range); + } + None => { + self.emit( + OpKind::Import { + module, + item: ImportItem::Root, + dst, + }, + a.range, + ); + self.name(&module.first_component(), Access::Store(dst), a.name.range); + } + } + } + } + Stmt::ImportFrom(x) => { + let module = self.module.new_maybe_relative( + self.is_init, + x.level, + x.module.as_ref().map(|m| &m.id), + ); + for a in &x.names { + let dst = self.reg(); + match module { + None => self.gap(GapKind::InvalidRelativeImport, Some(dst), a.range), + Some(module) if &a.name == "*" => { + self.emit( + OpKind::Import { + module, + item: ImportItem::Module, + dst, + }, + a.range, + ); + self.gap(GapKind::StarImport, None, a.range); + } + Some(module) => { + self.emit( + OpKind::Import { + module, + item: ImportItem::Name(a.name.id.clone()), + dst, + }, + a.range, + ); + } + } + if &a.name != "*" { + let bound = a.asname.as_ref().unwrap_or(&a.name); + self.name(&bound.id, Access::Store(dst), bound.range); + } + } + } + Stmt::Expr(x) => { + self.expr(&x.value); + } + Stmt::If(_) | Stmt::While(_) | Stmt::For(_) | Stmt::Break(_) | Stmt::Continue(_) => { + unreachable!("the structured driver decomposes `if`, loops and jumps") + } + Stmt::IpyEscapeCommand(x) => self.gap(GapKind::IpyEscape, None, x.range), + Stmt::Global(_) | Stmt::Nonlocal(_) | Stmt::Pass(_) => {} + } + } + + /// `try`: the body may raise at any statement boundary. Exceptions go to a dispatch + /// that tries each handler in order; an unhandled exception runs `finally` and + /// propagates. `finally` also runs on every normal exit, including `return`, + /// `break` and `continue`. + fn try_(&mut self, x: &'a StmtTry) { + self.gap(GapKind::Exceptions, None, x.range); + let depth = self.r.unwind.len(); + // An exception escaping the body and handlers runs a copy of `finally`, then + // propagates. Pushing the handler above `Finally` means that a jump to it does + // not also unwind through `Finally`, and that an exception inside a `finally` + // copy goes to the outer handlers. + let propagate = (!x.finalbody.is_empty()).then(|| self.new_block()); + if let Some(propagate) = propagate { + self.r.unwind.push(Unwind::Finally(&x.finalbody)); + self.r.unwind.push(Unwind::Handler(propagate)); + } + let protected = self.r.unwind.len(); + let dispatch = (!x.handlers.is_empty()).then(|| self.new_block()); + if let Some(dispatch) = dispatch { + self.r.unwind.push(Unwind::Handler(dispatch)); + } + structured::suite(self, &x.body); + self.r.unwind.truncate(protected); + structured::suite(self, &x.orelse); + let finish = self.new_block(); + self.jump(finish); + if let Some(dispatch) = dispatch { + self.start(dispatch); + for ExceptHandler::ExceptHandler(h) in &x.handlers { + if let Some(t) = &h.type_ { + self.expr(t); + } + let this = self.new_block(); + let next = self.new_block(); + self.terminate(Terminator::Choice(vec![this, next])); + self.start(this); + let handler_depth = self.r.unwind.len(); + if let Some(name) = &h.name { + let dst = self.reg(); + self.gap(GapKind::Exceptions, Some(dst), name.range); + self.name(&name.id, Access::Store(dst), name.range); + // Python deletes the bound exception however the handler is left. + self.r.unwind.push(Unwind::DeleteName(name)); + } + structured::suite(self, &h.body); + self.unwind_to(handler_depth); + self.r.unwind.truncate(handler_depth); + self.jump(finish); + self.start(next); + if h.type_.is_none() { + // A bare `except` catches everything. + self.terminate(Terminator::Choice(Vec::new())); + } + } + // No handler matched. + self.raise(None, RaiseCause::Reraise, x.range); + } + self.r.unwind.truncate(depth); + self.start(finish); + if let Some(propagate) = propagate { + self.finally_body(&x.finalbody); + let after = self.new_block(); + self.jump(after); + self.start(propagate); + self.finally_body(&x.finalbody); + self.raise(None, RaiseCause::Reraise, x.range); + self.start(after); + } + } + + /// Lower one copy of a `finally` body. Each exit path gets its own copy, so nested + /// `finally` blocks multiply copies; past a budget, the copy is replaced by a gap. + fn finally_body(&mut self, body: &'a [Stmt]) { + if self.r.current.is_none() { + return; + } + if self.r.finally_copies >= MAX_FINALLY_COPIES { + self.gap(GapKind::FinallyLimit, None, self.r.stmt); + return; + } + self.r.finally_copies += 1; + structured::suite(self, body); + } + + fn decorators( + &mut self, + decorators: &'a [ruff_python_ast::Decorator], + ) -> Vec<(Reg, TextRange)> { + decorators + .iter() + .map(|d| (self.expr(&d.expression), d.range)) + .collect() + } + + /// Apply already-evaluated decorators bottom-up. + fn apply_decorators( + &mut self, + decorators: Vec<(Reg, TextRange)>, + mut value: Reg, + _: TextRange, + ) -> Reg { + for (func, range) in decorators.into_iter().rev() { + let dst = self.reg(); + self.emit( + OpKind::Call { + func, + args: vec![Arg::Positional(value)], + dst, + }, + range, + ); + value = dst; + } + value + } + + // Deferred regions. + + /// Evaluate defaults, lower the deferred region, and emit its `MakeFunction` into `dst`. + fn function( + &mut self, + range: TextRange, + parameters: Option<&'a Parameters>, + body: Body<'a>, + dst: Reg, + ) { + let defaults = parameters + .into_iter() + .flat_map(|p| p.iter_non_variadic_params()) + .filter_map(|p| p.default.as_deref()) + .map(|d| self.expr(d)) + .collect(); + let scope = self.table.scope_of_node(range); + let region = self.table.scopes[scope.index()].region; + let make = self.emit( + OpKind::MakeFunction { + region, + defaults, + dst, + }, + range, + ); + // A `finally` body is lowered once per exit path, so the same definition can be + // reached again; its region is lowered only the first time. + if self.regions[region.index()].is_some() { + return; + } + let def_scope = self.r.scope; + let mut params = Vec::new(); + if let Some(parameters) = parameters { + let kinds = parameters + .posonlyargs + .iter() + .map(|p| (&p.parameter, ParamKind::PositionalOnly, p.default.is_some())) + .chain( + parameters + .args + .iter() + .map(|p| (&p.parameter, ParamKind::Positional, p.default.is_some())), + ) + .chain( + parameters + .vararg + .iter() + .map(|p| (&**p, ParamKind::VarArgs, false)), + ) + .chain( + parameters + .kwonlyargs + .iter() + .map(|p| (&p.parameter, ParamKind::KeywordOnly, p.default.is_some())), + ) + .chain( + parameters + .kwarg + .iter() + .map(|p| (&**p, ParamKind::KwArgs, false)), + ); + for (p, kind, has_default) in kinds { + let annotation = p + .annotation + .as_deref() + .map(|a| self.static_expr(def_scope, a)); + params.push(Param { + name: p.name.id.clone(), + symbol: self.table.resolve(scope, &p.name.id).0, + kind, + annotation, + has_default, + range: p.range, + }); + } + } + let (name, trigger, returns) = match &body { + Body::Function(f) => { + let generator = Ast::body_contains_yield(&f.body); + let trigger = if generator { + Trigger::OnIteration + } else if f.is_async { + Trigger::OnAwait + } else { + Trigger::OnCall + }; + let returns = f.returns.as_deref().map(|r| self.static_expr(def_scope, r)); + (f.name.id.clone(), trigger, returns) + } + Body::Lambda(e) => { + let trigger = if Ast::expr_contains_yield(e) { + Trigger::OnIteration + } else { + Trigger::OnCall + }; + (Name::new_static(""), trigger, None) + } + Body::Generator(..) => { + let symbol = self + .table + .resolve(scope, &Name::new_static(GENERATOR_ARG)) + .0; + params.push(Param { + name: Name::new_static(GENERATOR_ARG), + symbol, + kind: ParamKind::PositionalOnly, + annotation: None, + has_default: false, + range, + }); + (Name::new_static(""), Trigger::OnIteration, None) + } + }; + let outer = std::mem::replace(&mut self.r, RegionState::new(scope)); + for (index, p) in params.iter().enumerate() { + let dst = self.reg(); + self.emit(OpKind::Param { index, dst }, p.range); + self.emit( + OpKind::Name { + symbol: p.symbol, + access: Access::Store(dst), + }, + p.range, + ); + } + match body { + Body::Function(f) => { + structured::region(self, &f.body); + if self.r.current.is_some() { + let none = self.literal(Literal::None, f.range); + self.terminate(Terminator::Return(Some(none))); + } + } + Body::Lambda(e) => { + self.r.stmt = e.range(); + let value = self.expr(e); + self.terminate(Terminator::Return(Some(value))); + } + Body::Generator(generators, elt) => { + self.r.stmt = range; + let iterator = self.reg(); + self.emit( + OpKind::Name { + symbol: params[0].symbol, + access: Access::Load(iterator), + }, + range, + ); + self.comprehension_loops(generators, iterator, &mut |this| { + let value = this.expr(elt); + let dst = this.reg(); + this.emit( + OpKind::Yield { + value: Some(value), + dst, + from: false, + }, + elt.range(), + ); + }); + let none = self.literal(Literal::None, range); + self.terminate(Terminator::Return(Some(none))); + } + } + let state = std::mem::replace(&mut self.r, outer); + let (blocks, num_regs) = state.into_blocks(); + let parent = RegionId(self.regions_lowering_index()); + self.regions[region.index()] = Some(Region { + name, + trigger, + scope, + parent: Some((parent, make)), + params, + returns, + blocks, + entry: BlockId(0), + num_regs, + // Filled in once all symbols are known. + num_slots: 0, + range, + }); + } + + /// `iter(iterable)` for one comprehension generator. + fn iter(&mut self, iterable: Reg, g: &Comprehension) -> Reg { + let iterator = self.reg(); + self.emit( + OpKind::Iter { + iterable, + dst: iterator, + is_async: g.is_async, + }, + g.iter.range(), + ); + iterator + } + + /// The loops of a comprehension. The first iterator is already created; `body` runs + /// once per innermost iteration. Leaves the exit block current. + fn comprehension_loops( + &mut self, + generators: &'a [Comprehension], + iterator: Reg, + body: &mut dyn FnMut(&mut Self), + ) { + let (g, rest) = generators + .split_first() + .expect("a comprehension has at least one generator"); + let head = self.new_block(); + let next = self.new_block(); + let exit = self.new_block(); + self.jump(head); + self.start(head); + self.terminate(Terminator::Choice(vec![next, exit])); + self.start(next); + let item = self.reg(); + self.emit( + OpKind::Next { + iterator, + dst: item, + is_async: g.is_async, + }, + g.target.range(), + ); + self.assign(&g.target, item); + for cond in &g.ifs { + let c = self.expr(cond); + let pass = self.new_block(); + self.terminate(Terminator::Branch { + cond: c, + then: pass, + orelse: head, + }); + self.start(pass); + } + match rest.first() { + Some(inner) => { + let inner_iterable = self.expr(&inner.iter); + let inner_iterator = self.iter(inner_iterable, inner); + self.comprehension_loops(rest, inner_iterator, body); + } + None => body(self), + } + self.jump(head); + self.start(exit); + } + + /// A list, set or dict comprehension, lowered inline in its own scope. + fn comprehension( + &mut self, + range: TextRange, + kind: BuildKind, + generators: &'a [Comprehension], + element: &mut dyn FnMut(&mut Self) -> Element, + dst: Reg, + ) { + let first = self.expr(&generators[0].iter); + let iterator = self.iter(first, &generators[0]); + self.emit( + OpKind::Build { + kind, + elements: Vec::new(), + dst, + }, + range, + ); + let scope = self.table.scope_of_node(range); + let outer = std::mem::replace(&mut self.r.scope, scope); + self.comprehension_loops(generators, iterator, &mut |this| { + let element = element(this); + this.emit( + OpKind::Append { + collection: dst, + element, + }, + range, + ); + }); + self.r.scope = outer; + } + + // Targets. + + fn assign(&mut self, target: &'a Expr, value: Reg) { + match target { + Expr::Name(n) => self.name(&n.id, Access::Store(value), n.range), + Expr::Attribute(a) => { + let object = self.expr(&a.value); + self.emit( + OpKind::Attribute { + object, + attr: a.attr.id.clone(), + access: Access::Store(value), + }, + a.range, + ); + } + Expr::Subscript(s) => { + let object = self.expr(&s.value); + let index = self.expr(&s.slice); + self.emit( + OpKind::Subscript { + object, + index, + access: Access::Store(value), + }, + s.range, + ); + } + Expr::Tuple(_) | Expr::List(_) => { + let elts = match target { + Expr::Tuple(t) => &t.elts, + Expr::List(l) => &l.elts, + _ => unreachable!("matched a tuple or list above"), + }; + let dsts: Vec = elts.iter().map(|_| self.reg()).collect(); + let star = elts.iter().position(|e| matches!(e, Expr::Starred(_))); + self.emit( + OpKind::Unpack { + value, + dsts: dsts.clone(), + star, + }, + target.range(), + ); + for (e, dst) in elts.iter().zip(dsts) { + let e = match e { + Expr::Starred(s) => &*s.value, + e => e, + }; + self.assign(e, dst); + } + } + Expr::Starred(s) => self.assign(&s.value, value), + // Any other target is a syntax error that the parser reports. + _ => {} + } + } + + fn delete(&mut self, target: &'a Expr) { + match target { + Expr::Name(n) => self.name(&n.id, Access::Delete, n.range), + Expr::Attribute(a) => { + let object = self.expr(&a.value); + self.emit( + OpKind::Attribute { + object, + attr: a.attr.id.clone(), + access: Access::Delete, + }, + a.range, + ); + } + Expr::Subscript(s) => { + let object = self.expr(&s.value); + let index = self.expr(&s.slice); + self.emit( + OpKind::Subscript { + object, + index, + access: Access::Delete, + }, + s.range, + ); + } + Expr::Tuple(t) => t.elts.iter().for_each(|e| self.delete(e)), + Expr::List(l) => l.elts.iter().for_each(|e| self.delete(e)), + _ => {} + } + } + + // Expressions. + + fn expr(&mut self, x: &'a Expr) -> Reg { + let dst = self.reg(); + self.expr_to(x, dst); + dst + } + + fn literal(&mut self, value: Literal, range: TextRange) -> Reg { + let dst = self.reg(); + self.emit(OpKind::Literal { value, dst }, range); + dst + } + + fn binary(&mut self, op: BinaryOp, left: Reg, right: Reg, dst: Reg, range: TextRange) { + self.emit( + OpKind::Binary { + op, + left, + right, + dst, + }, + range, + ); + } + + fn build(&mut self, kind: BuildKind, elts: &'a [Expr], dst: Reg, range: TextRange) { + let elements = elts + .iter() + .map(|e| match e { + Expr::Starred(s) => Element::Star(self.expr(&s.value)), + e => Element::Value(self.expr(e)), + }) + .collect(); + self.emit( + OpKind::Build { + kind, + elements, + dst, + }, + range, + ); + } + + /// Evaluate call arguments as CPython does: every positional and starred argument + /// first, then every keyword and `**` argument, each group in source order. This + /// differs from source order when `*args` follows a keyword argument. + fn arguments(&mut self, arguments: &'a Arguments) -> Vec { + let positional = arguments.args.iter().map(|a| match a { + Expr::Starred(s) => Arg::Star(self.expr(&s.value)), + e => Arg::Positional(self.expr(e)), + }); + let mut args: Vec = positional.collect(); + for k in &arguments.keywords { + let value = self.expr(&k.value); + args.push(match &k.arg { + Some(name) => Arg::Keyword(name.id.clone(), value), + None => Arg::DoubleStar(value), + }); + } + args + } + + /// Evaluate `x` into `dst`. + fn expr_to(&mut self, x: &'a Expr, dst: Reg) { + let range = x.range(); + match x { + Expr::BoolOp(b) => { + let after = self.new_block(); + let (last, init) = b + .values + .split_last() + .expect("a boolean operation has operands"); + for v in init { + self.expr_to(v, dst); + let next = self.new_block(); + let (then, orelse) = match b.op { + BoolOp::And => (next, after), + BoolOp::Or => (after, next), + }; + self.terminate(Terminator::Branch { + cond: dst, + then, + orelse, + }); + self.start(next); + } + self.expr_to(last, dst); + self.jump(after); + self.start(after); + } + Expr::Named(n) => { + self.expr_to(&n.value, dst); + self.assign(&n.target, dst); + } + Expr::BinOp(b) => { + let left = self.expr(&b.left); + let right = self.expr(&b.right); + self.binary(BinaryOp::Arith(b.op), left, right, dst, range); + } + Expr::UnaryOp(u) => { + let operand = self.expr(&u.operand); + self.emit( + OpKind::Unary { + op: u.op, + operand, + dst, + }, + range, + ); + } + Expr::Lambda(l) => { + self.function(l.range, l.parameters.as_deref(), Body::Lambda(&l.body), dst); + } + Expr::If(i) => { + let cond = self.expr(&i.test); + let then = self.new_block(); + let orelse = self.new_block(); + let after = self.new_block(); + self.terminate(Terminator::Branch { cond, then, orelse }); + self.start(then); + self.expr_to(&i.body, dst); + self.jump(after); + self.start(orelse); + self.expr_to(&i.orelse, dst); + self.jump(after); + self.start(after); + } + Expr::Dict(d) => { + let elements = d + .items + .iter() + .map(|item| match &item.key { + Some(k) => { + let k = self.expr(k); + Element::Pair(k, self.expr(&item.value)) + } + None => Element::DoubleStar(self.expr(&item.value)), + }) + .collect(); + self.emit( + OpKind::Build { + kind: BuildKind::Dict, + elements, + dst, + }, + range, + ); + } + Expr::Set(s) => self.build(BuildKind::Set, &s.elts, dst, range), + Expr::List(l) => self.build(BuildKind::List, &l.elts, dst, range), + Expr::Tuple(t) => self.build(BuildKind::Tuple, &t.elts, dst, range), + Expr::ListComp(c) => self.comprehension( + range, + BuildKind::List, + &c.generators, + &mut |this| Element::Value(this.expr(&c.elt)), + dst, + ), + Expr::SetComp(c) => self.comprehension( + range, + BuildKind::Set, + &c.generators, + &mut |this| Element::Value(this.expr(&c.elt)), + dst, + ), + Expr::DictComp(c) => self.comprehension( + range, + BuildKind::Dict, + &c.generators, + &mut |this| match &c.key { + Some(key) => { + let k = this.expr(key); + Element::Pair(k, this.expr(&c.value)) + } + None => Element::DoubleStar(this.expr(&c.value)), + }, + dst, + ), + Expr::Generator(g) => { + // The first iterator is created eagerly and passed to the generator. + let first = self.expr(&g.generators[0].iter); + let first = self.iter(first, &g.generators[0]); + let function = self.reg(); + self.function( + range, + None, + Body::Generator(&g.generators, &g.elt), + function, + ); + self.emit( + OpKind::Call { + func: function, + args: vec![Arg::Positional(first)], + dst, + }, + range, + ); + } + Expr::Await(a) => { + let value = self.expr(&a.value); + self.emit(OpKind::Await { value, dst }, range); + } + Expr::Yield(y) => { + let value = y.value.as_ref().map(|v| self.expr(v)); + self.emit( + OpKind::Yield { + value, + dst, + from: false, + }, + range, + ); + } + Expr::YieldFrom(y) => { + let value = Some(self.expr(&y.value)); + self.emit( + OpKind::Yield { + value, + dst, + from: true, + }, + range, + ); + } + Expr::Compare(c) => { + // `a < b < c` evaluates `b` once and stops at the first false comparison. + let after = self.new_block(); + let mut left = self.expr(c.first_operand()); + for (i, (op, comparator)) in c.ops.iter().zip(&c.operands[1..]).enumerate() { + let right = self.expr(comparator); + self.binary(BinaryOp::Compare(*op), left, right, dst, range); + if i + 1 < c.ops.len() { + let next = self.new_block(); + self.terminate(Terminator::Branch { + cond: dst, + then: next, + orelse: after, + }); + self.start(next); + } + left = right; + } + self.jump(after); + self.start(after); + } + Expr::Call(c) => { + let func = self.expr(&c.func); + let args = self.arguments(&c.arguments); + self.emit(OpKind::Call { func, args, dst }, range); + } + Expr::FString(_) | Expr::TString(_) => { + let mut interpolations = Vec::new(); + Interpolations(&mut interpolations).visit_expr(x); + let elements = interpolations + .into_iter() + .map(|e| Element::Value(self.expr(e))) + .collect(); + self.emit( + OpKind::Build { + kind: BuildKind::String, + elements, + dst, + }, + range, + ); + } + Expr::StringLiteral(s) => { + self.emit( + OpKind::Literal { + value: Literal::Str(s.value.to_str().into()), + dst, + }, + range, + ); + } + Expr::BytesLiteral(_) => { + self.emit( + OpKind::Literal { + value: Literal::Bytes, + dst, + }, + range, + ); + } + Expr::NumberLiteral(n) => { + let value = match &n.value { + Number::Int(i) => Literal::Int(i.as_i64()), + Number::Float(f) => Literal::Float(*f), + Number::Complex { .. } => Literal::Complex, + }; + self.emit(OpKind::Literal { value, dst }, range); + } + Expr::BooleanLiteral(b) => { + self.emit( + OpKind::Literal { + value: Literal::Bool(b.value), + dst, + }, + range, + ); + } + Expr::NoneLiteral(_) => { + self.emit( + OpKind::Literal { + value: Literal::None, + dst, + }, + range, + ); + } + Expr::EllipsisLiteral(_) => { + self.emit( + OpKind::Literal { + value: Literal::Ellipsis, + dst, + }, + range, + ); + } + Expr::Attribute(a) => { + let object = self.expr(&a.value); + self.emit( + OpKind::Attribute { + object, + attr: a.attr.id.clone(), + access: Access::Load(dst), + }, + range, + ); + } + Expr::Subscript(s) => { + let object = self.expr(&s.value); + let index = self.expr(&s.slice); + self.emit( + OpKind::Subscript { + object, + index, + access: Access::Load(dst), + }, + range, + ); + } + // A starred expression outside a display or call is a syntax error; lower + // its operand so its effects remain visible. + Expr::Starred(s) => self.expr_to(&s.value, dst), + Expr::Name(n) => self.name(&n.id, Access::Load(dst), range), + Expr::Slice(s) => { + let elements = [&s.lower, &s.upper, &s.step] + .into_iter() + .map(|b| match b { + Some(b) => Element::Value(self.expr(b)), + None => Element::Missing, + }) + .collect(); + self.emit( + OpKind::Build { + kind: BuildKind::Slice, + elements, + dst, + }, + range, + ); + } + Expr::IpyEscapeCommand(_) => self.gap(GapKind::IpyEscape, Some(dst), range), + } + } + + /// Make an unknown-to-the-frontend static description of an annotation. + fn static_expr(&mut self, scope: ScopeId, x: &Expr) -> StaticExpr { + match x { + Expr::Name(n) => StaticExpr::Name(n.id.clone(), self.table.resolve(scope, &n.id).0), + Expr::Attribute(a) => StaticExpr::Attribute( + Box::new(self.static_expr(scope, &a.value)), + a.attr.id.clone(), + ), + Expr::Subscript(s) => { + let base = Box::new(self.static_expr(scope, &s.value)); + let index = match &*s.slice { + Expr::Tuple(t) => t.elts.iter().map(|e| self.static_expr(scope, e)).collect(), + e => vec![self.static_expr(scope, e)], + }; + StaticExpr::Subscript(base, index) + } + Expr::Tuple(t) => { + StaticExpr::Sequence(t.elts.iter().map(|e| self.static_expr(scope, e)).collect()) + } + Expr::List(l) => { + StaticExpr::Sequence(l.elts.iter().map(|e| self.static_expr(scope, e)).collect()) + } + Expr::StringLiteral(s) => StaticExpr::Literal(Literal::Str(s.value.to_str().into())), + Expr::NumberLiteral(n) => match &n.value { + Number::Int(i) => StaticExpr::Literal(Literal::Int(i.as_i64())), + _ => StaticExpr::Other(n.range), + }, + Expr::UnaryOp(u) + if u.op == ruff_python_ast::UnaryOp::USub + && let Expr::NumberLiteral(n) = &*u.operand + && let Number::Int(i) = &n.value => + { + StaticExpr::Literal(Literal::Int(i.as_i64().map(|i| -i))) + } + Expr::BooleanLiteral(b) => StaticExpr::Literal(Literal::Bool(b.value)), + Expr::NoneLiteral(_) => StaticExpr::Literal(Literal::None), + Expr::EllipsisLiteral(_) => StaticExpr::Literal(Literal::Ellipsis), + _ => StaticExpr::Other(x.range()), + } + } +} + +/// Lowering as a structured backend: arms and loops become blocks and terminators. +/// Lowering never decides a test statically, so every arm is visited. +impl<'a> Structured for Lowerer<'a> { + type Syntax = Borrowed<'a>; + /// The enclosing statement range, restored after each statement. + type Suite = TextRange; + type If = IfBlocks; + type Loop = LoopBlocks; + type Target = LoopTarget; + + fn driver(&mut self) -> &mut Driver { + &mut self.driver + } + + fn observes_flow_points(&self) -> bool { + self.flow_points.is_some() + } + + fn flow_point(&mut self, point: FlowPointId) { + self.flow_points + .as_mut() + .expect("flow points are emitted only when observation is enabled") + .observe(point); + } + + fn suite_start(&mut self, _: &[Stmt]) -> TextRange { + self.r.stmt + } + + fn absorbs(&mut self, _: &Stmt, _: &Stmt) -> bool { + false + } + + fn stmt_before(&mut self, _: &mut TextRange, stmt: &Stmt, _: Option<&Stmt>) { + self.exception_edge(); + self.r.stmt = stmt.range(); + } + + fn stmt_after(&mut self, outer: &mut TextRange) { + self.r.stmt = *outer; + } + + fn suite_end(&mut self, _: TextRange) { + self.exception_edge(); + } + + fn other_stmt(&mut self, stmt: &'a Stmt) { + self.stmt(stmt); + } + + fn if_start(&mut self, _: TextRange, _: &mut &'a Expr) -> IfBlocks { + IfBlocks { + after: self.new_block(), + next: None, + } + } + + fn arm_start( + &mut self, + state: &mut IfBlocks, + _: TextRange, + _: bool, + test: Option<&mut &'a Expr>, + ) -> Option { + if let Some(test) = test { + let cond = self.expr(test); + let then = self.new_block(); + let next = self.new_block(); + self.terminate(Terminator::Branch { + cond, + then, + orelse: next, + }); + self.start(then); + state.next = Some(next); + } + None + } + + fn arm_skip(&mut self, _: &mut IfBlocks, _: Arm>) { + unreachable!("lowering never decides a test statically") + } + + fn arm_enter(&mut self, _: &mut IfBlocks, _: TextRange, _: Option<&'a Expr>, _: &[Stmt]) {} + + fn arm_exit(&mut self, state: &mut IfBlocks, _: TextRange) { + self.jump(state.after); + if let Some(next) = state.next.take() { + self.start(next); + } + } + + fn arm_dead(&mut self, _: &mut IfBlocks, _: Arm>) { + unreachable!("lowering never decides a test statically") + } + + fn if_end(&mut self, state: IfBlocks, _: bool) { + self.jump(state.after); + self.start(state.after); + } + + fn loop_start( + &mut self, + header: LoopHeader>, + _: &[Stmt], + ) -> (LoopBlocks, LoopTarget) { + let iterator = match header { + LoopHeader::For { iter, is_async, .. } => { + let iterable = self.expr(iter); + let iterator = self.reg(); + self.emit( + OpKind::Iter { + iterable, + dst: iterator, + is_async, + }, + iter.range(), + ); + Some(iterator) + } + LoopHeader::While { .. } => None, + }; + let head = self.new_block(); + let body = self.new_block(); + let orelse = self.new_block(); + let after = self.new_block(); + self.jump(head); + self.start(head); + match header { + LoopHeader::For { + target, is_async, .. + } => { + self.terminate(Terminator::Choice(vec![body, orelse])); + self.start(body); + let item = self.reg(); + self.emit( + OpKind::Next { + iterator: iterator.expect("a `for` header creates an iterator"), + dst: item, + is_async, + }, + target.range(), + ); + self.assign(target, item); + } + LoopHeader::While { test, .. } => { + if is_truthy_constant(test) { + self.terminate(Terminator::Jump(body)); + } else { + let cond = self.expr(test); + self.terminate(Terminator::Branch { + cond, + then: body, + orelse, + }); + } + self.start(body); + } + } + ( + LoopBlocks { + head, + orelse, + after, + }, + LoopTarget { + break_to: after, + continue_to: head, + depth: self.r.unwind.len(), + }, + ) + } + + fn loop_else(&mut self, state: &mut LoopBlocks) { + self.jump(state.head); + self.start(state.orelse); + } + + fn loop_end(&mut self, state: LoopBlocks) { + self.jump(state.after); + self.start(state.after); + } + + fn loop_jump(&mut self, jump: Jump, _: TextRange, target: Option) { + // Outside a loop this is a syntax error that the parser reports. + if let Some(target) = target { + self.unwind_to(target.depth); + self.jump(match jump { + Jump::Break => target.break_to, + Jump::Continue => target.continue_to, + }); + } + } +} + +/// Collects the interpolated expressions of an f-string or t-string in evaluation order, +/// including those nested in format specifications. +struct Interpolations<'a, 'b>(&'b mut Vec<&'a Expr>); + +impl<'a> Visitor<'a> for Interpolations<'a, '_> { + fn visit_interpolated_string_element(&mut self, x: &'a InterpolatedStringElement) { + if let InterpolatedStringElement::Interpolation(i) = x { + self.0.push(&i.expression); + if let Some(spec) = &i.format_spec { + for e in &spec.elements { + self.visit_interpolated_string_element(e); + } + } + } + } + + fn visit_expr(&mut self, x: &'a Expr) { + walk_expr(self, x) + } +} + +/// CPython compiles `while True:` and `while 1:` without a test. +fn is_truthy_constant(x: &Expr) -> bool { + match x { + Expr::BooleanLiteral(b) => b.value, + Expr::NumberLiteral(n) => matches!(&n.value, Number::Int(i) if i.as_i64() != Some(0)), + _ => false, + } +} diff --git a/static_analysis/src/scope.rs b/static_analysis/src/scope.rs new file mode 100644 index 0000000000..27bf4eac5c --- /dev/null +++ b/static_analysis/src/scope.rs @@ -0,0 +1,465 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! The scope prepass: which scope owns each name. +//! +//! The declarations come from the shared [`declarations`] walker, which Pyrefly's export +//! `Definitions` also consumes, but this sink answers a runtime question rather than a +//! type-binding one: Python decides ownership syntactically, so every expression position +//! is kept, no branch is pruned by platform or version checks, and a name bound only in +//! dead code is still local. Name lookup is the shared [`lexical`] resolution: +//! `global`/`nonlocal` redirect a name to the owning scope, and class scopes are invisible +//! to the scopes nested inside them. + +use ruff_python_ast::Expr; +use ruff_python_ast::Stmt; +use ruff_python_ast::name::Name; +use ruff_python_ast::visitor::Visitor; +use ruff_python_ast::visitor::walk_expr; +use ruff_text_size::TextRange; +use starlark_map::small_map::SmallMap; +use starlark_map::small_set::SmallSet; + +use crate::declarations; +use crate::declarations::ChildScope; +use crate::declarations::Declaration; +use crate::declarations::DeclarationSink; +use crate::ir::RegionId; +use crate::ir::Scope; +use crate::ir::ScopeId; +use crate::ir::ScopeKind; +use crate::ir::Symbol; +use crate::ir::SymbolId; +use crate::ir::SymbolOrigin; +use crate::lexical; +use crate::lexical::Category; +use crate::lexical::Probe; +use crate::lexical::Resolution; +use crate::lexical::ScopeChain; + +/// The parameter through which a generator expression receives its first iterator, +/// named as CPython names it. +pub const GENERATOR_ARG: &str = ".0"; + +/// Declarations collected for one scope before symbols are created. +#[derive(Default)] +struct Declarations { + bound: SmallMap, + globals: SmallSet, + nonlocals: SmallSet, +} + +/// The finished scope structure, consulted (and extended with builtins) by lowering. +pub struct ScopeTable { + pub scopes: Vec, + pub symbols: Vec, + pub num_regions: u32, + /// Scopes introduced by `def`, `lambda`, `class` and comprehension nodes, by node range. + by_node: SmallMap, + /// Names each scope declared `global` or `nonlocal`. + globals: Vec>, + nonlocals: Vec>, + builtins: ScopeId, +} + +impl ScopeTable { + pub fn new(body: &[Stmt], range: TextRange) -> Self { + let mut builder = Builder { + scopes: Vec::new(), + decls: Vec::new(), + by_node: SmallMap::new(), + current: ScopeId(0), + num_regions: 1, + }; + builder.push(ScopeKind::Module, None, RegionId(0), range); + declarations::walk_suite(body, &mut builder); + let mut table = builder.finish(); + // Module and class bodies look names up dynamically (`LOAD_NAME`/`LOAD_GLOBAL`): + // an unbound class-local name falls back to the module, and an unbound module + // name falls back to builtins. Function locals have no fallback. + for i in 0..table.symbols.len() { + let symbol = &table.symbols[i]; + let name = symbol.name.clone(); + let fallback = match table.scopes[symbol.scope.index()].kind { + ScopeKind::Class => match table.scopes[0].symbols.get(&name) { + Some(module) => Some(*module), + None => Some(table.builtin(&name)), + }, + ScopeKind::Module => Some(table.builtin(&name)), + ScopeKind::Function + | ScopeKind::Lambda + | ScopeKind::Comprehension + | ScopeKind::Builtins => None, + }; + table.symbols[i].fallback = fallback; + } + table + } + + pub fn module(&self) -> ScopeId { + ScopeId(0) + } + + /// The scope introduced by the `def`, `lambda`, `class` or comprehension at `range`. + pub fn scope_of_node(&self, range: TextRange) -> ScopeId { + *self + .by_node + .get(&range) + .expect("the prepass creates a scope for every scope-introducing node") + } + + fn scope(&self, id: ScopeId) -> &Scope { + &self.scopes[id.index()] + } + + /// Resolve `name` as code in `scope` sees it, creating a builtins symbol if no + /// scope defines it. The flag is true for a scope error such as an unresolvable + /// `nonlocal`, which resolves to a builtins symbol. + pub fn resolve(&mut self, scope: ScopeId, name: &Name) -> (SymbolId, bool) { + match lookup(&self.scopes, &self.globals, &self.nonlocals, scope, name) { + Lookup::Found(symbol) => (symbol, false), + Lookup::Builtin => (self.builtin(name), false), + Lookup::ScopeError => (self.builtin(name), true), + } + } + + fn builtin(&mut self, name: &Name) -> SymbolId { + if let Some(symbol) = self.scope(self.builtins).symbols.get(name) { + return *symbol; + } + let id = SymbolId(self.symbols.len() as u32); + self.symbols.push(Symbol { + name: name.clone(), + scope: self.builtins, + origin: SymbolOrigin::Builtin, + slot: None, + captured: false, + fallback: None, + }); + self.scopes[self.builtins.index()] + .symbols + .insert(name.clone(), id); + id + } +} + +enum Lookup { + Found(SymbolId), + Builtin, + /// `nonlocal` with no enclosing function binding. + ScopeError, +} + +/// The scope tree as [`lexical`] navigates it. +#[derive(Clone, Copy)] +struct Tree<'s>(&'s [Scope]); + +impl ScopeChain for Tree<'_> { + type Id = ScopeId; + type Barrier = (); + + fn parent(&self, id: ScopeId) -> Option { + self.0[id.index()].parent + } + + fn category(&self, id: ScopeId) -> Category { + match self.0[id.index()].kind { + ScopeKind::Module => Category::Module, + ScopeKind::Class => Category::Class, + ScopeKind::Function | ScopeKind::Lambda => Category::Function, + ScopeKind::Comprehension => Category::Comprehension, + ScopeKind::Builtins => { + unreachable!("lookup from the builtins scope is answered directly") + } + } + } + + fn barrier(&self, _: ScopeId) {} + + fn join(_: (), _: ()) {} +} + +/// Python's lexical lookup. +fn lookup( + scopes: &[Scope], + globals: &[SmallSet], + nonlocals: &[SmallSet], + scope: ScopeId, + name: &Name, +) -> Lookup { + let symbols = |id: ScopeId| scopes[id.index()].symbols.get(name).copied(); + // The builtins scope sees only itself. + if scopes[scope.index()].kind == ScopeKind::Builtins { + return symbols(scope).map_or(Lookup::Builtin, Lookup::Found); + } + // Only the module can both own a name and declare it, and there the declaration is + // a no-op (`global`) or a syntax error (`nonlocal`). + let probe = |id: ScopeId| match symbols(id) { + Some(symbol) => Probe::Owned(symbol), + None if globals[id.index()].contains(name) => Probe::Global, + None if nonlocals[id.index()].contains(name) => Probe::Nonlocal, + None => Probe::Absent, + }; + match lexical::resolve(Tree(scopes), scope, probe) { + Resolution::Found(symbol) => Lookup::Found(symbol), + Resolution::Unbound => Lookup::Builtin, + Resolution::Nonlocal(_) => Lookup::ScopeError, + } +} + +impl ScopeTable { + /// The pieces [`crate::ir::Program`] keeps for public name resolution. + pub fn into_parts(self) -> ScopeParts { + ScopeParts { + scopes: self.scopes, + symbols: self.symbols, + lexical: Lexical { + globals: self.globals, + nonlocals: self.nonlocals, + builtins: self.builtins, + }, + } + } +} + +pub struct ScopeParts { + pub scopes: Vec, + pub symbols: Vec, + pub lexical: Lexical, +} + +/// The `global`/`nonlocal` declarations a finished program needs for name lookup. +pub struct Lexical { + globals: Vec>, + nonlocals: Vec>, + builtins: ScopeId, +} + +impl Lexical { + /// Lookup over a finished program; `None` for a builtin that no code referenced. + pub fn resolve(&self, scopes: &[Scope], scope: ScopeId, name: &Name) -> Option { + match lookup(scopes, &self.globals, &self.nonlocals, scope, name) { + Lookup::Found(s) => Some(s), + Lookup::Builtin | Lookup::ScopeError => { + scopes[self.builtins.index()].symbols.get(name).copied() + } + } + } +} + +struct Builder { + scopes: Vec, + decls: Vec, + by_node: SmallMap, + current: ScopeId, + num_regions: u32, +} + +impl Builder { + fn push( + &mut self, + kind: ScopeKind, + parent: Option, + region: RegionId, + range: TextRange, + ) -> ScopeId { + let id = ScopeId(self.scopes.len() as u32); + self.scopes.push(Scope { + kind, + parent, + region, + range, + symbols: SmallMap::new(), + }); + self.decls.push(Declarations::default()); + id + } + + /// Create the scope for the node at `range`, with its own region if it is deferred. + fn enter(&mut self, kind: ScopeKind, range: TextRange, deferred: bool) -> ScopeId { + let region = if deferred { + self.num_regions += 1; + RegionId(self.num_regions - 1) + } else { + self.scopes[self.current.index()].region + }; + let id = self.push(kind, Some(self.current), region, range); + self.by_node.insert(range, id); + id + } + + fn bind(&mut self, name: &Name, origin: SymbolOrigin) { + if name.is_empty() { + return; + } + let bound = &mut self.decls[self.current.index()].bound; + if !bound.contains_key(name) { + bound.insert(name.clone(), origin); + } + } + + fn finish(mut self) -> ScopeTable { + // `global x` anywhere creates the module symbol `x`. + let declared: Vec = self.decls[1..] + .iter() + .flat_map(|d| d.globals.iter().cloned()) + .collect(); + for name in declared { + let module = &mut self.decls[0].bound; + if !module.contains_key(&name) { + module.insert(name, SymbolOrigin::GlobalDeclaration); + } + } + let builtins = ScopeId(self.scopes.len() as u32); + self.scopes.push(Scope { + kind: ScopeKind::Builtins, + parent: None, + region: RegionId(0), + range: TextRange::default(), + symbols: SmallMap::new(), + }); + self.scopes[0].parent = None; + let mut symbols = Vec::new(); + let mut slots = vec![0u32; self.num_regions as usize]; + let mut globals = Vec::new(); + let mut nonlocals = Vec::new(); + for (i, decls) in self.decls.into_iter().enumerate() { + let scope = &mut self.scopes[i]; + for (name, origin) in decls.bound { + // In the module, `global x` is a no-op; elsewhere it removes ownership. + if i > 0 && (decls.globals.contains(&name) || decls.nonlocals.contains(&name)) { + continue; + } + let slot = &mut slots[scope.region.index()]; + let id = SymbolId(symbols.len() as u32); + symbols.push(Symbol { + name: name.clone(), + scope: ScopeId(i as u32), + origin, + slot: Some(*slot), + captured: false, + fallback: None, + }); + *slot += 1; + scope.symbols.insert(name, id); + } + globals.push(if i > 0 { + decls.globals + } else { + SmallSet::new() + }); + nonlocals.push(decls.nonlocals); + } + globals.push(SmallSet::new()); + nonlocals.push(SmallSet::new()); + ScopeTable { + scopes: self.scopes, + symbols, + num_regions: self.num_regions, + by_node: self.by_node, + globals, + nonlocals, + builtins, + } + } +} + +impl<'a> DeclarationSink<'a> for Builder { + fn declare(&mut self, declaration: Declaration<'a>) { + match declaration { + Declaration::Import(a) => match &a.asname { + Some(alias) => self.bind(&alias.id, SymbolOrigin::Bound), + None => { + let root = a.name.id.split('.').next().unwrap_or_default(); + self.bind(&Name::new(root), SymbolOrigin::Bound) + } + }, + Declaration::ImportFrom(x) => { + // Star imports are reported as a gap by lowering. + for a in x.names.iter().filter(|a| &a.name != "*") { + self.bind( + &a.asname.as_ref().unwrap_or(&a.name).id, + SymbolOrigin::Bound, + ) + } + } + Declaration::Function(x) => self.bind(&x.name.id, SymbolOrigin::Bound), + Declaration::Class(x) => self.bind(&x.name.id, SymbolOrigin::Bound), + Declaration::Assign(x, _) + | Declaration::AugAssign(x, _) + | Declaration::Annotated(x, _) + | Declaration::Store(x) + | Declaration::Delete(x) + | Declaration::Walrus(x) => self.bind(&x.id, SymbolOrigin::Bound), + Declaration::Handler(x) | Declaration::Capture(x) => { + self.bind(&x.id, SymbolOrigin::Bound) + } + Declaration::Parameter(x) => self.bind(&x.id, SymbolOrigin::Parameter), + Declaration::Global(x) => { + self.decls[self.current.index()] + .globals + .insert(x.id.clone()); + } + Declaration::Nonlocal(x) => { + self.decls[self.current.index()] + .nonlocals + .insert(x.id.clone()); + } + } + } + + /// Functions, lambdas and generator expressions get their own region; other scopes run + /// in the current region. + fn child(&mut self, child: ChildScope<'a>) { + let (kind, range, deferred) = match child { + ChildScope::Function(x) => (ScopeKind::Function, x.range, true), + ChildScope::Lambda(x) => (ScopeKind::Lambda, x.range, true), + ChildScope::Class(x) => (ScopeKind::Class, x.range, false), + ChildScope::Comprehension { + range, + is_generator, + .. + } => (ScopeKind::Comprehension, range, is_generator), + }; + let scope = self.enter(kind, range, deferred); + let outer = self.current; + self.current = scope; + if matches!( + child, + ChildScope::Comprehension { + is_generator: true, + .. + } + ) { + // A generator expression receives its first iterator as parameter `.0`. + self.bind(&Name::new_static(GENERATOR_ARG), SymbolOrigin::Parameter); + } + declarations::walk_child(child, self); + self.current = outer; + } +} + +/// Expressions in `x` that would be evaluated at definition time if annotations were +/// eager, and that could have effects. +pub fn annotation_may_have_effects(x: &Expr) -> bool { + struct Finder(bool); + impl<'a> Visitor<'a> for Finder { + fn visit_expr(&mut self, x: &'a Expr) { + match x { + Expr::Call(_) + | Expr::Named(_) + | Expr::Await(_) + | Expr::Yield(_) + | Expr::YieldFrom(_) => self.0 = true, + _ => walk_expr(self, x), + } + } + } + let mut f = Finder(false); + f.visit_expr(x); + f.0 +} diff --git a/static_analysis/src/structured.rs b/static_analysis/src/structured.rs new file mode 100644 index 0000000000..40645e3e33 --- /dev/null +++ b/static_analysis/src/structured.rs @@ -0,0 +1,510 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! A structured statement driver over the parsed AST. +//! +//! The driver owns the statement-level structure of Python code: the order of every +//! suite, the decomposition of `if`/`elif`/`else`, `while` and `for` (with `else`), the +//! attribution of `break` and `continue` to their loop, and the loop-free boundary of a +//! function body. A [`Structured`] backend owns the values that flow along that structure +//! (control-flow blocks, type-flow environments, inference keys) and the policy deciding +//! whether a test is statically known. +//! +//! The driver applies a backend's static truth uniformly: a statically false arm is +//! skipped, a statically true arm (or an `else`) ends the chain, makes the fork exhaustive +//! and leaves the remaining arms dead. A backend that answers `None` for every test, such +//! as runtime lowering, gets an unpruned walk. +//! +//! Every other statement goes to [`Structured::other_stmt`]. Nested suites of such statements +//! (`with`, `try`, `match`, class bodies) are expected to come back through [`suite`], and +//! function bodies through [`region`], so that loop attribution and coverage stay with the +//! driver. Expression-level control flow is not decomposed. + +use std::marker::PhantomData; +use std::mem; + +use pyrefly_python::ast::Ast; +use ruff_python_ast::Expr; +use ruff_python_ast::Stmt; +use ruff_python_ast::Suite; +use ruff_text_size::Ranged; +use ruff_text_size::TextRange; + +use crate::flow_point::FlowPointId; +use crate::flow_point::FlowPointKind; + +/// How a backend holds the AST: by value, so payloads can move into its output without +/// cloning, or by reference. +pub trait Syntax: Sized { + type Stmt; + type Expr; + type Suite: IntoIterator; + fn stmt(stmt: &Self::Stmt) -> &Stmt; + fn suite(suite: &Self::Suite) -> &[Stmt]; + fn shape(stmt: Self::Stmt) -> Shape; +} + +/// The AST is consumed. +pub struct Owned; + +/// The AST is borrowed for `'a`. +pub struct Borrowed<'a>(PhantomData<&'a ()>); + +/// One arm of an `if` statement: the `if` itself, an `elif`, or an `else` (no test). +pub struct Arm { + pub range: TextRange, + pub test: Option, + pub body: S::Suite, +} + +/// The loop header, whose evaluation precedes the body. +pub enum LoopHeader { + While { + range: TextRange, + test: S::Expr, + }, + For { + range: TextRange, + target: S::Expr, + iter: S::Expr, + is_async: bool, + }, +} + +/// The statement-level structure the driver decomposes. +pub enum Shape { + If { + range: TextRange, + /// Never empty; the first arm always has a test. + arms: Vec>, + }, + Loop { + header: LoopHeader, + body: S::Suite, + orelse: S::Suite, + }, + Jump(Jump, TextRange), + /// Handled entirely by the backend. + Other(S::Stmt), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Jump { + Break, + Continue, +} + +impl Syntax for Owned { + type Stmt = Stmt; + type Expr = Expr; + type Suite = Suite; + + fn stmt(stmt: &Stmt) -> &Stmt { + stmt + } + + fn suite(suite: &Suite) -> &[Stmt] { + suite + } + + fn shape(stmt: Stmt) -> Shape { + match stmt { + Stmt::If(x) => Shape::If { + range: x.range, + arms: Ast::if_branches_owned(x) + .map(|(range, test, body)| Arm { range, test, body }) + .collect(), + }, + Stmt::While(x) => Shape::Loop { + header: LoopHeader::While { + range: x.range, + test: *x.test, + }, + body: x.body, + orelse: x.orelse, + }, + Stmt::For(x) => Shape::Loop { + header: LoopHeader::For { + range: x.range, + target: *x.target, + iter: *x.iter, + is_async: x.is_async, + }, + body: x.body, + orelse: x.orelse, + }, + Stmt::Break(x) => Shape::Jump(Jump::Break, x.range), + Stmt::Continue(x) => Shape::Jump(Jump::Continue, x.range), + x => Shape::Other(x), + } + } +} + +impl<'a> Syntax for Borrowed<'a> { + type Stmt = &'a Stmt; + type Expr = &'a Expr; + type Suite = &'a [Stmt]; + + fn stmt<'b>(stmt: &'b &'a Stmt) -> &'b Stmt { + stmt + } + + fn suite<'b>(suite: &'b &'a [Stmt]) -> &'b [Stmt] { + suite + } + + fn shape(stmt: &'a Stmt) -> Shape { + match stmt { + Stmt::If(x) => Shape::If { + range: x.range, + arms: Ast::if_branches(x) + .zip( + [x.range] + .into_iter() + .chain(x.elif_else_clauses.iter().map(|c| c.range)), + ) + .map(|((test, body), range)| Arm { range, test, body }) + .collect(), + }, + Stmt::While(x) => Shape::Loop { + header: LoopHeader::While { + range: x.range, + test: &x.test, + }, + body: &x.body, + orelse: &x.orelse, + }, + Stmt::For(x) => Shape::Loop { + header: LoopHeader::For { + range: x.range, + target: &x.target, + iter: &x.iter, + is_async: x.is_async, + }, + body: &x.body, + orelse: &x.orelse, + }, + Stmt::Break(x) => Shape::Jump(Jump::Break, x.range), + Stmt::Continue(x) => Shape::Jump(Jump::Continue, x.range), + x => Shape::Other(x), + } + } +} + +/// What the driver did, for coverage reports. Statements are counted once, by shape. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Coverage { + pub suites: usize, + pub regions: usize, + /// Statements handed to [`Structured::other_stmt`]. + pub other: usize, + pub ifs: usize, + pub arms_visited: usize, + /// Arms whose test the backend decided is statically false. + pub arms_skipped: usize, + /// Arms after a statically true arm. + pub arms_dead: usize, + pub loops: usize, + pub jumps: usize, + /// `break`/`continue` statements outside any loop of their region. + pub jumps_outside_loop: usize, +} + +/// Driver state kept by the backend: the loops enclosing the current point of the +/// current region, innermost last, and the coverage counters. +pub struct Driver { + loops: Vec, + /// The lexical source suite, populated only while an observer is active. + source_region: Option, + /// A definition's body suite, distinguished from suites synthesized for its headers. + scope_body: Option, + pub coverage: Coverage, +} + +impl Default for Driver { + fn default() -> Self { + Self { + loops: Vec::new(), + source_region: None, + scope_body: None, + coverage: Coverage::default(), + } + } +} + +type StmtOf = <::Syntax as Syntax>::Stmt; +type ExprOf = <::Syntax as Syntax>::Expr; +type SuiteOf = <::Syntax as Syntax>::Suite; + +/// A consumer of the driver. Hooks are called in source order; between `arm_enter` and +/// `arm_exit`, and between `loop_start` and `loop_else`, the driver drives the body. +pub trait Structured: Sized { + type Syntax: Syntax; + /// Per-suite state. + type Suite; + /// Per-`if` state, from `if_start` to `if_end`. + type If; + /// Per-loop state, from `loop_start` to `loop_end`. + type Loop; + /// What a `break` or `continue` needs to know about its loop. + type Target: Clone; + + fn driver(&mut self) -> &mut Driver; + + /// Enable source-point observation for this traversal. Keep this choice fixed while + /// driving a suite; the default path allocates no observer storage. + fn observes_flow_points(&self) -> bool { + false + } + + /// Observe a source boundary in traversal order, bracketing the corresponding + /// backend step. No reachability or type-dependent policy is implied by its presence. + fn flow_point(&mut self, _point: FlowPointId) {} + + fn suite_start(&mut self, suite: &[Stmt]) -> Self::Suite; + /// Whether `stmt` absorbs the statement that follows it, dropping `next` from the + /// suite. The backend may record whatever it needs from `next`. + fn absorbs(&mut self, stmt: &Stmt, next: &Stmt) -> bool; + /// Called before each statement of a suite, with the statement that follows it once + /// any absorbed statement is dropped. + fn stmt_before(&mut self, suite: &mut Self::Suite, stmt: &Stmt, next: Option<&Stmt>); + fn stmt_after(&mut self, suite: &mut Self::Suite); + fn suite_end(&mut self, suite: Self::Suite); + + /// A statement the driver does not decompose. + fn other_stmt(&mut self, stmt: StmtOf); + + /// Before any arm. A backend may evaluate the first test here, before forking, or in + /// `arm_start` like the other tests. + fn if_start(&mut self, range: TextRange, first_test: &mut ExprOf) -> Self::If; + /// Start an arm and evaluate its test, which is `None` for an `else`. Returns the + /// test's static truth if the backend knows it; ignored for an `else`, which is true. + fn arm_start( + &mut self, + state: &mut Self::If, + range: TextRange, + first: bool, + test: Option<&mut ExprOf>, + ) -> Option; + /// The started arm is statically false: its body is not driven. + fn arm_skip(&mut self, state: &mut Self::If, arm: Arm); + fn arm_enter( + &mut self, + state: &mut Self::If, + range: TextRange, + test: Option>, + body: &[Stmt], + ); + fn arm_exit(&mut self, state: &mut Self::If, range: TextRange); + /// An arm after a statically true arm. It was not started. + fn arm_dead(&mut self, state: &mut Self::If, arm: Arm); + /// `exhaustive` is true if some visited arm was statically true (or an `else`), so + /// control cannot pass the statement without running an arm. + fn if_end(&mut self, state: Self::If, exhaustive: bool); + + fn loop_start( + &mut self, + header: LoopHeader, + body: &[Stmt], + ) -> (Self::Loop, Self::Target); + /// After the body, before the `else` suite. + fn loop_else(&mut self, state: &mut Self::Loop); + fn loop_end(&mut self, state: Self::Loop); + /// `target` is the innermost enclosing loop of the current region, if any. + fn loop_jump(&mut self, jump: Jump, range: TextRange, target: Option); +} + +/// The span is computed from the complete suite, before any backend pruning or absorption. +fn suite_range(body: &[Stmt]) -> Option { + Some(TextRange::new( + body.first()?.range().start(), + body.last()?.range().end(), + )) +} + +fn point(b: &mut B, range: TextRange, kind: FlowPointKind) { + if b.observes_flow_points() { + let region = b + .driver() + .source_region + .expect("an observed nonempty suite has a lexical source region"); + b.flow_point(FlowPointId { + region, + range, + kind, + }); + } +} + +/// Drive a suite. Empty suites emit no points because they have no source occurrence. +pub fn suite(b: &mut B, suite: SuiteOf) { + let outer_region = b.driver().source_region; + let range = b + .observes_flow_points() + .then(|| suite_range(B::Syntax::suite(&suite))) + .flatten(); + if let Some(range) = range { + if outer_region.is_none() || b.driver().scope_body == Some(range) { + b.driver().source_region = Some(range); + } + point(b, range, FlowPointKind::SuiteEntry); + } + b.driver().coverage.suites += 1; + let mut state = b.suite_start(B::Syntax::suite(&suite)); + let mut iter = suite.into_iter().peekable(); + while let Some(x) = iter.next() { + if let Some(next) = iter.peek() + && b.absorbs(B::Syntax::stmt(&x), B::Syntax::stmt(next)) + { + iter.next(); + } + b.stmt_before( + &mut state, + B::Syntax::stmt(&x), + iter.peek().map(B::Syntax::stmt), + ); + statement(b, x); + b.stmt_after(&mut state); + } + b.suite_end(state); + if let Some(range) = range { + point(b, range, FlowPointKind::SuiteExit); + } + b.driver().source_region = outer_region; +} + +/// Drive the body of a function: a suite with no enclosing loop. +pub fn region(b: &mut B, body: SuiteOf) { + b.driver().coverage.regions += 1; + let outer = mem::take(&mut b.driver().loops); + let outer_region = b.driver().source_region; + if b.observes_flow_points() { + b.driver().source_region = suite_range(B::Syntax::suite(&body)); + } + suite(b, body); + b.driver().source_region = outer_region; + b.driver().loops = outer; +} + +fn statement(b: &mut B, x: StmtOf) { + match B::Syntax::shape(x) { + Shape::If { range, arms } => if_(b, range, arms), + Shape::Loop { + header, + body, + orelse, + } => { + b.driver().coverage.loops += 1; + let range = match &header { + LoopHeader::While { range, .. } | LoopHeader::For { range, .. } => *range, + }; + point(b, range, FlowPointKind::LoopHeader); + let (mut state, target) = b.loop_start(header, B::Syntax::suite(&body)); + b.driver().loops.push(target); + point(b, range, FlowPointKind::LoopBody); + suite(b, body); + b.driver().loops.pop(); + point(b, range, FlowPointKind::LoopBackedge); + b.loop_else(&mut state); + point(b, range, FlowPointKind::LoopExit); + if !B::Syntax::suite(&orelse).is_empty() { + point(b, range, FlowPointKind::LoopElse); + } + suite(b, orelse); + point(b, range, FlowPointKind::LoopJoin); + b.loop_end(state); + } + Shape::Jump(jump, range) => { + let driver = b.driver(); + driver.coverage.jumps += 1; + let target = driver.loops.last().cloned(); + if target.is_none() { + driver.coverage.jumps_outside_loop += 1; + } + point( + b, + range, + match jump { + Jump::Break => FlowPointKind::Break, + Jump::Continue => FlowPointKind::Continue, + }, + ); + b.loop_jump(jump, range, target); + } + Shape::Other(x) => { + b.driver().coverage.other += 1; + let scope = if b.observes_flow_points() { + match B::Syntax::stmt(&x) { + Stmt::FunctionDef(f) => Some((f.range, suite_range(&f.body))), + Stmt::ClassDef(c) => Some((c.range, suite_range(&c.body))), + _ => None, + } + } else { + None + }; + let outer_scope_body = b.driver().scope_body; + if let Some((range, body_region)) = scope { + point(b, range, FlowPointKind::ScopeEntry); + b.driver().scope_body = body_region; + } + b.other_stmt(x); + if let Some((range, _)) = scope { + b.driver().scope_body = outer_scope_body; + point(b, range, FlowPointKind::ScopeExit); + } + } + } +} + +fn if_(b: &mut B, range: TextRange, arms: Vec>) { + b.driver().coverage.ifs += 1; + let mut arms = arms.into_iter(); + let mut first = arms.next().expect("an `if` statement has a first arm"); + point(b, range, FlowPointKind::IfFork); + let mut state = b.if_start( + range, + first + .test + .as_mut() + .expect("the first arm of an `if` statement has a test"), + ); + let mut exhaustive = false; + let mut is_first = true; + let mut next = Some(first); + while let Some(mut arm) = next.take().or_else(|| arms.next()) { + if arm.test.is_some() { + point(b, arm.range, FlowPointKind::ArmTest); + } + let truth = b.arm_start(&mut state, arm.range, is_first, arm.test.as_mut()); + is_first = false; + let truth = if arm.test.is_none() { + Some(true) + } else { + truth + }; + if truth == Some(false) { + b.driver().coverage.arms_skipped += 1; + b.arm_skip(&mut state, arm); + continue; + } + b.driver().coverage.arms_visited += 1; + point(b, arm.range, FlowPointKind::ArmEntry); + b.arm_enter(&mut state, arm.range, arm.test, B::Syntax::suite(&arm.body)); + suite(b, arm.body); + point(b, arm.range, FlowPointKind::ArmExit); + b.arm_exit(&mut state, arm.range); + if truth == Some(true) { + exhaustive = true; + for rest in arms.by_ref() { + b.driver().coverage.arms_dead += 1; + b.arm_dead(&mut state, rest); + } + } + } + point(b, range, FlowPointKind::IfJoin); + b.if_end(state, exhaustive); +} diff --git a/static_analysis/tests/declarations.rs b/static_analysis/tests/declarations.rs new file mode 100644 index 0000000000..4e5610927c --- /dev/null +++ b/static_analysis/tests/declarations.rs @@ -0,0 +1,535 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Tests of the shared declaration walker, the lexical navigation engine, and the runtime +//! scope table built on them. + +use pyrefly_python::ast::Ast; +use pyrefly_static_analysis::declarations::ChildScope; +use pyrefly_static_analysis::declarations::Context; +use pyrefly_static_analysis::declarations::Declaration; +use pyrefly_static_analysis::declarations::DeclarationSink; +use pyrefly_static_analysis::declarations::NameSite; +use pyrefly_static_analysis::declarations::Position; +use pyrefly_static_analysis::declarations::TestSite; +use pyrefly_static_analysis::declarations::walk_child; +use pyrefly_static_analysis::declarations::walk_suite; +use pyrefly_static_analysis::lexical; +use pyrefly_static_analysis::lexical::Category; +use pyrefly_static_analysis::lexical::NonlocalError; +use pyrefly_static_analysis::lexical::Probe; +use pyrefly_static_analysis::lexical::Resolution; +use pyrefly_static_analysis::lexical::ScopeChain; +use pyrefly_static_analysis::*; +use ruff_python_ast::Expr; +use ruff_python_ast::ExprName; +use ruff_python_ast::PySourceType; +use ruff_python_ast::Stmt; + +/// Records every event, walking nested scopes when `runtime`. Otherwise it declines every +/// optional position and treats a test named `STATIC_FALSE` as statically false. +struct Trace { + runtime: bool, + events: Vec, +} + +impl<'a> DeclarationSink<'a> for Trace { + fn declare(&mut self, d: Declaration<'a>) { + let event = match d { + Declaration::Import(a) => format!("import {}", a.name), + Declaration::ImportFrom(x) => format!( + "from {}", + x.names + .iter() + .map(|a| a.name.as_str()) + .collect::>() + .join(" ") + ), + Declaration::Function(x) => format!("def {}", x.name), + Declaration::Class(x) => format!("class {}", x.name), + Declaration::Assign(x, _) => format!("assign {}", x.id), + Declaration::AugAssign(x, _) => format!("aug {}", x.id), + Declaration::Annotated(x, _) => format!("annotated {}", x.id), + Declaration::Store(x) => format!("store {}", x.id), + Declaration::Delete(x) => format!("del {}", x.id), + Declaration::Walrus(x) => format!("walrus {}", x.id), + Declaration::Handler(x) => format!("handler {}", x.id), + Declaration::Capture(x) => format!("capture {}", x.id), + Declaration::Global(x) => format!("global {}", x.id), + Declaration::Nonlocal(x) => format!("nonlocal {}", x.id), + Declaration::Parameter(x) => format!("param {}", x.id), + }; + self.events.push(event); + } + + fn name(&mut self, name: &'a ExprName, site: NameSite) { + let site = match site { + NameSite::Expression => "read", + NameSite::AnnAssignTarget => "ann", + NameSite::AugAssignTarget => "aug", + NameSite::WalrusTarget => "target", + NameSite::Pattern => "pattern", + }; + self.events.push(format!("{site} {}", name.id)); + } + + fn keeps(&mut self, _: Position) -> bool { + self.runtime + } + + fn enter(&mut self, context: Context<'a>) { + if let Context::Comprehension = context { + self.events.push("(".to_owned()); + } + } + + fn exit(&mut self, context: Context<'a>) { + if let Context::Comprehension = context { + self.events.push(")".to_owned()); + } + } + + fn static_truth(&mut self, test: &'a Expr, _: TestSite) -> Option { + match test { + Expr::Name(x) if !self.runtime && x.id.as_str() == "STATIC_FALSE" => Some(false), + _ => None, + } + } + + fn statement(&mut self, stmt: &'a Stmt) { + if let Stmt::Expr(_) = stmt { + self.events.push(";".to_owned()); + } + } + + fn child(&mut self, child: ChildScope<'a>) { + let kind = match child { + ChildScope::Function(_) => "function", + ChildScope::Lambda(_) => "lambda", + ChildScope::Class(_) => "class", + ChildScope::Comprehension { .. } => "comprehension", + }; + self.events.push(format!("{kind} {{")); + if self.runtime { + walk_child(child, self); + } + self.events.push("}".to_owned()); + } +} + +fn trace(source: &str, runtime: bool) -> String { + let body = Ast::parse(source, PySourceType::Python).0.body; + let mut sink = Trace { + runtime, + events: Vec::new(), + }; + walk_suite(&body, &mut sink); + sink.events.join(", ") +} + +#[test] +fn comprehension_walrus_binds_in_the_enclosing_scope() { + let source = "[(a := f(x)) for x in (b := xs) if (c := x) for y in [(d := 1) for _ in z]]\n"; + // The enclosing scope reads only the first iterable, and declares every walrus target, target target + // in source order, even inside the nested comprehension. + assert_eq!( + trace(source, false), + "(, walrus a, target a, walrus b, target b, read xs, walrus c, target c, walrus d, target d, ), comprehension {, }, ;" + ); + // The comprehension declares its targets and evaluates the rest, without redeclaring + // the walrus targets, target targets; the inner comprehension's first iterable runs in the outer one. + assert_eq!( + trace(source, true), + "(, walrus a, target a, walrus b, target b, read xs, walrus c, target c, walrus d, target d, ), comprehension {, \ + store x, read x, read x, (, read z, ), comprehension {, store _, read _, }, store y, \ + read y, read f, read x, }, ;" + ); +} + +#[test] +fn lambdas_stop_walrus_but_not_their_defaults() { + let source = "[lambda a=(d := 1): (body := 2) for _ in xs]\n"; + assert_eq!( + trace(source, true), + "(, walrus d, target d, read xs, ), comprehension {, store _, read _, lambda {, param a, \ + walrus body, target body, }, }, ;" + ); + // A sink that declines parameter defaults sees neither the default nor its walrus. + assert_eq!(trace(source, false), "(, read xs, ), comprehension {, }, ;"); +} + +#[test] +fn headers_and_optional_positions() { + let source = "\ +@dec +def f(a, b=(w1 := 1), *args, c=(w2 := 2), **kw): + return a +class C(Base, metaclass=(w3 := M)): + x: int = 1 +if t: + pass +elif (w4 := u): + pass +try: + pass +except (w5 := E) as err: + pass +match s: + case [p] if (w6 := p): + pass + case Point(x=Color.RED): + pass +target[(w7 := 0)] += 1 +total += 1 +label: str = s +"; + let runtime = trace(source, true); + for name in ["w1", "w2", "w3", "w4", "w5", "w6", "w7"] { + assert!(runtime.contains(&format!("walrus {name}")), "{runtime}"); + } + assert!(runtime.contains("function {, param a, param b, param args, param c, param kw")); + // Every name expression is reported, with its site. + for event in ["pattern Point", "pattern Color", "aug total", "ann label"] { + assert!(runtime.contains(event), "{runtime}"); + } + let typing = trace(source, false); + for name in ["w1", "w2", "w3", "w4", "w5", "w6", "w7"] { + assert!(!typing.contains(&format!("walrus {name}")), "{typing}"); + } + // Decorators run in the enclosing scope, then the name is bound, then the scope opens. + assert!(typing.starts_with("read dec, def f, function {, }, class C, class {, }")); +} + +#[test] +fn statement_order_and_static_pruning() { + let source = "\ +import a.b, c as d +from m import *, x as y +try: + t = 1 +except E as e: + pass +match v: + case [q, *r]: + pass +del (z := w)[0], gone +first, second[(s := 0)] = __all__ = value +if STATIC_FALSE: + dead = 1 +elif other: + live = 1 +else: + rest = 1 +pick = p if STATIC_FALSE else (o := 1) +type Alias[T] = list[T] +global g +nonlocal n +"; + assert_eq!( + trace(source, false), + "import a.b, import c, from * x, handler e, assign t, read t, read v, capture q, \ + capture r, walrus z, target z, read w, read gone, del gone, read value, store first, \ + read first, read second, walrus s, target s, assign __all__, read __all__, read STATIC_FALSE, \ + assign live, read live, assign rest, read rest, read STATIC_FALSE, walrus o, target o, \ + assign pick, read pick, store Alias, read Alias, global g, nonlocal n" + ); + // The runtime prunes nothing. + assert!(trace(source, true).contains("assign dead")); +} + +/// A chain of scopes given by their categories, innermost last. +#[derive(Clone, Copy)] +struct Chain<'a>(&'a [Category]); + +impl ScopeChain for Chain<'_> { + type Id = usize; + type Barrier = usize; + + fn parent(&self, id: usize) -> Option { + id.checked_sub(1) + } + + fn category(&self, id: usize) -> Category { + self.0[id] + } + + fn barrier(&self, id: usize) -> usize { + id + } + + fn join(a: usize, b: usize) -> usize { + a.max(b) + } +} + +#[test] +fn visible_scopes_skip_classes_except_from_their_annotation_scopes() { + use Category::*; + let views = |categories: &[Category]| { + lexical::visible(Chain(categories), categories.len() - 1) + .map(|v| (v.id, v.depth, v.barrier)) + .collect::>() + }; + // A method does not see its class, which contributes no barrier. + assert_eq!( + views(&[Module, Class, Function]), + vec![(2, 0, 0), (0, 2, 2)] + ); + // A class body sees itself, and its annotation and `type` alias scopes see it. + assert_eq!(views(&[Module, Class]), vec![(1, 0, 0), (0, 1, 1)]); + assert_eq!( + views(&[Module, Class, Annotation { class_scope: false }]), + vec![(2, 0, 0), (1, 1, 2), (0, 2, 2)] + ); + assert_eq!( + views(&[Module, Class, TypeAlias]), + vec![(2, 0, 0), (1, 1, 2), (0, 2, 2)] + ); + // Scopes nested further in do not. + assert_eq!( + views(&[Module, Class, Annotation { class_scope: false }, Function]), + vec![(3, 0, 0), (2, 1, 3), (0, 3, 3)] + ); +} + +#[test] +fn nonlocal_resolution() { + use Category::*; + let resolve = |categories: &[Category], binds: &[Probe<()>]| { + let probe = |id: usize| match binds[id] { + Probe::Owned(()) => Probe::Owned(id), + Probe::Absent => Probe::Absent, + Probe::Global => Probe::Global, + Probe::Nonlocal => Probe::Nonlocal, + }; + lexical::resolve_nonlocal(Chain(categories), Some(categories.len() - 1), probe) + }; + let (o, a, g, n) = ( + || Probe::Owned(()), + || Probe::Absent, + || Probe::Global, + || Probe::Nonlocal, + ); + assert_eq!(resolve(&[Module, Function], &[a(), o()]), Ok(1)); + // Classes are skipped, enclosing `nonlocal` declarations are followed. + assert_eq!( + resolve(&[Module, Function, Class, Function], &[a(), o(), o(), n()]), + Ok(1) + ); + assert_eq!( + resolve(&[Module, Function, Function], &[a(), o(), g()]), + Err(NonlocalError::Global) + ); + assert_eq!( + resolve(&[Module, Function], &[o(), a()]), + Err(NonlocalError::Global) + ); + // A `nonlocal` declaration at the root refers to nothing. + assert_eq!( + resolve(&[Module, Function], &[n(), a()]), + Err(NonlocalError::NotFound) + ); + assert_eq!( + resolve(&[Module, Function], &[a(), a()]), + Err(NonlocalError::NotFound) + ); +} + +#[test] +fn resolution_redirects_global_and_nonlocal() { + use Category::*; + let categories = [Module, Function, Function]; + let probe = |binds: [Probe; 3]| { + let mut binds = binds.map(Some); + move |id: usize| binds[id].take().unwrap_or(Probe::Absent) + }; + let found = |r: Resolution| match r { + Resolution::Found(id) => Some(id), + Resolution::Unbound | Resolution::Nonlocal(_) => None, + }; + let chain = Chain(&categories); + assert_eq!( + found(lexical::resolve( + chain, + 2, + probe([Probe::Owned(0), Probe::Owned(1), Probe::Global]) + )), + Some(0) + ); + assert_eq!( + found(lexical::resolve( + chain, + 2, + probe([Probe::Owned(0), Probe::Owned(1), Probe::Nonlocal]) + )), + Some(1) + ); + assert!(matches!( + lexical::resolve( + chain, + 2, + probe([Probe::Absent, Probe::Absent, Probe::Absent]) + ), + Resolution::Unbound + )); +} + +fn program(source: &str) -> Program { + analyze_python(source, ModuleName::from_str("pkg.mod"), false) +} + +fn scope_kinds(p: &Program) -> Vec<(ScopeKind, Option, u32)> { + p.scopes + .iter() + .map(|s| (s.kind, s.parent.map(|x| x.0), s.region.0)) + .collect() +} + +#[test] +fn runtime_ownership_is_unpruned() { + let p = program( + "\ +import sys +def f(): + if False: + dead = 1 + if sys.version_info < (3, 0): + old = 1 + return dead, old +", + ); + let function = p + .scopes + .iter() + .position(|s| s.kind == ScopeKind::Function) + .unwrap(); + for name in ["dead", "old"] { + let symbol = p + .resolve(ScopeId(function as u32), &Name::new(name)) + .unwrap(); + assert_eq!(p.symbol(symbol).scope.0, function as u32, "{name}"); + } +} + +#[test] +fn runtime_scope_tree_and_regions() { + let p = program( + "\ +def f(a=lambda: 1): + return [x for x in a] +g = (y for y in ()) +class C: + z = lambda: 2 +", + ); + // Default lambdas belong to the module and are created before the function they + // parameterize; the eager comprehension shares the function's region; the generator + // and lambdas get their own regions; the class runs in the module's region. + assert_eq!( + scope_kinds(&p) + .into_iter() + .filter(|(k, ..)| *k != ScopeKind::Builtins) + .collect::>(), + vec![ + (ScopeKind::Module, None, 0), + (ScopeKind::Lambda, Some(0), 1), + (ScopeKind::Function, Some(0), 2), + (ScopeKind::Comprehension, Some(2), 2), + (ScopeKind::Comprehension, Some(0), 3), + (ScopeKind::Class, Some(0), 0), + (ScopeKind::Lambda, Some(5), 4), + ] + ); + let generator = &p.scopes[4]; + assert!(generator.symbols.contains_key(&Name::new(".0"))); + assert!(!p.scopes[3].symbols.contains_key(&Name::new(".0"))); +} + +#[test] +fn runtime_class_body_in_function_falls_back_to_the_module() { + let p = program( + "\ +def f(): + x = 1 + class C: + x = x + y = x + return C +", + ); + let class = p + .scopes + .iter() + .position(|s| s.kind == ScopeKind::Class) + .unwrap(); + let class_x = *p.scopes[class].symbols.get(&Name::new("x")).unwrap(); + // `LOAD_NAME` in a class body skips the enclosing function, so an unbound class `x` + // falls back to the (absent) module `x`, which is the builtin. + let fallback = p.symbol(class_x).fallback.unwrap(); + assert_eq!(p.symbol(fallback).origin, SymbolOrigin::Builtin); +} + +#[test] +fn runtime_builtins_and_module_declarations() { + let p = program( + "\ +global g +g = 1 +nonlocal n +n = 2 +def f(): + global n + return print(g, n) +", + ); + let print = p.resolve(ScopeId(0), &Name::new("print")).unwrap(); + // The builtins scope resolves its own names, and nothing else. + let builtins = p.symbol(print).scope; + assert_eq!(p.resolve(builtins, &Name::new("print")), Some(print)); + assert_eq!(p.resolve(builtins, &Name::new("missing")), None); + // `global` at module level is a no-op, and a module-level `nonlocal` (a syntax error) + // does not hide the module's own binding. + let function = p + .scopes + .iter() + .position(|s| s.kind == ScopeKind::Function) + .unwrap(); + for name in ["g", "n"] { + let module = *p.scopes[0].symbols.get(&Name::new(name)).unwrap(); + assert_eq!( + p.resolve(ScopeId(0), &Name::new(name)), + Some(module), + "{name}" + ); + } + let n = *p.scopes[0].symbols.get(&Name::new("n")).unwrap(); + assert_eq!( + p.resolve(ScopeId(function as u32), &Name::new("n")), + Some(n) + ); +} + +#[test] +fn runtime_nonlocal_cannot_cross_a_global_declaration() { + let p = program( + "\ +x = 1 +def f(): + global x + def g(): + nonlocal x + return x +", + ); + assert!(p.ops.iter().any(|op| matches!( + op.kind, + OpKind::Gap { + kind: GapKind::ScopeError, + .. + } + ))); +} diff --git a/static_analysis/tests/engine.rs b/static_analysis/tests/engine.rs new file mode 100644 index 0000000000..c864c29b61 --- /dev/null +++ b/static_analysis/tests/engine.rs @@ -0,0 +1,724 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Tests of the generic engine with small domains: value provenance through aliases, +//! a typestate domain with ambient context, convergence limits, branch pruning and +//! outer cells. + +use std::collections::BTreeMap; +use std::collections::BTreeSet; + +use pyrefly_static_analysis::*; + +fn program(source: &str) -> Program { + analyze_python(source, ModuleName::from_str("pkg.mod"), false) +} + +fn region(p: &Program, name: &str) -> RegionId { + RegionId( + p.regions + .iter() + .position(|r| r.name.as_str() == name) + .expect("region exists") as u32, + ) +} + +fn line(p: &Program, start: u32) -> usize { + p.line_col(TextRange::new(start.into(), start.into())) + .0 + .get() +} + +/// Tracks which operations created each value, so aliases share an origin. +/// Reports attribute stores into values created by an import. +struct Provenance; + +impl Domain for Provenance { + type Value = BTreeSet; + type State = (); + type Diagnostic = (u32, BTreeSet); + + fn top(&self) -> Self::Value { + BTreeSet::new() + } + + fn join(&self, a: &Self::Value, b: &Self::Value) -> Self::Value { + a.union(b).copied().collect() + } + + fn join_state(&self, _: &(), _: &()) {} + + fn transfer(&self, op: OpId, kind: &OpKind, cx: &mut Cx<'_, Self>) { + match kind { + // Names and parameters keep the engine's bookkeeping. + OpKind::Name { .. } | OpKind::Param { .. } => {} + OpKind::Attribute { + object, + access: Access::Store(_), + .. + } => { + let origins = cx.reg(*object).clone(); + if !origins.is_empty() { + cx.report((cx.range().start().into(), origins)); + } + } + _ => { + for dst in kind.writes() { + cx.set_reg(dst, BTreeSet::from([op])); + } + } + } + } +} + +#[test] +fn assignment_copies_values_not_name_links() { + let p = program( + "\ +import foo as b +a = b +b = {} +a.x = 1 +b.y = 2 +", + ); + let import = OpId( + p.ops + .iter() + .position(|op| matches!(op.kind, OpKind::Import { .. })) + .unwrap() as u32, + ); + let result = Engine::new(&p, &Provenance).analyze(Program::MODULE, Entry::new(())); + assert!(!result.is_incomplete()); + let lines: Vec<(usize, bool)> = result + .diagnostics + .iter() + .map(|(start, origins)| (line(&p, *start), origins.contains(&import))) + .collect(); + // `a` still holds the imported module after `b` is rebound. + assert_eq!(lines, vec![(4, true), (5, false)]); +} + +#[test] +fn parameter_rebinding_and_outer_cells() { + let p = program( + "\ +def f(p): + global g + q = p + p = {} + q.x = 1 + g = p + return h +", + ); + let f = region(&p, "f"); + let arg = OpId(1_000); + let outer = OpId(2_000); + // `h` is not defined in the module, so it resolves to a builtins symbol. + let h = p + .resolve(p.region(f).scope, &Name::new("h")) + .expect("h is referenced"); + let g = p.global("g").expect("g is declared global"); + let mut entry = Entry::new(()); + entry.args = vec![BTreeSet::from([arg])]; + entry + .outer + .insert(h, Binding::bound(BTreeSet::from([outer]))); + let result = Engine::new(&p, &Provenance).analyze(f, entry); + // `q` aliases the argument; rebinding `p` does not affect it. + assert_eq!(result.diagnostics.len(), 1); + assert_eq!(result.diagnostics[0].1, BTreeSet::from([arg])); + // A load of a global reads the cell the caller provided. + assert_eq!(result.returned, Some(BTreeSet::from([outer]))); + // A store to a global updates its cell, visible to the caller at exit. + let exit = result.exit.expect("f returns"); + let g_value = &exit.outer()[&g].value; + assert_eq!(g_value.len(), 1); + assert!(matches!( + p.op(*g_value.first().unwrap()).kind, + OpKind::Build { .. } + )); +} + +/// A small typestate domain for file handles, with lock ownership as ambient context. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Ref { + OpenFn, + LockFn, + Lock, + False, + Handle(OpId), + Method(OpId, Method), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +enum Method { + Read, + Write, + Close, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Status { + Open, + Closed, + MaybeClosed, +} + +#[derive(Debug, Clone, PartialEq)] +struct FileState { + files: BTreeMap, + /// The possible numbers of held locks: ambient context joined across paths. + held: BTreeSet, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +enum FileError { + UseAfterClose(u32), + MaybeUseAfterClose(u32), + WriteWithoutLock(u32), +} + +struct Files; + +impl Domain for Files { + type Value = BTreeSet; + type State = FileState; + type Diagnostic = FileError; + + fn top(&self) -> Self::Value { + BTreeSet::new() + } + + fn join(&self, a: &Self::Value, b: &Self::Value) -> Self::Value { + a.union(b).copied().collect() + } + + fn join_state(&self, a: &FileState, b: &FileState) -> FileState { + let mut files = a.files.clone(); + for (h, s) in &b.files { + let joined = match files.get(h) { + None => *s, + Some(old) if old == s => *s, + Some(_) => Status::MaybeClosed, + }; + files.insert(*h, joined); + } + FileState { + files, + held: a.held.union(&b.held).copied().collect(), + } + } + + fn transfer(&self, op: OpId, kind: &OpKind, cx: &mut Cx<'_, Self>) { + let start: u32 = cx.range().start().into(); + match kind { + OpKind::Name { + symbol, + access: Access::Load(dst), + } if cx.program().owner(*symbol).is_none() => { + let value = match cx.program().symbol(*symbol).name.as_str() { + "open" => BTreeSet::from([Ref::OpenFn]), + "lock" => BTreeSet::from([Ref::LockFn]), + _ => BTreeSet::new(), + }; + cx.set_reg(*dst, value); + } + OpKind::Literal { + value: Literal::Bool(false), + dst, + } => cx.set_reg(*dst, BTreeSet::from([Ref::False])), + OpKind::Attribute { + object, + attr, + access: Access::Load(dst), + } => { + let method = match attr.as_str() { + "read" => Method::Read, + "write" => Method::Write, + "close" => Method::Close, + _ => return, + }; + let value = cx + .reg(*object) + .iter() + .filter_map(|r| match r { + Ref::Handle(h) => Some(Ref::Method(*h, method)), + _ => None, + }) + .collect(); + cx.set_reg(*dst, value); + } + OpKind::Call { func, dst, .. } => { + let callee = cx.reg(*func).clone(); + let mut result = BTreeSet::new(); + for r in callee { + match r { + Ref::OpenFn => { + cx.state_mut().files.insert(op, Status::Open); + result.insert(Ref::Handle(op)); + } + Ref::LockFn => { + result.insert(Ref::Lock); + } + Ref::Method(h, Method::Close) => { + cx.state_mut().files.insert(h, Status::Closed); + } + Ref::Method(h, m) => { + match cx.state().files.get(&h) { + Some(Status::Closed) => cx.report(FileError::UseAfterClose(start)), + Some(Status::MaybeClosed) => { + cx.report(FileError::MaybeUseAfterClose(start)) + } + Some(Status::Open) | None => {} + } + if m == Method::Write && cx.state().held.contains(&0) { + cx.report(FileError::WriteWithoutLock(start)); + } + } + Ref::Lock | Ref::False | Ref::Handle(_) => {} + } + } + cx.set_reg(*dst, result); + } + OpKind::ContextEnter { manager, .. } if cx.reg(*manager).contains(&Ref::Lock) => { + let held = cx.state().held.iter().map(|n| n + 1).collect(); + cx.state_mut().held = held; + } + OpKind::ContextExit { manager, .. } if cx.reg(*manager).contains(&Ref::Lock) => { + let held = cx.state().held.iter().map(|n| n - 1).collect(); + cx.state_mut().held = held; + } + _ => {} + } + } + + fn assume(&self, cond: Reg, truth: bool, cx: &mut Cx<'_, Self>) -> bool { + !(truth && cx.reg(cond) == &BTreeSet::from([Ref::False])) + } +} + +fn files_entry() -> Entry { + Entry::new(FileState { + files: BTreeMap::new(), + held: BTreeSet::from([0]), + }) +} + +#[test] +fn typestate_with_ambient_context() { + let p = program( + "\ +f = open(p) +if c: + f.close() +f.read() +g = open(q) +with lock(): + g.write(1) +g.write(2) +g.close() +g.read() +if False: + g.read() +", + ); + let result = Engine::new(&p, &Files).analyze(Program::MODULE, files_entry()); + let found: Vec<(&str, usize)> = result + .diagnostics + .iter() + .map(|d| match d { + FileError::UseAfterClose(s) => ("closed", line(&p, *s)), + FileError::MaybeUseAfterClose(s) => ("maybe closed", line(&p, *s)), + FileError::WriteWithoutLock(s) => ("no lock", line(&p, *s)), + }) + .collect(); + // Diagnostics are sorted by their `Ord`. Line 12 is pruned by `assume`: + // diagnostics come only from reachable paths. + assert_eq!( + found, + vec![("closed", 10), ("maybe closed", 4), ("no lock", 8)] + ); + let exit = result.exit.expect("module completes"); + assert_eq!(exit.state.held, BTreeSet::from([0])); +} + +#[test] +fn ambient_context_is_restored_on_return() { + let p = program( + "\ +def h(c): + k = open(r) + with lock(): + if c: + return k + k.close() + return None +", + ); + let result = Engine::new(&p, &Files).analyze(region(&p, "h"), files_entry()); + let exit = result.exit.expect("h returns"); + assert_eq!(exit.state.held, BTreeSet::from([0])); + // One path returns the open handle, the other closes it. + assert_eq!( + exit.state.files.values().collect::>(), + vec![&Status::MaybeClosed] + ); + assert!(matches!( + result.returned.as_ref().map(|v| v.iter().next().copied()), + Some(Some(Ref::Handle(_))) + )); +} + +/// Tracks the exact set of integers a value may hold. +struct Counter { + widen: bool, +} + +impl Domain for Counter { + /// `None` is any integer. + type Value = Option>; + type State = (); + type Diagnostic = (); + + fn top(&self) -> Self::Value { + None + } + + fn unbound(&self) -> Self::Value { + Some(BTreeSet::new()) + } + + fn join(&self, a: &Self::Value, b: &Self::Value) -> Self::Value { + Some(a.as_ref()?.union(b.as_ref()?).copied().collect()) + } + + fn join_state(&self, _: &(), _: &()) {} + + fn widen(&self, old: &Self::Value, new: &Self::Value) -> Self::Value { + if self.widen && old != new { + None + } else { + new.clone() + } + } + + fn transfer(&self, _: OpId, kind: &OpKind, cx: &mut Cx<'_, Self>) { + match kind { + OpKind::Literal { + value: Literal::Int(Some(n)), + dst, + } => cx.set_reg(*dst, Some(BTreeSet::from([*n]))), + OpKind::Binary { + op: BinaryOp::Arith(Operator::Add), + left, + right, + dst, + } => { + let sum = match (cx.reg(*left), cx.reg(*right)) { + (Some(a), Some(b)) => Some( + a.iter() + .flat_map(|x| b.iter().map(move |y| x + y)) + .collect(), + ), + _ => None, + }; + cx.set_reg(*dst, sum); + } + _ => {} + } + } +} + +#[test] +fn loop_convergence_needs_widening() { + let p = program("i = 0\nwhile c:\n i = i + 1\n"); + let i = p.global("i").unwrap(); + let limits = Limits { + widen_after: 2, + max_block_visits: 20, + }; + let unbounded = Engine::new(&p, &Counter { widen: false }) + .with_limits(limits) + .analyze(Program::MODULE, Entry::new(())); + assert_eq!(unbounded.convergence, Convergence::LimitReached); + assert!(unbounded.is_incomplete()); + + let widened = Engine::new(&p, &Counter { widen: true }) + .with_limits(limits) + .analyze(Program::MODULE, Entry::new(())); + assert_eq!(widened.convergence, Convergence::Converged); + let exit = widened.exit.expect("the loop can exit"); + assert_eq!(exit.symbol(&p, i), Some(&None)); + + // Without a loop, values stay exact. + let p = program("i = 1\nif c:\n i = i + 1\n"); + let result = Engine::new(&p, &Counter { widen: true }).analyze(Program::MODULE, Entry::new(())); + let i = p.global("i").unwrap(); + assert_eq!( + result.exit.unwrap().symbol(&p, i), + Some(&Some(BTreeSet::from([1, 2]))) + ); +} + +#[test] +fn reachable_gaps_and_replay() { + let p = program( + "\ +try: + x = 1 +except E: + x = 2 +match x: + case 1: + pass +", + ); + let engine = Engine::new(&p, &Provenance); + let result = engine.analyze(Program::MODULE, Entry::new(())); + let kinds: Vec = result.gaps.iter().map(|(k, _)| *k).collect(); + assert_eq!(kinds, vec![GapKind::Exceptions, GapKind::Match]); + assert!(result.is_incomplete()); + // Replay visits each reachable operation once, in a deterministic order. + let mut seen = Vec::new(); + engine.replay(&result, &mut |op, _| seen.push(op)); + let unique: BTreeSet = seen.iter().copied().collect(); + assert_eq!(unique.len(), seen.len()); + let mut again = Vec::new(); + engine.replay(&result, &mut |op, _| again.push(op)); + assert_eq!(seen, again); +} + +fn origins_by_line(p: &Program, result: &RegionResult) -> Vec<(usize, Vec)> { + result + .diagnostics + .iter() + .map(|(start, origins)| { + ( + line(p, *start), + origins.iter().map(|op| tag(p, *op)).collect(), + ) + }) + .collect() +} + +#[derive(Debug, PartialEq, Eq)] +enum OpKindTag { + Import, + Call(String), + External(u32), + Other, +} + +fn tag(p: &Program, op: OpId) -> OpKindTag { + match p.ops.get(op.index()).map(|o| &o.kind) { + Some(OpKind::Import { .. }) => OpKindTag::Import, + Some(OpKind::Call { .. }) => OpKindTag::Call(p.text(p.op(op).range).to_owned()), + Some(_) => OpKindTag::Other, + None => OpKindTag::External(op.0), + } +} + +#[test] +fn class_body_names_fall_back_to_the_module() { + let p = program( + "\ +import registry +y = make() +class C: + registry = registry + registry.entries = 1 + if c: + y = other() + y.attr = 1 + del y + y.z = 2 +", + ); + let result = Engine::new(&p, &Provenance).analyze(Program::MODULE, Entry::new(())); + assert_eq!( + origins_by_line(&p, &result), + vec![ + // The class `registry` is unbound on the right-hand side: the module's is read. + (5, vec![OpKindTag::Import]), + // Possibly bound in the class: the join of the class and module bindings. + ( + 8, + vec![ + OpKindTag::Call("make()".to_owned()), + OpKindTag::Call("other()".to_owned()) + ] + ), + // After `del`, the module binding is visible again. + (10, vec![OpKindTag::Call("make()".to_owned())]), + ] + ); +} + +#[test] +fn class_fallback_skips_enclosing_function_and_module_falls_back_to_builtins() { + let p = program( + "\ +x = 0 +def f(): + x = 1 + class D: + x = x + x.q = 1 +len = len +len.q = 2 +", + ); + let f = region(&p, "f"); + let module_x = p.global("x").unwrap(); + let mut entry = Entry::new(()); + entry + .outer + .insert(module_x, Binding::bound(BTreeSet::from([OpId(5_000)]))); + let result = Engine::new(&p, &Provenance).analyze(f, entry); + // `x = x` in a class body reads the module's `x`, not `f`'s local. + assert_eq!( + origins_by_line(&p, &result), + vec![(6, vec![OpKindTag::External(5_000)])] + ); + + let builtin_len = p.symbol(p.global("len").unwrap()).fallback.unwrap(); + assert_eq!(p.symbol(builtin_len).origin, SymbolOrigin::Builtin); + let mut entry = Entry::new(()); + entry + .outer + .insert(builtin_len, Binding::bound(BTreeSet::from([OpId(6_000)]))); + let result = Engine::new(&p, &Provenance).analyze(Program::MODULE, entry); + assert_eq!( + origins_by_line(&p, &result), + vec![(8, vec![OpKindTag::External(6_000)])] + ); + let exit = result.exit.unwrap(); + assert_eq!( + exit.presence(&p, p.global("len").unwrap()), + Some(Presence::Bound) + ); +} + +/// Provenance plus calls: reports each call's callee origins, and applies a callee's +/// effect on outer cells at calls of one function. +struct Calls { + function: Option, + effects: Vec<(SymbolId, Binding>)>, +} + +impl Domain for Calls { + type Value = BTreeSet; + type State = (); + type Diagnostic = (u32, BTreeSet); + + fn top(&self) -> Self::Value { + BTreeSet::new() + } + + fn join(&self, a: &Self::Value, b: &Self::Value) -> Self::Value { + a.union(b).copied().collect() + } + + fn join_state(&self, _: &(), _: &()) {} + + fn transfer(&self, op: OpId, kind: &OpKind, cx: &mut Cx<'_, Self>) { + match kind { + OpKind::Name { .. } | OpKind::Param { .. } => {} + OpKind::Call { func, dst, .. } => { + let callee = cx.reg(*func).clone(); + if self.function.is_some_and(|f| callee.contains(&f)) { + for (symbol, binding) in &self.effects { + cx.set_binding(*symbol, binding.clone()); + } + } + cx.report((cx.range().start().into(), callee)); + cx.set_reg(*dst, BTreeSet::from([op])); + } + _ => { + for dst in kind.writes() { + cx.set_reg(dst, BTreeSet::from([op])); + } + } + } + } +} + +#[test] +fn deleted_outer_cells_fall_back_and_propagate_to_callers() { + let p = program( + "\ +open = lambda *args: None +def f(): + global open + del open + open('inside') +f() +open('after') +", + ); + let f = region(&p, "f"); + let module_open = p.global("open").unwrap(); + let builtin_open = p.symbol(module_open).fallback.unwrap(); + let builtin = BTreeSet::from([OpId(7_000)]); + let no_calls = Calls { + function: None, + effects: Vec::new(), + }; + + // Inside `f`, the deleted global falls back to the builtin cell. + let mut entry = Entry::new(()); + entry + .outer + .insert(module_open, Binding::bound(BTreeSet::from([OpId(1)]))); + entry + .outer + .insert(builtin_open, Binding::bound(builtin.clone())); + let result = Engine::new(&p, &no_calls).analyze(f, entry); + let callees: Vec<&BTreeSet> = result.diagnostics.iter().map(|(_, c)| c).collect(); + assert_eq!(callees, vec![&builtin]); + // The exit cell records the deletion: the caller sees `open` unbound, not unknown. + let exit = result.exit.unwrap(); + assert_eq!(exit.outer()[&module_open].presence, Presence::Unbound); + + // Applying `f`'s effect at the module's call makes the later load fall back too. + let make_f = OpId( + p.ops + .iter() + .position(|op| matches!(op.kind, OpKind::MakeFunction { region, .. } if region == f)) + .unwrap() as u32, + ); + let calls = Calls { + function: Some(make_f), + effects: vec![(module_open, exit.outer()[&module_open].clone())], + }; + let mut entry = Entry::new(()); + entry + .outer + .insert(builtin_open, Binding::bound(builtin.clone())); + let result = Engine::new(&p, &calls).analyze(Program::MODULE, entry); + let last = result + .diagnostics + .iter() + .max_by_key(|(start, _)| *start) + .unwrap(); + assert_eq!(line(&p, last.0), 7); + assert_eq!(last.1, builtin); + let exit = result.exit.unwrap(); + assert_eq!(exit.presence(&p, module_open), Some(Presence::Unbound)); +} + +#[test] +fn unknown_outer_cells_are_not_unbound() { + // With no cell for the global, a load is unknown (`top`), not a fallback. + let p = program("def f():\n return g\n"); + let f = region(&p, "f"); + let result = Engine::new(&p, &Provenance).analyze(f, Entry::new(())); + assert_eq!(result.returned, Some(BTreeSet::new())); + assert!(result.exit.unwrap().outer().is_empty()); +} diff --git a/static_analysis/tests/flow_points.rs b/static_analysis/tests/flow_points.rs new file mode 100644 index 0000000000..98dc355954 --- /dev/null +++ b/static_analysis/tests/flow_points.rs @@ -0,0 +1,280 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use std::collections::HashSet; +use std::marker::PhantomData; + +use pyrefly_python::ast::Ast; +use pyrefly_static_analysis::ModuleName; +use pyrefly_static_analysis::analyze_python; +use pyrefly_static_analysis::analyze_python_with_flow_points; +use pyrefly_static_analysis::flow_point::FlowPointCollector; +use pyrefly_static_analysis::flow_point::FlowPointId; +use pyrefly_static_analysis::flow_point::FlowPointKind; +use pyrefly_static_analysis::structured; +use pyrefly_static_analysis::structured::Arm; +use pyrefly_static_analysis::structured::Borrowed; +use pyrefly_static_analysis::structured::Driver; +use pyrefly_static_analysis::structured::Jump; +use pyrefly_static_analysis::structured::LoopHeader; +use pyrefly_static_analysis::structured::Owned; +use pyrefly_static_analysis::structured::Structured; +use pyrefly_static_analysis::structured::Syntax; +use ruff_python_ast::Expr; +use ruff_python_ast::PySourceType; +use ruff_python_ast::Stmt; +use ruff_text_size::TextRange; + +trait TestSyntax: Syntax { + fn expr(expr: &Self::Expr) -> &Expr; + fn nested(trace: &mut Trace, stmt: Self::Stmt); +} + +impl TestSyntax for Owned { + fn expr(expr: &Expr) -> &Expr { + expr + } + + fn nested(trace: &mut Trace, stmt: Stmt) { + match stmt { + Stmt::FunctionDef(f) => structured::region(trace, f.body), + Stmt::ClassDef(c) => structured::suite(trace, c.body), + Stmt::With(w) => structured::suite(trace, w.body), + _ => {} + } + } +} + +impl<'a> TestSyntax for Borrowed<'a> { + fn expr<'b>(expr: &'b &'a Expr) -> &'b Expr { + expr + } + + fn nested(trace: &mut Trace, stmt: &'a Stmt) { + match stmt { + Stmt::FunctionDef(f) => structured::region(trace, &f.body), + Stmt::ClassDef(c) => structured::suite(trace, &c.body), + Stmt::With(w) => structured::suite(trace, &w.body), + _ => {} + } + } +} + +struct Trace { + driver: Driver<()>, + points: FlowPointCollector, + prune: bool, + observe: bool, + syntax: PhantomData, +} + +impl Trace { + fn new(prune: bool, observe: bool) -> Self { + Self { + driver: Driver::default(), + points: FlowPointCollector::default(), + prune, + observe, + syntax: PhantomData, + } + } +} + +impl Structured for Trace { + type Syntax = S; + type Suite = (); + type If = (); + type Loop = (); + type Target = (); + + fn driver(&mut self) -> &mut Driver<()> { + &mut self.driver + } + fn observes_flow_points(&self) -> bool { + self.observe + } + fn flow_point(&mut self, point: FlowPointId) { + self.points.observe(point); + } + fn suite_start(&mut self, _: &[Stmt]) {} + fn absorbs(&mut self, _: &Stmt, _: &Stmt) -> bool { + false + } + fn stmt_before(&mut self, _: &mut (), _: &Stmt, _: Option<&Stmt>) {} + fn stmt_after(&mut self, _: &mut ()) {} + fn suite_end(&mut self, _: ()) {} + fn other_stmt(&mut self, stmt: S::Stmt) { + S::nested(self, stmt); + } + fn if_start(&mut self, _: TextRange, _: &mut S::Expr) {} + fn arm_start( + &mut self, + _: &mut (), + _: TextRange, + _: bool, + test: Option<&mut S::Expr>, + ) -> Option { + if self.prune { + test.and_then(|test| match S::expr(test) { + Expr::BooleanLiteral(b) => Some(b.value), + _ => None, + }) + } else { + None + } + } + fn arm_skip(&mut self, _: &mut (), _: Arm) {} + fn arm_enter(&mut self, _: &mut (), _: TextRange, _: Option, _: &[Stmt]) {} + fn arm_exit(&mut self, _: &mut (), _: TextRange) {} + fn arm_dead(&mut self, _: &mut (), _: Arm) {} + fn if_end(&mut self, _: (), _: bool) {} + fn loop_start(&mut self, _: LoopHeader, _: &[Stmt]) -> ((), ()) { + ((), ()) + } + fn loop_else(&mut self, _: &mut ()) {} + fn loop_end(&mut self, _: ()) {} + fn loop_jump(&mut self, _: Jump, _: TextRange, _: Option<()>) {} +} + +const SOURCE: &str = r#" +if False: + def omitted(): + while c: + break +elif flag: + value = 1 +else: + value = 2 + +def outer(): + while running: + def inner(): + for item in items: + if item: + continue + else: + break + else: + pass + if True: + pass + elif c: + while other: + pass + break + else: + pass + class C: + if c: + pass + def sibling(): + while c: + pass +"#; + +fn owned(source: &str, prune: bool) -> Vec { + let (ast, errors, _) = Ast::parse(source, PySourceType::Python); + assert!(errors.is_empty(), "{errors:?}"); + let mut trace = Trace::::new(prune, true); + structured::suite(&mut trace, ast.body); + trace.points.points +} + +#[test] +fn owned_and_borrowed_source_ids_agree_and_repeat_parse() { + let (ast, errors, _) = Ast::parse(SOURCE, PySourceType::Python); + assert!(errors.is_empty()); + let mut trace = Trace::::new(false, true); + structured::suite(&mut trace, &ast.body); + let full = owned(SOURCE, false); + assert_eq!(trace.points.points, full); + assert_eq!(owned(SOURCE, false), full); + assert_eq!(full.iter().collect::>().len(), full.len()); + for kind in [ + FlowPointKind::ScopeEntry, + FlowPointKind::ScopeExit, + FlowPointKind::ArmTest, + FlowPointKind::ArmEntry, + FlowPointKind::ArmExit, + FlowPointKind::IfJoin, + FlowPointKind::LoopBody, + FlowPointKind::LoopBackedge, + FlowPointKind::LoopExit, + FlowPointKind::LoopElse, + FlowPointKind::LoopJoin, + FlowPointKind::Continue, + FlowPointKind::Break, + ] { + assert!(full.iter().any(|p| p.kind == kind), "{kind:?}"); + } + assert!(full.iter().map(|p| p.region).collect::>().len() >= 5); +} + +#[test] +fn pruning_keeps_surviving_source_ids_without_visit_ordinals() { + let full = owned(SOURCE, false); + let pruned = owned(SOURCE, true); + assert!(pruned.len() < full.len()); + let full: HashSet<_> = full.into_iter().collect(); + assert!(pruned.iter().all(|p| full.contains(p))); + let (ast, _, _) = Ast::parse(SOURCE, PySourceType::Python); + let mut trace = Trace::::new(true, true); + structured::suite(&mut trace, &ast.body); + assert_eq!(trace.points.points, pruned); +} + +#[test] +fn actual_cfg_backend_emits_the_shared_ids_without_changing_lowering() { + let observed = analyze_python_with_flow_points(SOURCE, ModuleName::from_str("main"), false); + assert_eq!(observed.flow_points, owned(SOURCE, false)); + let plain = analyze_python(SOURCE, ModuleName::from_str("main"), false); + assert_eq!( + format!("{:?}", observed.program.ops), + format!("{:?}", plain.ops) + ); + assert_eq!( + format!("{:?}", observed.program.regions), + format!("{:?}", plain.regions) + ); +} + +#[test] +fn finally_clones_repeat_source_occurrences_not_runtime_instance_ids() { + let source = r#" +def f(flag): + try: + if flag: + return + finally: + if flag: + pass +"#; + let observed = analyze_python_with_flow_points(source, ModuleName::from_str("main"), false); + let exits: Vec<_> = observed + .flow_points + .iter() + .filter(|p| p.kind == FlowPointKind::ArmExit) + .collect(); + assert!(exits.iter().collect::>().len() < exits.len()); + let repeated = analyze_python_with_flow_points(source, ModuleName::from_str("main"), false); + assert_eq!(observed.flow_points, repeated.flow_points); +} + +#[test] +fn unobserved_and_empty_suites_emit_nothing() { + let (ast, _, _) = Ast::parse(SOURCE, PySourceType::Python); + let mut trace = Trace::::new(false, false); + structured::suite(&mut trace, ast.body); + assert!(trace.points.points.is_empty()); + assert_eq!(trace.points.points.capacity(), 0); + assert!(owned("", false).is_empty()); + // Driver state is restored after the outermost suite, even on reuse. + trace.observe = true; + let (ast, _, _) = Ast::parse("if c:\n pass\n", PySourceType::Python); + structured::suite(&mut trace, ast.body); + assert_eq!(trace.points.points, owned("if c:\n pass\n", false)); +} diff --git a/static_analysis/tests/frontend.rs b/static_analysis/tests/frontend.rs new file mode 100644 index 0000000000..2c1a2e6efd --- /dev/null +++ b/static_analysis/tests/frontend.rs @@ -0,0 +1,861 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! Tests of the shared frontend: scopes, evaluation order, control flow, and the +//! domain-free history queries. + +use std::collections::BTreeSet; + +use pyrefly_static_analysis::*; + +fn program(source: &str) -> Program { + analyze_python(source, ModuleName::from_str("pkg.mod"), false) +} + +fn region(p: &Program, name: &str) -> RegionId { + let i = p + .regions + .iter() + .position(|r| r.name.as_str() == name) + .unwrap_or_else(|| panic!("no region named {name}")); + RegionId(i as u32) +} + +/// The `Name` operations on symbols called `name` that run in `region`, in emission order. +fn name_ops(p: &Program, region: RegionId, name: &str) -> Vec<(OpId, SymbolId, Access)> { + p.ops + .iter() + .enumerate() + .filter_map(|(i, op)| match op.kind { + OpKind::Name { symbol, access } + if op.region == region && p.symbol(symbol).name.as_str() == name => + { + Some((OpId(i as u32), symbol, access)) + } + _ => None, + }) + .collect() +} + +fn loads(p: &Program, region: RegionId, name: &str) -> Vec { + name_ops(p, region, name) + .into_iter() + .filter(|(_, _, a)| matches!(a, Access::Load(_))) + .map(|(op, _, _)| op) + .collect() +} + +fn stores(p: &Program, region: RegionId, name: &str) -> Vec { + name_ops(p, region, name) + .into_iter() + .filter(|(_, _, a)| matches!(a, Access::Store(_))) + .map(|(op, _, _)| op) + .collect() +} + +fn defs(ops: &[OpId]) -> BTreeSet { + ops.iter().map(|op| Def::Op(*op)).collect() +} + +fn reaching_last_load(p: &Program, h: &History, region: RegionId, name: &str) -> BTreeSet { + let load = *loads(p, region, name).last().expect("a load"); + h.reaching(load).expect("a reachable owned load").clone() +} + +fn gaps(p: &Program) -> Vec { + p.ops + .iter() + .filter_map(|op| match op.kind { + OpKind::Gap { kind, .. } => Some(kind), + _ => None, + }) + .collect() +} + +#[test] +fn assignment_evaluates_value_before_targets() { + let p = program("a.b[c] = d = f(x)\n"); + assert_eq!( + p.dump(Program::MODULE), + "\ +b0: + r1 = f@builtins + r2 = x@builtins + r0 = r1(r2) + r4 = a@builtins + r3 = r4.b + r5 = c@builtins + r3[r5] = r0 + d = r0 + return +" + ); +} + +#[test] +fn decorators_then_defaults_then_function_then_application() { + let p = program("@d1\n@d2\ndef g(p=h()):\n pass\n"); + assert_eq!( + p.dump(Program::MODULE), + "\ +b0: + r0 = d1@builtins + r1 = d2@builtins + r4 = h@builtins + r3 = r4() + r2 = function g(r3) + r5 = r1(r2) + r6 = r0(r5) + g = r6 + return +" + ); + let g = p.region(region(&p, "g")); + assert_eq!(g.trigger, Trigger::OnCall); + assert_eq!(g.parent.map(|(r, _)| r), Some(Program::MODULE)); + assert!(g.params[0].has_default); +} + +#[test] +fn augmented_assignment_reads_target_then_value_then_writes() { + let p = program("a.b += v()\n"); + assert_eq!( + p.dump(Program::MODULE), + "\ +b0: + r1 = a@builtins + r2 = r1.b + r4 = v@builtins + r3 = r4() + r0 = r2 InPlace(Add) r3 + r1.b = r0 + return +" + ); +} + +#[test] +fn short_circuit_and_conditional_branches_join() { + let p = program( + "\ +x = 0 +y = (x := 1) and (x := 2) +z = x +w = (x := 3) if c else 4 +v = x +", + ); + let h = History::new(&p); + let x = stores(&p, Program::MODULE, "x"); + let x_loads = loads(&p, Program::MODULE, "x"); + // `and` may stop after its first operand. + assert_eq!(h.reaching(x_loads[0]).unwrap(), &defs(&[x[1], x[2]])); + // The walrus runs only on the `if` arm of the conditional expression. + assert_eq!(h.reaching(x_loads[1]).unwrap(), &defs(&[x[1], x[2], x[3]])); +} + +#[test] +fn chained_comparison_evaluates_middle_once_and_short_circuits() { + let p = program("r = a < f() < c\n"); + let dump = p.dump(Program::MODULE); + assert_eq!(dump.matches("f@builtins").count(), 1); + assert_eq!(dump.matches("c@builtins").count(), 1); + // `c` is evaluated only on the branch where the first comparison holds. + let c = loads(&p, Program::MODULE, "c")[0]; + let a = loads(&p, Program::MODULE, "a")[0]; + assert_ne!(p.op(c).block, p.op(a).block); +} + +#[test] +fn loop_else_break_and_continue() { + let p = program( + "\ +x = 0 +for i in it: + if c: + x = 1 + break + if d: + x = 2 + continue + x = 3 +else: + x = 4 +y = x +", + ); + let h = History::new(&p); + let x = stores(&p, Program::MODULE, "x"); + // `else` runs exactly when the loop was not broken, so only `break` and `else` reach. + assert_eq!( + reaching_last_load(&p, &h, Program::MODULE, "x"), + defs(&[x[1], x[4]]) + ); +} + +#[test] +fn while_loop_paths() { + let p = program( + "\ +x = 0 +while c: + x = 1 + if d: + continue + x = 2 +y = x +while True: + x = 5 + if e: + break +z = x +while True: + pass +after = 1 +", + ); + let h = History::new(&p); + let x = stores(&p, Program::MODULE, "x"); + let x_loads = loads(&p, Program::MODULE, "x"); + assert_eq!(h.reaching(x_loads[0]).unwrap(), &defs(&[x[0], x[1], x[2]])); + // `while True` exits only through `break`. + assert_eq!(h.reaching(x_loads[1]).unwrap(), &defs(&[x[3]])); + // Code after an infinite loop without `break` is unreachable. + let after = stores(&p, Program::MODULE, "after")[0]; + assert!(h.uses(after).is_empty()); + struct Unit; + impl Domain for Unit { + type Value = (); + type State = (); + type Diagnostic = (); + fn top(&self) {} + fn join(&self, _: &(), _: &()) {} + fn join_state(&self, _: &(), _: &()) {} + } + let result = Engine::new(&p, &Unit).analyze(Program::MODULE, Entry::new(())); + assert!(!result.is_reachable(after, &p)); + assert!(result.exit.is_none()); +} + +#[test] +fn deferred_function_regions_and_outer_defaults() { + let p = program( + "\ +def f(a, b=(y := 1), *args, c, **kw) -> R: + z = a + lam = lambda q=y: q + return z +", + ); + let f = region(&p, "f"); + // Body operations live only in the function region. + assert!(stores(&p, Program::MODULE, "z").is_empty()); + assert_eq!(stores(&p, f, "z").len(), 1); + // The default's walrus binds in the module, where it is evaluated. + let y = stores(&p, Program::MODULE, "y"); + assert_eq!(y.len(), 1); + assert_eq!( + p.symbol(name_ops(&p, Program::MODULE, "y")[0].1).scope, + p.region(Program::MODULE).scope + ); + // The lambda default is evaluated in `f`, reading the module's `y`. + let y_in_f = loads(&p, f, "y"); + assert_eq!(y_in_f.len(), 1); + assert_eq!(p.owner(name_ops(&p, f, "y")[0].1), Some(Program::MODULE)); + let kinds: Vec = p.region(f).params.iter().map(|p| p.kind).collect(); + assert_eq!( + kinds, + vec![ + ParamKind::Positional, + ParamKind::Positional, + ParamKind::VarArgs, + ParamKind::KeywordOnly, + ParamKind::KwArgs + ] + ); + assert!(matches!(&p.region(f).returns, Some(StaticExpr::Name(n, _)) if n.as_str() == "R")); + let lam = p.region(region(&p, "")); + assert_eq!(lam.parent.map(|(r, _)| r), Some(f)); +} + +#[test] +fn imported_aliases_and_local_shadowing() { + let p = program( + "\ +import os.path +import os.path as osp +from . import sibling +from .sub import thing as t +def f(): + t = 1 + return t, osp +def g(): + print(t) + if False: + t = 2 +", + ); + let imports: Vec<(String, ImportItem)> = p + .ops + .iter() + .filter_map(|op| match &op.kind { + OpKind::Import { module, item, .. } => Some((module.to_string(), item.clone())), + _ => None, + }) + .collect(); + assert_eq!( + imports, + vec![ + ("os.path".to_owned(), ImportItem::Root), + ("os.path".to_owned(), ImportItem::Module), + ("pkg".to_owned(), ImportItem::Name(Name::new("sibling"))), + ("pkg.sub".to_owned(), ImportItem::Name(Name::new("thing"))), + ] + ); + // `import os.path` binds `os`. + assert!(p.global("os").is_some()); + let f = region(&p, "f"); + let g = region(&p, "g"); + let module_t = p.global("t").unwrap(); + assert!(name_ops(&p, f, "t").iter().all(|(_, s, _)| *s != module_t)); + assert_eq!(name_ops(&p, f, "osp")[0].1, p.global("osp").unwrap()); + // A binding in dead code still makes `t` local to `g`: the read is unbound. + let h = History::new(&p); + assert_eq!( + h.reaching(loads(&p, g, "t")[0]).unwrap(), + &BTreeSet::from([Def::Entry]) + ); +} + +#[test] +fn global_nonlocal_and_closure_ownership() { + let p = program( + "\ +counter = 0 +def outer(): + v = 1 + def inner(): + nonlocal v + global counter, fresh + v = v + 1 + counter = 2 + fresh = 3 + return w + w = 4 + return inner +", + ); + let outer = region(&p, "outer"); + let inner = region(&p, "inner"); + let v = name_ops(&p, inner, "v"); + assert!(v.iter().all(|(_, s, _)| p.owner(*s) == Some(outer))); + assert!(p.symbol(v[0].1).captured); + let counter = name_ops(&p, inner, "counter")[0].1; + assert_eq!(counter, p.global("counter").unwrap()); + let fresh = p.global("fresh").unwrap(); + assert_eq!(p.symbol(fresh).origin, SymbolOrigin::GlobalDeclaration); + assert_eq!(name_ops(&p, inner, "fresh")[0].1, fresh); + // A free read in `inner` resolves to `outer`'s `w`, even though it is assigned later. + assert_eq!(p.owner(name_ops(&p, inner, "w")[0].1), Some(outer)); + // History lists accesses from every region. + let h = History::new(&p); + assert_eq!(h.accesses(counter).len(), 2); + // Loads of another region's symbol have no reaching definitions. + assert!(h.reaching(loads(&p, inner, "v")[0]).is_none()); +} + +#[test] +fn class_and_comprehension_scopes() { + let p = program( + "\ +x = 'module' +class C(Base): + x = 'class' + y = x + def m(self): + return x + z = [x for _ in range(3)] +def f(items): + total = [last := i for i in items] + return last +", + ); + let module_x = p.global("x").unwrap(); + // The class body reads its own `x`. + let class_scope = p + .scopes + .iter() + .position(|s| s.kind == ScopeKind::Class) + .unwrap(); + let class_x = *p.scopes[class_scope].symbols.get(&Name::new("x")).unwrap(); + let x_loads: Vec = name_ops(&p, Program::MODULE, "x") + .into_iter() + .filter(|(_, _, a)| matches!(a, Access::Load(_))) + .map(|(_, s, _)| s) + .collect(); + // `y = x` reads the class `x`; the comprehension element cannot see class scope. + assert_eq!(x_loads, vec![class_x, module_x]); + let m = region(&p, "m"); + assert_eq!(name_ops(&p, m, "x")[0].1, module_x); + // The class body is lowered inline, followed by class creation and binding. + assert!(p.dump(Program::MODULE).contains("= class C(")); + // A walrus in a comprehension binds in the enclosing function. + let f = region(&p, "f"); + let last = name_ops(&p, f, "last"); + assert!(last.iter().all(|(_, s, _)| p.owner(*s) == Some(f) + && p.scope(p.symbol(*s).scope).kind == ScopeKind::Function)); + let i = name_ops(&p, f, "i")[0].1; + assert_eq!(p.scope(p.symbol(i).scope).kind, ScopeKind::Comprehension); +} + +#[test] +fn generator_expression_is_deferred_except_first_iterable() { + let p = program("g = (f(e) for e in xs if e)\n"); + let genexpr = region(&p, ""); + assert_eq!(p.region(genexpr).trigger, Trigger::OnIteration); + assert_eq!(loads(&p, Program::MODULE, "xs").len(), 1); + assert!(loads(&p, Program::MODULE, "f").is_empty()); + assert_eq!(loads(&p, genexpr, "f").len(), 1); +} + +#[test] +fn triggers() { + let p = program( + "\ +def plain(): pass +def gen(): yield 1 +async def coro(): await x +async def agen(): yield 1 +", + ); + let trigger = |n| p.region(region(&p, n)).trigger; + assert_eq!(trigger(""), Trigger::OnImport); + assert_eq!(trigger("plain"), Trigger::OnCall); + assert_eq!(trigger("gen"), Trigger::OnIteration); + assert_eq!(trigger("coro"), Trigger::OnAwait); + assert_eq!(trigger("agen"), Trigger::OnIteration); +} + +#[test] +fn context_exit_and_finally_run_on_every_normal_exit() { + let p = program( + "\ +def f(): + for i in xs: + with m() as c: + if c: + break + try: + return c + finally: + cleanup() +", + ); + let dump = p.dump(region(&p, "f")); + // `return c` runs `finally`, then exits the context, then returns. + assert!( + dump.contains(" r7 = c\n r9 = cleanup@builtins\n r8 = r9()\n exit r3\n return r7\n"), + "{dump}" + ); + // `break` exits the context before leaving the loop. + assert!(dump.contains(" exit r3\n jump b4\n"), "{dump}"); + assert!(gaps(&p).contains(&GapKind::Exceptions)); +} + +#[test] +fn try_except_models_exceptions_at_statement_boundaries() { + let p = program( + "\ +x = 0 +try: + x = 1 + x = 2 +except E as e: + y = x +else: + x = 3 +z = x +", + ); + let h = History::new(&p); + let x = stores(&p, Program::MODULE, "x"); + let x_loads = loads(&p, Program::MODULE, "x"); + // The handler sees the state before, between and after the body statements. + assert_eq!(h.reaching(x_loads[0]).unwrap(), &defs(&[x[0], x[1], x[2]])); + // After the statement: the handler path or the `else` path. + assert_eq!( + h.reaching(x_loads[1]).unwrap(), + &defs(&[x[0], x[1], x[2], x[3]]) + ); + // The handler deletes its exception name. + assert!( + name_ops(&p, Program::MODULE, "e") + .iter() + .any(|(_, _, a)| *a == Access::Delete) + ); +} + +#[test] +fn explicit_gaps() { + let p = program( + "\ +from os import * +from .... import too_far +match v: + case [a, b]: + pass +type Alias = int +x: make_type() = 1 +def f[T](a: T): pass +", + ); + let found: BTreeSet = gaps(&p).into_iter().collect(); + for kind in [ + GapKind::StarImport, + GapKind::InvalidRelativeImport, + GapKind::Match, + GapKind::TypeParams, + GapKind::AnnotationEvaluation, + ] { + assert!(found.contains(&kind), "missing {kind:?} in {found:?}"); + } + let bad = program("def f():\n nonlocal q\n q = 1\n"); + assert!(gaps(&bad).contains(&GapKind::ScopeError)); + let syntax = program("x = (\n"); + assert!(!syntax.syntax_errors.is_empty()); + assert!(gaps(&syntax).contains(&GapKind::SyntaxError)); +} + +#[test] +fn history_def_use_chains() { + let p = program( + "\ +x = 1 +if c: + x = 2 +print(x) +del x +", + ); + let h = History::new(&p); + let x = stores(&p, Program::MODULE, "x"); + let load = loads(&p, Program::MODULE, "x")[0]; + assert_eq!(h.uses(x[0]), &[load]); + assert_eq!(h.uses(x[1]), &[load]); + let x_symbol = p.global("x").unwrap(); + assert_eq!(h.accesses(x_symbol).len(), 4); +} + +#[test] +fn raise_terminators_carry_cause_and_range() { + let p = program( + "\ +def f(x): + assert x, 'msg' + try: + g() + except ValueError: + pass + raise KeyError(x) +", + ); + let f = p.region(region(&p, "f")); + let mut raises: Vec<(RaiseCause, String)> = f + .blocks + .iter() + .filter_map(|b| match &b.terminator { + Terminator::Raise { cause, range, .. } => { + Some((*cause, p.text(*range).lines().next().unwrap().to_owned())) + } + _ => None, + }) + .collect(); + raises.sort(); + assert!(f.blocks.iter().all(|b| !matches!( + b.terminator, + Terminator::Raise { + cause: RaiseCause::AssertFailure, + exc: Some(_), + .. + } + ))); + assert_eq!( + raises, + vec![ + (RaiseCause::Explicit, "raise KeyError(x)".to_owned()), + (RaiseCause::AssertFailure, "assert x, 'msg'".to_owned()), + (RaiseCause::Reraise, "try:".to_owned()), + ] + ); +} + +#[test] +fn free_symbols_and_operation_scopes() { + let p = program( + "\ +import m +def outer(): + v = 1 + def inner(): + return v, m, len + class C: + m = m + return inner +", + ); + let names = |r: RegionId| -> Vec { + p.free_symbols(r) + .iter() + .map(|s| { + let owner = p + .owner(*s) + .map_or("builtins".to_owned(), |o| p.region(o).name.to_string()); + format!("{}@{owner}", p.symbol(*s).name) + }) + .collect() + }; + let inner = region(&p, "inner"); + let outer = region(&p, "outer"); + // `inner` reads `outer`'s cell, a module global (falling back to the builtin while + // unbound) and a builtin. + let mut inner_free = names(inner); + inner_free.sort(); + assert_eq!( + inner_free, + vec!["len@builtins", "m@", "m@builtins", "v@outer"] + ); + // `outer`'s free symbols include the module `m` read through the class fallback, + // and exclude `v`, which `outer` owns. + let mut outer_free = names(outer); + outer_free.sort(); + assert_eq!(outer_free, vec!["len@builtins", "m@", "m@builtins"]); + let module_symbols: Vec = p + .module_symbols() + .map(|s| p.symbol(s).name.to_string()) + .collect(); + assert_eq!(module_symbols, vec!["m", "outer"]); + // Operations in the class body record the class scope. + let class_store = stores(&p, outer, "m")[0]; + assert_eq!(p.scope(p.op(class_store).scope).kind, ScopeKind::Class); +} + +#[test] +fn call_arguments_evaluate_positional_before_keyword() { + // CPython evaluates `*args` before an earlier keyword argument. + let p = program("f(k=kw(), *star(), **more())\nclass C(meta=kw(), *star()): pass\n"); + let calls: Vec = p + .ops + .iter() + .filter_map(|op| match &op.kind { + OpKind::Call { args, .. } if args.is_empty() => Some(p.text(op.range).to_owned()), + _ => None, + }) + .collect(); + assert_eq!(calls, vec!["star()", "kw()", "more()", "star()", "kw()"]); + let shapes: Vec> = p + .ops + .iter() + .filter_map(|op| match &op.kind { + OpKind::Call { args, .. } if !args.is_empty() => Some(args), + OpKind::MakeClass { bases, .. } => Some(bases), + _ => None, + }) + .map(|args| { + args.iter() + .map(|a| match a { + Arg::Positional(_) => "positional", + Arg::Star(_) => "star", + Arg::Keyword(..) => "keyword", + Arg::DoubleStar(_) => "double star", + }) + .collect() + }) + .collect(); + assert_eq!( + shapes, + vec![ + vec!["star", "keyword", "double star"], + vec!["star", "keyword"] + ] + ); +} + +#[test] +fn annotated_targets_without_values_evaluate_their_receivers() { + let p = program( + "\ +effect().attr: int +table[key()]: int +obj().field: int = value() +def f(): + local: int +", + ); + let calls: Vec = p + .ops + .iter() + .filter_map(|op| match &op.kind { + OpKind::Call { .. } => Some(p.text(op.range).to_owned()), + _ => None, + }) + .collect(); + // With a value, the value is evaluated before the target's object. + assert_eq!(calls, vec!["effect()", "key()", "value()", "obj()"]); + // Nothing is stored for annotation-only targets. + let stores = p + .ops + .iter() + .filter(|op| { + matches!( + op.kind, + OpKind::Attribute { + access: Access::Store(_), + .. + } | OpKind::Subscript { + access: Access::Store(_), + .. + } + ) + }) + .count(); + assert_eq!(stores, 1); + // A bare annotation still makes the name local, without a store. + let f = region(&p, "f"); + assert!(name_ops(&p, f, "local").is_empty()); + assert!( + p.scope(p.region(f).scope) + .symbols + .contains_key(&Name::new("local")) + ); +} + +/// Every operation appears exactly once, in the block it records. +fn assert_ops_in_blocks(p: &Program) { + let mut seen = vec![0; p.ops.len()]; + for (r, region) in p.regions.iter().enumerate() { + for (b, block) in region.blocks.iter().enumerate() { + for op in &block.ops { + seen[op.index()] += 1; + assert_eq!(p.op(*op).region, RegionId(r as u32)); + assert_eq!(p.op(*op).block, BlockId(b as u32)); + } + } + } + assert!(seen.iter().all(|n| *n == 1), "{seen:?}"); +} + +#[test] +fn definitions_in_finally_are_lowered_once() { + let p = program( + "\ +def outer(c): + try: + if c: + return 1 + finally: + g = lambda: c + return 2 +", + ); + assert_ops_in_blocks(&p); + let lambdas = p + .regions + .iter() + .filter(|r| r.name.as_str() == "") + .count(); + assert_eq!(lambdas, 1); + // Each exit path creates the function from the same region. + let creations = p + .ops + .iter() + .filter(|op| matches!(op.kind, OpKind::MakeFunction { .. }) && op.region != Program::MODULE) + .count(); + assert_eq!(creations, 3); +} + +#[test] +fn nested_finally_expansion_is_bounded() { + // Each level's `finally` body holds the next `try`, so copies multiply per level. + let depth = 14; + let mut source = String::from("def f(c):\n"); + for i in 0..depth { + let indent = " ".repeat(2 * i + 1); + source.push_str(&format!( + "{indent}try:\n{indent} if c: return {i}\n{indent}finally:\n{indent} g{i}()\n" + )); + } + let p = program(&source); + assert_ops_in_blocks(&p); + assert!(p.ops.len() < 100_000, "{}", p.ops.len()); + assert!(gaps(&p).contains(&GapKind::FinallyLimit)); +} + +#[test] +fn external_writers_make_reaching_definitions_inexact() { + let p = program( + "\ +x = 1 +def set_global(): + global x + x = 2 +set_global() +print(x) +def outer(): + y = 1 + def clear(): + nonlocal y + del y + clear() + return y +", + ); + let h = History::new(&p); + let x = stores(&p, Program::MODULE, "x"); + let load = loads(&p, Program::MODULE, "x")[0]; + assert_eq!( + h.reaching(load).unwrap(), + &BTreeSet::from([Def::Op(x[0]), Def::External]) + ); + let outer = region(&p, "outer"); + let y_load = loads(&p, outer, "y")[0]; + assert!(h.reaching(y_load).unwrap().contains(&Def::External)); +} + +#[test] +fn handler_name_is_deleted_on_every_exit() { + let p = program( + "\ +def f(): + for i in xs: + try: + g() + except E as e: + if c: + break + return e +", + ); + let f = region(&p, "f"); + let deletes = name_ops(&p, f, "e") + .into_iter() + .filter(|(_, _, a)| *a == Access::Delete) + .count(); + // `break`, `return` (after evaluating `e`) and fallthrough (unreachable here but + // still lowered) each delete the name. + assert!(deletes >= 2, "{}", p.dump(f)); + let dump = p.dump(f); + assert!(dump.contains("del e\n return r"), "{dump}"); +} + +#[test] +fn generator_expression_receives_an_iterator() { + let p = program("g = (e for e in xs)\n"); + let module = p.dump(Program::MODULE); + assert!(module.contains("= iter r"), "{module}"); + let genexpr = p.dump(region(&p, "")); + assert!(!genexpr.contains("= iter "), "{genexpr}"); +} diff --git a/static_analysis/tests/scaling.rs b/static_analysis/tests/scaling.rs new file mode 100644 index 0000000000..e96a2e012a --- /dev/null +++ b/static_analysis/tests/scaling.rs @@ -0,0 +1,49 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! A scaling probe for the domain-absent paths: lowering, an engine run with a trivial +//! domain, and history. Run with +//! `cargo test --release -p pyrefly_static_analysis --test scaling -- --ignored --nocapture`. + +use std::time::Instant; + +use pyrefly_static_analysis::*; + +struct Unit; + +impl Domain for Unit { + type Value = (); + type State = (); + type Diagnostic = (); + fn top(&self) {} + fn join(&self, _: &(), _: &()) {} + fn join_state(&self, _: &(), _: &()) {} +} + +#[test] +#[ignore = "timing probe; run explicitly in release mode"] +fn scaling_probe() { + for n in [100, 800, 3200] { + let source: String = (0..n) + .map(|i| format!("if c{i}:\n x{} = f(a, b) + g(c)\n", i % 50)) + .collect(); + let start = Instant::now(); + let p = analyze_python(&source, ModuleName::from_str("m"), false); + let lower = start.elapsed(); + let start = Instant::now(); + let result = Engine::new(&p, &Unit).analyze(Program::MODULE, Entry::new(())); + let engine = start.elapsed(); + let start = Instant::now(); + History::new(&p); + let history = start.elapsed(); + assert_eq!(result.convergence, Convergence::Converged); + println!( + "branches={n} ops={} lower={lower:?} engine={engine:?} history={history:?}", + p.ops.len() + ); + } +} diff --git a/static_analysis/tests/scope_regressions.rs b/static_analysis/tests/scope_regressions.rs new file mode 100644 index 0000000000..6862fa26f8 --- /dev/null +++ b/static_analysis/tests/scope_regressions.rs @@ -0,0 +1,29 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use pyrefly_static_analysis::GapKind; +use pyrefly_static_analysis::ModuleName; +use pyrefly_static_analysis::OpKind; +use pyrefly_static_analysis::analyze_python; + +#[test] +fn nonlocal_global_barrier() { + let source = "def outer():\n x = 1\n def middle():\n global x\n def inner():\n nonlocal x\n return x\n"; + let program = analyze_python(source, ModuleName::from_str("main"), false); + let has_scope_error = program.ops.iter().any(|op| { + matches!( + op.kind, + OpKind::Gap { + kind: GapKind::ScopeError, + .. + } + ) + }); + // An enclosing global directive blocks nonlocal lookup, even when a farther + // enclosing function owns a name with the same spelling. + assert!(has_scope_error); +} diff --git a/static_analysis/tests/structured.rs b/static_analysis/tests/structured.rs new file mode 100644 index 0000000000..0414110139 --- /dev/null +++ b/static_analysis/tests/structured.rs @@ -0,0 +1,262 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! The structured driver with a tracing backend that owns the AST and decides tests +//! named `T` and `F` statically. + +use pyrefly_python::ast::Ast; +use pyrefly_static_analysis::structured; +use pyrefly_static_analysis::structured::Arm; +use pyrefly_static_analysis::structured::Coverage; +use pyrefly_static_analysis::structured::Driver; +use pyrefly_static_analysis::structured::Jump; +use pyrefly_static_analysis::structured::LoopHeader; +use pyrefly_static_analysis::structured::Owned; +use pyrefly_static_analysis::structured::Structured; +use ruff_python_ast::Expr; +use ruff_python_ast::PySourceType; +use ruff_python_ast::Stmt; +use ruff_text_size::TextRange; + +#[derive(Default)] +struct Trace { + driver: Driver, + events: Vec, + /// Loop ids are assigned in source order. + next_loop: usize, +} + +fn text(x: &Expr) -> String { + match x { + Expr::Name(n) => n.id.to_string(), + Expr::Call(c) => format!("{}()", text(&c.func)), + _ => "?".to_owned(), + } +} + +impl Trace { + fn log(&mut self, event: impl Into) { + self.events.push(event.into()); + } +} + +impl Structured for Trace { + type Syntax = Owned; + type Suite = (); + type If = (); + type Loop = usize; + type Target = usize; + + fn driver(&mut self) -> &mut Driver { + &mut self.driver + } + + fn suite_start(&mut self, _: &[Stmt]) {} + + fn absorbs(&mut self, stmt: &Stmt, _: &Stmt) -> bool { + matches!(stmt, Stmt::Expr(e) if text(&e.value) == "merge_next()") + } + + fn stmt_before(&mut self, _: &mut (), stmt: &Stmt, next: Option<&Stmt>) { + if matches!(stmt, Stmt::Expr(e) if text(&e.value) == "merge_next()") { + self.log(format!("next after merge: {}", next.is_some())); + } + } + + fn stmt_after(&mut self, _: &mut ()) {} + + fn suite_end(&mut self, _: ()) {} + + fn other_stmt(&mut self, stmt: Stmt) { + match stmt { + Stmt::Expr(e) => { + let event = text(&e.value); + self.log(event); + } + Stmt::FunctionDef(f) => { + self.log(format!("def {}", f.name.id)); + structured::region(self, f.body); + self.log(format!("end def {}", f.name.id)); + } + Stmt::With(w) => { + self.log("with"); + structured::suite(self, w.body); + } + _ => self.log("other"), + } + } + + fn if_start(&mut self, _: TextRange, first_test: &mut Expr) { + self.log(format!("if {}", text(first_test))); + } + + fn arm_start( + &mut self, + _: &mut (), + _: TextRange, + first: bool, + test: Option<&mut Expr>, + ) -> Option { + let test = test.map(|t| text(t)); + self.log(format!("arm {test:?} first={first}")); + match test.as_deref() { + Some("T") => Some(true), + Some("F") => Some(false), + _ => None, + } + } + + fn arm_skip(&mut self, _: &mut (), arm: Arm) { + self.log(format!("skip {:?}", arm.test.as_ref().map(text))); + } + + fn arm_enter(&mut self, _: &mut (), _: TextRange, _: Option, _: &[Stmt]) {} + + fn arm_exit(&mut self, _: &mut (), _: TextRange) { + self.log("arm end"); + } + + fn arm_dead(&mut self, _: &mut (), arm: Arm) { + self.log(format!("dead {:?}", arm.test.as_ref().map(text))); + } + + fn if_end(&mut self, _: (), exhaustive: bool) { + self.log(format!("end if exhaustive={exhaustive}")); + } + + fn loop_start(&mut self, header: LoopHeader, _: &[Stmt]) -> (usize, usize) { + let id = self.next_loop; + self.next_loop += 1; + let kind = match header { + LoopHeader::While { test, .. } => format!("while {}", text(&test)), + LoopHeader::For { iter, .. } => format!("for in {}", text(&iter)), + }; + self.log(format!("{kind} #{id}")); + (id, id) + } + + fn loop_else(&mut self, id: &mut usize) { + self.log(format!("else #{id}")); + } + + fn loop_end(&mut self, id: usize) { + self.log(format!("end loop #{id}")); + } + + fn loop_jump(&mut self, jump: Jump, _: TextRange, target: Option) { + self.log(format!("{jump:?} -> {target:?}")); + } +} + +fn drive(source: &str) -> (Vec, Coverage) { + let (ast, errors, _) = Ast::parse(source, PySourceType::Python); + assert!(errors.is_empty(), "{errors:?}"); + let mut trace = Trace::default(); + structured::suite(&mut trace, ast.body); + (trace.events, trace.driver.coverage) +} + +#[test] +fn static_truth_prunes_arms_and_decides_exhaustiveness() { + let (events, coverage) = drive( + r#" +if c: + a() +elif F: + b() +elif T: + d() +elif e: + f() +else: + g() +if c: + h() +"#, + ); + assert_eq!( + events, + [ + "if c", + "arm Some(\"c\") first=true", + "a()", + "arm end", + "arm Some(\"F\") first=false", + "skip Some(\"F\")", + "arm Some(\"T\") first=false", + "d()", + "arm end", + "dead Some(\"e\")", + "dead None", + "end if exhaustive=true", + "if c", + "arm Some(\"c\") first=true", + "h()", + "arm end", + "end if exhaustive=false", + ] + ); + assert_eq!(coverage.arms_visited, 3); + assert_eq!(coverage.arms_skipped, 1); + assert_eq!(coverage.arms_dead, 2); +} + +#[test] +fn jumps_target_the_innermost_loop_of_their_region() { + let (events, coverage) = drive( + r#" +break +while c: + for x in xs: + with m: + continue + break + def f(): + break +else: + break +"#, + ); + assert_eq!( + events, + [ + "Break -> None", + "while c #0", + "for in xs #1", + "with", + "Continue -> Some(1)", + "else #1", + "end loop #1", + "Break -> Some(0)", + "def f", + "Break -> None", + "end def f", + "else #0", + "Break -> None", + "end loop #0", + ] + ); + assert_eq!(coverage.jumps, 5); + assert_eq!(coverage.jumps_outside_loop, 3); + assert_eq!(coverage.regions, 1); +} + +#[test] +fn a_backend_can_absorb_the_next_statement() { + let (events, coverage) = drive("merge_next()\ndropped()\nkept()\nmerge_next()\ndropped()\n"); + assert_eq!( + events, + [ + "next after merge: true", + "merge_next()", + "kept()", + "next after merge: false", + "merge_next()", + ] + ); + assert_eq!(coverage.other, 3); +}