From 7f62027aaa42eb997f60c18608e62e4e5e1e58e7 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 02:43:35 -0400 Subject: [PATCH 1/5] Add authenticated transport rules registry Prepare isolated source codecs and shared population, geography, target, calibration, gate and terminal kernels. Route all pinned rules bindings through one simulate.rules_by_ref kernel, validate adapter roots and relations, and bind injected Python adapters to explicit cache configuration identities. Add engine-free property and differential coverage for registry dispatch. Full composer and CLI work remains blocked on the absent AS bridge, gap and bands kernels; no stand-in policy calculations are introduced. Co-Authored-By: GPT-6.1 Sol --- changelog.d/transport-rules-registry.added.md | 4 + .../src/microcosm/build/transport/registry.py | 321 ++++++++++- .../shared/test_transport_registry.py | 544 ++++++++++++++++++ 3 files changed, 867 insertions(+), 2 deletions(-) create mode 100644 changelog.d/transport-rules-registry.added.md create mode 100644 packages/microcosm-build/tests/engine_free/shared/test_transport_registry.py diff --git a/changelog.d/transport-rules-registry.added.md b/changelog.d/transport-rules-registry.added.md new file mode 100644 index 0000000000..12ea85c07d --- /dev/null +++ b/changelog.d/transport-rules-registry.added.md @@ -0,0 +1,4 @@ +Add explicit transport registry preparation with pinned Axiom bindings, one +multi-engine rules kernel, shared calibration and terminal kernels, and the +transport gate bindings. Engine-free adapters can exercise registry dispatch +without installing a country engine. diff --git a/packages/microcosm-build/src/microcosm/build/transport/registry.py b/packages/microcosm-build/src/microcosm/build/transport/registry.py index 3c969e2a74..b1666ea308 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/registry.py +++ b/packages/microcosm-build/src/microcosm/build/transport/registry.py @@ -1,12 +1,329 @@ -"""Explicit registration for the population side of a transport graph.""" +"""Explicit transport registrations and authenticated rules-engine bindings. +Registries are local to a run. All rules bindings share one routing kernel; +adding a binding against the same RuleSpec tree and pins leaves existing +references unchanged. Kernel implementation identity binds adapter classes, +never spec data, so adding a binding of an existing class leaves it unchanged. +The AS bridge and gap kernels are registered by their owning package when +that package is available; this module +does not substitute computations for kernels that have not been implemented. +""" + +from __future__ import annotations + +import hashlib +import re +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from types import MappingProxyType + +from microcosm.build.graph_atomic_geography import register_atomic_geography_kernels +from microcosm.calibrate.ordered_kernels import CALIBRATE_ORDERED_ADAM +from microcosm.frame.adapters.axiom import ( + NZ_SCHEMA, + AxiomEngine, + AxiomPeriod, + assert_no_relations, + axiom_engine_ref, + rulespec_tree_digest, +) +from microcosm.frame.rules import RulesEngine +from microcosm.frame.rules_kernels import SimulateRulesByRefKernel from microcosm.graph import KernelRegistry +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import SourceCodecRegistry +from .codecs import register_transport_codecs from .column_kernels import register_column_kernels +from .gate_bindings import TRANSPORT_GATE_REGISTRY +from .gate_kernels import register_gate_kernel from .geography_kernels import register_geography_kernels from .population_kernels import register_population_kernels +from .target_kernels import register_target_kernels +from .terminal_kernels import register_terminal_kernels + +__all__ = [ + "TransportRegistry", + "build_transport_registry", + "register_transport_population_kernels", +] + +_SHA256 = re.compile(r"[0-9a-f]{64}") +_COMMIT = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})") + + +@dataclass(frozen=True) +class TransportRegistry: + """One run's codecs, kernels, adapters and binding-to-reference lookup. + + ``engines`` is the same mapping bridge kernels must receive when added. + ``engine_refs`` lets composition resolve a spec binding without depending + on the adapter's internal state. Both mappings are immutable snapshots. + """ + + kernels: KernelRegistry + codecs: SourceCodecRegistry + engines: Mapping[str, RulesEngine] + engine_refs: Mapping[str, str] + + +def _mapping(value: object, label: str) -> Mapping: + if not isinstance(value, Mapping): + raise TypeError(f"{label} must be a mapping.") + return value + + +def _text(value: object, label: str) -> str: + if not isinstance(value, str) or not value: + raise ValueError(f"{label} must be a non-empty string.") + return value -__all__ = ["register_transport_population_kernels"] + +def _pin(value: object, label: str, pattern: re.Pattern) -> str: + if not isinstance(value, str) or pattern.fullmatch(value) is None: + raise ValueError(f"{label} is missing or is not a lowercase content pin.") + return value + + +def _binding_rows(document: Mapping) -> tuple[Mapping, ...]: + rows = document.get("bindings") + if isinstance(rows, str | bytes) or not isinstance(rows, Sequence) or not rows: + raise ValueError("axiom_rules_bindings.bindings must be a non-empty list.") + result = tuple(_mapping(row, "rules binding") for row in rows) + ids = [_text(row.get("id"), "rules binding.id") for row in result] + if len(ids) != len(set(ids)): + raise ValueError("axiom_rules_bindings.bindings repeats a binding id.") + return result + + +def _binding_semantics( + row: Mapping, root: Path, periods: Mapping, entity_names: Mapping +) -> dict: + label = f"rules binding {row['id']!r}" + relative = _text(row.get("rulespec_path"), f"{label}.rulespec_path") + path = PurePosixPath(relative) + if path.is_absolute() or ".." in path.parts or str(path) != relative: + raise ValueError(f"{label}.rulespec_path must be a canonical relative path.") + module = (root / relative).resolve() + if not module.is_file() or not module.is_relative_to(root): + raise ValueError(f"{label}.rulespec_path is not a file under rulespec_nz.") + digest = _pin(row.get("sha256"), f"{label}.sha256", _SHA256) + if hashlib.sha256(module.read_bytes()).hexdigest() != digest: + raise ValueError(f"{label}.sha256 does not match rulespec_path {relative!r}.") + entity = _text(row.get("entity"), f"{label}.entity") + if entity not in NZ_SCHEMA.entities: + raise ValueError(f"{label}.entity {entity!r} is absent from the schema.") + engine_entity = _text(row.get("engine_entity"), f"{label}.engine_entity") + if engine_entity != entity_names[entity]: + raise ValueError(f"{label}.engine_entity disagrees with entity_names.") + period = _text(row.get("period"), f"{label}.period") + if period not in periods: + raise ValueError(f"{label}.period {period!r} has no explicit period bounds.") + variables = row.get("variables") + if isinstance(variables, str | bytes) or not isinstance(variables, Sequence): + raise TypeError(f"{label}.variables must be a list of strings.") + variables = tuple(_text(item, f"{label}.variables") for item in variables) + if not variables or len(variables) != len(set(variables)): + raise ValueError(f"{label}.variables must be non-empty and distinct.") + # Metadata prose and declarations do not decide engine outputs. + return { + "id": row["id"], + "rulespec_path": relative, + "sha256": digest, + "entity": entity, + "engine_entity": engine_entity, + "period": period, + "variables": variables, + } + + +def _check_engine( + engine: RulesEngine, + semantics: Mapping, + root: Path, + periods: Mapping[str, AxiomPeriod], + entity_names: Mapping[str, str], +) -> None: + label = f"rules binding {semantics['id']!r}" + if not isinstance(engine, RulesEngine): + raise TypeError(f"{label} does not bind a RulesEngine.") + if engine.entity_schema() != NZ_SCHEMA: + raise ValueError(f"{label} engine schema does not match the transport schema.") + if isinstance(engine, AxiomEngine): + roots = tuple(path.resolve() for path in engine._rulespec_roots) + if roots != (root,): + raise ValueError(f"{label} engine root must equal resolved rulespec_nz.") + if engine._module.resolve() != (root / semantics["rulespec_path"]).resolve(): + raise ValueError(f"{label} engine module does not match rulespec_path.") + if ( + engine._periods != periods + or engine._entity_names != entity_names + or engine._output_dtypes != "graph" + ): + raise ValueError( + f"{label} engine must use declared periods and graph dtypes." + ) + else: + declared_root = getattr(engine, "rulespec_root", None) + if ( + not isinstance(declared_root, str | Path) + or Path(declared_root).resolve() != root + ): + raise ValueError(f"{label} engine root must equal resolved rulespec_nz.") + if not callable(getattr(engine, "assert_no_relations", None)): + raise TypeError( + f"{label} injected engine must expose assert_no_relations(entity)." + ) + if not callable(getattr(engine, "cache_identity", None)): + raise TypeError(f"{label} injected engine must expose cache_identity().") + + +def build_transport_registry( + rules_bindings: Mapping, + rulespec_root: str | Path, + *, + engines_by_binding: Mapping[str, RulesEngine] | None = None, + dependencies: tuple[str, ...] = (), +) -> TransportRegistry: + """Build a run registry with one rules router and the available kernels. + + Real adapters use ``NZ_SCHEMA``, explicit periods, graph output dtypes, + authenticated module bytes and :func:`axiom_engine_ref`. References are + pinned before any relation check compiles a program. + + Engine-free fixtures may inject pure-Python adapters by binding id. Each + must implement ``RulesEngine``, declare ``rulespec_root`` and expose an + ``assert_no_relations(entity)`` check and ``cache_identity()`` mapping. + That mapping must describe every output-affecting adapter configuration + value; the caller is responsible for its completeness and for the adapter + implementing the pinned fixture tree. Output-affecting configuration must + remain fixed after preparation. Their references explicitly name a + Python adapter and bind its declared identity and spec bytes, without + claiming an Axiom execution. Injection must cover exactly the bindings. + + This registers the kernels implemented in this package; it does not imply + readiness of a graph that also needs AS bridge or gap kernels. + """ + document = _mapping(rules_bindings, "axiom_rules_bindings") + root = Path(rulespec_root).resolve() + if not root.is_dir(): + raise ValueError("rulespec_nz must resolve to a directory.") + # A symlink can point at bytes outside the tree the source claims to name. + if any(path.is_symlink() for path in root.rglob("*")): + raise ValueError("rulespec_nz must not contain symbolic links.") + engine_pins = _mapping(document.get("engine"), "axiom_rules_bindings.engine") + rulespec_pins = _mapping(document.get("rulespec"), "axiom_rules_bindings.rulespec") + pins = { + "engine_commit": _pin( + engine_pins.get("commit"), "axiom_rules_bindings.engine.commit", _COMMIT + ), + "wheel_sha256": _pin( + engine_pins.get("wheel_sha256"), + "axiom_rules_bindings.engine.wheel_sha256", + _SHA256, + ), + "rulespec_commit": _pin( + rulespec_pins.get("commit"), "axiom_rules_bindings.rulespec.commit", _COMMIT + ), + } + raw_periods = _mapping(document.get("periods"), "axiom_rules_bindings.periods") + if not raw_periods: + raise ValueError("axiom_rules_bindings.periods must supply explicit bounds.") + periods = {} + for label, bounds in raw_periods.items(): + _text(label, "period label") + values = _mapping(bounds, f"period {label!r}") + if set(values) != {"start", "end", "kind"}: + raise ValueError( + f"period {label!r} must supply exactly start, end and kind." + ) + periods[label] = AxiomPeriod(**values) + entity_names = dict( + _mapping(document.get("entity_names"), "axiom_rules_bindings.entity_names") + ) + if set(entity_names) != set(NZ_SCHEMA.entities): + raise ValueError( + "axiom_rules_bindings.entity_names must cover the transport schema." + ) + for entity, name in entity_names.items(): + _text(name, f"entity_names.{entity}") + if len(set(entity_names.values())) != len(entity_names): + raise ValueError("axiom_rules_bindings.entity_names must be distinct.") + rows = _binding_rows(document) + if engines_by_binding is not None: + injected = _mapping(engines_by_binding, "engines_by_binding") + if set(injected) != {row["id"] for row in rows}: + raise ValueError( + "engines_by_binding must cover exactly the declared binding ids." + ) + engines = {} + refs = {} + tree_sha256, tree_bytes = rulespec_tree_digest(root) + for row in rows: + semantics = _binding_semantics(row, root, periods, entity_names) + binding_periods = {semantics["period"]: periods[semantics["period"]]} + engine = ( + AxiomEngine( + root / semantics["rulespec_path"], + schema=NZ_SCHEMA, + rulespec_roots=(root,), + periods=binding_periods, + entity_names=entity_names, + output_dtypes="graph", + ) + if engines_by_binding is None + else engines_by_binding[row["id"]] + ) + _check_engine(engine, semantics, root, binding_periods, entity_names) + if isinstance(engine, AxiomEngine): + reference = axiom_engine_ref(engine, rulespec_root=root, **pins) + assert_no_relations(engine, semantics["entity"]) + else: + identity = _mapping( + engine.cache_identity(), + f"rules binding {row['id']!r} adapter cache identity", + ) + reference = canonical_json( + { + "format": "microcosm.transport.python-rules-engine-ref/1", + "engine": "python-rules-engine", + "adapter": f"{type(engine).__module__}.{type(engine).__qualname__}", + "adapter_identity": identity, + "binding": semantics, + "pins": pins, + "rulespec_tree_sha256": tree_sha256, + "rulespec_tree_bytes": tree_bytes, + "entity_names": entity_names, + "periods": {semantics["period"]: raw_periods[semantics["period"]]}, + } + ).decode("utf-8") + engine.assert_no_relations(semantics["entity"]) + for variable in semantics["variables"]: + if engine.variable_metadata(variable).entity != semantics["entity"]: + raise ValueError( + f"rules binding {row['id']!r} variable {variable!r} belongs to another entity." + ) + if isinstance(engine, AxiomEngine): + engine.graph_dtype(variable) + if reference in engines: + raise ValueError( + f"rules binding {row['id']!r} repeats an engine reference." + ) + refs[row["id"]] = reference + engines[reference] = engine + engine_mapping = MappingProxyType(dict(engines)) + kernels = KernelRegistry() + codecs = SourceCodecRegistry() + register_transport_codecs(codecs) + register_transport_population_kernels(kernels) + register_atomic_geography_kernels(kernels) + register_target_kernels(kernels) + kernels.register(CALIBRATE_ORDERED_ADAM) + register_gate_kernel(kernels, TRANSPORT_GATE_REGISTRY) + register_terminal_kernels(kernels, codecs=codecs) + kernels.register(SimulateRulesByRefKernel(engine_mapping, dependencies)) + return TransportRegistry(kernels, codecs, engine_mapping, MappingProxyType(refs)) def register_transport_population_kernels(registry: KernelRegistry) -> KernelRegistry: diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_registry.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_registry.py new file mode 100644 index 0000000000..40a89b8952 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_registry.py @@ -0,0 +1,544 @@ +"""Authenticated transport bindings with three engine-free adapters in one run.""" + +from __future__ import annotations + +import copy +import hashlib +import json +from collections.abc import Mapping, Sequence +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +import microcosm.build.transport.registry as registry_module +from microcosm.build.transport.codecs import ( + DONOR_SOURCE_CODEC, + FACTS_SOURCE_CODEC, + RULESPEC_SOURCE_CODEC, +) +from microcosm.build.transport.gate_bindings import TRANSPORT_GATE_REGISTRY +from microcosm.build.transport.registry import build_transport_registry +from microcosm.build.transport.terminal_kernels import EXPORT_SOURCE_CODEC +from microcosm.frame import ( + EntitySchema, + ExportContract, + Frame, + VariableMetadata, + WeightKind, + Weights, +) +from microcosm.frame.adapters.axiom import NZ_SCHEMA, AxiomEngine, AxiomPeriod +from microcosm.frame.kernels import SimulateRulesKernel +from microcosm.frame.rules_kernels import SimulateRulesByRefKernel +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + Graph, + KernelBase, + KernelContext, + KernelRegistry, + KernelResult, + Node, + Numeric, + Owned, + SeedSource, + Slice, + SourceRef, + StructuralDelta, + compile_graph, + run_graph, +) +from microcosm.graph.codecs import load_raw_bytes + + +class _ToyEngine: + """Pure adapter whose formula and configuration have explicit identities.""" + + def __init__(self, root: Path, binding: Mapping, *, offset: float = 0.0): + self.rulespec_root = root + self.binding = binding + self.offset = offset + self.relation_checks = [] + + def cache_identity(self) -> Mapping: + return {"offset": self.offset} + + def assert_no_relations(self, entity: str) -> None: + self.relation_checks.append(entity) + if self._program()["relations"]: + raise NotImplementedError(f"{self.binding['id']} declares relations") + + def _program(self) -> Mapping: + return json.loads( + (self.rulespec_root / self.binding["rulespec_path"]).read_text() + ) + + def variable_metadata(self, name: str) -> VariableMetadata: + if name not in self.binding["variables"]: + raise ValueError(f"Unknown toy output {name!r}") + return VariableMetadata(name, self.binding["entity"], "float", "year") + + def variables(self) -> Sequence[str]: + return (self._program()["input"],) + + def entity_schema(self) -> EntitySchema: + return NZ_SCHEMA + + def materialize( + self, bundle: Frame, variables: Sequence[str], period: int | str + ) -> Mapping[str, np.ndarray]: + program = self._program() + values = bundle.table(self.binding["entity"])[program["input"]].to_numpy( + dtype=np.float64 + ) + return { + name: values * program["coefficient"] + self.offset for name in variables + } + + def export_contract(self) -> ExportContract: + return ExportContract.empty() + + def write_dataset(self, bundle: Frame, path: str | Path, period: int | str): + raise NotImplementedError("The toy fixture does not export datasets") + + +def _fixture(tmp_path: Path): + root = tmp_path / "rulespec" + root.mkdir(exist_ok=True) + rows = [] + for name, entity, column, coefficient in ( + ("main_benefit", "person", "age", 2.0), + ("pension", "person", "age", 3.0), + ("housing", "family", "cost", 4.0), + ): + payload = json.dumps( + {"input": column, "coefficient": coefficient, "relations": []}, + sort_keys=True, + ).encode() + module = root / f"{name}.yaml" + module.write_bytes(payload) + rows.append( + { + "id": name, + "rulespec_path": module.name, + "sha256": hashlib.sha256(payload).hexdigest(), + "entity": entity, + "engine_entity": entity.capitalize(), + "period": "2026-27", + "variables": [f"toy_{name}"], + } + ) + document = { + "engine": {"commit": "a" * 40, "wheel_sha256": "b" * 64}, + "rulespec": {"commit": "c" * 40}, + "entity_names": { + "person": "Person", + "household": "Household", + "family": "Family", + }, + "periods": { + "2026-27": { + "kind": "tax_year", + "start": "2026-04-01", + "end": "2027-03-31", + } + }, + "bindings": rows, + } + engines = {row["id"]: _ToyEngine(root, row) for row in rows} + return root, document, engines + + +def test_registers_available_kernels_and_codecs(tmp_path): + root, document, engines = _fixture(tmp_path) + result = build_transport_registry(document, root, engines_by_binding=engines) + assert isinstance( + result.kernels.get("simulate.rules_by_ref@1"), SimulateRulesByRefKernel + ) + assert "simulate.rules@1" not in result.kernels.refs() + assert { + "transport.create@1", + "transport.boundary@1", + "geography.assign_atomic@1", + "targets.compile@1", + "targets.problem@1", + "calibrate.ordered_adam@1", + "diagnostics.calibration@1", + "export.prepare@1", + "export.readback@1", + "transport.package@1", + } <= set(result.kernels.refs()) + gate = result.kernels.get("gates.battery@1") + assert all( + gate.registry[name] is binding + for name, binding in TRANSPORT_GATE_REGISTRY.items() + ) + for codec in ( + DONOR_SOURCE_CODEC, + FACTS_SOURCE_CODEC, + RULESPEC_SOURCE_CODEC, + EXPORT_SOURCE_CODEC, + ): + assert callable(result.codecs.get(codec)) + assert all( + engine.relation_checks == [engine.binding["entity"]] + for engine in engines.values() + ) + assert len(result.engines) == 3 + assert all( + json.loads(ref)["engine"] == "python-rules-engine" for ref in result.engines + ) + with pytest.raises(TypeError): + result.engine_refs["other"] = "unused" + with pytest.raises(TypeError): + result.engines["other"] = engines["housing"] + + +@settings( + max_examples=12, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given(st.integers(min_value=-20, max_value=20)) +def test_added_binding_and_prose_leave_existing_refs_and_hashes_unchanged( + tmp_path, offset +): + root, document, engines = _fixture(tmp_path) + engines["main_benefit"].offset = float(offset) + first_document = copy.deepcopy(document) + first_document["bindings"] = first_document["bindings"][:2] + first = build_transport_registry( + first_document, + root, + engines_by_binding={ + name: engines[name] for name in ("main_benefit", "pension") + }, + ) + document["description"] = "Different explanatory prose" + document["bindings"][0]["notes"] = "This is not an engine configuration" + document["periods"]["unused_period"] = { + "kind": "year", + "start": "2028-01-01", + "end": "2028-12-31", + } + second = build_transport_registry(document, root, engines_by_binding=engines) + assert all( + second.engine_refs[name] == ref for name, ref in first.engine_refs.items() + ) + assert first.kernels.implementation_hash( + "simulate.rules_by_ref@1" + ) == second.kernels.implementation_hash("simulate.rules_by_ref@1") + + +@settings( + max_examples=12, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given(st.integers(min_value=1, max_value=30)) +def test_adapter_configuration_changes_only_its_reference(tmp_path, offset): + root, document, engines = _fixture(tmp_path) + first = build_transport_registry(document, root, engines_by_binding=engines) + engines["housing"].offset = float(offset) + second = build_transport_registry(document, root, engines_by_binding=engines) + assert first.engine_refs["housing"] != second.engine_refs["housing"] + assert first.engine_refs["main_benefit"] == second.engine_refs["main_benefit"] + assert first.engine_refs["pension"] == second.engine_refs["pension"] + assert first.kernels.implementation_hash( + "simulate.rules_by_ref@1" + ) == second.kernels.implementation_hash("simulate.rules_by_ref@1") + + +@settings( + max_examples=8, + deadline=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +@given(st.binary(min_size=1, max_size=8)) +def test_rulespec_tree_bytes_change_refs_without_changing_kernel_hash(tmp_path, edit): + root, document, engines = _fixture(tmp_path) + shared = root / "shared-resource.txt" + shared.write_bytes(b"before") + first = build_transport_registry(document, root, engines_by_binding=engines) + shared.write_bytes(b"after" + edit) + second = build_transport_registry(document, root, engines_by_binding=engines) + assert all(first.engine_refs[name] != second.engine_refs[name] for name in engines) + assert first.kernels.implementation_hash( + "simulate.rules_by_ref@1" + ) == second.kernels.implementation_hash("simulate.rules_by_ref@1") + + +def test_native_engines_use_pinned_refs_before_relation_compilation( + tmp_path, monkeypatch +): + root, document, _ = _fixture(tmp_path) + document["periods"]["unused_period"] = { + "kind": "year", + "start": "2028-01-01", + "end": "2028-12-31", + } + checks = [] + + def check(engine, entity): + assert engine._pinned_tree_sha256 is not None + assert engine._schema == NZ_SCHEMA + assert engine._rulespec_roots == (root.resolve(),) + assert engine._output_dtypes == "graph" + assert engine._periods == { + "2026-27": AxiomPeriod("2026-04-01", "2027-03-31", "tax_year") + } + checks.append((engine._module.name, entity)) + + def metadata(engine, name): + row = next(row for row in document["bindings"] if name in row["variables"]) + return VariableMetadata(name, row["entity"], "float", "year") + + monkeypatch.setattr(registry_module, "assert_no_relations", check) + monkeypatch.setattr(AxiomEngine, "variable_metadata", metadata) + monkeypatch.setattr(AxiomEngine, "graph_dtype", lambda engine, name: "float64") + result = build_transport_registry(document, root) + assert len(checks) == 3 + assert all(isinstance(engine, AxiomEngine) for engine in result.engines.values()) + for row in document["bindings"]: + ref = json.loads(result.engine_refs[row["id"]]) + assert ref["engine"] == "axiom-rules-engine" + assert ref["module_sha256"] == row["sha256"] + assert ref["module"] == row["rulespec_path"] + assert ref["periods"] == {row["period"]: document["periods"][row["period"]]} + + +@pytest.mark.parametrize( + "problem", ["root", "module", "schema", "periods", "names", "dtypes"] +) +def test_injected_axiom_configuration_must_match_the_binding(tmp_path, problem): + root, document, _ = _fixture(tmp_path) + periods = {"2026-27": AxiomPeriod("2026-04-01", "2027-03-31", "tax_year")} + engines = { + row["id"]: AxiomEngine( + root / row["rulespec_path"], + schema=NZ_SCHEMA, + rulespec_roots=(root,), + periods=periods, + entity_names=document["entity_names"], + output_dtypes="graph", + ) + for row in document["bindings"] + } + engine = engines["main_benefit"] + if problem == "root": + engine._rulespec_roots = (tmp_path,) + elif problem == "module": + engine._module = root / "pension.yaml" + elif problem == "schema": + engine._schema = EntitySchema(group_entities=("household",)) + elif problem == "periods": + engine._periods = None + elif problem == "names": + engine._entity_names = {"person": "DifferentPerson"} + else: + engine._output_dtypes = "native" + with pytest.raises(ValueError, match="rules binding 'main_benefit'"): + build_transport_registry(document, root, engines_by_binding=engines) + + +@pytest.mark.parametrize( + ("section", "field"), + [("engine", "commit"), ("engine", "wheel_sha256"), ("rulespec", "commit")], +) +def test_missing_pin_names_the_activation_gap(tmp_path, section, field): + root, document, engines = _fixture(tmp_path) + document[section][field] = None + with pytest.raises(ValueError, match=rf"axiom_rules_bindings\.{section}\.{field}"): + build_transport_registry(document, root, engines_by_binding=engines) + + +@pytest.mark.parametrize("problem", ["hash", "path", "period", "duplicate", "entity"]) +def test_malformed_binding_is_refused(tmp_path, problem): + root, document, engines = _fixture(tmp_path) + first = document["bindings"][0] + if problem == "hash": + first["sha256"] = "d" * 64 + elif problem == "path": + first["rulespec_path"] = "../outside.yaml" + elif problem == "period": + first["period"] = "unmapped" + elif problem == "duplicate": + document["bindings"].append(dict(first)) + else: + first["engine_entity"] = "Family" + with pytest.raises(ValueError): + build_transport_registry(document, root, engines_by_binding=engines) + + +@pytest.mark.parametrize("problem", ["root", "schema", "relations", "identity", "keys"]) +def test_injected_adapter_contract_is_checked(tmp_path, problem): + root, document, engines = _fixture(tmp_path) + engine = engines["housing"] + if problem == "root": + engine.rulespec_root = tmp_path + elif problem == "schema": + engine.entity_schema = lambda: EntitySchema(group_entities=("household",)) + elif problem == "relations": + engine.assert_no_relations = lambda entity: (_ for _ in ()).throw( + NotImplementedError("relations") + ) + elif problem == "identity": + engine.cache_identity = None + else: + engines.pop("housing") + with pytest.raises((TypeError, ValueError, NotImplementedError)): + build_transport_registry(document, root, engines_by_binding=engines) + + +@pytest.mark.parametrize("value", [float("nan"), {1: "key"}, lambda: None]) +def test_adapter_identity_uses_closed_canonical_data(tmp_path, value): + root, document, engines = _fixture(tmp_path) + engines["housing"].cache_identity = lambda: {"configuration": value} + with pytest.raises((TypeError, ValueError)): + build_transport_registry(document, root, engines_by_binding=engines) + + +def _frame(document: Mapping) -> Frame: + return Frame( + { + "person": pd.DataFrame(document["person"]).astype("int64"), + "household": pd.DataFrame(document["household"]).astype("int64"), + "family": pd.DataFrame(document["family"]).astype( + {"family_id": "int64", "cost": "float64"} + ), + }, + NZ_SCHEMA, + { + "household": Weights( + np.asarray(document["weights"], dtype=np.float64), WeightKind.DESIGN + ) + }, + ) + + +class _Population(KernelBase): + ref = "test.registry_population@1" + capabilities = Capabilities( + determinism=Determinism.DETERMINISTIC, + numeric=Numeric.BITWISE, + seed_source=SeedSource.NONE, + structural=StructuralDelta.CREATE, + ) + + def run(self, context: KernelContext) -> KernelResult: + return KernelResult( + frame=_frame(json.loads(context.sources["population"].read_text())) + ) + + +def _node( + row: Mapping, reference: str, kernel: str = SimulateRulesByRefKernel.ref +) -> Node: + inputs = (Slice("person", ("age",)),) + if row["entity"] == "family": + inputs += (Slice("family", ("cost",)),) + return Node( + f"toy.rules.{row['id']}", + kernel, + sources=("rulespec_nz",), + inputs=inputs, + outputs=tuple( + Owned(row["entity"], name, "float64") for name in row["variables"] + ), + params={ + "engine_ref": reference, + "period": row["period"], + "variables": tuple(row["variables"]), + }, + ) + + +def test_three_engines_run_together_and_equal_single_engine_dispatch(tmp_path): + root, document, engines = _fixture(tmp_path) + result = build_transport_registry(document, root, engines_by_binding=engines) + result.codecs.register_bytes("raw-bytes-v1", load_raw_bytes) + result.kernels.register(_Population()) + population_path = tmp_path / "population.json" + population_path.write_text( + json.dumps( + { + "person": { + "person_id": [1, 2, 3], + "person_household_id": [10, 10, 20], + "person_family_id": [100, 100, 200], + "age": [20, 5, 70], + }, + "household": {"household_id": [10, 20]}, + "family": {"family_id": [100, 200], "cost": [12.5, 32.0]}, + "weights": [1.5, 2.5], + } + ) + ) + population = Node( + "toy.create", + _Population.ref, + sources=("population",), + structural=StructuralDelta.CREATE, + outputs=(Owned("person", "age", "int64"), Owned("family", "cost", "float64")), + ) + sources = ( + SourceRef("population", "raw-bytes-v1"), + SourceRef("rulespec_nz", RULESPEC_SOURCE_CODEC), + ) + nodes = tuple( + _node(row, result.engine_refs[row["id"]]) for row in document["bindings"] + ) + graph = Graph("toy.registry", sources, (population, *nodes)) + store = ContentStore(tmp_path / "combined", codecs=result.codecs) + manifest = run_graph( + compile_graph(graph), + sources={"population": population_path, "rulespec_nz": root}, + store=store, + kernels=result.kernels, + resume="forbid", + ) + assert len(manifest.nodes) == 4 + assert all(not node.hit for node in manifest.nodes.values()) + warm = run_graph( + compile_graph(graph), + sources={"population": population_path, "rulespec_nz": root}, + store=store, + kernels=result.kernels, + resume="require", + ) + assert all(node.hit for node in warm.nodes.values()) + for row in document["bindings"]: + ref = result.engine_refs[row["id"]] + registry = KernelRegistry() + registry.register(_Population()) + registry.register(SimulateRulesKernel(ref, engines[row["id"]])) + direct_node = _node(row, ref, SimulateRulesKernel.ref) + direct_graph = Graph("toy.registry.direct", sources, (population, direct_node)) + direct_store = ContentStore( + tmp_path / f"direct-{row['id']}", codecs=result.codecs + ) + direct = run_graph( + compile_graph(direct_graph), + sources={"population": population_path, "rulespec_nz": root}, + store=direct_store, + kernels=registry, + resume="forbid", + ) + for variable in row["variables"]: + coordinate = (row["entity"], variable) + combined = store.load_column( + manifest.nodes[direct_node.id].artifacts[coordinate] + ) + single = direct_store.load_column( + direct.nodes[direct_node.id].artifacts[coordinate] + ) + assert combined.to_numpy().tobytes() == single.to_numpy().tobytes() + assert combined.dtype == single.dtype == np.dtype("float64") + assert dict(manifest.nodes[direct_node.id].receipt) == dict( + direct.nodes[direct_node.id].receipt + ) From a0adb354bba002a7ce5e66132cb5b51c096c57cb Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 05:30:27 -0400 Subject: [PATCH 2/5] Add the transport skeleton composer, local build driver and shim - compose.py: compose_transport_graph(spec, config) builds the skeleton a country's transport_graph.json spec resource declares. A node parameter that is an object must be an explicit resource selection (value, json or the resource's SHA-256) or the prepared engine-reference map, so a node binds only the resources it selects; no whole-spec fingerprint enters any node. Rules nodes name a binding and get its authenticated engine_ref and period, all through the one simulate.rules_by_ref@1 router. Kernels outside the skeleton vocabulary enter only through TransportGraphConfig.extensions, the typed extension point for the entitlement package; composition refuses an extension that changes any skeleton node's predecessors. validate_transport_activation refuses unresolved evidence with every item named (the benefit-unit age limit, placeholder references, the receipt contract, engine pins, the S4 rent factor, a missing declaration or selected resource). - cli.py: prepare (registry; CREATE's column inventory, read by running transport.create@1 once on the declared params), compile, then run ancestor-closed checkpoints on /.graph-store: geography, calibration, export preparation (then the skeleton H5), each terminal endpoint, the whole graph. Writes graph.json, manifest.json, checkpoint manifests, artifacts, the H5 and descriptor, calibration diagnostics, explorer.html and build-status.json (with the outputs still owed) under --out only. --resume require refuses a cold store before creating anything; an --out that overlaps an input is refused. No upload or staging path. - registry.py: build the engines' schema and unit nesting from the benefit-unit rule (the schema CREATE builds) instead of naming a country schema, check injected and Axiom adapters against both, and drop country words from messages, so the transport source rules pass. - tools/build_transport.py: the six-line shim. - Tests on an engine-free toy package (synthetic donor, facts for both roles, toy RuleSpec tree, three pure-Python engines in one run_graph): purity; a Hypothesis resource-locality property (one resource edit re-keys exactly its selectors and their descendants); A1, A2, A4, A5; A3 for a calibration reference; C2 for a gates.json threshold and a hold-out reference; hold-out ancestry; B1; extension isolation; E3; CLI outputs, warm resume, H5 rewrite and corruption refusal; differentials (router vs stock simulate.rules@1, prepared vs independent CREATE inventory, registry vs CREATE schema, declared vs executed keys); the packaged spec's named refusals; no policy literal or remote import. Invariants: the graph is a pure function of the spec resources and the prepared config; editing one resource re-keys exactly the nodes that select it and their descendants; the extension point never re-keys a skeleton node; no calibration ancestor reads the hold-out facts or a family-engine column; every rules node runs through the one router; every output lies under --out. Co-Authored-By: Claude Opus 5.5 --- changelog.d/transport-compose-cli.added.md | 8 + changelog.d/transport-rules-registry.added.md | 6 +- docs/agent-guide.md | 9 + .../src/microcosm/build/transport/cli.py | 487 +++++++ .../src/microcosm/build/transport/compose.py | 770 +++++++++++ .../src/microcosm/build/transport/registry.py | 78 +- .../shared/test_transport_composed_graph.py | 900 +++++++++++++ .../shared/test_transport_registry.py | 58 +- .../microcosm_build/transport_composed.py | 1166 +++++++++++++++++ tools/build_transport.py | 6 + 10 files changed, 3454 insertions(+), 34 deletions(-) create mode 100644 changelog.d/transport-compose-cli.added.md create mode 100644 packages/microcosm-build/src/microcosm/build/transport/cli.py create mode 100644 packages/microcosm-build/src/microcosm/build/transport/compose.py create mode 100644 packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py create mode 100644 test_support/microcosm_build/transport_composed.py create mode 100644 tools/build_transport.py diff --git a/changelog.d/transport-compose-cli.added.md b/changelog.d/transport-compose-cli.added.md new file mode 100644 index 0000000000..23fad8d752 --- /dev/null +++ b/changelog.d/transport-compose-cli.added.md @@ -0,0 +1,8 @@ +Add the transport skeleton composer, its local-only build driver and +`tools/build_transport.py`. A country's `transport_graph.json` spec resource +declares the skeleton graph; each node binds only the spec resources it +selects, every rules node runs through the one `simulate.rules_by_ref@1` +router, and later packages append branches through a typed extension point +that cannot re-key the skeleton. The driver runs ancestor-closed checkpoints +on `/.graph-store` and writes every output under `--out`. Unresolved +country evidence is refused with each missing item named. diff --git a/changelog.d/transport-rules-registry.added.md b/changelog.d/transport-rules-registry.added.md index 12ea85c07d..746f6d42ac 100644 --- a/changelog.d/transport-rules-registry.added.md +++ b/changelog.d/transport-rules-registry.added.md @@ -1,4 +1,6 @@ Add explicit transport registry preparation with pinned Axiom bindings, one multi-engine rules kernel, shared calibration and terminal kernels, and the -transport gate bindings. Engine-free adapters can exercise registry dispatch -without installing a country engine. +transport gate bindings. Engines use the persons-households-benefit-units +schema and nesting that the benefit-unit rule declares (the schema CREATE +builds). Engine-free adapters can exercise registry dispatch without +installing a country engine. diff --git a/docs/agent-guide.md b/docs/agent-guide.md index ca981de83d..5bbce3feef 100644 --- a/docs/agent-guide.md +++ b/docs/agent-guide.md @@ -339,6 +339,15 @@ Update this guide in the same PR whenever the workspace layout, test commands, or release flow change. If you find it contradicting the repo, trust the repo and fix this file. +The transport build driver (`tools/build_transport.py`, a shim over +`microcosm.build.transport.cli`) composes a country's graph from its +`transport_graph.json` spec resource and runs it locally: every file it +writes, including its `.graph-store`, lies under `--out`, and it has no +upload or staging path. It refuses before reading any graph source while the +country spec still has unresolved evidence, and names each missing item. +Tests for it run on the engine-free toy package in +`test_support/microcosm_build/transport_composed.py`. + UK size experiments use `microcosm-build-uk --release-role dense --dataset-households` (`tools/build_uk_full.py`; `tools/build_uk_rowwise_candidate.py` is a stub over it) with the same pool inputs as the dense candidate. The flag changes exported diff --git a/packages/microcosm-build/src/microcosm/build/transport/cli.py b/packages/microcosm-build/src/microcosm/build/transport/cli.py new file mode 100644 index 0000000000..c35d861f41 --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/transport/cli.py @@ -0,0 +1,487 @@ +"""Prepare, compile and run a transport skeleton graph, writing local files only. + +``main`` loads a country spec, prepares the run (rules-engine registry, +CREATE's column inventory), composes and compiles the one graph, then runs +ancestor-closed checkpoints of it on ``/.graph-store``: through the +geography endpoint, through the calibration endpoint, through export +preparation (after which the skeleton H5 is written outside the graph), +through each terminal endpoint, and finally the whole graph. Every run shares +the store, so a later checkpoint reuses an earlier one's work. + +Everything is written under ``--out``: ``graph.json``, ``manifest.json``, a +manifest per checkpoint, each opaque artifact and an index of them, the +skeleton H5 and its descriptor, the calibration diagnostics file, the graph +explorer page and ``build-status.json``. The status file lists the outputs +the spec declares as still owed by a later package. There is no upload, +staging or publication path; ``--local-only`` is the only mode. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, replace +from pathlib import Path +from types import MappingProxyType + +import numpy as np +import pandas as pd + +from microcosm.build.artifact_files import file_artifact, materialize_bytes +from microcosm.frame.rules import RulesEngine +from microcosm.frame.unit_construction import BenefitUnitRule +from microcosm.graph import ( + ContentStore, + Graph, + GraphRuntimeError, + KernelContext, + Node, + Owned, + ResumePolicy, + RunManifest, + StoreMissError, + compile_graph, + explain_html, + graph_to_json, + run_graph, +) +from microcosm.graph.canonical import canonical_json + +from .codecs import RULESPEC_SOURCE_CODEC +from .compose import ( + TransportGraphConfig, + TransportGraphExtension, + compose_transport_graph, + load_transport_spec, + skeleton_create_declaration, + transport_endpoints, + transport_pending_outputs, + transport_resource, + validate_transport_activation, +) +from .population_kernels import TRANSPORT_CREATE +from .registry import TransportRegistry, build_transport_registry +from .terminal_kernels import materialize_export, validate_export + +__all__ = [ + "EXPORT_FILENAME", + "PreparedTransportBuild", + "execute_transport_graph", + "main", + "parse_args", + "prepare_create_outputs", + "prepare_transport_build", + "through", +] + +#: The skeleton's exported dataset, written under ``--out``. +EXPORT_FILENAME = "skeleton.h5" +_STORE = ".graph-store" +_CONTENT_BASIS = ( + "Transported donor support records reweighted to destination facts; " + "not observed destination microdata." +) + + +@dataclass(frozen=True) +class PreparedTransportBuild: + """A compiled-ready graph, its run registry and its local inputs.""" + + graph: Graph + registry: TransportRegistry + sources: Mapping[str, Path] + endpoints: Mapping[str, object] + pending_outputs: tuple[str, ...] + + +def prepare_create_outputs( + spec: Mapping[str, object], sources: Mapping[str, str | Path] +) -> tuple[Owned, ...]: + """CREATE's column inventory, read by running its kernel once. + + The executor requires CREATE to declare every column it loads, and the + columns depend on the pinned donor file. This runs ``transport.create@1`` + on the declared parameters and sources (the kernel itself checks the + donor's size and SHA-256) and declares every non-structural column of + the frame it returns, with its dtype. The frame is then discarded; the + graph's own CREATE recomputes it under the executor. + """ + + declaration = skeleton_create_declaration(spec) + missing = sorted(set(declaration.sources) - set(sources)) + if missing: + raise ValueError(f"CREATE needs local paths for sources {missing}.") + # A probe node: CREATE reads only its params and declared sources. It + # declares no outputs, so it is never part of a graph. + probe = Node( + declaration.id, + declaration.kernel, + params=declaration.params, + sources=declaration.sources, + ) + context = KernelContext( + node=probe, + tables={}, + weights={}, + strata=pd.Series(dtype="float64"), + params=probe.params, + rng=np.random.default_rng(0), + sources={name: Path(sources[name]).resolve() for name in probe.sources}, + ) + frame = TRANSPORT_CREATE.run(context).frame + schema = frame.schema + structural = { + schema.person_entity: { + schema.person_id_column, + *(schema.membership_column(group) for group in schema.group_entities), + }, + **{group: {schema.id_column(group)} for group in schema.group_entities}, + } + return tuple( + Owned(entity, column, str(table[column].dtype)) + for entity in schema.entities + for table in (frame.table(entity),) + for column in table.columns + if column not in structural[entity] + ) + + +def _rules_source(spec: Mapping[str, object]) -> str: + rows = transport_resource(spec, "transport_graph").get("sources", ()) + names = [ + row["name"] + for row in rows + if isinstance(row, Mapping) and row.get("codec") == RULESPEC_SOURCE_CODEC + ] + if len(names) != 1: + raise ValueError("The skeleton must declare exactly one RuleSpec tree source.") + return names[0] + + +def prepare_transport_build( + spec: Mapping[str, object], + *, + sources: Mapping[str, str | Path], + engines_by_binding: Mapping[str, RulesEngine] | None = None, + extensions: tuple[TransportGraphExtension, ...] = (), +) -> PreparedTransportBuild: + """Check activation, build the registry and CREATE inventory, then compose. + + Activation is checked before any engine is built or donor read. + ``sources`` maps every declared source name except the exported dataset + (which the run writes) to a local path. ``engines_by_binding`` is the + registry's seam for pure-Python adapters; without it, Axiom adapters are + built for every binding. The registry's schema comes from the same + benefit-unit rule CREATE is given. + """ + + validate_transport_activation(spec) + paths = {name: Path(path).resolve() for name, path in sources.items()} + rules_source = _rules_source(spec) + if rules_source not in paths: + raise ValueError(f"Source {rules_source!r} (the RuleSpec tree) needs a path.") + create = skeleton_create_declaration(spec) + rule = BenefitUnitRule.from_dict(json.loads(create.params["unit_rule"])) + registry = build_transport_registry( + transport_resource(spec, "axiom_rules_bindings"), + paths[rules_source], + unit_rule=rule, + engines_by_binding=engines_by_binding, + ) + graph = compose_transport_graph( + spec, + TransportGraphConfig( + engine_refs=registry.engine_refs, + create_outputs=prepare_create_outputs(spec, paths), + extensions=extensions, + ), + ) + compile_graph(graph) + declared = {source.name for source in graph.sources} + unknown = sorted(set(paths) - declared) + if unknown: + raise ValueError(f"Source paths were supplied for undeclared names {unknown}.") + return PreparedTransportBuild( + graph, + registry, + MappingProxyType(paths), + MappingProxyType(transport_endpoints(spec)), + transport_pending_outputs(spec), + ) + + +def through(graph: Graph, endpoint: str) -> Graph: + """The ancestor-closed subgraph of ``graph`` that ends at ``endpoint``.""" + + compiled = compile_graph(graph) + if endpoint not in compiled.predecessors: + raise ValueError(f"Unknown transport endpoint {endpoint!r}.") + needed, pending = {endpoint}, [endpoint] + while pending: + for parent in compiled.predecessors[pending.pop()]: + if parent not in needed: + needed.add(parent) + pending.append(parent) + return replace( + graph, nodes=tuple(node for node in graph.nodes if node.id in needed) + ) + + +def _destination(out: Path, filename: str) -> Path: + """A file directly under ``out``; refuse names or links that leave it.""" + + if Path(filename).name != filename or filename in {"", ".", ".."}: + raise ValueError(f"Output name {filename!r} must be a simple filename.") + path = out / filename + if path.is_symlink() or not path.resolve().is_relative_to(out): + raise ValueError(f"Output {filename!r} would be written outside --out.") + return path + + +def _check_disjoint(out: Path, sources: Mapping[str, Path]) -> None: + """Refuse an output directory that overlaps an input. + + Writing inside an input tree would change the bytes a source names (a + RuleSpec directory is hashed whole), and an input inside ``--out`` could + be overwritten by an output. + """ + + for name, path in sources.items(): + if path.is_relative_to(out) or out.is_relative_to(path): + raise ValueError(f"--out overlaps the input of source {name!r}: {path}.") + + +def _store_is_cold(root: Path) -> bool: + objects = root / "objects" + return not objects.is_dir() or not any(objects.iterdir()) + + +def _evidence(manifest: RunManifest, store: ContentStore, out: Path) -> dict: + inventory = {} + for node_id, receipt in sorted(manifest.nodes.items()): + for name, key in sorted(receipt.opaque_artifacts.items()): + payload = store.load_bytes(key) + suffix = ".json" if payload.startswith((b"{", b"[")) else ".artifact" + written = materialize_bytes( + payload, _destination(out, f"{node_id}.{name}{suffix}") + ) + inventory[f"{node_id}/{name}"] = {"key": key, **written} + materialize_bytes( + canonical_json(inventory), _destination(out, "evidence-index.json") + ) + return inventory + + +def _run(graph: Graph, registry, sources, store, resume) -> RunManifest: + used = {name for node in graph.nodes for name in node.sources} + return run_graph( + compile_graph(graph), + sources={name: path for name, path in sources.items() if name in used}, + store=store, + kernels=registry.kernels, + resume=resume, + ) + + +def execute_transport_graph( + graph: Graph, + registry: TransportRegistry, + sources: Mapping[str, str | Path], + *, + out: str | Path, + endpoints: Mapping[str, object], + pending_outputs: Sequence[str] = (), + resume: ResumePolicy = "auto", +) -> RunManifest: + """Run the graph's checkpoints in order and write the local outputs. + + ``require`` refuses a cold store before anything is created, and every + checkpoint then runs with ``require``, so a partly warm store is refused + at its first missing node. ``forbid`` recomputes the first checkpoint; + later checkpoints reuse this invocation's work. The skeleton H5 is + rewritten only when the export descriptor changed; an unchanged H5 must + pass readback, and under ``require`` it must already exist. + """ + + if resume not in {"auto", "require", "forbid"}: + raise ValueError("resume must be 'auto', 'require' or 'forbid'.") + compiled = compile_graph(graph) + output = Path(out).resolve() + supplied = {name: Path(path).resolve() for name, path in sources.items()} + _check_disjoint(output, supplied) + prepare = [node for node in graph.nodes if node.kernel == "export.prepare@1"] + readback = [node for node in graph.nodes if node.kernel == "export.readback@1"] + if len(prepare) != 1 or len(readback) != 1 or len(readback[0].sources) != 1: + raise ValueError( + "A transport skeleton has one export.prepare@1 and one export.readback@1 " + "reading one dataset source." + ) + prepare, export_source = prepare[0], readback[0].sources[0] + if export_source in supplied: + raise ValueError( + f"Source {export_source!r} is written by the run, not supplied." + ) + checkpoints = [ + ("geography", endpoints["geography"]), + ("calibration", endpoints["calibration"]), + ] + terminals = [("terminal", endpoint) for endpoint in endpoints["terminal"]] + for _, endpoint in (*checkpoints, *terminals): + through(graph, endpoint) + store_root = output / _STORE + if resume == "require" and _store_is_cold(store_root): + raise StoreMissError("--resume require refuses a cold transport graph store.") + + output.mkdir(parents=True, exist_ok=True) + store_root = _destination(output, _STORE) + exported = _destination(output, EXPORT_FILENAME) + supplied[export_source] = exported + store = ContentStore(store_root, codecs=registry.codecs) + later = "require" if resume == "require" else "auto" + current = resume + materialize_bytes( + graph_to_json(graph).encode("utf-8"), _destination(output, "graph.json") + ) + for label, endpoint in checkpoints: + manifest = _run(through(graph, endpoint), registry, supplied, store, current) + materialize_bytes( + manifest.to_json_bytes(), _destination(output, f"checkpoint-{label}.json") + ) + current = later + + prepared = _run(through(graph, prepare.id), registry, supplied, store, current) + descriptor_bytes = store.load_bytes( + prepared.nodes[prepare.id].opaque_artifacts["export_descriptor"] + ) + descriptor_path = _destination(output, "export-descriptor.json") + previous = descriptor_path.read_bytes() if descriptor_path.is_file() else None + if exported.exists() and previous == descriptor_bytes: + report = validate_export(exported, json.loads(descriptor_bytes)) + if report["passed"] is not True: + raise ValueError( + f"The existing skeleton H5 failed readback: {report['failures']}." + ) + elif resume == "require": + raise StoreMissError( + "--resume require needs the skeleton H5 of the stored run." + ) + else: + version = compiled.versions[prepare.id] + materialize_export( + prepared.populations[version], json.loads(descriptor_bytes), exported + ) + materialize_bytes(descriptor_bytes, descriptor_path) + materialize_bytes( + prepared.to_json_bytes(), + _destination(output, "checkpoint-export-prepared.json"), + ) + for index, (label, endpoint) in enumerate(terminals): + manifest = _run(through(graph, endpoint), registry, supplied, store, later) + materialize_bytes( + manifest.to_json_bytes(), + _destination(output, f"checkpoint-{label}-{index}.json"), + ) + + manifest = _run(graph, registry, supplied, store, later) + inventory = _evidence(manifest, store, output) + produced = { + "graph": file_artifact(_destination(output, "graph.json")), + "skeleton_dataset": file_artifact(exported), + "export_descriptor": file_artifact(descriptor_path), + "artifacts": inventory, + } + diagnostics = [ + node.id for node in graph.nodes if node.kernel == "diagnostics.calibration@1" + ] + if len(diagnostics) == 1: + produced["calibration_diagnostics"] = materialize_bytes( + store.load_bytes( + manifest.nodes[diagnostics[0]].opaque_artifacts["diagnostics"] + ), + _destination(output, "calibration-diagnostics.json"), + ) + produced["explorer"] = materialize_bytes( + explain_html( + compiled, manifest, title=f"{graph.country} transport skeleton" + ).encode("utf-8"), + _destination(output, "explorer.html"), + ) + materialize_bytes(manifest.to_json_bytes(), _destination(output, "manifest.json")) + materialize_bytes( + canonical_json( + { + "schema_version": 1, + "country": graph.country, + "local_only": True, + "content_basis": _CONTENT_BASIS, + "produced": produced, + "not_yet_produced": list(pending_outputs), + } + ), + _destination(output, "build-status.json"), + ) + return manifest + + +def _source_argument(value: str) -> tuple[str, Path]: + name, separator, path = value.partition("=") + if not separator or not name or not path: + raise argparse.ArgumentTypeError("--source takes NAME=PATH.") + return name, Path(path) + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + origin = parser.add_mutually_exclusive_group(required=True) + origin.add_argument("--country", help="Packaged country spec code.") + origin.add_argument("--spec-dir", type=Path, help="Local country spec directory.") + parser.add_argument( + "--source", + action="append", + default=[], + type=_source_argument, + metavar="NAME=PATH", + help="Local path of one declared graph source (repeat per source).", + ) + parser.add_argument("--out", type=Path, required=True, help="Output directory.") + parser.add_argument( + "--resume", choices=("auto", "require", "forbid"), default="auto" + ) + parser.add_argument( + "--local-only", + action="store_true", + default=True, + help="Write local files only. This is the default and the only mode.", + ) + args = parser.parse_args(argv) + names = [name for name, _ in args.source] + if len(names) != len(set(names)): + parser.error("--source names a source more than once.") + return args + + +def main(argv: Sequence[str] | None = None) -> int: + """Run the CLI; return 0 on success and 1 on a refused build.""" + + args = parse_args(argv) + try: + spec = load_transport_spec(args.spec_dir if args.spec_dir else args.country) + prepared = prepare_transport_build(spec, sources=dict(args.source)) + execute_transport_graph( + prepared.graph, + prepared.registry, + prepared.sources, + out=args.out, + endpoints=prepared.endpoints, + pending_outputs=prepared.pending_outputs, + resume=args.resume, + ) + except (GraphRuntimeError, ValueError, TypeError, OSError) as error: + print(f"transport build refused: {error}", file=sys.stderr) + return 1 + print(f"Transport skeleton written under {args.out}.") + if prepared.pending_outputs: + print("Not yet produced: " + "; ".join(prepared.pending_outputs) + ".") + return 0 diff --git a/packages/microcosm-build/src/microcosm/build/transport/compose.py b/packages/microcosm-build/src/microcosm/build/transport/compose.py new file mode 100644 index 0000000000..bfd6cb4c1e --- /dev/null +++ b/packages/microcosm-build/src/microcosm/build/transport/compose.py @@ -0,0 +1,770 @@ +"""Compose the transport skeleton graph from a country's spec data. + +The country's ``transport_graph.json`` (a ``legacy_json`` spec resource) +declares the skeleton: its sources, its nodes, the execution endpoints and +the outputs a later package still owes. This module turns that declaration +into a :class:`~microcosm.graph.Graph`; it holds no country value. + +Node parameters are pure data. A parameter that is a JSON object must be one +explicit selection: + +- ``{"resource": name, "path": [...], "encoding": e}`` reads the value at + ``path`` (a list of object keys) inside the named spec resource. ``e`` is + ``"value"`` (the JSON value, lists as tuples), ``"json"`` (an object as + canonical JSON text) or ``"sha256"`` (the SHA-256 of the whole resource's + bytes, so the node's key moves with that one resource only). +- ``{"prepared": "engine_refs"}`` is the canonical JSON of the authenticated + binding-id -> engine reference map that preparation built. + +So a node binds exactly the resources it selects. The graph resource's own +digest and the country spec's fingerprint enter no node: editing one +resource re-keys only the nodes that select it and their descendants. + +A rules node names its binding (``rules_binding``) instead of an engine +reference. Composition supplies the authenticated ``engine_ref`` and the +binding's period, and every rules node runs through the one +``simulate.rules_by_ref@1`` router. + +``TransportGraphConfig.extensions`` is the extension point for the package +that adds the entitlement chain. Each extension is a factory called with +the spec, the config and the composed skeleton; it returns new nodes only. +Composition refuses an extension whose nodes change any skeleton node's +compiled predecessors (for example a new member of a calibration base +version), so an extension can never re-key the skeleton. + +Composition reads no donor, no facts and no engine, and writes nothing. +Unresolved country evidence is refused up front with every missing item +named (:func:`validate_transport_activation`); no proposed value is chosen. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from importlib.resources import files +from pathlib import Path +from types import MappingProxyType +from typing import Protocol + +from microcosm.build.country_spec import load_country_spec +from microcosm.build.graph_atomic_geography import ( + ATOMIC_GEOGRAPHY_VALIDATION_TYPE, + ATOMIC_SUPPORT_TYPE, +) +from microcosm.calibrate.ordered_kernels import OUTPUT_TYPES +from microcosm.graph import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Graph, + Node, + Owned, + Ownership, + Slice, + SourceRef, + StructuralDelta, + WeightTransition, + compile_graph, +) +from microcosm.graph.canonical import canonical_json + +from .artifact_types import KERNEL_OUTPUTS + +__all__ = [ + "GRAPH_RESOURCE", + "RULES_ROUTER", + "SKELETON_KERNELS", + "TransportGraphConfig", + "TransportGraphExtension", + "compose_transport_graph", + "CreateDeclaration", + "load_transport_spec", + "skeleton_create_declaration", + "transport_endpoints", + "transport_graph_document", + "transport_pending_outputs", + "transport_resource", + "validate_transport_activation", +] + +#: The spec resource that declares the skeleton. +GRAPH_RESOURCE = "transport_graph" +#: The single kernel every rules node runs through (one kernel per ref). +RULES_ROUTER = "simulate.rules_by_ref@1" +#: The kernels a skeleton declaration may use. A kernel outside this set +#: (for example an entitlement bridge) enters only through an extension. +SKELETON_KERNELS = frozenset( + { + "transport.create@1", + "transport.boundary@1", + "transport.currency@1", + "transport.quantile_map@1", + "transport.unit_attributes@1", + "concepts.encode@1", + "concepts.encode_groups@1", + RULES_ROUTER, + "takeup.assign@1", + "targets.compile@1", + "targets.problem@1", + "calibrate.ordered_adam@1", + "geography.support_from_facts@1", + "geography.assign_atomic@1", + "geography.derive@1", + "geography.gate@1", + "diagnostics.calibration@1", + "gates.battery@1", + "export.prepare@1", + "export.readback@1", + "transport.package@1", + } +) +_DOCUMENT_FIELDS = frozenset( + { + "schema_version", + "country", + "description", + "sources", + "nodes", + "endpoints", + "pending_outputs", + "mass_partition", + } +) +_NODE_FIELDS = frozenset( + { + "id", + "kernel", + "inputs", + "outputs", + "params", + "population", + "structural", + "base", + "sources", + "weights", + "mass", + "description", + "citation", + "artifact_inputs", + "artifact_outputs", + "rules_binding", + } +) +_ENCODINGS = frozenset({"value", "json", "sha256"}) +_PREPARED = frozenset({"engine_refs"}) +#: Typed outputs each skeleton kernel must declare, when the declaration +#: leaves them implicit. +_OUTPUT_TYPES: Mapping[str, Mapping[str, ArtifactType]] = MappingProxyType( + { + **KERNEL_OUTPUTS, + "calibrate.ordered_adam@1": OUTPUT_TYPES, + "geography.support_from_facts@1": {"support": ATOMIC_SUPPORT_TYPE}, + "geography.gate@1": {"validation": ATOMIC_GEOGRAPHY_VALIDATION_TYPE}, + } +) + + +class TransportGraphExtension(Protocol): + """A factory of new nodes appended to the composed skeleton. + + It receives the spec, the config and the skeleton, and returns a tuple + of new :class:`Node` objects. Ordinary extension nodes belong in + populations the extension creates (a FILTER may branch from any + skeleton population). Composition compares every skeleton node's + compiled predecessors with and without the extension and refuses any + change, so a member added to a calibration base version is refused. + """ + + def __call__( + self, + spec: Mapping[str, object], + config: TransportGraphConfig, + skeleton: Graph, + ) -> tuple[Node, ...]: ... + + +@dataclass(frozen=True) +class TransportGraphConfig: + """What preparation supplies; every country value stays in the spec. + + Attributes: + engine_refs: Binding id -> authenticated engine reference, from + :func:`microcosm.build.transport.registry.build_transport_registry`. + create_outputs: CREATE's declared column inventory, prepared from + the pinned donor (every loaded column must be declared). + extensions: Node factories appended after the skeleton, in order. + """ + + engine_refs: Mapping[str, str] + create_outputs: tuple[Owned, ...] + extensions: tuple[TransportGraphExtension, ...] = () + + def __post_init__(self) -> None: + if not isinstance(self.engine_refs, Mapping) or any( + not isinstance(key, str) + or not key + or not isinstance(value, str) + or not value + for key, value in self.engine_refs.items() + ): + raise ValueError("engine_refs must map binding ids to authenticated refs.") + if ( + not isinstance(self.create_outputs, tuple) + or not self.create_outputs + or any(not isinstance(item, Owned) for item in self.create_outputs) + ): + raise ValueError("create_outputs must declare CREATE's prepared inventory.") + if not isinstance(self.extensions, tuple) or any( + not callable(extension) for extension in self.extensions + ): + raise TypeError("extensions must be a tuple of node factories.") + object.__setattr__( + self, "engine_refs", MappingProxyType(dict(self.engine_refs)) + ) + + +def _mapping(value: object, label: str) -> Mapping: + if not isinstance(value, Mapping): + raise TypeError(f"{label} must be an object.") + return value + + +def _sequence(value: object, label: str) -> Sequence: + if isinstance(value, str | bytes) or not isinstance(value, Sequence): + raise TypeError(f"{label} must be a list.") + return value + + +def _stem(name: str) -> str: + return name.removesuffix(".json") + + +def _resources(spec: Mapping[str, object]) -> Mapping: + return _mapping(_mapping(spec, "transport spec").get("resources"), "resources") + + +def _has_resource(spec: Mapping[str, object], name: str) -> bool: + documents = _resources(spec) + stem = _stem(name) + return stem in documents or f"{stem}.json" in documents + + +def transport_resource(spec: Mapping[str, object], name: str) -> Mapping: + """One JSON resource of a normalized transport spec. + + A spec is ``{"country", "resources", "resource_hashes"?}``. A resource + key may be the JSON filename or its stem. ``resource_hashes`` keeps each + resource's exact file digest (:func:`load_transport_spec` supplies it); + without it a resource's digest is that of its canonical JSON. + """ + + documents = _resources(spec) + stem = _stem(name) + value = documents.get(stem, documents.get(f"{stem}.json")) + if value is None: + raise ValueError(f"Transport spec is missing resource {stem}.json.") + return _mapping(value, f"{stem}.json") + + +def _resource_digest(spec: Mapping[str, object], name: str) -> str: + hashes = _mapping(spec.get("resource_hashes", {}), "resource_hashes") + stem = _stem(name) + digest = hashes.get(stem, hashes.get(f"{stem}.json")) + if digest is None: + return hashlib.sha256( + canonical_json(transport_resource(spec, name)) + ).hexdigest() + if ( + not isinstance(digest, str) + or len(digest) != 64 + or not set(digest) <= set("0123456789abcdef") + ): + raise ValueError(f"Resource hash for {name!r} must be lowercase SHA-256.") + return digest + + +def load_transport_spec(source: str | Path) -> dict[str, object]: + """Load a country spec's JSON resources and their exact byte digests. + + A string names a packaged country; a :class:`~pathlib.Path` is a local + spec directory. The country spec loader validates the package first. + Only local files are read. + """ + + country = load_country_spec(source) + root = ( + Path(source) + if isinstance(source, Path) + else files("microcosm.build").joinpath(source) + ) + resources = {} + hashes = {} + for row in country.resource_rows: + if row.kind != "legacy_json": + continue + payload = root.joinpath(row.path).read_bytes() + name = _stem(row.path) + resources[name] = _mapping(json.loads(payload), row.path) + hashes[name] = hashlib.sha256(payload).hexdigest() + return { + "country": country.country, + "resources": resources, + "resource_hashes": hashes, + } + + +def _selections(value: object) -> Iterator[Mapping]: + """Every explicit selection inside a declared parameter value.""" + + if isinstance(value, Mapping): + yield value + elif isinstance(value, list | tuple): + for item in value: + yield from _selections(item) + + +def _selected_resources(document: Mapping) -> set[str]: + names = set() + for row in document.get("nodes", ()): + if not isinstance(row, Mapping): + continue + params = row.get("params", {}) + if not isinstance(params, Mapping): + continue + for value in params.values(): + for selection in _selections(value): + if isinstance(selection.get("resource"), str): + names.add(_stem(selection["resource"])) + return names + + +def validate_transport_activation(spec: Mapping[str, object]) -> None: + """Refuse unresolved country evidence, naming every missing item at once. + + The items are the skeleton declaration and every resource it selects, + the dependent-child age limit of the benefit-unit rule (``max_age``, or + the proposal's ``age_limit_years``), any reference not marked active, + an executable receipt contract (``receipt_contract.json``), the engine + commit and wheel pins, and a scenario knob still awaiting evidence + (``rent_stock_factor``). Nothing here picks a value for a missing item. + """ + + missing = [] + for name in ("benefit_unit_rule", "axiom_rules_bindings", "scenarios"): + if not _has_resource(spec, name): + missing.append(f"{name}.json") + if _has_resource(spec, "benefit_unit_rule"): + unit = transport_resource(spec, "benefit_unit_rule") + child = _mapping( + unit.get("dependent_child", {}), "benefit_unit_rule.dependent_child" + ) + age_key = "max_age" if "max_age" in child else "age_limit_years" + if child.get(age_key) is None: + missing.append(f"benefit_unit_rule.dependent_child.{age_key}") + for resource in ("precal_references", "target_references", "holdout_references"): + if not _has_resource(spec, resource): + missing.append(f"{resource}.json") + continue + rows = transport_resource(spec, resource).get("target_references") + for row in _sequence(rows, f"{resource}.target_references"): + reference = _mapping(row, f"{resource} reference") + metadata = _mapping(reference.get("metadata", {}), f"{resource}.metadata") + if metadata.get("activation_status", "active") != "active": + missing.append( + f"placeholder reference {resource}.{reference.get('name', '?')}" + ) + receipt = ( + transport_resource(spec, "receipt_contract") + if _has_resource(spec, "receipt_contract") + else None + ) + contracts = None if receipt is None else receipt.get("receipts") + if not isinstance(contracts, Mapping): + missing.append( + "receipt_contract.json (the input closure's receipt_requirements " + "are prose, not an executable contract)" + ) + elif any( + not isinstance(row.get("programs"), list) or not row["programs"] + for row in ((contracts,) if "programs" in contracts else contracts.values()) + if isinstance(row, Mapping) + ): + missing.append("receipt_contract.json receipts.programs") + if _has_resource(spec, "axiom_rules_bindings"): + bindings = transport_resource(spec, "axiom_rules_bindings") + pins = _mapping(bindings.get("engine", {}), "axiom_rules_bindings.engine") + for name in ("commit", "wheel_sha256"): + if pins.get(name) is None: + missing.append(f"axiom_rules_bindings.engine.{name}") + if _has_resource(spec, "scenarios"): + rows = transport_resource(spec, "scenarios").get("scenarios", ()) + for row in _sequence(rows, "scenarios.scenarios"): + scenario = _mapping(row, "scenario") + knobs = _mapping(scenario.get("knobs", {}), "scenario.knobs") + if "rent_stock_factor" in knobs and knobs["rent_stock_factor"] is None: + missing.append(f"scenario {scenario.get('id', '?')} rent_stock_factor") + if not _has_resource(spec, GRAPH_RESOURCE): + missing.append(f"{GRAPH_RESOURCE}.json (the skeleton declaration)") + else: + selected = _selected_resources(transport_resource(spec, GRAPH_RESOURCE)) + missing.extend( + f"{name}.json (selected by {GRAPH_RESOURCE}.json)" + for name in sorted(selected) + if not _has_resource(spec, name) + ) + if missing: + raise ValueError("Transport spec is not activated: " + "; ".join(missing) + ".") + + +def transport_graph_document(spec: Mapping[str, object]) -> Mapping: + """The skeleton declaration, checked for its schema and country.""" + + document = transport_resource(spec, GRAPH_RESOURCE) + unknown = set(document) - _DOCUMENT_FIELDS + if unknown: + raise ValueError(f"{GRAPH_RESOURCE}.json has unknown fields {sorted(unknown)}.") + if document.get("schema_version") != 1 or document.get("country") != spec.get( + "country" + ): + raise ValueError( + f"{GRAPH_RESOURCE}.json must declare schema_version 1 and the spec country." + ) + return document + + +def transport_endpoints(spec: Mapping[str, object]) -> dict[str, object]: + """The ordered checkpoints: geography, calibration, then the terminals.""" + + values = _mapping( + transport_graph_document(spec).get("endpoints"), "transport endpoints" + ) + if set(values) != {"geography", "calibration", "terminal"}: + raise ValueError( + "Transport endpoints must be exactly geography, calibration and terminal." + ) + for name in ("geography", "calibration"): + if not isinstance(values[name], str) or not values[name]: + raise ValueError(f"Transport endpoint {name!r} must name one node.") + terminals = tuple(_sequence(values["terminal"], "terminal endpoints")) + if not terminals or any( + not isinstance(name, str) or not name for name in terminals + ): + raise ValueError("Terminal endpoints must be a nonempty list of node ids.") + return { + "geography": values["geography"], + "calibration": values["calibration"], + "terminal": terminals, + } + + +def transport_pending_outputs(spec: Mapping[str, object]) -> tuple[str, ...]: + """Outputs of the full build that this skeleton does not yet produce.""" + + rows = transport_graph_document(spec).get("pending_outputs", ()) + values = tuple(_sequence(rows, "pending_outputs")) + if any(not isinstance(value, str) or not value for value in values): + raise ValueError("pending_outputs must be nonempty strings.") + return values + + +def _param( + spec: Mapping[str, object], + value: object, + engine_refs: Mapping[str, str] | None, +) -> object: + if isinstance(value, list | tuple): + return tuple(_param(spec, item, engine_refs) for item in value) + if not isinstance(value, Mapping): + return value + if set(value) == {"prepared"}: + if value["prepared"] not in _PREPARED: + raise ValueError(f"Unknown prepared parameter {value['prepared']!r}.") + if engine_refs is None: + raise ValueError("Prepared engine references are not available here.") + return canonical_json( + {"engine_refs": dict(sorted(engine_refs.items()))} + ).decode("utf-8") + if set(value) - {"resource", "path", "encoding"} or "resource" not in value: + raise ValueError( + "An object parameter must be a resource or prepared selection." + ) + name = value["resource"] + encoding = value.get("encoding", "value") + if encoding not in _ENCODINGS: + raise ValueError(f"Unknown resource encoding {encoding!r}.") + document: object = transport_resource(spec, name) + path = tuple(_sequence(value.get("path", ()), "resource selection path")) + if encoding == "sha256": + if path: + raise ValueError("A sha256 selection names a whole resource (no path).") + return _resource_digest(spec, name) + for component in path: + if not isinstance(document, Mapping) or component not in document: + raise ValueError(f"Resource {name!r} has no selected path {list(path)!r}.") + document = document[component] + if document is None: + raise ValueError(f"Resource {name!r} path {list(path)!r} is not activated.") + if encoding == "json": + if not isinstance(document, Mapping): + raise ValueError(f"Resource {name!r} json selection must be an object.") + return canonical_json(document).decode("utf-8") + if isinstance(document, Mapping): + raise ValueError( + f"Resource {name!r} value selection is an object; use json encoding." + ) + return _param(spec, document, engine_refs) + + +def _artifact_type(value: object) -> ArtifactType: + document = _mapping(value, "artifact type") + if set(document) != {"name", "schema_version"}: + raise ValueError("An artifact type declares exactly name and schema_version.") + return ArtifactType(document["name"], document["schema_version"]) + + +def _artifact_outputs(row: Mapping) -> tuple[ArtifactOutput, ...]: + if "artifact_outputs" not in row: + return tuple( + ArtifactOutput(name, kind) + for name, kind in _OUTPUT_TYPES.get(row["kernel"], {}).items() + ) + return tuple( + ArtifactOutput(value["name"], _artifact_type(value["type"])) + for value in _sequence(row["artifact_outputs"], "artifact outputs") + ) + + +def _rules_params( + spec: Mapping[str, object], + row: Mapping, + params: dict, + config: TransportGraphConfig, +) -> dict: + binding_id = row["rules_binding"] + if row["kernel"] != RULES_ROUTER: + raise ValueError(f"Rules bindings run through the one {RULES_ROUTER} router.") + rows = transport_resource(spec, "axiom_rules_bindings").get("bindings", ()) + matches = [ + binding + for binding in _sequence(rows, "axiom_rules_bindings.bindings") + if isinstance(binding, Mapping) and binding.get("id") == binding_id + ] + if len(matches) != 1 or binding_id not in config.engine_refs: + raise ValueError( + f"Rules binding {binding_id!r} needs one declaration and one prepared " + "engine reference." + ) + binding = matches[0] + if set(params) - {"variables"}: + raise ValueError( + f"Rules node {row['id']!r} may set only variables; its engine_ref and " + "period come from its binding." + ) + declared = tuple(binding["variables"]) + variables = tuple(params.get("variables", declared)) + if ( + not variables + or len(set(variables)) != len(variables) + or not set(variables) <= set(declared) + ): + raise ValueError( + f"Rules node {row['id']!r} variables must be distinct binding variables." + ) + columns = {item["column"] for item in _sequence(row.get("outputs", ()), "outputs")} + if columns != set(variables): + raise ValueError(f"Rules node {row['id']!r} must own exactly its variables.") + return { + "engine_ref": config.engine_refs[binding_id], + "period": binding["period"], + "variables": variables, + } + + +def _node( + spec: Mapping[str, object], + row: Mapping, + config: TransportGraphConfig, + types: Mapping[tuple[str, str], ArtifactType], +) -> Node: + unknown = set(row) - _NODE_FIELDS + if unknown: + raise ValueError( + f"Transport node {row.get('id')!r} has unknown fields {sorted(unknown)}." + ) + kernel = row.get("kernel") + if kernel not in SKELETON_KERNELS: + raise ValueError( + f"Kernel {kernel!r} is not a skeleton kernel; add it through an extension." + ) + params = { + name: _param(spec, value, config.engine_refs) + for name, value in _mapping(row.get("params", {}), "node params").items() + } + if "rules_binding" in row: + params = _rules_params(spec, row, params, config) + elif kernel == RULES_ROUTER: + raise ValueError("Every skeleton rules node must name its rules_binding.") + structural = StructuralDelta(row.get("structural", "none")) + if structural is StructuralDelta.CREATE: + if row.get("outputs"): + raise ValueError("CREATE's outputs come from preparation, not the spec.") + outputs = config.create_outputs + else: + outputs = tuple( + Owned( + **{ + **value, + "ownership": Ownership(value.get("ownership", "produced")), + } + ) + for value in _sequence(row.get("outputs", ()), "owned outputs") + ) + inputs = tuple( + Slice(**{**value, "columns": tuple(value["columns"])}) + for value in _sequence(row.get("inputs", ()), "input slices") + ) + artifacts = [] + for value in _sequence(row.get("artifact_inputs", ()), "artifact inputs"): + edge = _mapping(value, "artifact input") + if set(edge) - {"name", "producer", "artifact"}: + raise ValueError( + f"Node {row['id']!r}: an artifact input declares name, producer " + "and optionally artifact; its type is the producer's." + ) + producer, artifact = edge["producer"], edge.get("artifact", edge["name"]) + if (producer, artifact) not in types: + raise ValueError( + f"Node {row['id']!r} reads undeclared artifact {producer}.{artifact}." + ) + artifacts.append( + ArtifactInput(edge["name"], producer, artifact, types[producer, artifact]) + ) + weights = row.get("weights") + return Node( + id=row["id"], + kernel=kernel, + inputs=inputs, + outputs=outputs, + params=params, + population=row.get("population"), + structural=structural, + base=row.get("base"), + sources=tuple(_sequence(row.get("sources", ()), "node sources")), + weights=None if weights is None else WeightTransition(**weights), + mass=row.get("mass", "conserve"), + description=row.get("description", ""), + citation=row.get("citation", ""), + artifact_inputs=tuple(artifacts), + artifact_outputs=_artifact_outputs(row), + ) + + +@dataclass(frozen=True) +class CreateDeclaration: + """The declared CREATE node's id, kernel, resolved params and sources.""" + + id: str + kernel: str + params: Mapping[str, object] + sources: tuple[str, ...] + + +def skeleton_create_declaration(spec: Mapping[str, object]) -> CreateDeclaration: + """The declared CREATE with its parameters resolved, for preparation. + + Preparation runs this kernel once to read CREATE's column inventory, + which the composed CREATE node must then declare. + """ + + validate_transport_activation(spec) + rows = [ + row + for row in _sequence( + transport_graph_document(spec).get("nodes"), "transport nodes" + ) + if isinstance(row, Mapping) and row.get("structural") == "create" + ] + if len(rows) != 1 or rows[0].get("kernel") != "transport.create@1": + raise ValueError( + "A transport skeleton declares exactly one transport.create@1." + ) + row = rows[0] + return CreateDeclaration( + id=row["id"], + kernel=row["kernel"], + params=MappingProxyType( + { + name: _param(spec, value, None) + for name, value in _mapping(row.get("params", {}), "params").items() + } + ), + sources=tuple(_sequence(row.get("sources", ()), "node sources")), + ) + + +def compose_transport_graph( + spec: Mapping[str, object], config: TransportGraphConfig +) -> Graph: + """The skeleton declared by the spec, plus any extension branches. + + The result is a pure function of the spec's resources and ``config``: + composing twice gives equal ``graph_to_json``. Activation is checked + first, so pending country evidence fails with its items named. + """ + + if not isinstance(config, TransportGraphConfig): + raise TypeError("config must be a TransportGraphConfig.") + validate_transport_activation(spec) + document = transport_graph_document(spec) + rows = tuple( + _mapping(row, "transport node") + for row in _sequence(document.get("nodes"), "transport nodes") + ) + types = { + (row["id"], output.name): output.type + for row in rows + for output in _artifact_outputs(row) + } + sources = tuple( + SourceRef(**_mapping(row, "transport source")) + for row in _sequence(document.get("sources"), "transport sources") + ) + partition = document.get("mass_partition") + skeleton = Graph( + country=spec["country"], + sources=sources, + nodes=tuple(_node(spec, row, config, types) for row in rows), + mass_partition=None if partition is None else tuple(partition), + ) + compiled = compile_graph(skeleton) + endpoints = transport_endpoints(spec) + named = {endpoints["geography"], endpoints["calibration"], *endpoints["terminal"]} + if not named <= set(compiled.order): + raise ValueError("Transport endpoints must name declared skeleton nodes.") + transport_pending_outputs(spec) + additional: list[Node] = [] + for factory in config.extensions: + nodes = factory(spec, config, skeleton) + if not isinstance(nodes, tuple) or any( + not isinstance(node, Node) for node in nodes + ): + raise TypeError("A transport extension must return a tuple of Nodes.") + additional.extend(nodes) + if not additional: + return skeleton + graph = Graph( + skeleton.country, + skeleton.sources, + (*skeleton.nodes, *additional), + skeleton.mass_partition, + ) + extended = compile_graph(graph) + for node in skeleton.nodes: + if extended.predecessors[node.id] != compiled.predecessors[node.id]: + raise ValueError( + f"An extension would re-key skeleton node {node.id!r}; " + "branch a new FILTER population instead." + ) + return graph diff --git a/packages/microcosm-build/src/microcosm/build/transport/registry.py b/packages/microcosm-build/src/microcosm/build/transport/registry.py index b1666ea308..da82f22654 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/registry.py +++ b/packages/microcosm-build/src/microcosm/build/transport/registry.py @@ -20,8 +20,8 @@ from microcosm.build.graph_atomic_geography import register_atomic_geography_kernels from microcosm.calibrate.ordered_kernels import CALIBRATE_ORDERED_ADAM +from microcosm.frame import EntitySchema from microcosm.frame.adapters.axiom import ( - NZ_SCHEMA, AxiomEngine, AxiomPeriod, assert_no_relations, @@ -30,6 +30,7 @@ ) from microcosm.frame.rules import RulesEngine from microcosm.frame.rules_kernels import SimulateRulesByRefKernel +from microcosm.frame.unit_construction import BenefitUnitRule from microcosm.graph import KernelRegistry from microcosm.graph.canonical import canonical_json from microcosm.graph.codecs import SourceCodecRegistry @@ -47,6 +48,7 @@ "TransportRegistry", "build_transport_registry", "register_transport_population_kernels", + "transport_entity_schema", ] _SHA256 = re.compile(r"[0-9a-f]{64}") @@ -66,6 +68,23 @@ class TransportRegistry: codecs: SourceCodecRegistry engines: Mapping[str, RulesEngine] engine_refs: Mapping[str, str] + schema: EntitySchema + nesting: Mapping[str, str] + + +def transport_entity_schema( + rule: BenefitUnitRule, +) -> tuple[EntitySchema, Mapping[str, str]]: + """The persons, their households and the rule's benefit units. + + This is the schema ``transport.create@1`` gives its frame. Each unit is + built inside one household (``build_benefit_units`` returns + ``_household_id``), so the unit nests in the household. + """ + if not isinstance(rule, BenefitUnitRule): + raise TypeError("transport_entity_schema requires a BenefitUnitRule.") + schema = EntitySchema(group_entities=("household", rule.entity)) + return schema, MappingProxyType({rule.entity: "household"}) def _mapping(value: object, label: str) -> Mapping: @@ -98,7 +117,11 @@ def _binding_rows(document: Mapping) -> tuple[Mapping, ...]: def _binding_semantics( - row: Mapping, root: Path, periods: Mapping, entity_names: Mapping + row: Mapping, + root: Path, + periods: Mapping, + entity_names: Mapping, + schema: EntitySchema, ) -> dict: label = f"rules binding {row['id']!r}" relative = _text(row.get("rulespec_path"), f"{label}.rulespec_path") @@ -107,12 +130,14 @@ def _binding_semantics( raise ValueError(f"{label}.rulespec_path must be a canonical relative path.") module = (root / relative).resolve() if not module.is_file() or not module.is_relative_to(root): - raise ValueError(f"{label}.rulespec_path is not a file under rulespec_nz.") + raise ValueError( + f"{label}.rulespec_path is not a file under the RuleSpec tree." + ) digest = _pin(row.get("sha256"), f"{label}.sha256", _SHA256) if hashlib.sha256(module.read_bytes()).hexdigest() != digest: raise ValueError(f"{label}.sha256 does not match rulespec_path {relative!r}.") entity = _text(row.get("entity"), f"{label}.entity") - if entity not in NZ_SCHEMA.entities: + if entity not in schema.entities: raise ValueError(f"{label}.entity {entity!r} is absent from the schema.") engine_entity = _text(row.get("engine_entity"), f"{label}.engine_entity") if engine_entity != entity_names[entity]: @@ -144,25 +169,30 @@ def _check_engine( root: Path, periods: Mapping[str, AxiomPeriod], entity_names: Mapping[str, str], + schema: EntitySchema, + nesting: Mapping[str, str], ) -> None: label = f"rules binding {semantics['id']!r}" if not isinstance(engine, RulesEngine): raise TypeError(f"{label} does not bind a RulesEngine.") - if engine.entity_schema() != NZ_SCHEMA: + if engine.entity_schema() != schema: raise ValueError(f"{label} engine schema does not match the transport schema.") if isinstance(engine, AxiomEngine): roots = tuple(path.resolve() for path in engine._rulespec_roots) if roots != (root,): - raise ValueError(f"{label} engine root must equal resolved rulespec_nz.") + raise ValueError( + f"{label} engine root must equal the resolved RuleSpec tree." + ) if engine._module.resolve() != (root / semantics["rulespec_path"]).resolve(): raise ValueError(f"{label} engine module does not match rulespec_path.") if ( engine._periods != periods or engine._entity_names != entity_names + or engine._nesting != dict(nesting) or engine._output_dtypes != "graph" ): raise ValueError( - f"{label} engine must use declared periods and graph dtypes." + f"{label} engine must use declared periods, unit nesting and graph dtypes." ) else: declared_root = getattr(engine, "rulespec_root", None) @@ -170,7 +200,9 @@ def _check_engine( not isinstance(declared_root, str | Path) or Path(declared_root).resolve() != root ): - raise ValueError(f"{label} engine root must equal resolved rulespec_nz.") + raise ValueError( + f"{label} engine root must equal the resolved RuleSpec tree." + ) if not callable(getattr(engine, "assert_no_relations", None)): raise TypeError( f"{label} injected engine must expose assert_no_relations(entity)." @@ -183,14 +215,18 @@ def build_transport_registry( rules_bindings: Mapping, rulespec_root: str | Path, *, + unit_rule: BenefitUnitRule, engines_by_binding: Mapping[str, RulesEngine] | None = None, dependencies: tuple[str, ...] = (), ) -> TransportRegistry: """Build a run registry with one rules router and the available kernels. - Real adapters use ``NZ_SCHEMA``, explicit periods, graph output dtypes, - authenticated module bytes and :func:`axiom_engine_ref`. References are - pinned before any relation check compiles a program. + Real adapters use the schema and unit nesting of + :func:`transport_entity_schema` for ``unit_rule`` (the frame CREATE + builds), explicit periods, graph output dtypes, authenticated module + bytes and :func:`axiom_engine_ref`. References are pinned before any + relation check compiles a program. An injected adapter must report the + same schema. Engine-free fixtures may inject pure-Python adapters by binding id. Each must implement ``RulesEngine``, declare ``rulespec_root`` and expose an @@ -206,12 +242,13 @@ def build_transport_registry( readiness of a graph that also needs AS bridge or gap kernels. """ document = _mapping(rules_bindings, "axiom_rules_bindings") + schema, nesting = transport_entity_schema(unit_rule) root = Path(rulespec_root).resolve() if not root.is_dir(): - raise ValueError("rulespec_nz must resolve to a directory.") + raise ValueError("The RuleSpec tree source must resolve to a directory.") # A symlink can point at bytes outside the tree the source claims to name. if any(path.is_symlink() for path in root.rglob("*")): - raise ValueError("rulespec_nz must not contain symbolic links.") + raise ValueError("The RuleSpec tree source must not contain symbolic links.") engine_pins = _mapping(document.get("engine"), "axiom_rules_bindings.engine") rulespec_pins = _mapping(document.get("rulespec"), "axiom_rules_bindings.rulespec") pins = { @@ -242,7 +279,7 @@ def build_transport_registry( entity_names = dict( _mapping(document.get("entity_names"), "axiom_rules_bindings.entity_names") ) - if set(entity_names) != set(NZ_SCHEMA.entities): + if set(entity_names) != set(schema.entities): raise ValueError( "axiom_rules_bindings.entity_names must cover the transport schema." ) @@ -261,21 +298,24 @@ def build_transport_registry( refs = {} tree_sha256, tree_bytes = rulespec_tree_digest(root) for row in rows: - semantics = _binding_semantics(row, root, periods, entity_names) + semantics = _binding_semantics(row, root, periods, entity_names, schema) binding_periods = {semantics["period"]: periods[semantics["period"]]} engine = ( AxiomEngine( root / semantics["rulespec_path"], - schema=NZ_SCHEMA, + schema=schema, rulespec_roots=(root,), periods=binding_periods, entity_names=entity_names, + nesting=nesting, output_dtypes="graph", ) if engines_by_binding is None else engines_by_binding[row["id"]] ) - _check_engine(engine, semantics, root, binding_periods, entity_names) + _check_engine( + engine, semantics, root, binding_periods, entity_names, schema, nesting + ) if isinstance(engine, AxiomEngine): reference = axiom_engine_ref(engine, rulespec_root=root, **pins) assert_no_relations(engine, semantics["entity"]) @@ -323,7 +363,9 @@ def build_transport_registry( register_gate_kernel(kernels, TRANSPORT_GATE_REGISTRY) register_terminal_kernels(kernels, codecs=codecs) kernels.register(SimulateRulesByRefKernel(engine_mapping, dependencies)) - return TransportRegistry(kernels, codecs, engine_mapping, MappingProxyType(refs)) + return TransportRegistry( + kernels, codecs, engine_mapping, MappingProxyType(refs), schema, nesting + ) def register_transport_population_kernels(registry: KernelRegistry) -> KernelRegistry: diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py new file mode 100644 index 0000000000..1d31cd0a0b --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -0,0 +1,900 @@ +"""The composed transport skeleton, its CLI driver and its charter properties. + +Everything runs on the engine-free toy package in +``test_support/microcosm_build/transport_composed.py``: six invented donor +records, invented facts for both Chronicle roles, a toy RuleSpec tree and +three pure-Python rules engines (two person engines in the skeleton, a +family engine on the test extension) routed by one +``simulate.rules_by_ref@1`` kernel in one ``run_graph``. + +Invariants (each a property over every input the strategy draws, or an +executed check): + +- Purity: the graph is a function of the spec's resources and the prepared + config; composing twice gives equal ``graph_to_json``. +- Resource locality: adding a key to one spec resource re-keys exactly the + nodes that select that resource whole (or its digest) and their + descendants. A3 (a calibration reference) and C2 (a ``gates.json`` + threshold, a hold-out reference) are executed instances. +- A1 determinism, A2 memoization, A4 inert prose, A5 (one changed RuleSpec + byte re-keys exactly the rules nodes and their descendants). +- Hold-out ancestry: no calibration ancestor reads the hold-out facts, the + family engine, or a family-engine input or output column. +- B1: a non-structural node that owns a structural or engine column is + refused. +- Extension: an extension adds nodes without re-keying any skeleton node; + one that would add a member to a calibration base version is refused. +- E3: ``--resume require`` refuses a cold store and creates nothing. +- Differentials: each routed rules node equals the stock single-engine + ``simulate.rules@1``; preparation's CREATE inventory equals the + independent composition of the CREATE functions; the registry's schema + equals CREATE's frame schema; declared node keys equal executed keys. +- Real spec: the packaged country spec refuses with every activation gap + named. +""" + +from __future__ import annotations + +import ast +import copy +import json +import re +import shutil +import time +from dataclasses import replace +from pathlib import Path + +import numpy as np +import pytest +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from microcosm.build.transport import cli as cli_module +from microcosm.build.transport.cli import ( + EXPORT_FILENAME, + execute_transport_graph, + prepare_create_outputs, + prepare_transport_build, + through, +) +from microcosm.build.transport.compose import ( + TransportGraphConfig, + compose_transport_graph, + load_transport_spec, + validate_transport_activation, +) +from microcosm.build.transport.terminal_kernels import materialize_export +from microcosm.frame.kernels import SimulateRulesKernel +from microcosm.graph import ( + Capabilities, + ContentStore, + Determinism, + GraphError, + KernelBase, + KernelRegistry, + Node, + NodeRejectedError, + Owned, + Slice, + StoreMissError, + compile_graph, + graph_to_json, + run_graph, +) +from microcosm.graph.keys import node_key, source_content_key +from test_support.microcosm_build.transport_composed import ( + FAMILY_INPUTS, + PROGRAMS, + SHARED_PATH, + make_composed_fixture, +) +from test_support.microcosm_build.transport_graph import descendants +from test_support.paths import paths_for + +TRANSPORT = paths_for("microcosm-build").package / "src/microcosm/build/transport" +PROPERTY = settings( + max_examples=6, + deadline=None, + database=None, + suppress_health_check=[HealthCheck.function_scoped_fixture], +) +EXECUTED = settings(PROPERTY, max_examples=2) +CALIBRATION = "nz.calibrate" +RULES = "simulate.rules_by_ref@1" + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _keys(manifest) -> dict[str, str]: + return {name: receipt.key for name, receipt in manifest.nodes.items()} + + +def _artifacts(manifest, store) -> dict: + return { + (name, artifact): store.load_bytes(key) + for name, receipt in manifest.nodes.items() + for artifact, key in receipt.opaque_artifacts.items() + } + + +def _declared_keys(graph, fixture, exported: Path | None = None) -> dict[str, str]: + """Node keys from the public key formula, without executing anything. + + The exported dataset is an outer input; without a file its identity is + held fixed, which suffices to compare two declarations. + """ + compiled = compile_graph(graph) + sources = { + name: source_content_key(name, path) + for name, path in fixture.source_mapping(exported).items() + } + for source in graph.sources: + sources.setdefault(source.name, "outer-input-held-fixed") + keys: dict[str, str] = {} + for name in compiled.order: + kernel = fixture.registry.kernels.get(graph.node(name).kernel) + keys[name] = node_key( + compiled, + name, + keys, + kernel.implementation_hash(), + sources, + kernel_capabilities=kernel.capabilities, + ) + return keys + + +def _export(root: Path, fixture, graph) -> Path: + """Write the skeleton H5 from a separate store's export preparation.""" + store = ContentStore(root / "preparation", codecs=fixture.registry.codecs) + compiled = compile_graph(graph) + prepared = run_graph( + compile_graph(through(graph, "nz.export.prepare")), + sources=fixture.sources, + store=store, + kernels=fixture.registry.kernels, + ) + receipt = prepared.nodes["nz.export.prepare"] + descriptor = json.loads( + store.load_bytes(receipt.opaque_artifacts["export_descriptor"]) + ) + exported = root / EXPORT_FILENAME + version = compiled.versions["nz.export.prepare"] + materialize_export(prepared.populations[version], descriptor, exported) + return exported + + +def _full_run(root: Path, fixture, graph, *, store=None, exported=None, resume="auto"): + """One ``run_graph`` of the whole graph; the H5 is written beforehand. + + Preparing the H5 in a separate store keeps checkpoint hits from hiding + which nodes of the full run miss. Reusing one H5's bytes keeps the HDF5 + writer's timestamps out of determinism comparisons. + """ + root.mkdir(parents=True, exist_ok=True) + exported = _export(root, fixture, graph) if exported is None else exported + store = ( + ContentStore(root / "store", codecs=fixture.registry.codecs) + if store is None + else store + ) + manifest = run_graph( + compile_graph(graph), + sources=fixture.source_mapping(exported), + store=store, + kernels=fixture.registry.kernels, + resume=resume, + ) + return manifest, store, exported + + +def _copied_store(base, root: Path, fixture) -> ContentStore: + shutil.copytree(base.root, root / "store-copy") + return ContentStore(root / "store-copy", codecs=fixture.registry.codecs) + + +def _misses(manifest) -> set[str]: + return {name for name, receipt in manifest.nodes.items() if not receipt.hit} + + +def _changed(before, after) -> set[str]: + return { + name for name in after.nodes if before.nodes[name].key != after.nodes[name].key + } + + +def _selectors(spec: dict, resource: str) -> set[str]: + """Nodes whose resolved params change when ``resource``'s bytes change. + + A whole-resource selection (empty path) or a digest selection moves + with any edit; a path selection moves only with its own subtree. + """ + found = set() + for row in spec["resources"]["transport_graph"]["nodes"]: + stack = list(row.get("params", {}).values()) + while stack: + value = stack.pop() + if isinstance(value, list): + stack.extend(value) + elif isinstance(value, dict) and value.get("resource") == resource: + if value.get("encoding") == "sha256" or not value.get("path"): + found.add(row["id"]) + return found + + +# --------------------------------------------------------------------------- +# Shared fixtures (built once; tests that edit inputs copy what they change) +# --------------------------------------------------------------------------- + + +@pytest.fixture(scope="module") +def composed(tmp_path_factory): + return make_composed_fixture(tmp_path_factory.mktemp("composed")) + + +@pytest.fixture(scope="module") +def cold(composed, tmp_path_factory): + """One timed cold run of the extended graph.""" + graph = composed.graph(extended=True) + started = time.perf_counter() + manifest, store, exported = _full_run( + tmp_path_factory.mktemp("cold"), composed, graph + ) + return graph, manifest, store, exported, time.perf_counter() - started + + +# --------------------------------------------------------------------------- +# Composition invariants +# --------------------------------------------------------------------------- + + +@PROPERTY +@given(prose=st.text(alphabet="abc XYZ.", max_size=24)) +def test_composition_is_pure_and_prose_moves_no_key(composed, prose): + graph = composed.graph(extended=True) + again = compose_transport_graph( + copy.deepcopy(composed.spec), composed.extended_config() + ) + assert graph_to_json(graph) == graph_to_json(again) + edited = replace( + graph, + nodes=tuple( + replace(node, description=prose, citation=prose) for node in graph.nodes + ), + sources=tuple(replace(source, description=prose) for source in graph.sources), + ) + assert _declared_keys(edited, composed) == _declared_keys(graph, composed) + + +def _selected_resources(spec) -> list[str]: + names = set() + for row in spec["resources"]["transport_graph"]["nodes"]: + stack = list(row.get("params", {}).values()) + while stack: + value = stack.pop() + if isinstance(value, list): + stack.extend(value) + elif isinstance(value, dict) and "resource" in value: + names.add(value["resource"]) + return sorted(names) + + +@PROPERTY +@given(data=st.data(), note=st.text(alphabet="abcdef", min_size=1, max_size=8)) +def test_one_resource_edit_rekeys_exactly_its_selectors_and_descendants( + composed, data, note +): + """The graph binds each resource only through the nodes that select it.""" + resource = data.draw(st.sampled_from(_selected_resources(composed.spec))) + spec = copy.deepcopy(composed.spec) + spec["resources"][resource]["zz_hypothesis_note"] = note + graph = composed.graph(extended=True) + edited = compose_transport_graph(spec, composed.extended_config()) + before = _declared_keys(graph, composed) + after = _declared_keys(edited, composed) + moved = {name for name in before if before[name] != after[name]} + assert moved == descendants(graph, _selectors(spec, resource)) + + +def test_extension_adds_nodes_without_rekeying_the_skeleton(composed): + skeleton = composed.graph() + extended = composed.graph(extended=True) + first = _declared_keys(skeleton, composed) + second = _declared_keys(extended, composed) + assert set(second) - set(first) == { + "nz.test.scn", + "nz.test.scn.rules", + "nz.test.validate", + "nz.test.validate.compare", + } + assert {name: second[name] for name in first} == first + + +def test_an_extension_member_of_a_calibration_base_is_refused(composed): + def member_of_open(spec, config, skeleton): + return ( + Node( + "nz.test.member", + "transport.unit_attributes@1", + population="nz.open", + inputs=(Slice("person", ("age",)),), + outputs=(Owned("person", "toy_extra", "int64"),), + ), + ) + + config = replace(composed.config, extensions=(member_of_open,)) + with pytest.raises(ValueError, match="re-key skeleton node 'nz.calibrate'"): + compose_transport_graph(composed.spec, config) + + +def test_skeleton_refuses_a_kernel_outside_its_vocabulary(composed): + spec = copy.deepcopy(composed.spec) + spec["resources"]["transport_graph"]["nodes"][-1]["kernel"] = "takeup.gap@1" + with pytest.raises(ValueError, match="through an extension"): + compose_transport_graph(spec, composed.config) + + +def test_rules_nodes_take_engine_ref_and_period_only_from_their_binding(composed): + spec = copy.deepcopy(composed.spec) + row = next( + row + for row in spec["resources"]["transport_graph"]["nodes"] + if row.get("rules_binding") == "main_benefit" + ) + row["params"] = {"period": "2030"} + with pytest.raises(ValueError, match="may set only variables"): + compose_transport_graph(spec, composed.config) + graph = composed.graph() + routed = [node for node in graph.nodes if node.kernel == RULES] + assert {node.params["engine_ref"] for node in routed} == { + composed.registry.engine_refs["main_benefit"], + composed.registry.engine_refs["pension"], + } + + +# --------------------------------------------------------------------------- +# Executed charter properties +# --------------------------------------------------------------------------- + + +def test_a1_a2_three_engines_in_one_run_under_sixty_seconds(composed, cold, tmp_path): + graph, first, store, exported, seconds = cold + assert seconds < 60 + routed = [node for node in graph.nodes if node.kernel == RULES] + assert len({node.params["engine_ref"] for node in routed}) == 3 + assert all(not first.nodes[node.id].hit for node in routed) + assert all(engine.calls for engine in composed.engines_by_binding.values()) + gates = [ + receipt for receipt in first.nodes.values() if receipt.receipt.get("outcome") + ] + assert gates and all(receipt.receipt["outcome"] == "pass" for receipt in gates) + assert all( + composed.registry.kernels.get(node.kernel).capabilities.role.value != "release" + for node in graph.nodes + ) + second, second_store, _ = _full_run( + tmp_path / "second", composed, graph, exported=exported + ) + assert _keys(first) == _keys(second) + assert _artifacts(first, store) == _artifacts(second, second_store) + reloaded = ContentStore(second_store.root, codecs=composed.registry.codecs) + warm, _, _ = _full_run( + tmp_path / "warm", + composed, + graph, + store=reloaded, + exported=exported, + resume="require", + ) + assert _misses(warm) == set() + assert _keys(warm) == _keys(first) + + +def test_declared_keys_equal_executed_keys(composed, cold): + graph, manifest, _, exported, _ = cold + assert _declared_keys(graph, composed, exported) == _keys(manifest) + + +def test_a4_inert_fields_execute_as_store_hits(composed, cold, tmp_path): + graph, before, store, exported, _ = cold + edited = replace( + graph, + nodes=tuple( + replace(node, description="reviewed", citation="toy") + for node in graph.nodes + ), + ) + after, _, _ = _full_run( + tmp_path, + composed, + edited, + store=_copied_store(store, tmp_path, composed), + exported=exported, + ) + assert _keys(after) == _keys(before) + assert _misses(after) == set() + + +@EXECUTED +@given(marker=st.integers(min_value=1, max_value=10**6)) +def test_a3_calibration_reference_edit_is_descendant_exact( + composed, cold, tmp_path_factory, marker +): + graph, before, store, _, _ = cold + root = tmp_path_factory.mktemp("a3") + spec = copy.deepcopy(composed.spec) + spec["resources"]["target_references"]["target_references"][0]["metadata"][ + "review_marker" + ] = marker + changed = replace(composed, spec=spec) + edited = changed.graph(extended=True) + after, _, _ = _full_run( + root, changed, edited, store=_copied_store(store, root, composed) + ) + expected = descendants(edited, {"nz.targets.compile"}) + assert _changed(before, after) == expected + assert _misses(after) == expected + calibration_side_hits = { + name + for name in after.nodes + if name.startswith( + ("nz.transport.", "nz.geo.", "nz.takeup.", "nz.rules.", "nz.encode.") + ) + } + assert calibration_side_hits and calibration_side_hits.isdisjoint(expected) + + +@EXECUTED +@given(byte=st.integers(min_value=0, max_value=255), position=st.integers(0, 30)) +def test_a5_one_rulespec_byte_rekeys_exactly_rules_nodes_and_descendants( + composed, cold, tmp_path_factory, byte, position +): + graph, before, store, _, _ = cold + root = tmp_path_factory.mktemp("a5") + fixture = composed.relocated(root / "fixture") + assert _declared_keys(fixture.graph(extended=True), fixture) == _declared_keys( + graph, composed + ) + shared = fixture.sources["rulespec_nz"] / SHARED_PATH + payload = bytearray(shared.read_bytes()) + index = position % len(payload) + payload[index] = byte if byte != payload[index] else (byte + 1) % 256 + shared.write_bytes(bytes(payload)) + changed = fixture.reprepare() + edited = changed.graph(extended=True) + after, _, _ = _full_run( + root, changed, edited, store=_copied_store(store, root, composed) + ) + bound = {node.id for node in edited.nodes if node.kernel == RULES} + assert len(bound) == 3 + expected = descendants(edited, bound) + assert _changed(before, after) == expected + assert _misses(after) == expected + + +@EXECUTED +@given(which=st.sampled_from(("gate", "holdout")), marker=st.integers(1, 10**6)) +def test_c2_gate_and_holdout_edits_move_no_calibration_key( + composed, cold, tmp_path_factory, which, marker +): + graph, before, store, exported, _ = cold + root = tmp_path_factory.mktemp("c2") + spec = copy.deepcopy(composed.spec) + if which == "gate": + gate = spec["resources"]["gates"]["gates"][0]["parameters"] + gate["maximum_max_to_median_ratio"] = 4.0 + marker + start, reuse = {"nz.gates.terminal"}, None + else: + spec["resources"]["holdout_references"]["target_references"][0]["metadata"][ + "review_marker" + ] = marker + start, reuse = {"nz.test.validate.compare"}, exported + changed = replace(composed, spec=spec) + edited = changed.graph(extended=True) + after, _, _ = _full_run( + root, + changed, + edited, + store=_copied_store(store, root, composed), + exported=reuse, + ) + expected = descendants(edited, start) + assert _changed(before, after) == expected + assert _misses(after) == expected + calibration = set(compile_graph(through(edited, CALIBRATION)).order) + assert calibration.isdisjoint(expected) + + +def test_holdout_and_family_semantics_have_no_calibration_ancestry(composed): + graph = composed.graph(extended=True) + family_ref = composed.registry.engine_refs["family_housing"] + family_columns = set(FAMILY_INPUTS) | set(PROGRAMS["family_housing"]["outputs"]) + ancestors = through(graph, CALIBRATION).nodes + for node in ancestors: + assert "nz_holdout_facts" not in node.sources + assert node.params.get("engine_ref") != family_ref + columns = {column for item in node.inputs for column in item.columns} + columns |= {item.column for item in node.outputs} + assert columns.isdisjoint(family_columns), node.id + # The hold-out source and the family engine are in the graph, elsewhere. + assert any("nz_holdout_facts" in node.sources for node in graph.nodes) + assert any(node.params.get("engine_ref") == family_ref for node in graph.nodes) + calibration = composed.sources["nz_calibration_facts"].read_text().splitlines() + holdout = composed.sources["nz_holdout_facts"].read_text().splitlines() + assert {json.loads(row)["aggregate_fact_key"] for row in calibration}.isdisjoint( + json.loads(row)["aggregate_fact_key"] for row in holdout + ) + + +class _IllegalOwner(KernelBase): + ref = "test.illegal_owner@1" + capabilities = Capabilities(Determinism.DETERMINISTIC) + + def run(self, context): + raise AssertionError("A refused declaration never reaches its kernel.") + + +@pytest.mark.parametrize( + "column", ["person_id", "person_household_id", "person_family_id"] +) +def test_b1_a_non_structural_node_cannot_own_a_structural_column( + composed, tmp_path, column +): + graph = composed.graph() + compile_graph(graph) + kernels = KernelRegistry() + for ref in composed.registry.kernels.refs(): + kernels.register(composed.registry.kernels.get(ref)) + kernels.register(_IllegalOwner()) + mutation = Node( + "nz.test.illegal", + _IllegalOwner.ref, + population="nz.terminal", + inputs=(Slice("person", ("age",)),), + outputs=(Owned("person", column, "int64"),), + ) + mutated = replace(graph, nodes=(*graph.nodes, mutation)) + with pytest.raises(NodeRejectedError, match="structural column"): + run_graph( + compile_graph(through(mutated, mutation.id)), + sources=composed.sources, + store=ContentStore(tmp_path / "store", codecs=composed.registry.codecs), + kernels=kernels, + ) + + +@pytest.mark.parametrize( + ("population", "entity", "column"), + [ + ("nz.open", "person", "toy_main_eligible"), + ("nz.as", "family", "input_rent"), + ], +) +def test_b1_a_second_owner_of_an_engine_or_family_input_column_is_refused( + composed, population, entity, column +): + graph = composed.graph() + mutation = Node( + "nz.test.second_owner", + _IllegalOwner.ref, + population=population, + inputs=(Slice("person", ("age",)),), + outputs=(Owned(entity, column, "float64"),), + ) + with pytest.raises(GraphError, match="is owned by both"): + compile_graph(replace(graph, nodes=(*graph.nodes, mutation))) + + +# --------------------------------------------------------------------------- +# Differentials +# --------------------------------------------------------------------------- + + +def test_routed_rules_nodes_equal_the_stock_single_engine_kernel( + composed, cold, tmp_path +): + graph, combined, store, _, _ = cold + for node in (node for node in graph.nodes if node.kernel == RULES): + reference = node.params["engine_ref"] + kernels = KernelRegistry() + for ref in composed.registry.kernels.refs(): + kernels.register(composed.registry.kernels.get(ref)) + kernels.register( + SimulateRulesKernel(reference, composed.registry.engines[reference]) + ) + subset = through(graph, node.id) + subset = replace( + subset, + nodes=tuple( + replace(item, kernel="simulate.rules@1") if item.id == node.id else item + for item in subset.nodes + ), + ) + direct_store = ContentStore(tmp_path / node.id, codecs=composed.registry.codecs) + direct = run_graph( + compile_graph(subset), + sources=composed.sources, + store=direct_store, + kernels=kernels, + ) + for owned in node.outputs: + key = (owned.entity, owned.column) + np.testing.assert_array_equal( + store.load_column(combined.nodes[node.id].artifacts[key]), + direct_store.load_column(direct.nodes[node.id].artifacts[key]), + ) + + +def test_prepared_create_inventory_equals_independent_composition(composed): + assert ( + prepare_create_outputs(composed.spec, composed.sources) + == composed.config.create_outputs + ) + + +def test_registry_schema_equals_the_create_frame_schema(composed, cold): + graph, manifest, store, _, _ = cold + frame = store.load_frame(manifest.nodes["nz.create"].frame_key) + assert composed.registry.schema == frame.schema + assert dict(composed.registry.nesting) == {"family": "household"} + + +# --------------------------------------------------------------------------- +# CLI driver +# --------------------------------------------------------------------------- + + +def _inventory(root: Path) -> dict: + return { + path.relative_to(root): path.read_bytes() + for path in sorted(root.rglob("*")) + if path.is_file() + } + + +def test_e3_require_refuses_a_cold_store_and_creates_nothing(composed, tmp_path): + out = tmp_path / "out" + with pytest.raises(StoreMissError, match="cold"): + execute_transport_graph( + composed.graph(), + composed.registry, + composed.sources, + out=out, + endpoints=composed.endpoints, + resume="require", + ) + assert not out.exists() + + +def test_cli_checkpoints_write_only_under_out_and_resume_warm(composed, tmp_path): + graph = composed.graph(extended=True) + out = tmp_path / "out" + options = { + "out": out, + "endpoints": composed.endpoints, + "pending_outputs": composed.pending_outputs, + } + inputs = _inventory(composed.root) + manifest = execute_transport_graph( + graph, + composed.registry, + composed.sources, + out=out, + endpoints=composed.endpoints, + pending_outputs=composed.pending_outputs, + ) + assert _inventory(composed.root) == inputs + assert set(manifest.nodes) == {node.id for node in graph.nodes} + assert json.loads((out / "graph.json").read_text()) == json.loads( + graph_to_json(graph) + ) + assert set(json.loads((out / "manifest.json").read_text())["nodes"]) == set( + manifest.nodes + ) + status = json.loads((out / "build-status.json").read_text()) + assert status["local_only"] is True + assert status["not_yet_produced"] == list(composed.pending_outputs) + assert {"graph", "skeleton_dataset", "calibration_diagnostics", "explorer"} <= set( + status["produced"] + ) + for name in ( + "checkpoint-geography.json", + "checkpoint-calibration.json", + "checkpoint-export-prepared.json", + "checkpoint-terminal-0.json", + "checkpoint-terminal-1.json", + "explorer.html", + "calibration-diagnostics.json", + EXPORT_FILENAME, + ): + assert (out / name).is_file(), name + direct, _, _ = _full_run( + tmp_path / "direct", composed, graph, exported=out / EXPORT_FILENAME + ) + assert _keys(direct) == _keys(manifest) + written = _inventory(out) + warm = execute_transport_graph( + graph, + composed.registry, + composed.sources, + out=out, + endpoints=composed.endpoints, + pending_outputs=composed.pending_outputs, + resume="require", + ) + assert _misses(warm) == set() + assert _keys(warm) == _keys(manifest) + assert _inventory(out)[Path(EXPORT_FILENAME)] == written[Path(EXPORT_FILENAME)] + # A deleted H5 is rewritten from the warm store and passes readback; a + # corrupted one under an unchanged descriptor is refused, not replaced. + (out / EXPORT_FILENAME).unlink() + rewritten = execute_transport_graph( + graph, composed.registry, composed.sources, **options + ) + assert rewritten.nodes["nz.export.readback"].receipt["outcome"] == "pass" + (out / EXPORT_FILENAME).write_bytes(b"not an h5 file") + with pytest.raises(ValueError, match="cannot be read"): + execute_transport_graph(graph, composed.registry, composed.sources, **options) + + +def test_cli_refuses_an_output_directory_that_overlaps_an_input(composed): + for out in (composed.sources["rulespec_nz"] / "out", composed.root): + with pytest.raises(ValueError, match="overlaps"): + execute_transport_graph( + composed.graph(), + composed.registry, + composed.sources, + out=out, + endpoints=composed.endpoints, + ) + assert not (composed.sources["rulespec_nz"] / "out").exists() + + +def test_cli_refuses_a_supplied_export_path(composed, tmp_path): + sources = {**composed.sources, "nz_exported_dataset": tmp_path / "elsewhere.h5"} + with pytest.raises(ValueError, match="written by the run"): + execute_transport_graph( + composed.graph(), + composed.registry, + sources, + out=tmp_path / "out", + endpoints=composed.endpoints, + ) + + +def test_prepare_builds_the_same_graph_as_the_fixture(composed): + prepared = prepare_transport_build( + composed.spec, + sources=composed.sources, + engines_by_binding=composed.engines_by_binding, + extensions=(composed.extended_config().extensions), + ) + assert graph_to_json(prepared.graph) == graph_to_json(composed.graph(extended=True)) + assert prepared.endpoints == composed.endpoints + assert prepared.pending_outputs == composed.pending_outputs + + +def test_cli_arguments(): + args = cli_module.parse_args( + ["--country", "xx", "--out", "o", "--source", "a=one", "--source", "b=two"] + ) + assert args.local_only is True + assert dict(args.source) == {"a": Path("one"), "b": Path("two")} + for argv in ( + ["--out", "o"], + ["--country", "xx", "--spec-dir", "d", "--out", "o"], + ["--country", "xx", "--out", "o", "--source", "a=1", "--source", "a=2"], + ["--country", "xx", "--out", "o", "--source", "missing-separator"], + ): + with pytest.raises(SystemExit): + cli_module.parse_args(argv) + + +# --------------------------------------------------------------------------- +# The packaged country spec: activation gaps refuse, named +# --------------------------------------------------------------------------- + +_GAPS = ( + "benefit_unit_rule.dependent_child.age_limit_years", + "placeholder reference precal_references.", + "placeholder reference target_references.", + "placeholder reference holdout_references.", + "receipt_contract.json", + "axiom_rules_bindings.engine.commit", + "axiom_rules_bindings.engine.wheel_sha256", + "scenario S4 rent_stock_factor", + "transport_graph.json", +) + + +def test_packaged_spec_refuses_with_every_activation_gap_named(): + spec = load_transport_spec("nz") + with pytest.raises(ValueError) as error: + validate_transport_activation(spec) + for gap in _GAPS: + assert gap in str(error.value), gap + config = TransportGraphConfig( + engine_refs={"x": "y"}, create_outputs=(Owned("person", "age", "int64"),) + ) + with pytest.raises(ValueError, match="not activated"): + compose_transport_graph(spec, config) + + +def test_packaged_spec_refuses_before_reading_any_source(tmp_path): + missing = tmp_path / "absent" + with pytest.raises(ValueError, match="not activated"): + prepare_transport_build( + load_transport_spec("nz"), sources={"rulespec_nz": missing} + ) + + +def test_cli_main_refuses_the_packaged_spec_and_writes_nothing(tmp_path, capsys): + out = tmp_path / "out" + assert cli_module.main(["--country", "nz", "--out", str(out)]) == 1 + message = capsys.readouterr().err + for gap in _GAPS: + assert gap in message, gap + assert not out.exists() + + +def test_packaged_holdout_references_are_disjoint_from_calibration_references(): + spec = load_transport_spec("nz") + + def names(resource): + rows = spec["resources"][resource]["target_references"] + return {row["name"] for row in rows} + + calibration = names("target_references") | names("precal_references") + assert names("holdout_references").isdisjoint(calibration) + + +# --------------------------------------------------------------------------- +# Source rules for the new modules +# --------------------------------------------------------------------------- + + +def test_transport_python_has_no_policy_literal_or_remote_path(): + """No target, threshold or rate value is assigned in the transport Python. + + ``test_transport_graph.py`` already refuses float literals and the word + for any country; this also refuses integer-valued policy assignments, + and keeps the driver's imports local: no country runtime, no network or + hub client and no staged-bundle publication. + """ + number = r"[-+]?\d+(?:\.\d+)?" + patterns = ( + rf"\b(?:target_value|threshold|max_age|age_limit\w*|\w+_threshold|\w+_rate)\s*[=:]\s*{number}\b", + rf"[\"'](?:target_value|threshold|max_age|\w+_threshold|\w+_rate|rate)[\"']\s*:\s*{number}\b", + rf"\brate\s*=\s*{number}\b", + ) + for path in sorted(TRANSPORT.glob("*.py")): + source = path.read_text() + for pattern in patterns: + assert re.search(pattern, source) is None, (path.name, pattern) + remote = ( + "uk_runtime", + "us_runtime", + "huggingface", + "requests", + "urllib", + "http", + "boto", + ) + for name in ("compose.py", "cli.py", "registry.py"): + tree = ast.parse((TRANSPORT / name).read_text()) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + modules = [alias.name for alias in node.names] + imported = modules + elif isinstance(node, ast.ImportFrom): + modules = [node.module or ""] + imported = [alias.name for alias in node.names] + else: + continue + assert not any(part in module for module in modules for part in remote), ( + name + ) + assert "publish_staged_bundle" not in imported, name diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_registry.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_registry.py index 40a89b8952..f0cb3d488e 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_registry.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_registry.py @@ -54,6 +54,7 @@ run_graph, ) from microcosm.graph.codecs import load_raw_bytes +from test_support.microcosm_build.transport_population import UNIT_RULE class _ToyEngine: @@ -156,7 +157,9 @@ def _fixture(tmp_path: Path): def test_registers_available_kernels_and_codecs(tmp_path): root, document, engines = _fixture(tmp_path) - result = build_transport_registry(document, root, engines_by_binding=engines) + result = build_transport_registry( + document, root, unit_rule=UNIT_RULE, engines_by_binding=engines + ) assert isinstance( result.kernels.get("simulate.rules_by_ref@1"), SimulateRulesByRefKernel ) @@ -215,6 +218,7 @@ def test_added_binding_and_prose_leave_existing_refs_and_hashes_unchanged( first = build_transport_registry( first_document, root, + unit_rule=UNIT_RULE, engines_by_binding={ name: engines[name] for name in ("main_benefit", "pension") }, @@ -226,7 +230,9 @@ def test_added_binding_and_prose_leave_existing_refs_and_hashes_unchanged( "start": "2028-01-01", "end": "2028-12-31", } - second = build_transport_registry(document, root, engines_by_binding=engines) + second = build_transport_registry( + document, root, unit_rule=UNIT_RULE, engines_by_binding=engines + ) assert all( second.engine_refs[name] == ref for name, ref in first.engine_refs.items() ) @@ -243,9 +249,13 @@ def test_added_binding_and_prose_leave_existing_refs_and_hashes_unchanged( @given(st.integers(min_value=1, max_value=30)) def test_adapter_configuration_changes_only_its_reference(tmp_path, offset): root, document, engines = _fixture(tmp_path) - first = build_transport_registry(document, root, engines_by_binding=engines) + first = build_transport_registry( + document, root, unit_rule=UNIT_RULE, engines_by_binding=engines + ) engines["housing"].offset = float(offset) - second = build_transport_registry(document, root, engines_by_binding=engines) + second = build_transport_registry( + document, root, unit_rule=UNIT_RULE, engines_by_binding=engines + ) assert first.engine_refs["housing"] != second.engine_refs["housing"] assert first.engine_refs["main_benefit"] == second.engine_refs["main_benefit"] assert first.engine_refs["pension"] == second.engine_refs["pension"] @@ -264,9 +274,13 @@ def test_rulespec_tree_bytes_change_refs_without_changing_kernel_hash(tmp_path, root, document, engines = _fixture(tmp_path) shared = root / "shared-resource.txt" shared.write_bytes(b"before") - first = build_transport_registry(document, root, engines_by_binding=engines) + first = build_transport_registry( + document, root, unit_rule=UNIT_RULE, engines_by_binding=engines + ) shared.write_bytes(b"after" + edit) - second = build_transport_registry(document, root, engines_by_binding=engines) + second = build_transport_registry( + document, root, unit_rule=UNIT_RULE, engines_by_binding=engines + ) assert all(first.engine_refs[name] != second.engine_refs[name] for name in engines) assert first.kernels.implementation_hash( "simulate.rules_by_ref@1" @@ -287,6 +301,7 @@ def test_native_engines_use_pinned_refs_before_relation_compilation( def check(engine, entity): assert engine._pinned_tree_sha256 is not None assert engine._schema == NZ_SCHEMA + assert engine._nesting == {"family": "household"} assert engine._rulespec_roots == (root.resolve(),) assert engine._output_dtypes == "graph" assert engine._periods == { @@ -301,7 +316,7 @@ def metadata(engine, name): monkeypatch.setattr(registry_module, "assert_no_relations", check) monkeypatch.setattr(AxiomEngine, "variable_metadata", metadata) monkeypatch.setattr(AxiomEngine, "graph_dtype", lambda engine, name: "float64") - result = build_transport_registry(document, root) + result = build_transport_registry(document, root, unit_rule=UNIT_RULE) assert len(checks) == 3 assert all(isinstance(engine, AxiomEngine) for engine in result.engines.values()) for row in document["bindings"]: @@ -313,7 +328,7 @@ def metadata(engine, name): @pytest.mark.parametrize( - "problem", ["root", "module", "schema", "periods", "names", "dtypes"] + "problem", ["root", "module", "schema", "periods", "names", "nesting", "dtypes"] ) def test_injected_axiom_configuration_must_match_the_binding(tmp_path, problem): root, document, _ = _fixture(tmp_path) @@ -325,6 +340,7 @@ def test_injected_axiom_configuration_must_match_the_binding(tmp_path, problem): rulespec_roots=(root,), periods=periods, entity_names=document["entity_names"], + nesting={"family": "household"}, output_dtypes="graph", ) for row in document["bindings"] @@ -340,10 +356,14 @@ def test_injected_axiom_configuration_must_match_the_binding(tmp_path, problem): engine._periods = None elif problem == "names": engine._entity_names = {"person": "DifferentPerson"} + elif problem == "nesting": + engine._nesting = {} else: engine._output_dtypes = "native" with pytest.raises(ValueError, match="rules binding 'main_benefit'"): - build_transport_registry(document, root, engines_by_binding=engines) + build_transport_registry( + document, root, unit_rule=UNIT_RULE, engines_by_binding=engines + ) @pytest.mark.parametrize( @@ -354,7 +374,9 @@ def test_missing_pin_names_the_activation_gap(tmp_path, section, field): root, document, engines = _fixture(tmp_path) document[section][field] = None with pytest.raises(ValueError, match=rf"axiom_rules_bindings\.{section}\.{field}"): - build_transport_registry(document, root, engines_by_binding=engines) + build_transport_registry( + document, root, unit_rule=UNIT_RULE, engines_by_binding=engines + ) @pytest.mark.parametrize("problem", ["hash", "path", "period", "duplicate", "entity"]) @@ -372,7 +394,9 @@ def test_malformed_binding_is_refused(tmp_path, problem): else: first["engine_entity"] = "Family" with pytest.raises(ValueError): - build_transport_registry(document, root, engines_by_binding=engines) + build_transport_registry( + document, root, unit_rule=UNIT_RULE, engines_by_binding=engines + ) @pytest.mark.parametrize("problem", ["root", "schema", "relations", "identity", "keys"]) @@ -392,7 +416,9 @@ def test_injected_adapter_contract_is_checked(tmp_path, problem): else: engines.pop("housing") with pytest.raises((TypeError, ValueError, NotImplementedError)): - build_transport_registry(document, root, engines_by_binding=engines) + build_transport_registry( + document, root, unit_rule=UNIT_RULE, engines_by_binding=engines + ) @pytest.mark.parametrize("value", [float("nan"), {1: "key"}, lambda: None]) @@ -400,7 +426,9 @@ def test_adapter_identity_uses_closed_canonical_data(tmp_path, value): root, document, engines = _fixture(tmp_path) engines["housing"].cache_identity = lambda: {"configuration": value} with pytest.raises((TypeError, ValueError)): - build_transport_registry(document, root, engines_by_binding=engines) + build_transport_registry( + document, root, unit_rule=UNIT_RULE, engines_by_binding=engines + ) def _frame(document: Mapping) -> Frame: @@ -460,7 +488,9 @@ def _node( def test_three_engines_run_together_and_equal_single_engine_dispatch(tmp_path): root, document, engines = _fixture(tmp_path) - result = build_transport_registry(document, root, engines_by_binding=engines) + result = build_transport_registry( + document, root, unit_rule=UNIT_RULE, engines_by_binding=engines + ) result.codecs.register_bytes("raw-bytes-v1", load_raw_bytes) result.kernels.register(_Population()) population_path = tmp_path / "population.json" diff --git a/test_support/microcosm_build/transport_composed.py b/test_support/microcosm_build/transport_composed.py new file mode 100644 index 0000000000..8ffcd300de --- /dev/null +++ b/test_support/microcosm_build/transport_composed.py @@ -0,0 +1,1166 @@ +"""An engine-free toy country spec for the composed transport skeleton. + +Everything here is synthetic. The donor is the frame package's six invented +US support records (donor support, never destination microdata). Every +Chronicle fact, reference, rate, threshold, rule program and pin is made up. +The three rules engines are pure-Python adapters over a toy RuleSpec tree of +JSON programs; they share the one ``simulate.rules_by_ref@1`` router. + +The skeleton declaration (``transport_graph``) mirrors the transport layout: +CREATE, the keep-all ``nz.open`` version (transport, geography, encoding, +the person rules engines, the receipt layer, targets and problem), the +member-less ``nz.calibrate`` REWEIGHT, a ``nz.as`` FILTER off calibration +holding the family group encoding, and a ``nz.terminal`` FILTER (diagnostics, +gates, export and package). + +``toy_extension`` stands in for the later entitlement package's extension: +a scenario-like FILTER under ``nz.as`` running the third (family) engine, +and a validation FILTER whose comparison reads the hold-out facts. It is a +test probe of the extension point, not an entitlement chain. +""" + +from __future__ import annotations + +import json +import shutil +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, replace +from hashlib import sha256 +from math import fsum +from pathlib import Path + +import numpy as np +import pandas as pd + +from microcosm.build.transport.artifact_types import COMPARISON_TYPE +from microcosm.build.transport.codecs import ( + DONOR_SOURCE_CODEC, + FACTS_SOURCE_CODEC, + RULESPEC_SOURCE_CODEC, +) +from microcosm.build.transport.compose import ( + TransportGraphConfig, + compose_transport_graph, + transport_endpoints, + transport_pending_outputs, + transport_resource, +) +from microcosm.build.transport.registry import ( + TransportRegistry, + build_transport_registry, +) +from microcosm.build.transport.terminal_kernels import EXPORT_SOURCE_CODEC +from microcosm.frame import EntitySchema, ExportContract, Frame, VariableMetadata +from microcosm.frame.adapters.axiom import NZ_SCHEMA +from microcosm.frame.concept_mapping import ( + ConceptMapping, + Identity, + InputBinding, + InputDeclaration, +) +from microcosm.frame.concepts import CONCEPTS, ContentBasis, concept_schema_sha256 +from microcosm.frame.input_closure import ClosureEntry, InputClosure, UndeterminedPolicy +from microcosm.graph import ( + ArtifactOutput, + Graph, + Node, + Owned, + Slice, + StructuralDelta, +) +from microcosm.graph.canonical import canonical_json +from test_support.microcosm_build.transport_graph import ( + canonical_text, + reference_document, + reference_row, + toy_fact, + write_facts, +) +from test_support.microcosm_build.transport_population import ( + UNIT_RULE, + create_node, + direct_population, + donor_pin, + write_population_sources, +) + +COUNTRY = "nz" +PERIOD = "2026-27" +ENTITIES = ("person", "household", "family") +MAIN_PATH = "toy/main.yaml" +SUPER_PATH = "toy/super.yaml" +FAMILY_PATH = "toy/family.yaml" +#: A tree file no binding names: changing it changes the tree digest only. +SHARED_PATH = "shared.txt" + +#: The three invented rule programs, one per pure-Python engine. +PROGRAMS = { + "main_benefit": { + "path": MAIN_PATH, + "entity": "person", + "outputs": { + "toy_main_eligible": { + "input": "input_age", + "operation": "at_most", + "threshold": 40, + "dtype": "int64", + }, + "toy_main_payment": { + "input": "input_income", + "operation": "linear", + "coefficient": 0.05, + "offset": 2.0, + "dtype": "float64", + }, + }, + }, + "pension": { + "path": SUPER_PATH, + "entity": "person", + "outputs": { + "toy_pension_eligible": { + "input": "input_pension_age", + "operation": "at_least", + "threshold": 41, + "dtype": "int64", + }, + "toy_pension_payment": { + "input": "input_pension_age", + "operation": "linear", + "coefficient": 0.25, + "offset": 0.0, + "dtype": "float64", + }, + }, + }, + "family_housing": { + "path": FAMILY_PATH, + "entity": "family", + "outputs": { + "toy_family_weekly_amount": { + "input": "input_rent", + "operation": "linear", + "coefficient": 0.25, + "offset": 0.0, + "dtype": "float64", + }, + "toy_family_eligible": { + "input": "input_rent", + "operation": "at_least", + "threshold": 0, + "dtype": "int64", + }, + "toy_family_asset_test": { + "input": "input_assets", + "operation": "at_most", + "threshold": 1000, + "dtype": "int64", + }, + }, + }, +} +#: The family engine's inputs, encoded on the ``nz.as`` branch. +FAMILY_INPUTS = ("input_assets", "input_rent") +#: (name, entity, measure column) of the toy calibration targets. +CALIBRATION_REFERENCES = ( + ("toy_calibration_age", "person", "age"), + ("toy_calibration_units", "household", "n_family_units"), +) +_CALIBRATION_FACTORS = {"toy_calibration_age": 1.03, "toy_calibration_units": 0.97} +#: Dtypes the export descriptor writer round-trips. +_EXPORTABLE = frozenset({"bool", "int32", "int64", "float32", "float64", "string"}) + + +class ToyRulesEngine: + """A pure-Python ``RulesEngine`` over one JSON program of the toy tree. + + The program (inputs, coefficients, thresholds and dtypes) is read from + the tree on every call, so the outputs depend on the pinned bytes. + ``calls`` records each materialization so a test can see all three + engines ran; it affects neither outputs nor identity. + """ + + def __init__(self, root: Path, binding: Mapping): + self.rulespec_root = root + self.binding = binding + self.calls: list[tuple[tuple[str, ...], int | str]] = [] + + def program(self) -> dict: + return json.loads( + (self.rulespec_root / self.binding["rulespec_path"]).read_text() + ) + + def cache_identity(self) -> Mapping: + return {"formula": "synthetic-linear-and-threshold-v1"} + + def assert_no_relations(self, entity: str) -> None: + if self.program()["relations"]: + raise NotImplementedError("The toy adapter has no relation support.") + + def variable_metadata(self, name: str) -> VariableMetadata: + row = self.program()["outputs"][name] + kind = "float" if row["dtype"] == "float64" else "int" + return VariableMetadata(name, self.binding["entity"], kind, "year") + + def variables(self) -> Sequence[str]: + return tuple( + sorted({row["input"] for row in self.program()["outputs"].values()}) + ) + + def entity_schema(self) -> EntitySchema: + return NZ_SCHEMA + + def materialize( + self, bundle: Frame, variables: Sequence[str], period: int | str + ) -> Mapping[str, np.ndarray]: + self.calls.append((tuple(variables), period)) + table = bundle.table(self.binding["entity"]) + result = {} + for name in variables: + row = self.program()["outputs"][name] + values = table[row["input"]].to_numpy(dtype=np.float64) + if row["operation"] == "linear": + values = values * row["coefficient"] + row["offset"] + elif row["operation"] == "at_least": + values = np.where(values >= row["threshold"], 1, -1) + elif row["operation"] == "at_most": + values = np.where(values <= row["threshold"], 1, -1) + else: + raise ValueError(f"Unknown toy operation {row['operation']!r}.") + result[name] = values.astype(row["dtype"]) + return result + + def export_contract(self) -> ExportContract: + return ExportContract.empty() + + def write_dataset(self, bundle: Frame, path: str | Path, period: int | str) -> None: + raise NotImplementedError("The skeleton writes its H5 from the descriptor.") + + +def _select(resource: str, encoding: str = "json", *path: str) -> dict: + return {"resource": resource, "path": list(path), "encoding": encoding} + + +def _document(name: str, resource: str) -> dict: + """A whole resource as canonical JSON plus that resource's digest.""" + return { + name: _select(resource), + f"{name}_sha256": _select(resource, "sha256"), + } + + +def _edge(alias: str, producer: str, artifact: str | None = None) -> dict: + """An artifact input; its type is the producer's declared output type.""" + return { + "name": alias, + "producer": producer, + "artifact": alias if artifact is None else artifact, + } + + +def _slice(entity: str, columns: Sequence[str]) -> dict: + return {"entity": entity, "columns": list(columns)} + + +def _owned(entity: str, column: str, dtype: str, *, rewrite: bool = False) -> dict: + return {"entity": entity, "column": column, "dtype": dtype, "rewrite": rewrite} + + +def _node(id: str, kernel: str, **fields) -> dict: + return {"id": id, "kernel": kernel, **fields} + + +def _references(rows) -> dict: + document = reference_document(rows) + document["country"] = COUNTRY + return document + + +def _fact(name: str, value: float, entity: str = "person") -> dict: + row = toy_fact(name, value, entity=entity) + row["geography"].update(id="NZ", name="Synthetic destination") + return row + + +def _concept_mapping() -> ConceptMapping: + bindings = tuple( + InputBinding( + engine_input=name, + engine_entity=entity, + concepts=(concept,), + transform=Identity(), + relation="exact", + note="Invented fixture input, not a statutory definition.", + group_rule=group, + module=path, + canonical_input=f"toy:{entity}#input.{name}", + ) + for name, entity, concept, group, path in ( + ("input_age", "Person", "fact:person.age", None, MAIN_PATH), + ( + "input_income", + "Person", + "fact:person.employment_income", + None, + MAIN_PATH, + ), + ("input_pension_age", "Person", "fact:person.age", None, SUPER_PATH), + ( + "input_assets", + "Family", + "fact:person.liquid_financial_assets", + "sum_over_members", + FAMILY_PATH, + ), + ( + "input_rent", + "Family", + "fact:household.rent", + "allocate_to_reference_unit", + FAMILY_PATH, + ), + ) + ) + reads = {concept for binding in bindings for concept in binding.reads} + return ConceptMapping( + engine="axiom:toy", + engine_version="synthetic-v1", + entity_correspondence={"person": "Person", "household": "Household"}, + input_declaration=InputDeclaration.USAGE_INFERRED, + bindings=bindings, + unmapped={ + item.id: "Outside the invented fixture mapping." + for item in CONCEPTS + if item.id not in reads + }, + ) + + +def _calibration_totals(frame: Frame) -> list[tuple[str, str, float]]: + """Design-weighted totals of the target columns, times their factors. + + Read from the CREATE frame directly: age is never transported, and each + benefit unit lies in one household, so a household's unit count is its + number of unit rows. + """ + households = frame.table("household")["household_id"].to_numpy() + weights = pd.Series(frame.resolve_weights("household").values, index=households) + person = frame.table("person") + age = fsum( + person["age"].to_numpy(dtype=np.float64) + * person["person_household_id"].map(weights).to_numpy() + ) + units = ( + frame.table("family")["family_household_id"] + .value_counts() + .reindex(households, fill_value=0) + ) + totals = { + "toy_calibration_age": age, + "toy_calibration_units": fsum( + units.to_numpy(dtype=np.float64) * weights.to_numpy() + ), + } + entities = {name: entity for name, entity, _ in CALIBRATION_REFERENCES} + return [ + (name, entities[name], round(value * _CALIBRATION_FACTORS[name], 6)) + for name, value in totals.items() + ] + + +def _geography() -> tuple[dict, dict]: + code = { + "kind": "code", + "source": "synthetic-crosswalk", + "vintage": "toy-2026", + "relation": "exact", + } + columns = { + name: dict(code) for name in ("area", "ta", "region", "as_area", "as_area_alt") + } + columns["population"] = { + "kind": "weight", + "source": "synthetic-facts", + "basis": "persons", + } + support = { + "version": 1, + "level": "atomic", + "code_system": "toy", + "vintage": "toy-2026", + "columns": columns, + "share_tolerance": 1e-9, + "population_geography_level": "territorial_authority", + "population_code_column": "ta", + "rows": [ + { + "target": f"toy_population_{index}", + "codes": { + "area": f"TA{index}:1", + "ta": f"TA{index}", + "region": f"R{index}", + "as_area": "1", + "as_area_alt": "1", + }, + "population_share": 1.0, + } + for index in (1, 2) + ], + } + assignment = { + "version": 1, + "identity": ["household_id"], + "stream": ["sha256-u53-v1", "toy-geography", 0, 21], + "outputs": { + "area": "toy_atomic_area", + "system": "toy_area_system", + "basis": "toy_area_basis", + }, + "systems": [ + { + "id": "toy-atomic", + "level": "atomic", + "code_system": "toy", + "vintage": "toy-2026", + "source": "unused-toy-support", + "selector": {}, + "constraints": [], + "observed_area": None, + "stages": [{"level": "area", "weight": "population"}], + "layers": [ + { + "input": name, + "output": name, + "vintage": "toy-2026", + "relation": "exact", + "source": "synthetic-crosswalk", + } + for name in ("ta", "region", "as_area", "as_area_alt") + ], + } + ], + } + return support, assignment + + +def _blueprint(create_outputs: tuple[Owned, ...]) -> dict: + """The toy ``transport_graph`` declaration.""" + raw = { + entity: [item.column for item in create_outputs if item.entity == entity] + for entity in ENTITIES + } + raw_slices = [_slice(entity, columns) for entity, columns in raw.items()] + attributes = [ + _owned("family", "family_type", "string"), + _owned("family", "n_dependent_children", "int64"), + _owned("family", "is_sole_parent", "bool"), + _owned("household", "n_family_units", "int64"), + ] + encoded = [ + _owned("person", "input_age", "int64"), + _owned("person", "input_income", "float64"), + _owned("person", "input_pension_age", "int64"), + ] + main = [ + _owned("person", name, row["dtype"]) + for name, row in PROGRAMS["main_benefit"]["outputs"].items() + ] + pension = [ + _owned("person", name, row["dtype"]) + for name, row in PROGRAMS["pension"]["outputs"].items() + ] + receipts = [ + _owned("person", name, "bool") for name in ("receives_main", "receives_pension") + ] + geography = [ + _owned("household", name, "string") + for name in ( + "toy_atomic_area", + "toy_area_system", + "toy_area_basis", + "ta", + "region", + "as_area", + "as_area_alt", + ) + ] + open_columns = { + "person": raw["person"] + + [row["column"] for row in encoded + main + pension + receipts], + "household": raw["household"] + + ["n_family_units"] + + [row["column"] for row in geography], + "family": raw["family"] + + [row["column"] for row in attributes if row["entity"] == "family"], + } + # Nullable donor pointer columns stay in the population, but the export + # descriptor writer round-trips only plain dtypes. + unsupported = { + item.column for item in create_outputs if item.dtype not in _EXPORTABLE + } + population = [ + _slice(entity, [column for column in columns if column not in unsupported]) + for entity, columns in open_columns.items() + ] + rule = _document("unit_rule", "benefit_unit_rule") + encode = { + **_document("mapping", "concept_mapping"), + **_document("closure", "axiom_input_closure"), + } + support = _edge("toy-atomic", "nz.geo.support", "support") + geo = { + "definition": _select("geography_assignment"), + "stream": _select("geography_assignment", "value", "stream"), + } + nodes = [ + _node( + "nz.create", + "transport.create@1", + structural="create", + sources=["donor_pool", "nz_calibration_facts"], + params={ + "donor_source": "donor_pool", + "facts_source": "nz_calibration_facts", + "donor_sha256": _select("donor_pin", "value", "sha256"), + "donor_size": _select("donor_pin", "value", "size"), + "donor_pin_sha256": _select("donor_pin", "sha256"), + **rule, + **_document("mass_reference", "mass_references"), + "country": COUNTRY, + "concept_schema_sha256": _select( + "content", "value", "concept_schema_sha256" + ), + "seed_stream": _select("content", "value", "seed_stream"), + }, + ), + _node( + "nz.open", + "transport.boundary@1", + structural="filter", + base="nz.create", + inputs=[_slice("person", ["age"])], + ), + _node( + "nz.facts.precal", + "targets.compile@1", + population="nz.open", + sources=["nz_calibration_facts"], + params={"country": COUNTRY, **_document("references", "precal_references")}, + ), + _node( + "nz.transport.currency", + "transport.currency@1", + population="nz.open", + inputs=[_slice("person", ["interest_income"])], + outputs=[_owned("person", "interest_income", "float64", rewrite=True)], + params={ + "columns": _select("currency_bridge", "json", "columns"), + "rate": _select("currency_bridge", "value", "rate"), + "currency_sha256": _select("currency_bridge", "sha256"), + }, + ), + _node( + "nz.geo.support", + "geography.support_from_facts@1", + population="nz.open", + params={ + "definition": _select("as_area_crosswalk"), + "definition_sha256": _select("as_area_crosswalk", "sha256"), + "system": "toy-atomic", + }, + artifact_inputs=[_edge("surface", "nz.facts.precal")], + ), + _node( + "nz.geo.assign", + "geography.assign_atomic@1", + population="nz.open", + inputs=[_slice("household", [raw["household"][0]])], + outputs=geography[:3], + params=geo, + artifact_inputs=[support], + ), + _node( + "nz.geo.derive", + "geography.derive@1", + population="nz.open", + inputs=[_slice("household", [row["column"] for row in geography[:3]])], + outputs=geography[3:], + params=geo, + artifact_inputs=[support], + ), + _node( + "nz.geo.gate", + "geography.gate@1", + population="nz.open", + inputs=[_slice("household", [row["column"] for row in geography])], + params=geo, + artifact_inputs=[support], + ), + ] + for short, entity, column, aggregate in ( + ("employment_income", "person", "employment_income", None), + ("rent", "household", "rent", None), + ("liquid_assets", "person", "liquid_financial_assets", "household"), + ): + inputs = [_slice(entity, [column])] + params = { + "entity": entity, + "column": column, + "bands": _select("quantile_maps", "json", short), + "bands_sha256": _select("quantile_maps", "sha256"), + "interpolation": _select("quantile_maps", "json", "interpolation"), + } + if entity == "household": + inputs.append(_slice("person", ["age"])) + if aggregate is not None: + params["aggregate_entity"] = aggregate + inputs.append(_slice("household", [raw["household"][0]])) + nodes.append( + _node( + f"nz.transport.qmap.{short}", + "transport.quantile_map@1", + population="nz.open", + inputs=inputs, + outputs=[_owned(entity, column, "float64", rewrite=True)], + params=params, + artifact_inputs=[_edge("surface", "nz.facts.precal")], + ) + ) + terminal_inputs = [ + _edge("problem", "nz.targets.problem"), + _edge("solution", "nz.calibrate"), + _edge("diagnostics", "nz.calibration.diagnostics"), + _edge("surface", "nz.targets.compile"), + ] + nodes += [ + _node( + "nz.units.attributes", + "transport.unit_attributes@1", + population="nz.open", + inputs=raw_slices, + outputs=attributes, + params=rule, + ), + _node( + "nz.encode.person", + "concepts.encode@1", + population="nz.open", + inputs=raw_slices[:2], + outputs=encoded, + params={**encode, "rulespec_paths": [MAIN_PATH, SUPER_PATH]}, + ), + _node( + "nz.rules.main_benefits", + "simulate.rules_by_ref@1", + population="nz.open", + sources=["rulespec_nz"], + inputs=[_slice("person", ["input_age", "input_income"])], + outputs=main, + rules_binding="main_benefit", + ), + _node( + "nz.rules.pension", + "simulate.rules_by_ref@1", + population="nz.open", + sources=["rulespec_nz"], + inputs=[_slice("person", ["input_pension_age"])], + outputs=pension, + rules_binding="pension", + ), + _node( + "nz.takeup.assign", + "takeup.assign@1", + population="nz.open", + inputs=[ + _slice( + "person", + ["take_up_seed"] + [row["column"] for row in main + pension], + ) + ], + outputs=receipts, + params={ + "receipt_contract": _select("receipt_contract", "json", "receipts"), + "receipt_contract_sha256": _select("receipt_contract", "sha256"), + }, + ), + _node( + "nz.targets.compile", + "targets.compile@1", + population="nz.open", + sources=["nz_calibration_facts"], + params={"country": COUNTRY, **_document("references", "target_references")}, + ), + _node( + "nz.targets.problem", + "targets.problem@1", + population="nz.open", + inputs=population, + params={"entities": list(ENTITIES), "weight_entity": "household"}, + artifact_inputs=[_edge("surface", "nz.targets.compile")], + ), + _node( + "nz.calibrate", + "calibrate.ordered_adam@1", + structural="reweight", + base="nz.open", + mass="free", + inputs=[_slice("household", ["rent"])], + weights={"entity": "household", "to_kind": "calibrated", "mass": "free"}, + params={ + name: _select("calibration", "value", name) + for name in ( + "epochs", + "learning_rate", + "mass", + "max_weight_ratio", + "weight_anchor", + ) + }, + artifact_inputs=[_edge("problem", "nz.targets.problem")], + ), + _node( + "nz.as", + "transport.boundary@1", + structural="filter", + base="nz.calibrate", + inputs=[_slice("person", ["age"])], + ), + _node( + "nz.as.inputs", + "concepts.encode_groups@1", + population="nz.as", + inputs=raw_slices, + outputs=[_owned("family", name, "float64") for name in FAMILY_INPUTS], + params={ + **encode, + **rule, + "engine_entity": "Family", + "rulespec_paths": [FAMILY_PATH], + }, + ), + _node( + "nz.terminal", + "transport.boundary@1", + structural="filter", + base="nz.calibrate", + inputs=[_slice("person", ["age"])], + ), + _node( + "nz.calibration.diagnostics", + "diagnostics.calibration@1", + population="nz.terminal", + inputs=[_slice("household", ["rent"])], + params={"weight_entity": "household"}, + artifact_inputs=[ + _edge("problem", "nz.targets.problem"), + _edge("solution", "nz.calibrate"), + _edge("result", "nz.calibrate"), + _edge("surface", "nz.targets.compile"), + ], + ), + _node( + "nz.gates.terminal", + "gates.battery@1", + population="nz.terminal", + inputs=population, + params={ + "country": COUNTRY, + **_document("gates", "gates"), + "phase": "terminal", + "release_candidate": False, + "entities": list(ENTITIES), + "weight_entity": "household", + }, + artifact_inputs=terminal_inputs, + ), + _node( + "nz.export.prepare", + "export.prepare@1", + population="nz.terminal", + inputs=population, + params={ + "entities": list(ENTITIES), + "weight_entity": "household", + "time_period": _select("calibration", "value", "time_period"), + }, + artifact_inputs=[_edge("gate_report", "nz.gates.terminal")], + ), + _node( + "nz.export.readback", + "export.readback@1", + population="nz.terminal", + sources=["nz_exported_dataset"], + artifact_inputs=[_edge("export_descriptor", "nz.export.prepare")], + ), + _node( + "nz.package", + "transport.package@1", + population="nz.terminal", + params={"bindings": {"prepared": "engine_refs"}}, + artifact_inputs=[ + _edge("export_readback", "nz.export.readback"), + _edge("gate_report", "nz.gates.terminal"), + _edge("diagnostics", "nz.calibration.diagnostics"), + _edge("surface", "nz.targets.compile"), + ], + ), + ] + return { + "schema_version": 1, + "country": COUNTRY, + "description": "Synthetic toy skeleton; not a country declaration.", + "sources": [ + {"name": name, "codec": codec} + for name, codec in ( + ("donor_pool", DONOR_SOURCE_CODEC), + ("nz_calibration_facts", FACTS_SOURCE_CODEC), + ("nz_holdout_facts", FACTS_SOURCE_CODEC), + ("rulespec_nz", RULESPEC_SOURCE_CODEC), + ("nz_exported_dataset", EXPORT_SOURCE_CODEC), + ) + ], + "nodes": nodes, + "endpoints": { + "geography": "nz.geo.gate", + "calibration": "nz.calibrate", + "terminal": ["nz.export.prepare", "nz.package"], + }, + "pending_outputs": [ + "Toy family-entitlement chain and its diagnostics (a later package)" + ], + } + + +def toy_extension(spec, config, skeleton: Graph) -> tuple[Node, ...]: + """Test stand-in for the entitlement package's extension (see module doc).""" + branch = Node( + "nz.test.scn", + "transport.boundary@1", + structural=StructuralDelta.FILTER, + base="nz.as", + inputs=(Slice("person", ("age",)),), + ) + outputs = PROGRAMS["family_housing"]["outputs"] + rules = Node( + "nz.test.scn.rules", + "simulate.rules_by_ref@1", + population=branch.id, + sources=("rulespec_nz",), + # A family-only engine node still slices one person data column: + # the kernel context holds only the entities a node reads or owns. + inputs=(Slice("person", ("age",)), Slice("family", FAMILY_INPUTS)), + outputs=tuple( + Owned("family", name, row["dtype"]) for name, row in outputs.items() + ), + params={ + "engine_ref": config.engine_refs["family_housing"], + "period": PERIOD, + "variables": tuple(outputs), + }, + ) + validate = Node( + "nz.test.validate", + "transport.boundary@1", + structural=StructuralDelta.FILTER, + base="nz.calibrate", + inputs=(Slice("person", ("age",)),), + ) + holdout = transport_resource(spec, "holdout_references") + compare = Node( + "nz.test.validate.compare", + "takeup.compare@1", + population=validate.id, + sources=("nz_holdout_facts",), + inputs=(Slice("person", ("receives_main",)), Slice("household", ("rent",))), + params={ + "country": COUNTRY, + "references": canonical_text(holdout), + "references_sha256": sha256(canonical_json(holdout)).hexdigest(), + "entities": ("person", "household"), + "weight_entity": "household", + }, + artifact_outputs=(ArtifactOutput("comparison", COMPARISON_TYPE),), + ) + return branch, rules, validate, compare + + +def write_rulespec_tree(root: Path) -> list[dict]: + """Write the toy tree; return its binding rows (with module digests).""" + rows = [] + for name, program in PROGRAMS.items(): + path = root / program["path"] + path.parent.mkdir(parents=True, exist_ok=True) + payload = canonical_json({"outputs": program["outputs"], "relations": []}) + path.write_bytes(payload) + rows.append( + { + "id": name, + "rulespec_path": program["path"], + "sha256": sha256(payload).hexdigest(), + "entity": program["entity"], + "engine_entity": program["entity"].capitalize(), + "period": PERIOD, + "variables": list(program["outputs"]), + } + ) + (root / SHARED_PATH).write_text("synthetic shared rule resource\n") + return rows + + +@dataclass(frozen=True) +class ComposedFixture: + """A written toy package, its inputs and one prepared registry.""" + + root: Path + spec: dict + config: TransportGraphConfig + registry: TransportRegistry + sources: Mapping[str, Path] + engines_by_binding: Mapping[str, ToyRulesEngine] + + def graph(self, *, extended: bool = False) -> Graph: + return compose_transport_graph( + self.spec, self.extended_config() if extended else self.config + ) + + def extended_config(self) -> TransportGraphConfig: + return replace(self.config, extensions=(toy_extension,)) + + @property + def endpoints(self) -> dict: + return transport_endpoints(self.spec) + + @property + def pending_outputs(self) -> tuple[str, ...]: + return transport_pending_outputs(self.spec) + + def reprepare(self, spec: dict | None = None) -> ComposedFixture: + """A fresh registry over the current tree bytes and ``spec``.""" + spec = self.spec if spec is None else spec + bindings = spec["resources"]["axiom_rules_bindings"] + root = self.sources["rulespec_nz"] + engines = {row["id"]: ToyRulesEngine(root, row) for row in bindings["bindings"]} + registry = build_transport_registry( + bindings, root, unit_rule=UNIT_RULE, engines_by_binding=engines + ) + config = replace(self.config, engine_refs=registry.engine_refs) + return replace( + self, + spec=spec, + config=config, + registry=registry, + engines_by_binding=engines, + ) + + def relocated(self, root: Path) -> ComposedFixture: + """A byte copy of the inputs under ``root``, with a fresh registry. + + Source identities are path independent, so the copy's node keys + equal the original's until a copied input is edited. + """ + shutil.copytree(self.root, root) + sources = { + name: root / path.relative_to(self.root) + for name, path in self.sources.items() + } + return replace(self, root=root, sources=sources).reprepare() + + def source_mapping(self, exported: Path | None = None) -> dict[str, Path]: + values = dict(self.sources) + if exported is not None: + values["nz_exported_dataset"] = exported + return values + + +def make_composed_fixture(root: Path) -> ComposedFixture: + """Write the toy package's inputs below ``root`` and prepare one run.""" + root.mkdir(parents=True, exist_ok=True) + population = write_population_sources(root / "inputs") + frame, _ = direct_population(population) + create_outputs = create_node(population).outputs + mapping = _concept_mapping() + rules_root = root / "rulespec" + rows = write_rulespec_tree(rules_root) + bindings = { + "engine": {"commit": "a" * 40, "wheel_sha256": "b" * 64}, + "rulespec": {"commit": "c" * 40}, + "entity_names": {entity: entity.capitalize() for entity in ENTITIES}, + "periods": { + PERIOD: {"kind": "tax_year", "start": "2026-04-01", "end": "2027-03-31"} + }, + "bindings": rows, + } + closure = InputClosure( + country=COUNTRY, + content_basis=ContentBasis.TRANSPORT, + mapping_engine=mapping.engine, + mapping_engine_version=mapping.engine_version, + surface_rulespec_commit="synthetic", + surface_engine_repository="toy/engine", + surface_engine_commit="synthetic", + modules={row["rulespec_path"]: row["sha256"] for row in rows}, + engine_optional_evidence="The toy adapters declare no optional inputs.", + undetermined=UndeterminedPolicy( + action="refuse", + reason="Invented refusal policy.", + alternatives=("eligible", "not_eligible"), + new_default=False, + ), + knobs=(), + entries=tuple( + ClosureEntry( + module=binding.module, + entity=binding.engine_entity, + input=binding.engine_input, + closure_class="encoded", + encoded_by="concept_binding", + ) + for binding in mapping.bindings + ), + ) + # Each quantile map ranks one unit, so its band references count that + # unit (persons for wages; households for rent and aggregated assets). + band_units = {"person": "toy_person", "household": "toy_household"} + precal = [ + reference_row(f"{prefix}_{band}_band", entity=unit, measure=f"{unit}_count") + for unit, prefix in band_units.items() + for band in ("lower", "upper") + ] + facts = [_fact("toy_population", 120.0)] + [ + _fact(f"{prefix}_{band}_band", value, entity=unit) + for unit, prefix in band_units.items() + for band, value in (("lower", 1.0), ("upper", 3.0)) + ] + for index in (1, 2): + name = f"toy_population_{index}" + row = reference_row(name, entity="person", measure="person_count") + row["metadata"]["precal_use"] = "atomic_geography_support" + precal.append(row) + fact = _fact(name, 60.0) + fact["geography"].update( + level="territorial_authority", id=f"TA{index}", name=f"Synthetic TA {index}" + ) + facts.append(fact) + facts += [ + _fact(name, value, entity) for name, entity, value in _calibration_totals(frame) + ] + calibration_facts = write_facts(root / "inputs" / "calibration-facts.jsonl", facts) + holdout_facts = write_facts( + root / "inputs" / "holdout-facts.jsonl", [_fact("toy_holdout_receipts", 20.0)] + ) + support, assignment = _geography() + bands = { + name: { + "bands": [ + { + "lower": 100.0, + "upper": 200.0, + "reference": f"{band_units[unit]}_lower_band", + }, + { + "lower": 200.0, + "upper": 400.0, + "reference": f"{band_units[unit]}_upper_band", + }, + ] + } + for name, unit in ( + ("employment_income", "person"), + ("rent", "household"), + ("liquid_assets", "household"), + ) + } + resources = { + "benefit_unit_rule": UNIT_RULE.to_dict(), + "donor_pin": donor_pin(population.donor), + "content": { + "concept_schema_sha256": concept_schema_sha256(), + "seed_stream": "synthetic:composed:takeup", + }, + "mass_references": _references( + [reference_row("toy_population", entity="person", measure="person_count")] + ), + "precal_references": _references(precal), + "target_references": _references( + [ + reference_row(name, entity=entity, measure=measure) + for name, entity, measure in CALIBRATION_REFERENCES + ] + ), + "holdout_references": _references( + [ + reference_row( + "toy_holdout_receipts", entity="person", measure="receives_main" + ) + ] + ), + "currency_bridge": {"columns": {"person": ["interest_income"]}, "rate": 1.6}, + "quantile_maps": {**bands, "interpolation": {"method": "uniform"}}, + "as_area_crosswalk": support, + "geography_assignment": assignment, + "concept_mapping": mapping.to_dict(), + "axiom_input_closure": closure.to_dict(), + "axiom_rules_bindings": bindings, + "receipt_contract": { + "receipts": { + "seed_column": "take_up_seed", + "programs": [ + { + "program": "toy.main", + "output": "receives_main", + "judgment_column": "toy_main_eligible", + "payment_column": "toy_main_payment", + "rate": 0.65, + }, + { + "program": "toy.pension", + "output": "receives_pension", + "judgment_column": "toy_pension_eligible", + "payment_column": "toy_pension_payment", + "rate": 0.4, + }, + ], + "exclusion_groups": [["receives_main", "receives_pension"]], + } + }, + "calibration": { + "epochs": 8, + "learning_rate": 0.05, + "mass": "free", + "max_weight_ratio": 3.0, + "weight_anchor": "design", + "time_period": 2026, + }, + "gates": { + "country": COUNTRY, + "version": 1, + "policy": "Synthetic non-publication skeleton gates.", + "phases": ["terminal"], + "gates": [ + { + "id": "toy_weight_ratio", + "gate": "weight_ratio", + "phase": "terminal", + "criticality": "release_blocking", + "parameters": {"maximum_max_to_median_ratio": 4.0}, + } + ], + }, + "scenarios": {"scenarios": []}, + } + resources["transport_graph"] = _blueprint(create_outputs) + spec = {"country": COUNTRY, "resources": resources} + engines = {row["id"]: ToyRulesEngine(rules_root, row) for row in rows} + registry = build_transport_registry( + bindings, rules_root, unit_rule=UNIT_RULE, engines_by_binding=engines + ) + config = TransportGraphConfig( + engine_refs=registry.engine_refs, create_outputs=create_outputs + ) + sources = { + "donor_pool": population.donor, + "nz_calibration_facts": calibration_facts, + "nz_holdout_facts": holdout_facts, + "rulespec_nz": rules_root, + } + return ComposedFixture(root, spec, config, registry, sources, engines) diff --git a/tools/build_transport.py b/tools/build_transport.py new file mode 100644 index 0000000000..5234f39504 --- /dev/null +++ b/tools/build_transport.py @@ -0,0 +1,6 @@ +"""Build a local, spec-declared transport graph.""" + +from microcosm.build.transport.cli import main + +if __name__ == "__main__": + raise SystemExit(main()) From 5bd688cacb49a89f70a4ee69d6877b8dd076af5b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 05:41:31 -0400 Subject: [PATCH 3/5] Draw only valid node prose in the transport prose properties Both prose properties drew from alphabets that include a space, so Hypothesis could draw a whitespace-only string such as " ", which graph/decl.py's _description refuses. The G5a A4 test failed this way in 2 of 6 local runs on main (the same fix is microcosm#1160); the composed graph's purity/prose test has the same latent case. Map both strategies through str.strip: the result is empty or contains a non-whitespace character, which is what decl.py accepts. tests: composed prose property 4/4 runs passed. Co-Authored-By: Claude Opus 5.5 --- .../engine_free/shared/test_transport_composed_graph.py | 4 +++- .../engine_free/shared/test_transport_population_kernels.py | 6 +++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index 1d31cd0a0b..704e567586 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -252,7 +252,9 @@ def cold(composed, tmp_path_factory): @PROPERTY -@given(prose=st.text(alphabet="abc XYZ.", max_size=24)) +# Stripped: graph/decl.py accepts an empty description or text with a +# non-whitespace character, so a whitespace-only draw such as " " is invalid. +@given(prose=st.text(alphabet="abc XYZ.", max_size=24).map(str.strip)) def test_composition_is_pure_and_prose_moves_no_key(composed, prose): graph = composed.graph(extended=True) again = compose_transport_graph( diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_population_kernels.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_population_kernels.py index 5f3e9f5e12..08d7b0f52a 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_population_kernels.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_population_kernels.py @@ -214,7 +214,11 @@ def test_a1_a2_d2_create_contracts_on_a_synthetic_graph(tmp_path, mass): @PROPERTY @example(prose="Edited prose and citations.") -@given(prose=st.text(alphabet="abcdefghijklmnopqrstuvwxyz .", max_size=35)) +# Stripped: a node description must be empty or carry non-whitespace text +# (graph/decl.py), so a whitespace-only draw such as " " is not valid prose. +@given( + prose=st.text(alphabet="abcdefghijklmnopqrstuvwxyz .", max_size=35).map(str.strip) +) def test_a4_description_and_citation_change_no_key(tmp_path, prose): with TemporaryDirectory(dir=tmp_path) as temporary: root = Path(temporary) From e7953f31f1d9bc0867c57bc9ef52a44420767a1d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 10:03:00 -0400 Subject: [PATCH 4/5] Fix transport determinism, ancestry checks and extension preparation Write timestamp-free HDF5 exports without changing shared export kernels, validate calibration ancestry and activation selections, and preserve literal resource data. Cache CREATE inventories, refuse unsafe CLI paths and ids, isolate extension specs, and expose binding-driven rules nodes with additional sources and checkpoints. Narrow pin-provenance claims and expand regressions. Co-Authored-By: GPT-6.1 Sol --- changelog.d/transport-compose-cli.added.md | 15 +- changelog.d/transport-rules-registry.added.md | 5 +- docs/agent-guide.md | 21 +- .../src/microcosm/build/transport/cli.py | 185 ++++- .../src/microcosm/build/transport/compose.py | 472 +++++++++++- .../src/microcosm/build/transport/registry.py | 20 +- .../shared/test_transport_composed_graph.py | 727 +++++++++++++++++- .../shared/test_transport_registry.py | 80 +- .../microcosm_build/transport_composed.py | 44 +- 9 files changed, 1481 insertions(+), 88 deletions(-) diff --git a/changelog.d/transport-compose-cli.added.md b/changelog.d/transport-compose-cli.added.md index 23fad8d752..9f4e4a09b9 100644 --- a/changelog.d/transport-compose-cli.added.md +++ b/changelog.d/transport-compose-cli.added.md @@ -4,5 +4,16 @@ declares the skeleton graph; each node binds only the spec resources it selects, every rules node runs through the one `simulate.rules_by_ref@1` router, and later packages append branches through a typed extension point that cannot re-key the skeleton. The driver runs ancestor-closed checkpoints -on `/.graph-store` and writes every output under `--out`. Unresolved -country evidence is refused with each missing item named. +on `/.graph-store` and writes every output under `--out`. Activation checks every selected resource and reference set and every null +scenario knob before donor preparation. Calibration ancestry refuses hold-out +selections and sources, benefit-unit engine references, and graph digests. +Selected resource lists remain literal data. The driver writes deterministic +HDF5 container bytes while readback continues to hash the complete file. +CREATE inventories are cached by implementation, parameters and source bytes; +a cold build runs CREATE twice, and a warm inventory does not run the probe. +Cold `--resume require` builds refuse before preparation. Local spec directories +are protected from output overlap, node ids must be safe filenames, and +extensions receive independent spec copies. +Extensions can declare new sources and pre-export checkpoints, and use +`transport_rules_node` to obtain a binding's engine reference and period. +The locality property is parametrized over every selected resource. diff --git a/changelog.d/transport-rules-registry.added.md b/changelog.d/transport-rules-registry.added.md index 746f6d42ac..2d3d89c8a5 100644 --- a/changelog.d/transport-rules-registry.added.md +++ b/changelog.d/transport-rules-registry.added.md @@ -3,4 +3,7 @@ multi-engine rules kernel, shared calibration and terminal kernels, and the transport gate bindings. Engines use the persons-households-benefit-units schema and nesting that the benefit-unit rule declares (the schema CREATE builds). Engine-free adapters can exercise registry dispatch without -installing a country engine. +installing a country engine. Module hashes and tree digests bind the actual +RuleSpec bytes. Engine commit and wheel SHA-256 pins, and the RuleSpec commit +for exported trees, are format-checked caller declarations; installed engine +provenance is not checked. diff --git a/docs/agent-guide.md b/docs/agent-guide.md index 5bbce3feef..d9b294ff59 100644 --- a/docs/agent-guide.md +++ b/docs/agent-guide.md @@ -344,9 +344,26 @@ The transport build driver (`tools/build_transport.py`, a shim over `transport_graph.json` spec resource and runs it locally: every file it writes, including its `.graph-store`, lies under `--out`, and it has no upload or staging path. It refuses before reading any graph source while the -country spec still has unresolved evidence, and names each missing item. +country spec still has unresolved evidence: every selected resource/reference +set and every null scenario knob is checked before donor preparation. Tests for it run on the engine-free toy package in -`test_support/microcosm_build/transport_composed.py`. +`test_support/microcosm_build/transport_composed.py`. The rules registry checks +module hashes and binds the RuleSpec tree digest. Engine commit and wheel +SHA-256 pins, and the RuleSpec commit for exported trees, are format-checked +caller declarations; installed engine provenance is not checked. +Fresh driver runs write identical HDF5 bytes with object timestamps disabled; +readback keys continue to include the complete container bytes. CREATE's column +inventory is cached by implementation, parameters and source bytes. A cold +build runs CREATE twice (inventory probe and graph execution); warm inventory +lookups hash the inputs without running the probe. Cold `--resume require` +refuses before preparation. Output directories must also be disjoint from a +local `--spec-dir`. +G7 extensions can wrap a factory in `TransportExtension` to declare additional +sources and ordered checkpoints. Source names must be new; skeleton predecessor +sets must stay identical. Checkpoints run after calibration and before export, +and must not depend on the exported dataset. `transport_rules_node` supplies +the prepared engine reference and period from the binding, so extensions do +not repeat period values. UK size experiments use `microcosm-build-uk --release-role dense --dataset-households` (`tools/build_uk_full.py`; `tools/build_uk_rowwise_candidate.py` is a stub over it) diff --git a/packages/microcosm-build/src/microcosm/build/transport/cli.py b/packages/microcosm-build/src/microcosm/build/transport/cli.py index c35d861f41..f3a9a03e63 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/cli.py +++ b/packages/microcosm-build/src/microcosm/build/transport/cli.py @@ -24,18 +24,21 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass, replace from pathlib import Path +from tempfile import TemporaryDirectory from types import MappingProxyType import numpy as np import pandas as pd from microcosm.build.artifact_files import file_artifact, materialize_bytes +from microcosm.frame import Frame +from microcosm.frame.adapters.axiom import AxiomEntityTableDataset +from microcosm.frame.materialize import put_frame_table from microcosm.frame.rules import RulesEngine from microcosm.frame.unit_construction import BenefitUnitRule from microcosm.graph import ( ContentStore, Graph, - GraphRuntimeError, KernelContext, Node, Owned, @@ -47,7 +50,8 @@ graph_to_json, run_graph, ) -from microcosm.graph.canonical import canonical_json +from microcosm.graph.canonical import canonical_json, sha256_domain +from microcosm.graph.keys import source_content_key from .codecs import RULESPEC_SOURCE_CODEC from .compose import ( @@ -79,6 +83,7 @@ #: The skeleton's exported dataset, written under ``--out``. EXPORT_FILENAME = "skeleton.h5" _STORE = ".graph-store" +_CREATE_INVENTORY_FORMAT = "microcosm.transport.create-inventory/1" _CONTENT_BASIS = ( "Transported donor support records reweighted to destination facts; " "not observed destination microdata." @@ -97,22 +102,57 @@ class PreparedTransportBuild: def prepare_create_outputs( - spec: Mapping[str, object], sources: Mapping[str, str | Path] + spec: Mapping[str, object], + sources: Mapping[str, str | Path], + *, + inventory_store: ContentStore | None = None, + resume: ResumePolicy = "auto", ) -> tuple[Owned, ...]: - """CREATE's column inventory, read by running its kernel once. + """CREATE's column inventory, cached by its implementation and input bytes. The executor requires CREATE to declare every column it loads, and the columns depend on the pinned donor file. This runs ``transport.create@1`` on the declared parameters and sources (the kernel itself checks the donor's size and SHA-256) and declares every non-structural column of the frame it returns, with its dtype. The frame is then discarded; the - graph's own CREATE recomputes it under the executor. + graph's own CREATE recomputes it under the executor. A cold CLI build + therefore executes CREATE twice. A warm inventory lookup hashes every + CREATE source but does not construct the frame or run CREATE. Without an + ``inventory_store``, each call runs the probe; ``require`` needs a stored + inventory, and ``forbid`` recomputes it even if its key already exists. """ + if resume not in {"auto", "require", "forbid"}: + raise ValueError("resume must be 'auto', 'require' or 'forbid'.") + if resume == "require" and inventory_store is None: + raise StoreMissError("--resume require needs a CREATE inventory store.") declaration = skeleton_create_declaration(spec) missing = sorted(set(declaration.sources) - set(sources)) if missing: raise ValueError(f"CREATE needs local paths for sources {missing}.") + inventory_key = None + if inventory_store is not None: + inventory_key = sha256_domain( + _CREATE_INVENTORY_FORMAT, + canonical_json( + { + "kernel": declaration.kernel, + "implementation_hash": TRANSPORT_CREATE.implementation_hash(), + "params": declaration.params, + "sources": { + name: source_content_key(name, sources[name]) + for name in declaration.sources + }, + } + ), + ) + if resume != "forbid" and inventory_store.has(inventory_key): + cached = inventory_store.load_json( + inventory_key, kind=_CREATE_INVENTORY_FORMAT + ) + return tuple(Owned(**row) for row in cached["columns"]) + if resume == "require": + raise StoreMissError("--resume require needs this CREATE inventory.") # A probe node: CREATE reads only its params and declared sources. It # declares no outputs, so it is never part of a graph. probe = Node( @@ -139,13 +179,26 @@ def prepare_create_outputs( }, **{group: {schema.id_column(group)} for group in schema.group_entities}, } - return tuple( + outputs = tuple( Owned(entity, column, str(table[column].dtype)) for entity in schema.entities for table in (frame.table(entity),) for column in table.columns if column not in structural[entity] ) + if inventory_store is not None: + inventory_store.put_json( + inventory_key, + { + "columns": [ + {"entity": item.entity, "column": item.column, "dtype": item.dtype} + for item in outputs + ] + }, + kind=_CREATE_INVENTORY_FORMAT, + verify_existing=resume != "forbid", + ) + return outputs def _rules_source(spec: Mapping[str, object]) -> str: @@ -166,6 +219,8 @@ def prepare_transport_build( sources: Mapping[str, str | Path], engines_by_binding: Mapping[str, RulesEngine] | None = None, extensions: tuple[TransportGraphExtension, ...] = (), + inventory_store: ContentStore | str | Path | None = None, + resume: ResumePolicy = "auto", ) -> PreparedTransportBuild: """Check activation, build the registry and CREATE inventory, then compose. @@ -174,10 +229,30 @@ def prepare_transport_build( (which the run writes) to a local path. ``engines_by_binding`` is the registry's seam for pure-Python adapters; without it, Axiom adapters are built for every binding. The registry's schema comes from the same - benefit-unit rule CREATE is given. + benefit-unit rule CREATE is given. ``inventory_store`` may be a content + store or its local directory; a directory is created only after activation + succeeds. The CLI uses its graph store to cache the inventory. A cold build + runs a full CREATE probe before graph execution; a warm build hashes the + CREATE sources and reuses the stored inventory without running the probe. """ validate_transport_activation(spec) + if resume not in {"auto", "require", "forbid"}: + raise ValueError("resume must be 'auto', 'require' or 'forbid'.") + if resume == "require": + root = ( + inventory_store.root + if isinstance(inventory_store, ContentStore) + else Path(inventory_store) + if inventory_store is not None + else None + ) + if root is None or _store_is_cold(root): + raise StoreMissError( + "--resume require refuses a cold transport graph store." + ) + if isinstance(inventory_store, str | Path): + inventory_store = ContentStore(Path(inventory_store)) paths = {name: Path(path).resolve() for name, path in sources.items()} rules_source = _rules_source(spec) if rules_source not in paths: @@ -194,7 +269,9 @@ def prepare_transport_build( spec, TransportGraphConfig( engine_refs=registry.engine_refs, - create_outputs=prepare_create_outputs(spec, paths), + create_outputs=prepare_create_outputs( + spec, paths, inventory_store=inventory_store, resume=resume + ), extensions=extensions, ), ) @@ -207,7 +284,7 @@ def prepare_transport_build( graph, registry, MappingProxyType(paths), - MappingProxyType(transport_endpoints(spec)), + MappingProxyType(transport_endpoints(spec, extensions=extensions)), transport_pending_outputs(spec), ) @@ -240,17 +317,23 @@ def _destination(out: Path, filename: str) -> Path: return path -def _check_disjoint(out: Path, sources: Mapping[str, Path]) -> None: +def _check_disjoint( + out: Path, sources: Mapping[str, Path], *, spec_dir: Path | None = None +) -> None: """Refuse an output directory that overlaps an input. Writing inside an input tree would change the bytes a source names (a RuleSpec directory is hashed whole), and an input inside ``--out`` could - be overwritten by an output. + be overwritten by an output. A local ``--spec-dir`` is also an input tree. """ for name, path in sources.items(): if path.is_relative_to(out) or out.is_relative_to(path): raise ValueError(f"--out overlaps the input of source {name!r}: {path}.") + if spec_dir is not None: + directory = spec_dir.resolve() + if directory.is_relative_to(out) or out.is_relative_to(directory): + raise ValueError(f"--out overlaps --spec-dir: {directory}.") def _store_is_cold(root: Path) -> bool: @@ -285,6 +368,51 @@ def _run(graph: Graph, registry, sources, store, resume) -> RunManifest: ) +class _TimestampFreeHDFStore(pd.HDFStore): + """Disable object timestamps and unused pandas search indexes.""" + + def put(self, key: str, value: pd.DataFrame | pd.Series, **options) -> None: + options["track_times"] = False + options["index"] = False + super().put(key, value, **options) + + +def _materialize_deterministic_export( + frame: Frame, descriptor: Mapping[str, object], path: Path +) -> dict[str, object]: + """Validate through G5b, then rewrite the same tables without timestamps. + + The reader consumes whole tables, so query indexes are unnecessary. + Table order, row order, values, dtypes and period follow the validated + export, including its shared dtype boundary. Readback still hashes the + complete H5 container. A changed export costs one additional complete + H5 read and rewrite; unchanged exports use the existing readback path. + The intermediate H5 stays under --out and is removed on exit. + """ + + with TemporaryDirectory(prefix=".export-", dir=path.parent) as scratch: + staged = Path(scratch) / "validated.h5" + materialize_export(frame, descriptor, staged) + dataset = AxiomEntityTableDataset(file_path=staged) + path.unlink(missing_ok=True) + with _TimestampFreeHDFStore(str(path), mode="w") as store: + for entity in descriptor["entities"]: + put_frame_table( + store, + entity, + dataset.tables[entity], + preferred_format="table", + data_columns=True, + ) + store.put("_time_period", pd.Series([dataset.time_period]), format="table") + report = validate_export(path, descriptor) + if report["passed"] is not True: + raise ValueError( + f"The deterministic skeleton H5 failed readback: {report['failures']}." + ) + return file_artifact(path) + + def execute_transport_graph( graph: Graph, registry: TransportRegistry, @@ -326,10 +454,19 @@ def execute_transport_graph( checkpoints = [ ("geography", endpoints["geography"]), ("calibration", endpoints["calibration"]), + *( + (f"extension-{index}", endpoint) + for index, endpoint in enumerate(endpoints.get("checkpoints", ())) + ), ] terminals = [("terminal", endpoint) for endpoint in endpoints["terminal"]] for _, endpoint in (*checkpoints, *terminals): through(graph, endpoint) + for label, endpoint in checkpoints: + if label.startswith("extension-") and any( + export_source in node.sources for node in through(graph, endpoint).nodes + ): + raise ValueError("Extension checkpoints must precede export readback.") store_root = output / _STORE if resume == "require" and _store_is_cold(store_root): raise StoreMissError("--resume require refuses a cold transport graph store.") @@ -369,7 +506,7 @@ def execute_transport_graph( ) else: version = compiled.versions[prepare.id] - materialize_export( + _materialize_deterministic_export( prepared.populations[version], json.loads(descriptor_bytes), exported ) materialize_bytes(descriptor_bytes, descriptor_path) @@ -465,10 +602,28 @@ def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: def main(argv: Sequence[str] | None = None) -> int: """Run the CLI; return 0 on success and 1 on a refused build.""" - args = parse_args(argv) try: + args = parse_args(argv) + except SystemExit as error: + if error.code == 0: + return 0 + print( + "transport build refused: invalid command-line arguments.", file=sys.stderr + ) + return 1 + try: + output = args.out.resolve() + sources = {name: path.resolve() for name, path in args.source} + _check_disjoint(output, sources, spec_dir=args.spec_dir) + store_root = _destination(output, _STORE) + if args.resume == "require" and _store_is_cold(store_root): + raise StoreMissError( + "--resume require refuses a cold transport graph store." + ) spec = load_transport_spec(args.spec_dir if args.spec_dir else args.country) - prepared = prepare_transport_build(spec, sources=dict(args.source)) + prepared = prepare_transport_build( + spec, sources=sources, inventory_store=store_root, resume=args.resume + ) execute_transport_graph( prepared.graph, prepared.registry, @@ -478,7 +633,7 @@ def main(argv: Sequence[str] | None = None) -> int: pending_outputs=prepared.pending_outputs, resume=args.resume, ) - except (GraphRuntimeError, ValueError, TypeError, OSError) as error: + except Exception as error: print(f"transport build refused: {error}", file=sys.stderr) return 1 print(f"Transport skeleton written under {args.out}.") diff --git a/packages/microcosm-build/src/microcosm/build/transport/compose.py b/packages/microcosm-build/src/microcosm/build/transport/compose.py index bfd6cb4c1e..31d48bd445 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/compose.py +++ b/packages/microcosm-build/src/microcosm/build/transport/compose.py @@ -13,21 +13,30 @@ ``"value"`` (the JSON value, lists as tuples), ``"json"`` (an object as canonical JSON text) or ``"sha256"`` (the SHA-256 of the whole resource's bytes, so the node's key moves with that one resource only). -- ``{"prepared": "engine_refs"}`` is the canonical JSON of the authenticated +- ``{"prepared": "engine_refs"}`` is the canonical JSON of the prepared binding-id -> engine reference map that preparation built. So a node binds exactly the resources it selects. The graph resource's own digest and the country spec's fingerprint enter no node: editing one -resource re-keys only the nodes that select it and their descendants. +resource re-keys only the nodes that select the changed data and their +descendants. Selector-shaped objects inside selected resource data remain +literal data; they cannot introduce hidden resource dependencies. +Objects in selected lists remain objects and are subject to the graph's +scalar/tuple parameter boundary; encode an enclosing object as JSON when +structured data must enter a node. A rules node names its binding (``rules_binding``) instead of an engine -reference. Composition supplies the authenticated ``engine_ref`` and the +reference. Composition supplies the prepared ``engine_ref`` and the binding's period, and every rules node runs through the one ``simulate.rules_by_ref@1`` router. ``TransportGraphConfig.extensions`` is the extension point for the package that adds the entitlement chain. Each extension is a factory called with -the spec, the config and the composed skeleton; it returns new nodes only. +the spec, the config and the composed skeleton; it returns new nodes. +Wrap a factory in :class:`TransportExtension` to declare additional sources +and checkpoints, run after calibration and before export. The public +:func:`transport_rules_node` supplies a rules node's engine ref and period +from its binding, including for extension nodes. Composition refuses an extension whose nodes change any skeleton node's compiled predecessors (for example a new member of a calibration base version), so an extension can never re-key the skeleton. @@ -42,7 +51,7 @@ import hashlib import json from collections.abc import Iterator, Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, replace from importlib.resources import files from pathlib import Path from types import MappingProxyType @@ -78,6 +87,7 @@ "SKELETON_KERNELS", "TransportGraphConfig", "TransportGraphExtension", + "TransportExtension", "compose_transport_graph", "CreateDeclaration", "load_transport_spec", @@ -86,7 +96,9 @@ "transport_graph_document", "transport_pending_outputs", "transport_resource", + "transport_rules_node", "validate_transport_activation", + "validate_transport_calibration_ancestry", ] #: The spec resource that declares the skeleton. @@ -185,16 +197,52 @@ def __call__( ) -> tuple[Node, ...]: ... +@dataclass(frozen=True) +class TransportExtension: + """A node factory with additional local sources and ordered checkpoints. + + Sources must have new names, so they cannot replace a skeleton input. + Checkpoints name nodes in the extended graph and run after calibration, + before the skeleton H5 is written; they cannot read the exported dataset. + The factory receives a deep copy of the caller's spec. + """ + + factory: TransportGraphExtension + sources: tuple[SourceRef, ...] = () + checkpoints: tuple[str, ...] = () + + def __post_init__(self) -> None: + if not callable(self.factory): + raise TypeError("An extension factory must be callable.") + if not isinstance(self.sources, tuple) or any( + not isinstance(source, SourceRef) for source in self.sources + ): + raise TypeError("Extension sources must be a tuple of SourceRefs.") + if not isinstance(self.checkpoints, tuple): + raise TypeError("Extension checkpoints must be a tuple of node ids.") + for checkpoint in self.checkpoints: + _safe_node_id(checkpoint) + + def __call__( + self, + spec: Mapping[str, object], + config: TransportGraphConfig, + skeleton: Graph, + ) -> tuple[Node, ...]: + return self.factory(spec, config, skeleton) + + @dataclass(frozen=True) class TransportGraphConfig: """What preparation supplies; every country value stays in the spec. Attributes: - engine_refs: Binding id -> authenticated engine reference, from + engine_refs: Binding id -> prepared engine reference, from :func:`microcosm.build.transport.registry.build_transport_registry`. create_outputs: CREATE's declared column inventory, prepared from the pinned donor (every loaded column must be declared). extensions: Node factories appended after the skeleton, in order. + A :class:`TransportExtension` also declares sources and checkpoints. """ engine_refs: Mapping[str, str] @@ -209,7 +257,7 @@ def __post_init__(self) -> None: or not value for key, value in self.engine_refs.items() ): - raise ValueError("engine_refs must map binding ids to authenticated refs.") + raise ValueError("engine_refs must map binding ids to prepared refs.") if ( not isinstance(self.create_outputs, tuple) or not self.create_outputs @@ -238,6 +286,8 @@ def _sequence(value: object, label: str) -> Sequence: def _stem(name: str) -> str: + if not isinstance(name, str) or not name: + raise ValueError("A resource selection must name a nonempty string.") return name.removesuffix(".json") @@ -340,18 +390,32 @@ def _selected_resources(document: Mapping) -> set[str]: return names +def _null_paths(value: object, path: str = "") -> Iterator[str]: + if value is None: + yield path + elif isinstance(value, Mapping): + for key, item in value.items(): + yield from _null_paths(item, f"{path}.{key}" if path else str(key)) + elif isinstance(value, list | tuple): + for index, item in enumerate(value): + yield from _null_paths(item, f"{path}[{index}]") + + def validate_transport_activation(spec: Mapping[str, object]) -> None: - """Refuse unresolved country evidence, naming every missing item at once. - - The items are the skeleton declaration and every resource it selects, - the dependent-child age limit of the benefit-unit rule (``max_age``, or - the proposal's ``age_limit_years``), any reference not marked active, - an executable receipt contract (``receipt_contract.json``), the engine - commit and wheel pins, and a scenario knob still awaiting evidence - (``rent_stock_factor``). Nothing here picks a value for a missing item. + """Refuse unresolved declaration resources and country evidence before I/O. + + Check every selected reference resource, the mandatory pre-calibration, + target and hold-out reference sets, and every null scenario knob. Missing + resources are the declaration's explicit selections, including selections + inside parameter lists; selected resource data remains literal data. """ missing = [] + selected = ( + _selected_resources(transport_resource(spec, GRAPH_RESOURCE)) + if _has_resource(spec, GRAPH_RESOURCE) + else set() + ) for name in ("benefit_unit_rule", "axiom_rules_bindings", "scenarios"): if not _has_resource(spec, name): missing.append(f"{name}.json") @@ -363,7 +427,18 @@ def validate_transport_activation(spec: Mapping[str, object]) -> None: age_key = "max_age" if "max_age" in child else "age_limit_years" if child.get(age_key) is None: missing.append(f"benefit_unit_rule.dependent_child.{age_key}") - for resource in ("precal_references", "target_references", "holdout_references"): + reference_resources = { + "precal_references", + "target_references", + "holdout_references", + } + reference_resources.update( + name + for name in selected + if _has_resource(spec, name) + and "target_references" in transport_resource(spec, name) + ) + for resource in sorted(reference_resources): if not _has_resource(spec, resource): missing.append(f"{resource}.json") continue @@ -403,12 +478,11 @@ def validate_transport_activation(spec: Mapping[str, object]) -> None: for row in _sequence(rows, "scenarios.scenarios"): scenario = _mapping(row, "scenario") knobs = _mapping(scenario.get("knobs", {}), "scenario.knobs") - if "rent_stock_factor" in knobs and knobs["rent_stock_factor"] is None: - missing.append(f"scenario {scenario.get('id', '?')} rent_stock_factor") + for path in _null_paths(knobs): + missing.append(f"scenario {scenario.get('id', '?')} {path}") if not _has_resource(spec, GRAPH_RESOURCE): missing.append(f"{GRAPH_RESOURCE}.json (the skeleton declaration)") else: - selected = _selected_resources(transport_resource(spec, GRAPH_RESOURCE)) missing.extend( f"{name}.json (selected by {GRAPH_RESOURCE}.json)" for name in sorted(selected) @@ -434,8 +508,246 @@ def transport_graph_document(spec: Mapping[str, object]) -> Mapping: return document -def transport_endpoints(spec: Mapping[str, object]) -> dict[str, object]: - """The ordered checkpoints: geography, calibration, then the terminals.""" +def _ancestor_closure( + predecessors: Mapping[str, Sequence[str]], roots: set[str] +) -> set[str]: + ancestors = set() + pending = list(roots) + while pending: + name = pending.pop() + if name in ancestors: + continue + if name not in predecessors: + raise ValueError(f"Calibration endpoint {name!r} is not a declared node.") + ancestors.add(name) + pending.extend(predecessors[name]) + return ancestors + + +def _embeds_engine_ref( + value: object, + references: set[str], + unit_modules: frozenset[str] = frozenset(), +) -> bool: + if not references and not unit_modules: + return False + if isinstance(value, Mapping): + if unit_modules: + from microcosm.frame.adapters.axiom import _ENGINE_REF_SCHEMA + + from .registry import PYTHON_ENGINE_REF_FORMAT + + module = None + if value.get("format") == _ENGINE_REF_SCHEMA: + module = value.get("module") + elif value.get("format") == PYTHON_ENGINE_REF_FORMAT: + binding = value.get("binding") + if isinstance(binding, Mapping): + module = binding.get("rulespec_path") + if isinstance(module, str) and module in unit_modules: + return True + return any( + _embeds_engine_ref(item, references, unit_modules) + for pair in value.items() + for item in pair + ) + if isinstance(value, list | tuple): + return any(_embeds_engine_ref(item, references, unit_modules) for item in value) + if not isinstance(value, str): + return False + if any( + reference in value or json.dumps(reference, ensure_ascii=False)[1:-1] in value + for reference in references + ): + return True + try: + decoded = json.loads(value) + except (ValueError, TypeError): + decoder = json.JSONDecoder() + for index, character in enumerate(value): + if character not in {"{", '"'}: + continue + try: + embedded, _ = decoder.raw_decode(value, index) + except ValueError: + continue + if _embeds_engine_ref(embedded, references, unit_modules): + return True + return False + return decoded != value and _embeds_engine_ref(decoded, references, unit_modules) + + +def validate_transport_calibration_ancestry( + spec: Mapping[str, object], + *, + graph: Graph | None = None, + config: TransportGraphConfig | None = None, +) -> None: + """Refuse hold-out or benefit-unit engine semantics upstream of calibration. + + Without a graph, check the declaration's structural-base membership and + artifact ancestry. This permits a packaged declaration check before donor + preparation. Canonical engine-reference documents are matched to the + benefit-unit bindings' module paths even before references are prepared. + A composed graph also checks its exact compiled predecessors and the + prepared references, including references embedded in JSON text. + Benefit-unit attributes remain allowed; excluded columns are the unit + rules bindings' outputs and the group encoder's declared outputs. + """ + if not _has_resource(spec, GRAPH_RESOURCE): + return + document = transport_graph_document(spec) + rows = { + row["id"]: row + for row in ( + _mapping(item, "transport node") + for item in _sequence(document.get("nodes"), "transport nodes") + ) + } + unit_entity = transport_resource(spec, "benefit_unit_rule").get("entity") + bindings = transport_resource(spec, "axiom_rules_bindings").get("bindings", ()) + unit_bindings = { + row["id"]: row + for row in _sequence(bindings, "axiom_rules_bindings.bindings") + if isinstance(row, Mapping) and row.get("entity") == unit_entity + } + unit_modules = frozenset( + row["rulespec_path"] + for row in unit_bindings.values() + if isinstance(row.get("rulespec_path"), str) + ) + forbidden_columns = { + (unit_entity, variable) + for row in unit_bindings.values() + for variable in row.get("variables", ()) + } + for row in rows.values(): + if row.get("kernel") == "concepts.encode_groups@1": + forbidden_columns.update( + (output["entity"], output["column"]) + for output in row.get("outputs", ()) + if output.get("entity") == unit_entity + ) + references = { + row["engine_ref"] + for row in unit_bindings.values() + if isinstance(row.get("engine_ref"), str) and row["engine_ref"] + } + if config is not None: + references.update( + config.engine_refs[name] + for name in unit_bindings + if name in config.engine_refs + ) + endpoints = _mapping(document.get("endpoints"), "transport endpoints") + roots = { + row["id"] + for row in rows.values() + if isinstance(row.get("kernel"), str) and row["kernel"].startswith("calibrate.") + } + if isinstance(endpoints.get("calibration"), str): + roots.add(endpoints["calibration"]) + predecessors = {name: set() for name in rows} + for name, row in rows.items(): + base = row.get("base") + if isinstance(base, str): + predecessors[name].add(base) + predecessors[name].update( + item["id"] for item in rows.values() if item.get("population") == base + ) + elif isinstance(row.get("population"), str): + predecessors[name].add(row["population"]) + predecessors[name].update( + edge["producer"] for edge in row.get("artifact_inputs", ()) + ) + for name in _ancestor_closure(predecessors, roots): + row = rows[name] + reason = None + if row.get("rules_binding") in unit_bindings: + reason = "benefit-unit rules binding" + for value in _mapping(row.get("params", {}), "node params").values(): + for selection in _selections(value): + if selection.get("prepared") == "engine_refs": + reason = "prepared engine_refs map" + if isinstance(selection.get("resource"), str): + resource = _stem(selection["resource"]) + if resource == "holdout_references": + reason = "holdout_references selection" + if ( + resource == GRAPH_RESOURCE + and selection.get("encoding") == "sha256" + ): + reason = "graph resource's own digest" + if selection.get("encoding", "value") != "sha256" and _has_resource( + spec, resource + ): + selected = transport_resource(spec, resource) + for component in selection.get("path", ()): + if ( + not isinstance(selected, Mapping) + or component not in selected + ): + selected = None + break + selected = selected[component] + if _embeds_engine_ref(selected, references, unit_modules): + reason = "selected data embedding a benefit-unit engine reference" + columns = { + (item["entity"], column) + for item in row.get("inputs", ()) + for column in item["columns"] + } + columns.update( + (item["entity"], item["column"]) for item in row.get("outputs", ()) + ) + if columns & forbidden_columns: + reason = "benefit-unit engine input or output column" + if any( + isinstance(source, str) and "holdout" in source.lower() + for source in row.get("sources", ()) + ): + reason = "hold-out source" + if _embeds_engine_ref(row.get("params", {}), references, unit_modules): + reason = "embedded benefit-unit engine reference" + if reason: + raise ValueError( + f"Transport calibration ancestry: node {name!r} carries {reason}." + ) + if graph is None: + return + compiled = compile_graph(graph) + roots.update( + node.id for node in graph.nodes if node.kernel.startswith("calibrate.") + ) + for node in graph.nodes: + if node.kernel == "concepts.encode_groups@1": + forbidden_columns.update( + (item.entity, item.column) + for item in node.outputs + if item.entity == unit_entity + ) + for name in _ancestor_closure(compiled.predecessors, roots): + node = graph.node(name) + columns = { + (item.entity, column) for item in node.inputs for column in item.columns + } + columns.update((item.entity, item.column) for item in node.outputs) + if ( + any("holdout" in source.lower() for source in node.sources) + or _embeds_engine_ref(node.params, references, unit_modules) + or columns & forbidden_columns + ): + raise ValueError( + f"Transport calibration ancestry: node {name!r} carries hold-out or benefit-unit engine semantics." + ) + + +def transport_endpoints( + spec: Mapping[str, object], + *, + extensions: tuple[TransportGraphExtension, ...] = (), +) -> dict[str, object]: + """Geography, calibration, extension checkpoints, then export and terminals.""" values = _mapping( transport_graph_document(spec).get("endpoints"), "transport endpoints" @@ -452,10 +764,19 @@ def transport_endpoints(spec: Mapping[str, object]) -> dict[str, object]: not isinstance(name, str) or not name for name in terminals ): raise ValueError("Terminal endpoints must be a nonempty list of node ids.") + checkpoints = tuple( + checkpoint + for extension in extensions + if isinstance(extension, TransportExtension) + for checkpoint in extension.checkpoints + ) + if len(set(checkpoints)) != len(checkpoints): + raise ValueError("Extension checkpoints must name distinct nodes.") return { "geography": values["geography"], "calibration": values["calibration"], "terminal": terminals, + "checkpoints": checkpoints, } @@ -469,6 +790,15 @@ def transport_pending_outputs(spec: Mapping[str, object]) -> tuple[str, ...]: return values +def _literal_data(value: object) -> object: + """Convert selected JSON sequences without resolving selector-shaped data.""" + if isinstance(value, list | tuple): + return tuple(_literal_data(item) for item in value) + if isinstance(value, Mapping): + return {key: _literal_data(item) for key, item in value.items()} + return value + + def _param( spec: Mapping[str, object], value: object, @@ -479,6 +809,8 @@ def _param( if not isinstance(value, Mapping): return value if set(value) == {"prepared"}: + if value["prepared"] in {"fingerprint", "spec_fingerprint"}: + raise ValueError("The whole-spec fingerprint cannot enter node parameters.") if value["prepared"] not in _PREPARED: raise ValueError(f"Unknown prepared parameter {value['prepared']!r}.") if engine_refs is None: @@ -494,6 +826,10 @@ def _param( encoding = value.get("encoding", "value") if encoding not in _ENCODINGS: raise ValueError(f"Unknown resource encoding {encoding!r}.") + if _stem(name) == GRAPH_RESOURCE and encoding == "sha256": + raise ValueError( + "The graph resource's own digest cannot enter node parameters." + ) document: object = transport_resource(spec, name) path = tuple(_sequence(value.get("path", ()), "resource selection path")) if encoding == "sha256": @@ -514,7 +850,7 @@ def _param( raise ValueError( f"Resource {name!r} value selection is an object; use json encoding." ) - return _param(spec, document, engine_refs) + return _literal_data(document) def _artifact_type(value: object) -> ArtifactType: @@ -582,12 +918,74 @@ def _rules_params( } +def transport_rules_node( + spec: Mapping[str, object], + config: TransportGraphConfig, + binding_id: str, + node: Node, +) -> Node: + """Bind an extension's rules node to its prepared engine ref and period. + + ``node`` supplies its inputs, owned outputs, population and sources. Its + params may contain only ``variables`` (a subset of the binding's outputs); + omitting them selects every binding variable. The node must use the one + rules router and own exactly the selected variables. Callers supply no + engine reference or period: both come from the named binding. + """ + + if not isinstance(node, Node): + raise TypeError("A transport rules node must be a Node.") + _safe_node_id(node.id) + declaration = { + "id": node.id, + "kernel": node.kernel, + "rules_binding": binding_id, + "outputs": [{"column": output.column} for output in node.outputs], + } + return replace( + node, + params=_rules_params(spec, declaration, dict(node.params), config), + ) + + +def _check_extension_checkpoints(spec, config, graph, compiled) -> None: + checkpoints = transport_endpoints(spec, extensions=config.extensions)["checkpoints"] + if not set(checkpoints) <= set(compiled.order): + raise ValueError("Extension checkpoints must name nodes in the extended graph.") + exported = { + name + for node in graph.nodes + if node.kernel == "export.readback@1" + for name in node.sources + } + for name in _ancestor_closure(compiled.predecessors, set(checkpoints)): + if exported.intersection(graph.node(name).sources): + raise ValueError("Extension checkpoints must precede export readback.") + + +def _safe_node_id(value: object) -> str: + if ( + not isinstance(value, str) + or not value + or value == "." + or "/" in value + or "\\" in value + or "\x00" in value + or ".." in value + ): + raise ValueError( + f"Transport node id {value!r} must be a safe filename without path separators, NUL or '..'." + ) + return value + + def _node( spec: Mapping[str, object], row: Mapping, config: TransportGraphConfig, types: Mapping[tuple[str, str], ArtifactType], ) -> Node: + _safe_node_id(row.get("id")) unknown = set(row) - _NODE_FIELDS if unknown: raise ValueError( @@ -674,8 +1072,9 @@ class CreateDeclaration: def skeleton_create_declaration(spec: Mapping[str, object]) -> CreateDeclaration: """The declared CREATE with its parameters resolved, for preparation. - Preparation runs this kernel once to read CREATE's column inventory, - which the composed CREATE node must then declare. + Preparation uses the stored CREATE column inventory when its implementation, + parameters and source bytes match; a cold inventory runs the kernel once. + The composed CREATE node must declare that inventory. """ validate_transport_activation(spec) @@ -692,7 +1091,7 @@ def skeleton_create_declaration(spec: Mapping[str, object]) -> CreateDeclaration ) row = rows[0] return CreateDeclaration( - id=row["id"], + id=_safe_node_id(row.get("id")), kernel=row["kernel"], params=MappingProxyType( { @@ -718,6 +1117,7 @@ def compose_transport_graph( raise TypeError("config must be a TransportGraphConfig.") validate_transport_activation(spec) document = transport_graph_document(spec) + validate_transport_calibration_ancestry(spec, config=config) rows = tuple( _mapping(row, "transport node") for row in _sequence(document.get("nodes"), "transport nodes") @@ -739,28 +1139,44 @@ def compose_transport_graph( mass_partition=None if partition is None else tuple(partition), ) compiled = compile_graph(skeleton) + validate_transport_calibration_ancestry(spec, graph=skeleton, config=config) endpoints = transport_endpoints(spec) named = {endpoints["geography"], endpoints["calibration"], *endpoints["terminal"]} if not named <= set(compiled.order): raise ValueError("Transport endpoints must name declared skeleton nodes.") transport_pending_outputs(spec) additional: list[Node] = [] + additional_sources: list[SourceRef] = [] + source_names = {source.name for source in skeleton.sources} for factory in config.extensions: - nodes = factory(spec, config, skeleton) + if isinstance(factory, TransportExtension): + for source in factory.sources: + if source.name in source_names: + raise ValueError( + f"Extension source {source.name!r} must have a new name." + ) + source_names.add(source.name) + additional_sources.append(source) + nodes = factory(json.loads(canonical_json(spec)), config, skeleton) if not isinstance(nodes, tuple) or any( not isinstance(node, Node) for node in nodes ): raise TypeError("A transport extension must return a tuple of Nodes.") + for node in nodes: + _safe_node_id(node.id) additional.extend(nodes) - if not additional: + if not additional and not additional_sources: + _check_extension_checkpoints(spec, config, skeleton, compiled) return skeleton graph = Graph( skeleton.country, - skeleton.sources, + (*skeleton.sources, *additional_sources), (*skeleton.nodes, *additional), skeleton.mass_partition, ) extended = compile_graph(graph) + _check_extension_checkpoints(spec, config, graph, extended) + validate_transport_calibration_ancestry(spec, graph=graph, config=config) for node in skeleton.nodes: if extended.predecessors[node.id] != compiled.predecessors[node.id]: raise ValueError( diff --git a/packages/microcosm-build/src/microcosm/build/transport/registry.py b/packages/microcosm-build/src/microcosm/build/transport/registry.py index da82f22654..ea25ced9fa 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/registry.py +++ b/packages/microcosm-build/src/microcosm/build/transport/registry.py @@ -1,8 +1,11 @@ -"""Explicit transport registrations and authenticated rules-engine bindings. +"""Explicit transport registrations and checked rules-engine bindings. Registries are local to a run. All rules bindings share one routing kernel; adding a binding against the same RuleSpec tree and pins leaves existing -references unchanged. Kernel implementation identity binds adapter classes, +references unchanged. Engine commit and wheel SHA-256 pins, and the RuleSpec +commit for exported trees, are format-checked caller declarations. Module +bytes and tree contents are hashed; installed engine provenance is not checked. +Kernel implementation identity binds adapter classes, never spec data, so adding a binding of an existing class leaves it unchanged. The AS bridge and gap kernels are registered by their owning package when that package is available; this module @@ -45,12 +48,16 @@ from .terminal_kernels import register_terminal_kernels __all__ = [ + "PYTHON_ENGINE_REF_FORMAT", "TransportRegistry", "build_transport_registry", "register_transport_population_kernels", "transport_entity_schema", ] +#: Public format identity for an injected pure-Python rules-engine reference. +PYTHON_ENGINE_REF_FORMAT = "microcosm.transport.python-rules-engine-ref/1" + _SHA256 = re.compile(r"[0-9a-f]{64}") _COMMIT = re.compile(r"(?:[0-9a-f]{40}|[0-9a-f]{64})") @@ -223,8 +230,11 @@ def build_transport_registry( Real adapters use the schema and unit nesting of :func:`transport_entity_schema` for ``unit_rule`` (the frame CREATE - builds), explicit periods, graph output dtypes, authenticated module - bytes and :func:`axiom_engine_ref`. References are pinned before any + builds), explicit periods, graph output dtypes, checked module bytes and + :func:`axiom_engine_ref`. Engine commit and wheel SHA-256 pins, and the + RuleSpec commit for exported trees, are format-checked caller declarations. + The module hash and tree digest bind the actual RuleSpec bytes. References + are pinned before any relation check compiles a program. An injected adapter must report the same schema. @@ -326,7 +336,7 @@ def build_transport_registry( ) reference = canonical_json( { - "format": "microcosm.transport.python-rules-engine-ref/1", + "format": PYTHON_ENGINE_REF_FORMAT, "engine": "python-rules-engine", "adapter": f"{type(engine).__module__}.{type(engine).__qualname__}", "adapter_identity": identity, diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index 704e567586..d321a2e69f 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -37,6 +37,7 @@ import ast import copy +import hashlib import json import re import shutil @@ -50,6 +51,7 @@ from hypothesis import strategies as st from microcosm.build.transport import cli as cli_module +from microcosm.build.transport import compose as compose_module from microcosm.build.transport.cli import ( EXPORT_FILENAME, execute_transport_graph, @@ -76,6 +78,7 @@ NodeRejectedError, Owned, Slice, + SourceRef, StoreMissError, compile_graph, graph_to_json, @@ -86,7 +89,9 @@ FAMILY_INPUTS, PROGRAMS, SHARED_PATH, + _blueprint, make_composed_fixture, + toy_extension, ) from test_support.microcosm_build.transport_graph import descendants from test_support.paths import paths_for @@ -284,13 +289,25 @@ def _selected_resources(spec) -> list[str]: return sorted(names) +# Inventory changes no selectors; the blueprint needs one household column +# to construct its input slices, without running CREATE during collection. +SELECTED_RESOURCES = _selected_resources( + { + "resources": { + "transport_graph": _blueprint((Owned("household", "rent", "float64"),)) + } + } +) + + +@pytest.mark.parametrize("resource", SELECTED_RESOURCES) @PROPERTY -@given(data=st.data(), note=st.text(alphabet="abcdef", min_size=1, max_size=8)) +@given(note=st.text(alphabet="abcdef", min_size=1, max_size=8)) def test_one_resource_edit_rekeys_exactly_its_selectors_and_descendants( - composed, data, note + composed, resource, note ): """The graph binds each resource only through the nodes that select it.""" - resource = data.draw(st.sampled_from(_selected_resources(composed.spec))) + assert set(SELECTED_RESOURCES) == set(_selected_resources(composed.spec)) spec = copy.deepcopy(composed.spec) spec["resources"][resource]["zz_hypothesis_note"] = note graph = composed.graph(extended=True) @@ -531,6 +548,86 @@ def test_holdout_and_family_semantics_have_no_calibration_ancestry(composed): ) +@pytest.mark.parametrize( + "violation", + ["prepared", "embedded_ref", "holdout_resource", "holdout_source", "graph_digest"], +) +def test_compose_refuses_contaminated_calibration_ancestry(composed, violation): + spec = copy.deepcopy(composed.spec) + row = next( + row + for row in spec["resources"]["transport_graph"]["nodes"] + if row["id"] == "nz.targets.problem" + ) + if violation == "holdout_source": + row["sources"] = ["nz_holdout_facts"] + else: + row["params"]["contamination"] = { + "prepared": {"prepared": "engine_refs"}, + "embedded_ref": json.dumps( + {"nested": [composed.registry.engine_refs["family_housing"]]} + ), + "holdout_resource": { + "resource": "holdout_references", + "encoding": "sha256", + }, + "graph_digest": {"resource": "transport_graph", "encoding": "sha256"}, + }[violation] + with pytest.raises(ValueError, match="calibration ancestry|graph resource.*digest"): + compose_transport_graph(spec, composed.config) + + +@pytest.mark.parametrize( + "literal", + [ + {"resource": "holdout_references", "encoding": "sha256"}, + {"prepared": "engine_refs"}, + ], +) +def test_selected_lists_keep_selector_shaped_objects_literal(composed, literal): + spec = copy.deepcopy(composed.spec) + spec["resources"]["calibration"]["literal"] = [[literal]] + result = compose_module._param( + spec, + {"resource": "calibration", "path": ["literal"]}, + composed.config.engine_refs, + ) + assert result == ((literal,),) + + +@pytest.mark.parametrize("gap", ["mass", "scenario", "resource"]) +def test_activation_names_generic_selected_gaps_before_donor_read( + composed, monkeypatch, gap +): + spec = copy.deepcopy(composed.spec) + if gap == "mass": + spec["resources"]["mass_references"]["target_references"][0]["metadata"][ + "activation_status" + ] = "placeholder" + match = "placeholder reference mass_references" + elif gap == "scenario": + spec["resources"]["scenarios"]["scenarios"] = [ + {"id": "SX", "knobs": {"other_factor": None}} + ] + match = "scenario SX other_factor" + else: + spec["resources"]["transport_graph"]["nodes"][0]["params"]["missing"] = [ + {"resource": "absent_resource", "encoding": "sha256"} + ] + match = "absent_resource.json" + + def unread(*args, **kwargs): + pytest.fail("activation must refuse before CREATE reads the donor") + + monkeypatch.setattr(cli_module, "prepare_create_outputs", unread) + with pytest.raises(ValueError, match=match): + prepare_transport_build( + spec, + sources=composed.sources, + engines_by_binding=composed.engines_by_binding, + ) + + class _IllegalOwner(KernelBase): ref = "test.illegal_owner@1" capabilities = Capabilities(Determinism.DETERMINISTIC) @@ -671,12 +768,48 @@ def test_e3_require_refuses_a_cold_store_and_creates_nothing(composed, tmp_path) assert not out.exists() -def test_cli_checkpoints_write_only_under_out_and_resume_warm(composed, tmp_path): +def test_cli_fresh_builds_have_identical_h5_bytes_and_every_node_key( + composed, tmp_path +): graph = composed.graph(extended=True) + first_out, second_out = tmp_path / "first", tmp_path / "second" + first = execute_transport_graph( + graph, + composed.registry, + composed.sources, + out=first_out, + endpoints=composed.endpoints, + ) + # HDF5 object timestamps have second resolution; cross that boundary. + time.sleep(1.1) + second = execute_transport_graph( + graph, + composed.registry, + composed.sources, + out=second_out, + endpoints=composed.endpoints, + ) + assert (first_out / EXPORT_FILENAME).read_bytes() == ( + second_out / EXPORT_FILENAME + ).read_bytes() + assert _keys(first) == _keys(second) + + +def test_cli_checkpoints_write_only_under_out_and_resume_warm(composed, tmp_path): + extension = compose_module.TransportExtension( + toy_extension, + checkpoints=("nz.test.scn.rules", "nz.test.validate.compare"), + ) + graph = compose_transport_graph( + composed.spec, replace(composed.config, extensions=(extension,)) + ) + endpoints = compose_module.transport_endpoints( + composed.spec, extensions=(extension,) + ) out = tmp_path / "out" options = { "out": out, - "endpoints": composed.endpoints, + "endpoints": endpoints, "pending_outputs": composed.pending_outputs, } inputs = _inventory(composed.root) @@ -685,7 +818,7 @@ def test_cli_checkpoints_write_only_under_out_and_resume_warm(composed, tmp_path composed.registry, composed.sources, out=out, - endpoints=composed.endpoints, + endpoints=endpoints, pending_outputs=composed.pending_outputs, ) assert _inventory(composed.root) == inputs @@ -705,6 +838,8 @@ def test_cli_checkpoints_write_only_under_out_and_resume_warm(composed, tmp_path for name in ( "checkpoint-geography.json", "checkpoint-calibration.json", + "checkpoint-extension-0.json", + "checkpoint-extension-1.json", "checkpoint-export-prepared.json", "checkpoint-terminal-0.json", "checkpoint-terminal-1.json", @@ -723,7 +858,7 @@ def test_cli_checkpoints_write_only_under_out_and_resume_warm(composed, tmp_path composed.registry, composed.sources, out=out, - endpoints=composed.endpoints, + endpoints=endpoints, pending_outputs=composed.pending_outputs, resume="require", ) @@ -851,6 +986,7 @@ def names(resource): calibration = names("target_references") | names("precal_references") assert names("holdout_references").isdisjoint(calibration) + compose_module.validate_transport_calibration_ancestry(spec) # --------------------------------------------------------------------------- @@ -900,3 +1036,580 @@ def test_transport_python_has_no_policy_literal_or_remote_path(): name ) assert "publish_staged_bundle" not in imported, name + + +def _write_local_transport_spec(root, spec): + directory = root / spec["country"] + directory.mkdir(parents=True) + names = [] + for name, document in spec["resources"].items(): + filename = name + ".json" + names.append(filename) + (directory / filename).write_text(json.dumps(document), encoding="utf-8") + (directory / "country_package.json").write_text( + json.dumps( + { + "country": spec["country"], + "policy": "Synthetic transport test package.", + "resources": names, + } + ), + encoding="utf-8", + ) + return directory + + +def test_cli_main_require_refuses_before_preparing_or_reading_sources( + tmp_path, monkeypatch, capsys +): + calls = [] + + def prepare(*args, **kwargs): + calls.append("prepare") + raise ValueError("CREATE probe ran") + + monkeypatch.setattr(cli_module, "prepare_transport_build", prepare) + out = tmp_path / "out" + assert ( + cli_module.main(["--country", "nz", "--out", str(out), "--resume", "require"]) + == 1 + ) + assert "cold transport graph store" in capsys.readouterr().err + assert calls == [] + assert not out.exists() + + +@pytest.mark.parametrize("relation", ["inside", "contains", "same"]) +def test_cli_main_refuses_output_overlap_with_spec_dir( + tmp_path, relation, monkeypatch, capsys +): + spec_dir = tmp_path / "spec" + spec_dir.mkdir() + out = {"inside": spec_dir / "out", "contains": tmp_path, "same": spec_dir}[relation] + calls = [] + + def load(source): + calls.append(source) + raise ValueError("spec was read") + + monkeypatch.setattr(cli_module, "load_transport_spec", load) + assert cli_module.main(["--spec-dir", str(spec_dir), "--out", str(out)]) == 1 + assert "--out overlaps --spec-dir" in capsys.readouterr().err + assert calls == [] + + +@pytest.mark.parametrize("filename", ["../escape", "nested/file", ".", "..", ""]) +def test_cli_destination_refuses_escaping_names(tmp_path, filename): + out = tmp_path / "out" + out.mkdir() + with pytest.raises(ValueError, match="simple filename"): + cli_module._destination(out, filename) + + +@pytest.mark.parametrize("filename", ["graph.json", ".graph-store"]) +def test_cli_destination_refuses_symlinks(tmp_path, filename): + out = tmp_path / "out" + out.mkdir() + target = tmp_path / "outside" + target.write_bytes(b"preserved") + (out / filename).symlink_to(target) + with pytest.raises(ValueError, match="outside --out"): + cli_module._destination(out, filename) + assert target.read_bytes() == b"preserved" + + +@pytest.mark.parametrize("missing", ["unknown_variable", "unit_rule", "kernel"]) +def test_cli_main_turns_key_errors_into_refusals( + composed, tmp_path, monkeypatch, capsys, missing +): + spec = copy.deepcopy(composed.spec) + rows = spec["resources"]["transport_graph"]["nodes"] + if missing == "unknown_variable": + spec["resources"]["axiom_rules_bindings"]["bindings"][0]["variables"].append( + "unknown_variable" + ) + elif missing == "unit_rule": + create = next(row for row in rows if row["kernel"] == "transport.create@1") + create["params"]["renamed_unit_rule"] = create["params"].pop("unit_rule") + else: + next(row for row in rows if row["id"] == "nz.targets.problem").pop("kernel") + monkeypatch.setattr(cli_module, "load_transport_spec", lambda source: spec) + build_registry = cli_module.build_transport_registry + + def injected_registry(bindings, root, *, unit_rule, engines_by_binding=None): + return build_registry( + bindings, + root, + unit_rule=unit_rule, + engines_by_binding=composed.engines_by_binding, + ) + + monkeypatch.setattr(cli_module, "build_transport_registry", injected_registry) + out = tmp_path / "out" + argv = ["--country", "nz", "--out", str(out)] + for name, path in composed.sources.items(): + argv.extend(["--source", f"{name}={path}"]) + assert cli_module.main(argv) == 1 + error = capsys.readouterr().err + assert "transport build refused:" in error + assert missing in error + assert "Traceback" not in error + assert not (out / EXPORT_FILENAME).exists() + + +def test_cli_main_and_path_loader_success(composed, tmp_path, monkeypatch, capsys): + # Real Path loading, registry preparation and execution; only the injected + # engine seam replaces Axiom with the fixture's pure-Python adapters. + directory = _write_local_transport_spec(tmp_path / "spec", composed.spec) + loaded = load_transport_spec(directory) + assert loaded["country"] == composed.spec["country"] + assert loaded["resources"] == composed.spec["resources"] + for name in loaded["resources"]: + assert ( + loaded["resource_hashes"][name] + == hashlib.sha256((directory / (name + ".json")).read_bytes()).hexdigest() + ) + build_registry = cli_module.build_transport_registry + observe_warm = [] + calls = [] + + def injected_registry(bindings, root, *, unit_rule, engines_by_binding=None): + registry = build_registry( + bindings, + root, + unit_rule=unit_rule, + engines_by_binding=composed.engines_by_binding, + ) + if observe_warm: + for ref in registry.kernels.refs(): + kernel = registry.kernels.get(ref) + run = kernel.run + + def counted(context, _run=run, _ref=ref): + calls.append(_ref) + return _run(context) + + monkeypatch.setattr(kernel, "run", counted) + return registry + + monkeypatch.setattr(cli_module, "build_transport_registry", injected_registry) + out = tmp_path / "out" + argv = ["--spec-dir", str(directory), "--out", str(out)] + for name, path in composed.sources.items(): + argv.extend(["--source", f"{name}={path}"]) + assert cli_module.main(argv) == 0 + captured = capsys.readouterr() + assert captured.err == "" + assert "Transport skeleton written" in captured.out + assert "Not yet produced:" in captured.out + assert (out / EXPORT_FILENAME).is_file() + assert json.loads((out / "manifest.json").read_bytes())["nodes"] + observe_warm.append(True) + assert cli_module.main([*argv, "--resume", "require"]) == 0 + assert capsys.readouterr().err == "" + assert calls == [] + + +def test_cli_main_invalid_arguments_are_a_refusal(capsys): + assert cli_module.main(["--country", "nz"]) == 1 + assert "transport build refused:" in capsys.readouterr().err + + +def test_cli_main_help_succeeds(capsys): + assert cli_module.main(["--help"]) == 0 + captured = capsys.readouterr() + assert "usage:" in captured.out + assert captured.err == "" + + +def test_cli_main_refuses_store_symlink_before_loading_spec( + tmp_path, monkeypatch, capsys +): + out = tmp_path / "out" + out.mkdir() + target = tmp_path / "outside" + target.mkdir() + (target / "preserved").write_bytes(b"preserved") + (out / ".graph-store").symlink_to(target, target_is_directory=True) + calls = [] + + def load(source): + calls.append(source) + raise ValueError("spec was read") + + monkeypatch.setattr(cli_module, "load_transport_spec", load) + assert cli_module.main(["--country", "nz", "--out", str(out)]) == 1 + assert "outside --out" in capsys.readouterr().err + assert calls == [] + assert (target / "preserved").read_bytes() == b"preserved" + + +def test_create_inventory_cache_reuses_warm_without_running_create( + composed, tmp_path, monkeypatch +): + store = ContentStore(tmp_path / "inventory") + calls = [] + create = cli_module.TRANSPORT_CREATE.run + + def counted(context): + calls.append(context.node.id) + return create(context) + + monkeypatch.setattr(cli_module.TRANSPORT_CREATE, "run", counted) + first = prepare_create_outputs( + composed.spec, composed.sources, inventory_store=store + ) + assert calls == ["nz.create"] + for resume in ("auto", "require"): + assert ( + prepare_create_outputs( + composed.spec, composed.sources, inventory_store=store, resume=resume + ) + == first + ) + assert calls == ["nz.create"] + # Source paths are descriptive; moving identical bytes retains the hit. + copied = tmp_path / "moved-facts.jsonl" + copied.write_bytes(composed.sources["nz_calibration_facts"].read_bytes()) + moved = {**composed.sources, "nz_calibration_facts": copied} + assert ( + prepare_create_outputs( + composed.spec, moved, inventory_store=store, resume="require" + ) + == first + ) + assert calls == ["nz.create"] + assert ( + prepare_create_outputs( + composed.spec, composed.sources, inventory_store=store, resume="forbid" + ) + == first + ) + assert calls == ["nz.create", "nz.create"] + + +def test_create_inventory_cache_binds_actual_source_bytes( + composed, tmp_path, monkeypatch +): + store = ContentStore(tmp_path / "inventory") + calls = [] + create = cli_module.TRANSPORT_CREATE.run + + def counted(context): + calls.append(context.node.id) + return create(context) + + monkeypatch.setattr(cli_module.TRANSPORT_CREATE, "run", counted) + first = prepare_create_outputs( + composed.spec, composed.sources, inventory_store=store + ) + # An inert extra newline preserves facts and column inventory, but changes + # source bytes. A cache keyed only by declared pins would silently hit. + changed = tmp_path / "changed-facts.jsonl" + changed.write_bytes(composed.sources["nz_calibration_facts"].read_bytes() + b"\n") + sources = {**composed.sources, "nz_calibration_facts": changed} + with pytest.raises(StoreMissError, match="this CREATE inventory"): + prepare_create_outputs( + composed.spec, sources, inventory_store=store, resume="require" + ) + assert calls == ["nz.create"] + assert ( + prepare_create_outputs(composed.spec, sources, inventory_store=store) == first + ) + assert calls == ["nz.create", "nz.create"] + + +def test_prepare_does_not_create_inventory_store_before_activation(tmp_path): + inventory = tmp_path / "inventory" + with pytest.raises(ValueError, match="not activated"): + prepare_transport_build( + load_transport_spec("nz"), sources={}, inventory_store=inventory + ) + assert not inventory.exists() + + +def test_prepare_build_reuses_inventory_without_running_create( + composed, tmp_path, monkeypatch +): + inventory = tmp_path / "inventory" + calls = [] + create = cli_module.TRANSPORT_CREATE.run + + def counted(context): + calls.append(context.node.id) + return create(context) + + monkeypatch.setattr(cli_module.TRANSPORT_CREATE, "run", counted) + options = { + "sources": composed.sources, + "engines_by_binding": composed.engines_by_binding, + "inventory_store": inventory, + } + first = prepare_transport_build(composed.spec, **options) + second = prepare_transport_build(composed.spec, **options, resume="require") + assert graph_to_json(first.graph) == graph_to_json(second.graph) + assert calls == ["nz.create"] + + +@pytest.mark.parametrize( + "node_id", + [ + "../escape", + "nested/node", + "nested\\node", + "..", + "unsafe..node", + "unsafe\x00node", + ], +) +@pytest.mark.parametrize("extension", [False, True]) +def test_compose_refuses_unsafe_node_ids(composed, node_id, extension): + spec = copy.deepcopy(composed.spec) + config = composed.config + if extension: + + def factory(spec, config, skeleton): + return (replace(skeleton.node("nz.as"), id=node_id),) + + config = replace(config, extensions=(factory,)) + else: + rows = spec["resources"]["transport_graph"]["nodes"] + row = next(item for item in rows if item["id"] == "nz.targets.problem") + row["id"] = node_id + for item in rows: + for edge in item.get("artifact_inputs", ()): + if edge["producer"] == "nz.targets.problem": + edge["producer"] = node_id + with pytest.raises(ValueError, match="safe filename"): + compose_transport_graph(spec, config) + + +def test_extension_factories_receive_independent_spec_copies(composed): + before = copy.deepcopy(composed.spec) + observed = [] + + def first(spec, config, skeleton): + spec["resources"]["calibration"]["private_edit"] = "extension-local" + return () + + def second(spec, config, skeleton): + observed.append(copy.deepcopy(spec)) + return () + + compose_transport_graph( + composed.spec, replace(composed.config, extensions=(first, second)) + ) + assert composed.spec == before + assert observed == [before] + + +@pytest.mark.parametrize( + "selector", + [ + {"resource": "transport_graph", "encoding": "sha256"}, + {"prepared": "spec_fingerprint"}, + ], +) +def test_compose_refuses_global_declaration_identities(composed, selector): + spec = copy.deepcopy(composed.spec) + # Put it outside calibration ancestry, proving the global refusal. + row = next( + item + for item in spec["resources"]["transport_graph"]["nodes"] + if item["id"] == "nz.package" + ) + row["params"]["forbidden_identity"] = selector + with pytest.raises( + ValueError, match="graph resource.*digest|whole-spec fingerprint" + ): + compose_transport_graph(spec, composed.config) + + +@pytest.mark.parametrize("prefixed", [False, True, "escaped"]) +@pytest.mark.parametrize("adapter", ["python", "axiom"]) +@pytest.mark.parametrize("selected", [False, True]) +def test_public_declaration_check_refuses_unit_ref_without_config( + composed, selected, adapter, prefixed +): + spec = copy.deepcopy(composed.spec) + reference = composed.registry.engine_refs["family_housing"] + bindings = spec["resources"]["axiom_rules_bindings"]["bindings"] + if adapter == "axiom": + from microcosm.frame.adapters.axiom import _ENGINE_REF_SCHEMA + + binding = next(row for row in bindings if row["id"] == "family_housing") + reference = json.dumps( + {"format": _ENGINE_REF_SCHEMA, "module": binding["rulespec_path"]} + ) + if prefixed == "escaped": + reference = f"prefix {json.dumps(reference)} suffix" + elif prefixed: + reference = f"prefix {reference} suffix" + row = next( + item + for item in spec["resources"]["transport_graph"]["nodes"] + if item["id"] == "nz.targets.problem" + ) + if selected: + # Selector-shaped objects in the selected data are literal, but an + # actual embedded forbidden engine reference still contaminates it. + spec["resources"]["calibration"]["literal_ref"] = [{"nested": reference}] + row["params"]["contamination"] = { + "resource": "calibration", + "path": ["literal_ref"], + } + else: + row["params"]["contamination"] = json.dumps({"nested": [reference]}) + validator = getattr(compose_module, "validate_transport_calibration_ancestry", None) + assert callable(validator), "composition must expose its declaration ancestry check" + with pytest.raises(ValueError, match="calibration ancestry"): + validator(spec) + + +def test_extension_rules_helper_uses_the_binding_period_and_ref(composed): + spec = copy.deepcopy(composed.spec) + bindings = spec["resources"]["axiom_rules_bindings"] + family = next(row for row in bindings["bindings"] if row["id"] == "family_housing") + previous = family["period"] + family["period"] = "custom-bound-period" + bindings["periods"][family["period"]] = dict(bindings["periods"][previous]) + fixture = composed.reprepare(spec) + unbound = Node( + "nz.test.helper.rules", + "simulate.rules_by_ref@1", + population="nz.as", + sources=("rulespec_nz",), + inputs=(Slice("person", ("age",)), Slice("family", FAMILY_INPUTS)), + outputs=tuple( + Owned("family", name, row["dtype"]) + for name, row in PROGRAMS["family_housing"]["outputs"].items() + ), + ) + bound = compose_module.transport_rules_node( + spec, fixture.config, "family_housing", unbound + ) + assert bound.params["engine_ref"] == fixture.config.engine_refs["family_housing"] + assert bound.params["period"] == "custom-bound-period" + assert bound.params["variables"] == tuple(family["variables"]) + assert bound.inputs == unbound.inputs + assert bound.outputs == unbound.outputs + assert bound.sources == unbound.sources + # The fixture extension itself must also take the period from the binding. + assert ( + fixture.graph(extended=True).node("nz.test.scn.rules").params["period"] + == "custom-bound-period" + ) + with pytest.raises(ValueError, match="may set only variables"): + compose_module.transport_rules_node( + spec, + fixture.config, + "family_housing", + replace(unbound, params={"period": previous}), + ) + + +def test_extension_sources_rekey_only_their_own_nodes(composed, tmp_path): + from microcosm.build.transport.codecs import FACTS_SOURCE_CODEC + + def factory(spec, config, skeleton): + return tuple( + replace(node, sources=("scenario_facts",)) + if node.id == "nz.test.validate.compare" + else node + for node in toy_extension(spec, config, skeleton) + ) + + extension = compose_module.TransportExtension( + factory, sources=(SourceRef("scenario_facts", FACTS_SOURCE_CODEC),) + ) + config = replace(composed.config, extensions=(extension,)) + extra = tmp_path / "scenario-facts.jsonl" + extra.write_bytes(composed.sources["nz_holdout_facts"].read_bytes()) + fixture = replace(composed, sources={**composed.sources, "scenario_facts": extra}) + skeleton = _declared_keys(composed.graph(), fixture) + graph = compose_transport_graph(composed.spec, config) + first = _declared_keys(graph, fixture) + assert {source.name for source in graph.sources} == { + source.name for source in composed.graph().sources + } | {"scenario_facts"} + assert {name: first[name] for name in skeleton} == skeleton + before_source = source_content_key("scenario_facts", extra) + extra.write_bytes(extra.read_bytes() + b"\n") + assert source_content_key("scenario_facts", extra) != before_source + second = _declared_keys(graph, fixture) + assert {name for name in first if first[name] != second[name]} == { + "nz.test.validate.compare" + } + assert {name: second[name] for name in skeleton} == skeleton + + +@pytest.mark.parametrize("collision", ["skeleton", "extension"]) +def test_extension_sources_cannot_replace_declared_inputs(composed, collision): + def empty(spec, config, skeleton): + return () + + source = ( + composed.graph().sources[0] + if collision == "skeleton" + else SourceRef("scenario_facts", "raw-bytes-v1") + ) + extension = compose_module.TransportExtension(empty, sources=(source,)) + extensions = (extension,) if collision == "skeleton" else (extension, extension) + with pytest.raises(ValueError, match="must have a new name"): + compose_transport_graph( + composed.spec, replace(composed.config, extensions=extensions) + ) + + +@pytest.mark.parametrize( + "checkpoint,match", + [ + ("missing.checkpoint", "extended graph"), + ("nz.export.readback", "precede export readback"), + ], +) +def test_extension_checkpoints_are_checked_before_execution( + composed, checkpoint, match +): + extension = compose_module.TransportExtension( + lambda spec, config, skeleton: (), checkpoints=(checkpoint,) + ) + with pytest.raises(ValueError, match=match): + compose_transport_graph( + composed.spec, replace(composed.config, extensions=(extension,)) + ) + + +def test_extension_checkpoints_are_returned_in_declaration_order(composed): + extension = compose_module.TransportExtension( + toy_extension, checkpoints=("nz.test.scn.rules", "nz.test.validate.compare") + ) + extensions = (extension,) + endpoints = compose_module.transport_endpoints(composed.spec, extensions=extensions) + assert endpoints["checkpoints"] == extension.checkpoints + duplicate = replace( + extension, checkpoints=("nz.test.scn.rules", "nz.test.scn.rules") + ) + with pytest.raises(ValueError, match="distinct nodes"): + compose_module.transport_endpoints(composed.spec, extensions=(duplicate,)) + + +@pytest.mark.parametrize("failure_type", [ImportError, NotImplementedError]) +def test_cli_main_refuses_unavailable_rules_engine( + composed, tmp_path, monkeypatch, capsys, failure_type +): + monkeypatch.setattr(cli_module, "load_transport_spec", lambda source: composed.spec) + + def unavailable(*args, **kwargs): + raise failure_type("rules engine unavailable") + + monkeypatch.setattr(cli_module, "build_transport_registry", unavailable) + out = tmp_path / "out" + argv = ["--country", "nz", "--out", str(out)] + for name, path in composed.sources.items(): + argv.extend(["--source", f"{name}={path}"]) + assert cli_module.main(argv) == 1 + assert ( + "transport build refused: rules engine unavailable" in capsys.readouterr().err + ) + assert not (out / EXPORT_FILENAME).exists() diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_registry.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_registry.py index f0cb3d488e..7547bce961 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_registry.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_registry.py @@ -1,4 +1,4 @@ -"""Authenticated transport bindings with three engine-free adapters in one run.""" +"""Checked transport bindings with three engine-free adapters in one run.""" from __future__ import annotations @@ -54,6 +54,10 @@ run_graph, ) from microcosm.graph.codecs import load_raw_bytes +from test_support.microcosm_build.transport_composed import ( + ToyRulesEngine, + write_rulespec_tree, +) from test_support.microcosm_build.transport_population import UNIT_RULE @@ -82,7 +86,7 @@ def _program(self) -> Mapping: def variable_metadata(self, name: str) -> VariableMetadata: if name not in self.binding["variables"]: raise ValueError(f"Unknown toy output {name!r}") - return VariableMetadata(name, self.binding["entity"], "float", "year") + return VariableMetadata(name, self._program()["entity"], "float", "year") def variables(self) -> Sequence[str]: return (self._program()["input"],) @@ -94,7 +98,7 @@ def materialize( self, bundle: Frame, variables: Sequence[str], period: int | str ) -> Mapping[str, np.ndarray]: program = self._program() - values = bundle.table(self.binding["entity"])[program["input"]].to_numpy( + values = bundle.table(program["entity"])[program["input"]].to_numpy( dtype=np.float64 ) return { @@ -118,7 +122,12 @@ def _fixture(tmp_path: Path): ("housing", "family", "cost", 4.0), ): payload = json.dumps( - {"input": column, "coefficient": coefficient, "relations": []}, + { + "entity": entity, + "input": column, + "coefficient": coefficient, + "relations": [], + }, sort_keys=True, ).encode() module = root / f"{name}.yaml" @@ -379,8 +388,26 @@ def test_missing_pin_names_the_activation_gap(tmp_path, section, field): ) -@pytest.mark.parametrize("problem", ["hash", "path", "period", "duplicate", "entity"]) -def test_malformed_binding_is_refused(tmp_path, problem): +@pytest.mark.parametrize( + ("problem", "match"), + [ + ("hash", r"rules binding 'main_benefit'\.sha256 does not match"), + ( + "path", + r"rules binding 'main_benefit'\.rulespec_path must be a canonical relative path", + ), + ( + "period", + r"rules binding 'main_benefit'\.period 'unmapped' has no explicit period bounds", + ), + ("duplicate", r"axiom_rules_bindings\.bindings repeats a binding id"), + ( + "entity", + r"rules binding 'main_benefit'\.engine_entity disagrees with entity_names", + ), + ], +) +def test_malformed_binding_is_refused(tmp_path, problem, match): root, document, engines = _fixture(tmp_path) first = document["bindings"][0] if problem == "hash": @@ -393,14 +420,47 @@ def test_malformed_binding_is_refused(tmp_path, problem): document["bindings"].append(dict(first)) else: first["engine_entity"] = "Family" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match=match): + build_transport_registry( + document, root, unit_rule=UNIT_RULE, engines_by_binding=engines + ) + + +@pytest.mark.parametrize("toy", ["registry", "composed"]) +def test_injected_adapter_entity_is_independent_of_binding(tmp_path, toy): + root, document, engines = _fixture(tmp_path) + if toy == "composed": + document["bindings"] = write_rulespec_tree(root) + engines = {row["id"]: ToyRulesEngine(root, row) for row in document["bindings"]} + first = document["bindings"][0] + first["entity"] = "family" + first["engine_entity"] = "Family" + with pytest.raises(ValueError, match="belongs to another entity"): build_transport_registry( document, root, unit_rule=UNIT_RULE, engines_by_binding=engines ) -@pytest.mark.parametrize("problem", ["root", "schema", "relations", "identity", "keys"]) -def test_injected_adapter_contract_is_checked(tmp_path, problem): +@pytest.mark.parametrize( + ("problem", "match"), + [ + ( + "root", + "rules binding 'housing' engine root must equal the resolved RuleSpec tree", + ), + ( + "schema", + "rules binding 'housing' engine schema does not match the transport schema", + ), + ("relations", "relations"), + ( + "identity", + r"rules binding 'housing' injected engine must expose cache_identity\(\)", + ), + ("keys", "engines_by_binding must cover exactly the declared binding ids"), + ], +) +def test_injected_adapter_contract_is_checked(tmp_path, problem, match): root, document, engines = _fixture(tmp_path) engine = engines["housing"] if problem == "root": @@ -415,7 +475,7 @@ def test_injected_adapter_contract_is_checked(tmp_path, problem): engine.cache_identity = None else: engines.pop("housing") - with pytest.raises((TypeError, ValueError, NotImplementedError)): + with pytest.raises((TypeError, ValueError, NotImplementedError), match=match): build_transport_registry( document, root, unit_rule=UNIT_RULE, engines_by_binding=engines ) diff --git a/test_support/microcosm_build/transport_composed.py b/test_support/microcosm_build/transport_composed.py index 8ffcd300de..9b1d3b8b44 100644 --- a/test_support/microcosm_build/transport_composed.py +++ b/test_support/microcosm_build/transport_composed.py @@ -44,6 +44,7 @@ transport_endpoints, transport_pending_outputs, transport_resource, + transport_rules_node, ) from microcosm.build.transport.registry import ( TransportRegistry, @@ -200,7 +201,7 @@ def assert_no_relations(self, entity: str) -> None: def variable_metadata(self, name: str) -> VariableMetadata: row = self.program()["outputs"][name] kind = "float" if row["dtype"] == "float64" else "int" - return VariableMetadata(name, self.binding["entity"], kind, "year") + return VariableMetadata(name, self.program()["entity"], kind, "year") def variables(self) -> Sequence[str]: return tuple( @@ -214,7 +215,7 @@ def materialize( self, bundle: Frame, variables: Sequence[str], period: int | str ) -> Mapping[str, np.ndarray]: self.calls.append((tuple(variables), period)) - table = bundle.table(self.binding["entity"]) + table = bundle.table(self.program()["entity"]) result = {} for name in variables: row = self.program()["outputs"][name] @@ -841,22 +842,23 @@ def toy_extension(spec, config, skeleton: Graph) -> tuple[Node, ...]: inputs=(Slice("person", ("age",)),), ) outputs = PROGRAMS["family_housing"]["outputs"] - rules = Node( - "nz.test.scn.rules", - "simulate.rules_by_ref@1", - population=branch.id, - sources=("rulespec_nz",), - # A family-only engine node still slices one person data column: - # the kernel context holds only the entities a node reads or owns. - inputs=(Slice("person", ("age",)), Slice("family", FAMILY_INPUTS)), - outputs=tuple( - Owned("family", name, row["dtype"]) for name, row in outputs.items() + rules = transport_rules_node( + spec, + config, + "family_housing", + Node( + "nz.test.scn.rules", + "simulate.rules_by_ref@1", + population=branch.id, + sources=("rulespec_nz",), + # A family-only engine node still slices one person data column: + # the kernel context holds only the entities a node reads or owns. + inputs=(Slice("person", ("age",)), Slice("family", FAMILY_INPUTS)), + outputs=tuple( + Owned("family", name, row["dtype"]) for name, row in outputs.items() + ), + params={"variables": tuple(outputs)}, ), - params={ - "engine_ref": config.engine_refs["family_housing"], - "period": PERIOD, - "variables": tuple(outputs), - }, ) validate = Node( "nz.test.validate", @@ -890,7 +892,13 @@ def write_rulespec_tree(root: Path) -> list[dict]: for name, program in PROGRAMS.items(): path = root / program["path"] path.parent.mkdir(parents=True, exist_ok=True) - payload = canonical_json({"outputs": program["outputs"], "relations": []}) + payload = canonical_json( + { + "entity": program["entity"], + "outputs": program["outputs"], + "relations": [], + } + ) path.write_bytes(payload) rows.append( { From fe47f88bb52e1f14988fe7a549e65f199b98f958 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Fri, 9 Oct 2026 15:20:28 -0400 Subject: [PATCH 5/5] Refuse a graph store tree that contains a link The round-2 review of #1161 showed that a pre-existing `/.graph-store/objects` link sends ContentStore writes outside --out: ContentStore creates `objects`, `tmp` and shard directories with mkdir(exist_ok=True), which follows an existing link, and the driver only checked the store root itself. `_contained_store_root(out)` now checks the root with `_destination` and, when a store tree already exists, walks it without following links and refuses any entry that is a link (root, objects, tmp, shards, objects). Both store entry points use it: main(), before the spec is loaded, and execute_transport_graph(), before the ContentStore is opened. The agent guide and changelog now say what is guarded (links present when a build starts) and what is not (links planted while it runs). Tests: a link at objects, tmp or objects/aa is refused by _contained_store_root, by execute_transport_graph and by main(), and nothing is written to the link target; a plain store tree is accepted. With the old root-only check restored in memory (a pytest plugin, no tracked file changed), the 4 execution and main() cases fail (DID NOT RAISE; main() reached spec loading). test_transport_composed_graph.py 123 passed; test_transport_graph.py 15 passed; ruff check and format pass. Co-Authored-By: Claude Opus 5.5 --- changelog.d/transport-compose-cli.added.md | 4 +- docs/agent-guide.md | 5 +- .../src/microcosm/build/transport/cli.py | 35 ++++++++- .../shared/test_transport_composed_graph.py | 71 +++++++++++++++++++ 4 files changed, 111 insertions(+), 4 deletions(-) diff --git a/changelog.d/transport-compose-cli.added.md b/changelog.d/transport-compose-cli.added.md index 9f4e4a09b9..81b902e149 100644 --- a/changelog.d/transport-compose-cli.added.md +++ b/changelog.d/transport-compose-cli.added.md @@ -4,7 +4,9 @@ declares the skeleton graph; each node binds only the spec resources it selects, every rules node runs through the one `simulate.rules_by_ref@1` router, and later packages append branches through a typed extension point that cannot re-key the skeleton. The driver runs ancestor-closed checkpoints -on `/.graph-store` and writes every output under `--out`. Activation checks every selected resource and reference set and every null +on `/.graph-store` and writes every output under `--out`; it refuses an +existing store tree that contains a link, which could redirect store writes. +Activation checks every selected resource and reference set and every null scenario knob before donor preparation. Calibration ancestry refuses hold-out selections and sources, benefit-unit engine references, and graph digests. Selected resource lists remain literal data. The driver writes deterministic diff --git a/docs/agent-guide.md b/docs/agent-guide.md index d9b294ff59..d014313618 100644 --- a/docs/agent-guide.md +++ b/docs/agent-guide.md @@ -343,7 +343,10 @@ The transport build driver (`tools/build_transport.py`, a shim over `microcosm.build.transport.cli`) composes a country's graph from its `transport_graph.json` spec resource and runs it locally: every file it writes, including its `.graph-store`, lies under `--out`, and it has no -upload or staging path. It refuses before reading any graph source while the +upload or staging path. Before it opens the store it refuses any link in an +existing store tree (the root, `objects`, `tmp` or a shard), because the +store creates those directories by following links; it does not guard +against links planted while a build is running. It refuses before reading any graph source while the country spec still has unresolved evidence: every selected resource/reference set and every null scenario knob is checked before donor preparation. Tests for it run on the engine-free toy package in diff --git a/packages/microcosm-build/src/microcosm/build/transport/cli.py b/packages/microcosm-build/src/microcosm/build/transport/cli.py index f3a9a03e63..fac10c0cc1 100644 --- a/packages/microcosm-build/src/microcosm/build/transport/cli.py +++ b/packages/microcosm-build/src/microcosm/build/transport/cli.py @@ -20,6 +20,7 @@ import argparse import json +import os import sys from collections.abc import Mapping, Sequence from dataclasses import dataclass, replace @@ -317,6 +318,36 @@ def _destination(out: Path, filename: str) -> Path: return path +def _contained_store_root(out: Path) -> Path: + """The graph store under ``out``; refuse a store tree a link could redirect. + + ``ContentStore`` writes only under ``objects//`` and ``tmp/`` + and creates those directories with ``mkdir(exist_ok=True)``, which follows + an existing symbolic link. A link planted at ``objects``, ``tmp`` or any + shard would therefore send writes outside ``--out``. So the store root + must be a plain name under ``out`` (``_destination``), and no entry in an + existing store tree may be a link: the walk does not follow links. + """ + + root = _destination(out, _STORE) + if not root.exists(): + return root + pending = [root] + while pending: + directory = pending.pop() + with os.scandir(directory) as entries: + for entry in entries: + if entry.is_symlink(): + relative = Path(entry.path).relative_to(out) + raise ValueError( + f"Graph store entry {str(relative)!r} is a link, which " + "could send store writes outside --out." + ) + if entry.is_dir(follow_symlinks=False): + pending.append(Path(entry.path)) + return root + + def _check_disjoint( out: Path, sources: Mapping[str, Path], *, spec_dir: Path | None = None ) -> None: @@ -472,7 +503,7 @@ def execute_transport_graph( raise StoreMissError("--resume require refuses a cold transport graph store.") output.mkdir(parents=True, exist_ok=True) - store_root = _destination(output, _STORE) + store_root = _contained_store_root(output) exported = _destination(output, EXPORT_FILENAME) supplied[export_source] = exported store = ContentStore(store_root, codecs=registry.codecs) @@ -615,7 +646,7 @@ def main(argv: Sequence[str] | None = None) -> int: output = args.out.resolve() sources = {name: path.resolve() for name, path in args.source} _check_disjoint(output, sources, spec_dir=args.spec_dir) - store_root = _destination(output, _STORE) + store_root = _contained_store_root(output) if args.resume == "require" and _store_is_cold(store_root): raise StoreMissError( "--resume require refuses a cold transport graph store." diff --git a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py index d321a2e69f..e2b6272043 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_transport_composed_graph.py @@ -1244,6 +1244,77 @@ def load(source): assert (target / "preserved").read_bytes() == b"preserved" +def _plant_store_link(out, link, target): + """Make ``/.graph-store/`` a link to ``target``.""" + + path = out / ".graph-store" / link + path.parent.mkdir(parents=True, exist_ok=True) + path.symlink_to(target, target_is_directory=True) + + +@pytest.mark.parametrize("link", ["objects", "tmp", "objects/aa"]) +def test_cli_store_root_refuses_links_inside_the_store_tree(tmp_path, link): + out = tmp_path / "out" + out.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + _plant_store_link(out, link, outside) + with pytest.raises(ValueError, match="is a link"): + cli_module._contained_store_root(out) + assert list(outside.iterdir()) == [] + + +def test_cli_store_root_accepts_a_plain_store_tree(tmp_path): + out = tmp_path / "out" + (out / ".graph-store" / "objects" / "aa").mkdir(parents=True) + (out / ".graph-store" / "tmp").mkdir() + assert cli_module._contained_store_root(out) == out / ".graph-store" + assert cli_module._contained_store_root(tmp_path / "fresh") == ( + tmp_path / "fresh" / ".graph-store" + ) + + +@pytest.mark.parametrize("link", ["objects", "tmp", "objects/aa"]) +def test_cli_execution_refuses_a_redirected_store_and_writes_nothing_outside( + composed, tmp_path, link +): + out = tmp_path / "out" + out.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + _plant_store_link(out, link, outside) + with pytest.raises(ValueError, match="is a link"): + execute_transport_graph( + composed.graph(extended=True), + composed.registry, + composed.sources, + out=out, + endpoints=composed.endpoints, + ) + assert list(outside.iterdir()) == [] + + +def test_cli_main_refuses_a_nested_store_link_before_loading_spec( + tmp_path, monkeypatch, capsys +): + out = tmp_path / "out" + out.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + _plant_store_link(out, "objects", outside) + calls = [] + + def load(source): + calls.append(source) + raise ValueError("spec was read") + + monkeypatch.setattr(cli_module, "load_transport_spec", load) + assert cli_module.main(["--country", "nz", "--out", str(out)]) == 1 + assert "is a link" in capsys.readouterr().err + assert calls == [] + assert list(outside.iterdir()) == [] + + def test_create_inventory_cache_reuses_warm_without_running_create( composed, tmp_path, monkeypatch ):