From e06b0c90fd27ef0c7ac340f3ca80486fa186ade9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 15 Sep 2026 23:19:32 -0400 Subject: [PATCH 01/45] Open the verify-once lane journal Records the lane's base, goal, evidence base and the two local constraints: root out.md belongs to another lane, and the v5 cold run holds the machine. Co-Authored-By: Claude Opus --- PROGRESS.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/PROGRESS.md b/PROGRESS.md index 9b0b375e4..50dec777d 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -1413,3 +1413,32 @@ adversaries, T6 actual fixture validation and T7 GQ/zero-weight scientific fixtu remain open with concrete reasons in FINAL_REPORT.md. The independently confirmed old status/tax42 owner repair, inventories, CI/source closure, scientific/native capacity and release/default/PR893 decisions remain separately owned. + +--- + +# Lane: verify native sources once per run (`native-verify-once`) + +Branch `native-verify-once`, worktree `~/PolicyEngine/_worktrees/microcosm-verify-once`, +base `f7bb88525a78786f91bc3ebe2083ef4b1c85de18` (PR #893 head). Started +2026-09-15. + +**State (2026-09-15, opening):** investigation only. Nothing edited yet. + +**Goal.** The 9/15 pilot v5 measurement +(`~/PolicyEngine/_recovered/pilot-runs/native45-v5/out.md` §2) attributes ~79 % +of a native 1/1000 run to repeated admission and verification of 3.48 GiB of +staged source: node execution is 1,143 s of a 5,362 s run. Make source +authentication and identity verification happen once per run instead of once +per accessor use and once per executed node, without weakening any refusal and +without moving any digest value. + +**Done:** read the evidence base; located all five mechanisms at base HEAD. + +**Next:** pin-surface map, memo design note, implementation, tests, measurement, +draft PR. + +**Lane notes.** Root `out.md` is a tracked file holding the Amendment 19 lane's +committed report; this lane's report goes to +`experiments/native-verify-once/out.md`. The v5 cold run (`run_v5.py`, pid +81194) is live on this machine, so probe runs wait for the window the lane +brief allows (`pgrep -f run_v5.py` empty, or > 40 GB free). From 687dae4897bf993f51326f9cde4983283008bd20 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 15 Sep 2026 23:34:38 -0400 Subject: [PATCH 02/45] Build each column's context-digest stream with numpy, byte for byte _update_series boxed every value of a plain float, integer or boolean column into a Python object and framed them one at a time; on pandas 3.0.3 a plain column is a NumpyExtensionArray with no _data/_mask, so the fast masked path never applied to it. _object_stream now builds the identical framed bytes -- the same 'object' dtype header, the same shape header, the same per-value length prefixes and payloads -- with vectorised numpy, and returns None for every column kind it cannot reproduce exactly, including a Categorical whose values are an integer-coded view. No digest value moves. test_graph_executor_series_stream compares the live helper against a verbatim copy of the pre-change body at the byte level over every column kind, the float specials, non-canonical NaN payloads, both int64 endpoints and a 200-frame random sweep. Measured 0.47 s -> 0.09 s on a 6,928 x 240 frame. Co-Authored-By: Claude Opus --- .../src/microcosm/graph/executor.py | 74 ++++- .../test_graph_executor_series_stream.py | 291 ++++++++++++++++++ 2 files changed, 364 insertions(+), 1 deletion(-) create mode 100644 packages/microcosm-graph/tests/test_graph_executor_series_stream.py diff --git a/packages/microcosm-graph/src/microcosm/graph/executor.py b/packages/microcosm-graph/src/microcosm/graph/executor.py index 4e14efbec..8c0eada62 100644 --- a/packages/microcosm-graph/src/microcosm/graph/executor.py +++ b/packages/microcosm-graph/src/microcosm/graph/executor.py @@ -449,6 +449,69 @@ def _update_array(digest: hashlib._Hash, values: object) -> None: digest.update(np.ascontiguousarray(array).tobytes()) +def _framed(payloads: np.ndarray) -> bytes: + """Frame equal-width payload rows exactly as ``_update_scalar`` frames one.""" + + rows, width = payloads.shape + prefix = np.frombuffer( + np.full(rows, width, dtype=" bytes | None: + """The bytes ``_update_array`` writes for this column's object projection. + + ``to_numpy(dtype=object)`` boxes a plain numpy column into one Python object + per value, and ``_update_scalar`` then frames each box. For the three kinds + that dominate a survey frame the boxed value is always the same exact type, + so the whole framed stream is a pure function of the column's bytes and can + be built at C speed. Byte-for-byte identical output is the contract; any + column this cannot reproduce exactly returns None and takes the loop. + """ + + dtype = series.dtype + if not isinstance(dtype, np.dtype) or dtype.kind not in "fiub": + return None + values = np.ascontiguousarray(series.to_numpy(copy=False)) + if values.ndim != 1 or values.dtype != dtype: + return None + rows = values.shape[0] + if rows == 0: + return b"" + if dtype.kind == "b": + # bool -> b"b1" / b"b0"; np.bool_ boxes to bool, checked before int. + payloads = np.empty((rows, 2), dtype=np.uint8) + payloads[:, 0] = ord("b") + payloads[:, 1] = np.where(values, ord("1"), ord("0")) + return _framed(payloads) + if dtype.kind == "f": + # Every float width boxes to an exact Python float, which packs to the + # same eight native-order IEEE-754 bytes as the float64 cast, NaN + # payload, signed zero and infinities included. + wide = np.ascontiguousarray(values, dtype=" None: digest.update(str(series.dtype).encode("utf-8")) digest.update(b"\0") @@ -459,7 +522,16 @@ def _update_series(digest: hashlib._Hash, series: pd.Series) -> None: _update_array(digest, data) _update_array(digest, mask) else: - _update_array(digest, series.to_numpy(dtype=object, copy=False)) + stream = _object_stream(series) + if stream is None: + _update_array(digest, series.to_numpy(dtype=object, copy=False)) + else: + # The header the object projection would have written: its dtype is + # "object" and its shape is the column's own one-dimensional shape. + digest.update(b"object\0") + digest.update(f"({len(series)},)".encode("ascii")) + digest.update(b"\0") + digest.update(stream) _update_array(digest, series.index.to_numpy(copy=False)) diff --git a/packages/microcosm-graph/tests/test_graph_executor_series_stream.py b/packages/microcosm-graph/tests/test_graph_executor_series_stream.py new file mode 100644 index 000000000..d4786d490 --- /dev/null +++ b/packages/microcosm-graph/tests/test_graph_executor_series_stream.py @@ -0,0 +1,291 @@ +"""Byte-exact parity for the executor's vectorised column digest stream. + +Invented columns only: no data file, no store, no engine. ``_update_series`` +now builds the framed object-projection bytes for plain float, integer and +boolean columns with numpy instead of boxing every value into a Python object +and framing it one at a time. The emitted bytes -- the ``object`` dtype +header, the shape header, every per-value length prefix and every payload -- +must be identical to what the boxing loop wrote. Every test compares the live +helper against ``_reference_update_series`` below, a verbatim copy of the +pre-change body, at the byte level rather than only at the digest. +""" + +from __future__ import annotations + +import hashlib +import struct + +import numpy as np +import pandas as pd +import pytest + +from microcosm.graph.executor import _object_stream, _update_series + + +def _reference_update_scalar(digest, value): + """The pre-change scalar body, kept verbatim as the parity reference.""" + + if value is pd.NA: + payload = b"pd.NA" + elif value is pd.NaT: + payload = b"pd.NaT" + elif value is None: + payload = b"None" + elif type(value) is float: + payload = b"f" + struct.pack("=d", value) + elif isinstance(value, (float, np.floating)): + payload = b"f" + np.asarray([value], dtype=np.float64).tobytes() + elif isinstance(value, (bool, np.bool_)): + payload = b"b1" if bool(value) else b"b0" + elif isinstance(value, (int, np.integer)): + payload = b"i" + str(int(value)).encode("ascii") + elif isinstance(value, str): + payload = b"s" + value.encode("utf-8") + elif isinstance(value, (bytes, np.bytes_)): + payload = b"y" + bytes(value) + else: + payload = b"r" + repr(value).encode("utf-8") + digest.update(len(payload).to_bytes(8, "little")) + digest.update(payload) + + +def _reference_update_array(digest, values): + """The pre-change array body, kept verbatim as the parity reference.""" + + array = np.asarray(values) + digest.update(str(array.dtype).encode("utf-8")) + digest.update(b"\0") + digest.update(str(array.shape).encode("ascii")) + digest.update(b"\0") + if array.dtype.hasobject: + for value in array.ravel(order="C"): + _reference_update_scalar(digest, value) + else: + digest.update(np.ascontiguousarray(array).tobytes()) + + +def _reference_update_series(digest, series): + """The pre-change series body, kept verbatim as the parity reference.""" + + digest.update(str(series.dtype).encode("utf-8")) + digest.update(b"\0") + extension = series.array + data = getattr(extension, "_data", None) + mask = getattr(extension, "_mask", None) + if isinstance(data, np.ndarray) and isinstance(mask, np.ndarray): + _reference_update_array(digest, data) + _reference_update_array(digest, mask) + else: + _reference_update_array(digest, series.to_numpy(dtype=object, copy=False)) + _reference_update_array(digest, series.index.to_numpy(copy=False)) + + +class _Recorder: + """A digest-shaped sink that keeps every byte the helper writes.""" + + def __init__(self): + self.chunks = [] + + def update(self, payload): + self.chunks.append(bytes(payload)) + + def bytes(self): + return b"".join(self.chunks) + + +def _written(helper, series): + recorder = _Recorder() + helper(recorder, series) + return recorder.bytes() + + +def _digest(helper, series): + digest = hashlib.sha256() + helper(digest, series) + return digest.digest() + + +def _assert_identical(series): + assert _written(_update_series, series) == _written( + _reference_update_series, series + ) + assert _digest(_update_series, series) == _digest(_reference_update_series, series) + + +def _invented_columns(): + """One column of every kind a projected kernel context can carry.""" + + generator = np.random.default_rng(20260915) + columns = { + "float64": pd.Series(generator.normal(size=97)), + "float32": pd.Series(generator.normal(size=97).astype("float32")), + "float16": pd.Series(generator.normal(size=97).astype("float16")), + "longdouble": pd.Series(np.array([1.5, 1 / 3, np.nan], dtype=np.longdouble)), + "int64": pd.Series(generator.integers(-(10**15), 10**15, size=97)), + "int32": pd.Series( + generator.integers(-(2**30), 2**30, size=97).astype("int32") + ), + "int8": pd.Series(generator.integers(-128, 127, size=97).astype("int8")), + "uint64": pd.Series(np.array([0, 1, np.iinfo(np.uint64).max], dtype="uint64")), + "uint8": pd.Series(np.arange(256, dtype="uint8")), + "bool": pd.Series(generator.integers(0, 2, size=97).astype(bool)), + "complex": pd.Series(np.array([1 + 2j, 3 - 4j])), + "Int64": pd.Series(pd.array([1, None, -3], dtype="Int64")), + "Float64": pd.Series(pd.array([1.0, None, -0.0], dtype="Float64")), + "boolean": pd.Series(pd.array([True, None, False], dtype="boolean")), + "string": pd.Series(["a", "bb", None, "é\U0001f600"], dtype="str"), + "object": pd.Series( + ["x", 1, None, 2.5, True, b"z", pd.NA, pd.NaT, (1, 2)], dtype="object" + ), + "category": pd.Series(pd.Categorical(generator.choice(list("abcd"), size=97))), + "datetime": pd.Series(pd.to_datetime([0, 1_600_000_000], unit="s")), + "timedelta": pd.Series(pd.to_timedelta([0, 90_061], unit="s")), + } + for name in ("float64", "int64", "bool", "object", "str"): + columns[f"empty-{name}"] = pd.Series([], dtype=name) + columns["sliced-float"] = pd.Series(generator.normal(size=500))[13:400:7] + columns["strided-int"] = pd.Series(generator.integers(0, 10**6, size=500))[::11] + columns["labelled-index"] = pd.Series( + generator.normal(size=17), index=[f"k{i}" for i in range(17)] + ) + return columns + + +@pytest.mark.parametrize("name", sorted(_invented_columns())) +def test_every_column_kind_streams_identical_bytes(name): + _assert_identical(_invented_columns()[name]) + + +def test_float_specials_stream_identical_bytes(): + """Signed zero, both infinities, NaN and the subnormal extremes.""" + + _assert_identical( + pd.Series( + [ + 0.0, + -0.0, + np.inf, + -np.inf, + np.nan, + 1.7976931348623157e308, + -1.7976931348623157e308, + 5e-324, + -5e-324, + -1.5, + ] + ) + ) + + +def test_nan_payload_bits_survive_the_vectorised_path(): + """A non-canonical NaN keeps its exact payload, sign bit included.""" + + raw = np.array( + [0x7FF8000000ABCDEF, 0xFFF8000000ABCDEF, 0x7FF0000000000001], dtype="uint64" + ) + series = pd.Series(raw.view("float64")) + assert series.isna().all() + _assert_identical(series) + + +def test_integer_extremes_stream_identical_bytes(): + """Variable-width decimal payloads, including both int64 endpoints.""" + + _assert_identical( + pd.Series( + np.array( + [0, -1, 1, 9, 10, -10, np.iinfo(np.int64).min, np.iinfo(np.int64).max], + dtype="int64", + ) + ) + ) + + +def test_categorical_is_not_mistaken_for_its_integer_codes(): + """The fast path must refuse any column whose values are a coded view.""" + + series = pd.Series(pd.Categorical(list("abca"))) + assert series.array.dtype.kind not in "fiub" + assert _object_stream(series) is None + _assert_identical(series) + + +def test_datetime_and_string_columns_take_the_reference_loop(): + for series in ( + pd.Series(pd.to_datetime([0, 1], unit="s")), + pd.Series(pd.to_timedelta([0, 1], unit="s")), + pd.Series(["a", None], dtype="str"), + pd.Series([object()], dtype="object"), + ): + assert _object_stream(series) is None + _assert_identical(series) + + +def test_masked_columns_keep_their_existing_data_and_mask_path(): + """Nullable columns already streamed their raw buffers; nothing moves.""" + + for dtype in ("Int64", "Float64", "boolean"): + series = pd.Series(pd.array([1, None, 0], dtype=dtype)) + assert isinstance(series.array._data, np.ndarray) + _assert_identical(series) + + +def test_random_frames_of_every_kind_stream_identical_bytes(): + """The property sweep: random widths and contents, every column kind.""" + + generator = np.random.default_rng(20260916) + kinds = ( + "float64", + "float32", + "int64", + "int32", + "uint32", + "bool", + "Int64", + "str", + "category", + "object", + ) + for trial in range(200): + rows = int(generator.integers(0, 60)) + kind = kinds[trial % len(kinds)] + if kind.startswith("float"): + values = generator.normal(size=rows).astype(kind) + series = pd.Series(values) + if rows: + series.iloc[int(generator.integers(0, rows))] = np.nan + elif kind in ("int64", "int32", "uint32"): + series = pd.Series(generator.integers(0, 10**9, size=rows).astype(kind)) + elif kind == "bool": + series = pd.Series(generator.integers(0, 2, size=rows).astype(bool)) + elif kind == "Int64": + series = pd.Series( + pd.array( + [ + None if generator.random() < 0.3 else int(value) + for value in generator.integers(-99, 99, size=rows) + ], + dtype="Int64", + ) + ) + elif kind == "str": + series = pd.Series( + [ + None + if generator.random() < 0.2 + else "s" * int(generator.integers(0, 5)) + for _ in range(rows) + ], + dtype="str", + ) + elif kind == "category": + series = pd.Series(pd.Categorical(generator.choice(list("xyz"), size=rows))) + else: + series = pd.Series( + [ + [1, "a", None, 2.5, True][int(generator.integers(0, 5))] + for _ in range(rows) + ], + dtype="object", + ) + _assert_identical(series) From a65421bbf6e6c569841967c33a7a3ab19465e9e6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 15 Sep 2026 23:39:29 -0400 Subject: [PATCH 03/45] Record the verify-once design before changing any capsule docs/us-native-verification-once.md states, per mechanism, the guarantee today, the change, why the guarantee survives and the new cost class -- and the two residuals the epoch's unconditional final re-validation exists to close. Co-Authored-By: Claude Opus --- changelog.d/native-verify-once.changed.md | 1 + docs/us-native-verification-once.md | 291 ++++++++++++++++++++++ 2 files changed, 292 insertions(+) create mode 100644 changelog.d/native-verify-once.changed.md create mode 100644 docs/us-native-verification-once.md diff --git a/changelog.d/native-verify-once.changed.md b/changelog.d/native-verify-once.changed.md new file mode 100644 index 000000000..552bb3a66 --- /dev/null +++ b/changelog.d/native-verify-once.changed.md @@ -0,0 +1 @@ +verify-once: Verify native sources once per run instead of once per accessor use and once per executed node. diff --git a/docs/us-native-verification-once.md b/docs/us-native-verification-once.md new file mode 100644 index 000000000..46fbbc424 --- /dev/null +++ b/docs/us-native-verification-once.md @@ -0,0 +1,291 @@ +# Verifying native sources once per run + +A native US build spends most of its time re-proving things it already proved. +The 2026-09-15 pilot measurement on the nineteen-node financial graph at +`Fraction(1, 1000)` (3,168 expanded households, 5,362 s wall) attributes +**1,143 s to node execution and the remaining ~79 % to admission and +verification of the 3.48 GiB of staged source** — repeated. QRF fits inside +that run cost 0.06–0.07 s each. The graph should compile to roughly the +pre-graph runtime plus receipts; today it compiles to the pre-graph runtime +plus the source tree, once per accessor use and once per executed node. + +This note records the five mechanisms that cause it, what each one guarantees +today, what changes, why the guarantee survives, and the new cost class. + +## The rule the whole change obeys + +> Memoisation changes **when** a check runs. It never changes **what** the +> check computes. + +Every digest, key, seal and receipt value in this repository keeps its exact +value across this change. Nothing in `_validate`, `_validate_state` or +`source_content_key` produces a receipt field — all three only compare live +values against values frozen at issuance — so moving a comparison in time +cannot move a recorded value. The one place bytes are genuinely rebuilt +(`_update_series`, mechanism 5) is proven byte-for-byte against a verbatim copy +of the pre-change implementation. + +Two guarantees are stated up front because every section below leans on them: + +1. **Verified before first use.** Nothing is memoised until it has been + verified in full, exactly as today. +2. **Proven unchanged at the end.** Every memo is re-verified in full, + unconditionally, before the run hands anything back. A source or a live + object that changed while a memo was warm still makes the run refuse. + +## The verification epoch + +Memoisation is **opt-in and scoped**. `microcosm.build.us_runtime.verification_epoch` +provides a context manager. Outside an epoch there is no memo at all and every +capsule behaves byte-for-byte as it does today — which is why every existing +capsule test, including the two that trace `_producer` and `_file_identity` +firing inside a validation, keeps passing unchanged. + +Inside an epoch each capsule validation is split into two tiers: + +* **Tier A — every access, never memoised.** The issuance binding proof, the + producer encoding (`_encode(_producer())` / `_encode(_implementation())`), + and a *signature*: the stat identity `(st_dev, st_ino, st_size, st_mtime_ns, + st_ctime_ns)` of every file the memoised tier reads, plus the structural + witness of every live object the memoised tier digests (per table: the + column axis, dtypes, row count, and for each column and index the buffer + address, shape, strides and writeable flag). +* **Tier B — memoised on Tier A's signature.** Everything else: the source + catalogues, ACS native coverage, the nested ASEC native validation, the + whole-roster `_source_files` re-hash and the pure final seals. + +A signature mismatch is **not** a refusal. It is a memo miss: the complete +unmemoised validation runs, and whatever refusal it would have raised today it +raises now, with the same code, at the same access. + +On leaving an epoch — and on leaving every nested epoch, so a run that ends +early through an inner scope is still covered — every capsule the epoch +validated is re-validated **in full, with the memo bypassed**, and the result +is recorded. `verification_epoch.last_record()` reports the capsule count, the +number of memo hits, the number of signature misses and the number of +unconditional final re-validations. + +## The five mechanisms + +### 1. `AuthenticatedSurveyPopulationPreparation._checked()` + +`survey_population_preparation.py:1138-1162`. Every accessor — `.frame`, +`.context`, `.selection_plan`, `.receipt`, `.to_bytes()`, `.checked_view()`, +`validate()`, and the module-level `verify_survey_population_preparation` that +the graph registers as a per-node population observer — calls `_validate(state)`. + +**Guarantee today.** At the moment of every borrow: the capsule is the one that +was issued and its payload is unchanged; both source catalogues, the ACS native +coverage binding and the nested ASEC native population re-verify their own +source bytes; the ten-file source roster re-hashes to `state.files`; the +producer modules re-encode to `state.producer`; the roster's stat identities +re-derive to `state.file_stats`; and the pure seals (live authority, attached +evidence, nested seals, plan document, prepared frame identity, both source +frame seals) match. + +**Cost today.** One `_checked()` re-hashes the ACS archive pair fourteen times +(four in the ACS catalogue's source checks, eight across the native binding's +four snapshot dictionaries, two in `_source_files`) — about 11.1 GiB of +SHA-256 — plus three passes over the seven ASEC files, plus the whole roster. +It runs once per executed node through the population observer, plus once or +more inside every kernel that borrows the preparation. + +**Change.** `_validate` becomes Tier A + Tier B as above. Tier A keeps +`_encode(_producer())` and `_file_stats(state.root)` on every access, and adds +the paths that `_file_stats` does not cover but the I/O arm reads: the ACS +catalogue's `source_dir` entries, its two private snapshot copies and its +projection path, the native binding's four snapshot dictionaries, and the +`source_files` rosters of both nested ASEC capsules. Tier B is the rest. + +**Why the guarantee survives.** Producer tampering and live-callable rebinding +still refuse at the same access (`_producer()` calls `_live()`, and it is never +memoised). Every on-disk change to any file the I/O arm reads — truncation, +append, in-place rewrite, replacement, rename, roster addition or removal, +symlink substitution — moves at least one of `st_size`, `st_mtime_ns` or +`st_ctime_ns`, or the directory's own stat identity, so it is a memo miss and +the existing refusal fires at the same access it fires today. Every structural +change to a live frame — a replaced column, a reindex, a dtype change, a +rebound buffer, a flipped writeable flag — moves the witness and is likewise a +memo miss. What is deferred to the epoch's unconditional final re-validation is +exactly two things: an in-place value mutation of a live buffer, and an on-disk +rewrite that leaves all five stat fields identical. On this machine the second +is not reachable without privilege: an unprivileged same-length in-place +rewrite followed by `os.utime` restores `st_size`, `st_ino` and `st_mtime_ns` +but not `st_ctime_ns`, and `setattrlist(ATTR_CMN_CHGTIME)` refuses with +`EPERM` (measured 2026-09-15, APFS). That is an observation about one +filesystem, not a guarantee, which is why the final re-validation is +unconditional rather than conditional. + +**New cost class.** Per accessor use: O(modules) producer encoding + O(roster) +`lstat` + O(columns) witness — microseconds. Per run: one full validation at +first use plus one at each epoch exit, instead of one per accessor use. + +### 2. `AuthenticatedAsec2024NativePopulation.frame` + +`asec_2024_native_population.py:449-499`. `.frame`, `.context`, `.receipt`, +`.to_bytes()` and `validate()` all route through `_state()` → `_checked()` → +`_validate_state(state)`: every frame access is a full re-authentication, +including a full SHA-256 of all seven ASEC source files. + +**Guarantee today.** On every borrow: the implementation encoding matches, the +descendant and parent frame identities and the frame context match, the parent +capsule re-validates, the coverage/anchor/field owners re-verify, all seven +source files re-hash to their pinned digests with a before/after `fstat` guard, +and — because a file check or a foreign owner call may yield — the producer and +both frame identities are re-checked *after* the file loop as well as before. + +**Change.** Same two tiers. Tier A keeps `_encode(_implementation())` and adds +the stat identity of the seven `state.source_files` paths and of their parent +directories (which `_file_identity` does not cover: it opens `O_NOFOLLOW` on +the final component only). `_State` here is a `NamedTuple` and a test pins its +immutability, so the memo lives in a side table keyed by the identity of the +`_ISSUED` entry tuple and torn down by the same weakref callback that clears +`_ISSUED` — the entry tuple itself is never replaced, because about twenty-five +call sites compare it with `is`. + +**Why the guarantee survives.** As in mechanism 1. The documented +yield-tolerance pattern is preserved intact: a memo hit runs neither the file +loop nor the checks that bracket it, and a memo miss runs the whole of +`_validate_state` unchanged, so the bracketing pair is never split. + +**New cost class.** Per access: O(modules) + seven `lstat`s + O(columns) +witness. Per run: two full validations. + +### 3. Per-node source content keys in the executor + +`executor.py:2452-2457` re-derives `source_content_key(name, source_paths[name])` +for every source a cold-executed node declares, and `keys._directory_identity` +(`keys.py:31-51`) `read_bytes()`es every file of a directory source and hashes +the whole tree. Cost is O(executed source-declaring nodes × source bytes): the +3.48 GiB tree is re-read per node. + +**Guarantee today.** The check runs after the kernel returns and **before** +`_validate_result`, `_apply_result`, the population observer and `_write_node`. +Its real stake is the last one: no artifact computed from mutated source bytes +is ever persisted under a node key that claims the pre-mutation bytes, which +would poison the content-addressed store permanently. + +**Change.** `run_graph` gains a per-run cache in its own locals — never inside +`source_content_key`, which stays pure because four existing assertions call it +twice on one path with the bytes changed in between and require different +answers. The cache is keyed by source name and a *stat signature*: for a +regular file, its `_stat_identity`; for a directory, the sorted recursive +roster of relative names with each member's `_stat_identity` and the stat +identity of every directory in the tree. The per-node check derives the +signature (an `lstat` walk, no byte reads); on a match it reuses the cached +key, on a mismatch it re-derives the content key in full and raises the same +`NodeRejected` if it moved. Immediately before the manifest is built, every +source is re-derived **in full, cache bypassed**, and a mismatch raises. The +run-end re-derivation is written inline rather than through +`_source_paths_and_keys`, because a build test counts calls to that exact code +object. + +**Why the guarantee survives.** The refusal still precedes `_write_node` for +every mutation that moves any stat field — which is every ordinary mutation, +including one that adds, removes or renames a file inside a directory source, +since the roster is part of the signature. The narrowed case — a byte rewrite +that preserves all five stat fields — is caught at run end, before any caller +receives a manifest, but after intervening nodes have written store records. +That residual is stated rather than hidden: a run refused by the run-end check +may leave store entries behind, and those entries are self-consistent under +`_require_record_shape`, so the operator must clear the store for that run. +Against this, the executor gains a guarantee it does not have today: sources +are re-verified at run end even when the mutation happens during a node that +declares no source, which today is invisible. + +**Record.** `RunManifest` gains `source_identities`, an attached mapping modelled +exactly on `populations` and `mass_ledgers` — `repr=False`, `compare=False`, +excluded from `content_addressed` and absent from `to_json`. It carries the +run-end re-derived key of every source. It moves no manifest key, no manifest +JSON byte, no node receipt and no cache record. + +**New cost class.** Per node: an `lstat` walk of the source tree. Per run: two +full content derivations instead of one per executed source-declaring node. + +### 4. The ACS record fence + +`acs_person_coverage_authentication._records` fences raw logical CSV records +out of a zip member before any UTF-8 decoding or CSV allocation. It is a +pure-Python `for byte in block` loop over 4,096-byte blocks, and the member it +fences is the ACS person CSV inside `csv_pus.zip` — 2.4 GB expanded across +`psam_pusa.csv` and `psam_pusb.csv`. With `peek` and +`acs_person_coverage_columns.lines` it is 57–65 % of the admission phase in +every measured run. + +**Guarantee today.** For every byte, in this order: the record length must be +below its cap (64 KiB for the first record, 400 KB afterwards) or +`CSV_RECORD_BYTES`; an unquoted comma resets the token counter, any other byte +advances it and must stay within 64 KiB or `CSV_TOKEN_BYTES`; a double quote +toggles quote parity, which **persists across records**; an unquoted LF ends a +record, an unquoted CR ends it and absorbs a following LF. Record boundaries +are located by quote parity only; CSV validity remains the unchanged strict +literal parser's job. + +**Change.** The same state machine, driven by `bytes.find` and `bytes.count` +over 1 MiB blocks instead of a Python loop over individual bytes. Terminator +cursors are cached per block so an absent carriage return is searched for once +rather than once per record. Every completed record, and every in-flight record +at a block boundary, passes through a bound check: a record no longer than +64 KiB cannot have violated either cap, because the token counter resets at +each record start and so never exceeds the record's own length, and the record +cap is never below 64 KiB. Any record longer than that — which real ACS data +never produces — is replayed byte by byte through the original fence, so the +refusal code and the point at which it fires are exactly the original's. + +**Why the guarantee survives.** The yielded record sequence and the refusal +raised are proven identical, not argued: `test_us_acs_record_fence_scan.py` +compares the shipped `_records` against a verbatim copy of the pre-change loop +over 4,200 randomised byte strings, every boundary case (bare CR, CRLF split +across a block boundary, quoted terminators, odd quote parity carried across +records, BOM, empty lines, trailing CR at EOF) and every bound case on both +sides of every cap. The same comparison runs against the real staged +`csv_pus.zip`. + +**New cost class.** Linear in member bytes at C speed: measured 10.8 MB/s → +522.8 MB/s on the real archive, digests identical, a 48.5× reduction. + +**Pin.** `acs_person_coverage_authentication.py` is one of two modules whose raw +file SHA-256 is pinned in `acs_native_coverage_binding._ACCEPTED`, so this +change re-pins it. The new value is derived by `shasum -a 256` on the changed +file and recorded in the lane report. + +### 5. The kernel context digest + +`_context_digest` (`executor.py`) digests every column of every table twice per +node — once before the kernel runs and once after, to prove the kernel did not +mutate its input context. `_update_series` took a fast path only for masked +extension arrays exposing `_data` and `_mask` ndarrays. On pandas 3.0.3 an +ordinary float, integer or boolean column is a `NumpyExtensionArray` with +neither, so it fell through to `to_numpy(dtype=object)` and a per-value Python +loop with two `hashlib` updates per value. + +**Guarantee today.** The digest is a total function of the column: its dtype +string, the object projection's dtype and shape headers, and one framed payload +per value. + +**Change.** `_object_stream` builds exactly those bytes with numpy for the three +kinds whose boxed value is always one exact Python type — `float` packed to +eight native IEEE-754 bytes, `bool` rendered as `b"b1"`/`b"b0"`, and `int` +rendered as a variable-width decimal. It returns `None` for everything else, +including a `Categorical`, whose values array is an integer-coded view and +would otherwise be mistaken for an integer column. Both `_context_digest` calls +remain: the second one is the mutation guard, and reusing its result would +delete the guard. + +**Why the guarantee survives.** +`test_graph_executor_series_stream.py` compares the shipped helper against a +verbatim copy of the pre-change body **at the byte level**, not only at the +digest, across every column kind the projection can carry, the float specials +(signed zero, both infinities, the subnormal extremes), non-canonical NaN +payloads with their sign bit, both `int64` endpoints, empty, sliced and +strided columns, labelled indexes, and a 200-frame random sweep. + +**New cost class.** Measured 0.47 s → 0.09 s for a 6,928 × 240 frame; 183 → +18 ns per value for floats, 226 → 14 for booleans, 383 → 119 for integers. +Still linear in rows, but with no Python-level work per value. + +## What this note does not claim + +Nothing here is a statement about dataset quality, calibration or release +eligibility. The PR-CI / certification boundary is unchanged: these are code +contracts, and green checks do not certify any data artifact. From f8b4f471eb9b6f5b878b69dc46c20f53d16cbf0b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 15 Sep 2026 23:44:57 -0400 Subject: [PATCH 04/45] Fence ACS records with the C scanners instead of a byte loop _records read the 2.4 GB of ACS person CSV inside csv_pus.zip one Python byte at a time; with peek and columns.lines it was 57-65% of the admission phase in every measured native run. It now finds terminators with bytes.find, counts quote parity with bytes.count, and keeps each terminator cursor until the scan passes it -- so an absent carriage return costs one search per 1 MiB block rather than one per record, which is where the bulk of the win is. The two ceilings are charged by _fence. A record no longer than the smaller of the live ceilings cannot have violated either, because the token counter resets at every record start; anything longer is replayed byte by byte, so the refusal code and the byte it fires on stay the fence's own -- including the LF that closes a CRLF record, which the fence charges against the record ceiling only. test_us_acs_record_fence_scan compares the shipped fence against a verbatim copy of the byte loop: every boundary case, every block-boundary case, every ceiling on both sides, 4,500 randomised strings, 300 row-shaped inputs, and all 9,331 strings up to five bytes over the branching alphabet under each of five ceiling settings -- 46,655 exhaustive comparisons. Measured on the real staged archive: 10.8 MB/s -> 522.8 MB/s, digests identical. Re-pins acs_native_coverage_binding._ACCEPTED for the changed module: 475aa795c8a5b49a0dd3405a0877866dddd03012a1f2b9743447fab1fe85bcff -> 9ec68721a4cf480ef412c51ab354db9000eb7a7989e35e6e09b1574d88e8e49f (shasum -a 256 .../acs_person_coverage_authentication.py, after formatting) The module's graph_implementation_inventory contract is unchanged: the rewrite adds no _RESOURCE_CALLS name and no non-stdlib import. Co-Authored-By: Claude Opus --- .../us_runtime/acs_native_coverage_binding.py | 2 +- .../acs_person_coverage_authentication.py | 120 +++++-- .../tests/test_us_acs_record_fence_scan.py | 327 ++++++++++++++++++ 3 files changed, 416 insertions(+), 33 deletions(-) create mode 100644 packages/microcosm-build/tests/test_us_acs_record_fence_scan.py diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py index 770fdc9bb..98e9972ca 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py @@ -35,7 +35,7 @@ "acs_pums.py": "6ecf79f0dfb0c0bc0ad0af6be5fa65bd8c2d1e1968009402e4c8dee347de70dc", "acs_inputs.py": "aa4a8aeaba63dfef2f3e04fb89de59766deb088ed7f4d290aeba0425739916da", "acs_housing_universe_source.py": "beb46a4a05a13580a868be423809a77946441dcafc3a0f57160561157e93e9a3", - "acs_person_coverage_authentication.py": "475aa795c8a5b49a0dd3405a0877866dddd03012a1f2b9743447fab1fe85bcff", + "acs_person_coverage_authentication.py": "9ec68721a4cf480ef412c51ab354db9000eb7a7989e35e6e09b1574d88e8e49f", } _TOKEN = object() _ISSUED = WeakKeyDictionary() diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py index fe0346500..45d0f100e 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py @@ -38,6 +38,7 @@ MAX_RECORD_BYTES = 400_000 MAX_CSV_HEADER_BYTES = 64 * 1024 MAX_TOKEN_BYTES = 64 * 1024 +_SCAN_BLOCK = 1024**2 _MAGIC = b"ACS-COVERAGE/1\n" _TOKEN = object() _CSV_READER = _csv.reader @@ -185,46 +186,101 @@ def _producer(): } +def _fence(record, quoted, first): + """Charge one bounded record against the byte fence's own two ceilings. + + The token counter resets at every record start, so it never exceeds the + record's own length. A record no longer than the smaller of the two live + ceilings therefore cannot have violated either one, whatever those + ceilings are set to. Anything longer is replayed byte by byte so that the + refusal code, and the byte at which it fires, stay exactly the fence's own. + """ + cap = MAX_CSV_HEADER_BYTES if first else MAX_RECORD_BYTES + if len(record) <= min(cap, MAX_TOKEN_BYTES): + return + length, token_bytes, pending_cr = 0, 0, False + for byte in record: + if pending_cr: + # The LF closing a CRLF record is charged against the record + # ceiling only, exactly as the byte fence charges it. + _require(length < cap, "CSV_RECORD_BYTES") + length += 1 + pending_cr = False + continue + _require(length < cap, "CSV_RECORD_BYTES") + if byte == 44 and not quoted: + token_bytes = 0 + else: + token_bytes += 1 + _require(token_bytes <= MAX_TOKEN_BYTES, "CSV_TOKEN_BYTES") + length += 1 + if byte == 34: + quoted = not quoted + elif byte == 13 and not quoted: + pending_cr = True + + def _records(stream): """Fence raw logical records/tokens before UTF-8 decoding or CSV allocation. Quote parity only locates boundaries; the unchanged strict literal parser subsequently owns CSV validity. CR, LF and CRLF are preserved, including inside quotes. The conservative token ceiling includes raw CSV quoting. + + Boundaries are located with the C scanners rather than a byte loop: parity + is a count of quotes between terminators, and each terminator cursor is + kept until the scan passes it, so an absent carriage return costs one + search per block instead of one per record. Quote parity persists across + records, exactly as the byte fence carried it. A record is at most one + ceiling long, so it spans at most two blocks and the carried head is + rescanned rather than tracked, which is what keeps the pending-CR case a + plain continuation instead of a separate state. """ - record = bytearray() - quoted, pending_cr, first, token_bytes = False, False, True, 0 - cap = MAX_CSV_HEADER_BYTES - while block := stream.read(4096): - for byte in block: - if pending_cr: - if byte == 10: - _require(len(record) < cap, "CSV_RECORD_BYTES") - record.append(byte) - yield bytes(record) - record.clear() - first, pending_cr, token_bytes = False, False, 0 - if byte == 10: - continue - cap = MAX_CSV_HEADER_BYTES if first else MAX_RECORD_BYTES - _require(len(record) < cap, "CSV_RECORD_BYTES") - if byte == 44 and not quoted: - token_bytes = 0 + tail, tail_quoted, first = b"", False, True + while block := stream.read(_SCAN_BLOCK): + buffer = tail + block if tail else block + size = len(buffer) + pos, scan = 0, 0 + record_quoted = tail_quoted + parity = record_quoted + line, carriage = buffer.find(10), buffer.find(13) + while True: + if 0 <= line < scan: + line = buffer.find(10, scan) + if 0 <= carriage < scan: + carriage = buffer.find(13, scan) + if line < 0: + index = carriage + elif carriage < 0 or line < carriage: + index = line else: - token_bytes += 1 - _require(token_bytes <= MAX_TOKEN_BYTES, "CSV_TOKEN_BYTES") - record.append(byte) - if byte == 34: - quoted = not quoted - if not quoted and byte in (10, 13): - if byte == 13: - pending_cr = True - else: - yield bytes(record) - record.clear() - first, token_bytes = False, 0 - if record: - yield bytes(record) + index = carriage + if index < 0: + break + parity ^= buffer.count(34, scan, index) & 1 + if parity: + # A terminator inside quotes is literal record content. + scan = index + 1 + continue + end = index + 1 + if buffer[index] == 13: + if end == size: + # Whether a LF joins this record is the next block's first + # byte; carry the record and rescan it there. + break + if buffer[end] == 10: + end += 1 + record = buffer[pos:end] + _fence(record, record_quoted, first) + yield record + first = False + pos = scan = end + record_quoted = parity + tail = buffer[pos:] + tail_quoted = record_quoted + _fence(tail, tail_quoted, first) + if tail: + yield tail def _decode_record(raw, *, first): diff --git a/packages/microcosm-build/tests/test_us_acs_record_fence_scan.py b/packages/microcosm-build/tests/test_us_acs_record_fence_scan.py new file mode 100644 index 000000000..ee77c8d07 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_acs_record_fence_scan.py @@ -0,0 +1,327 @@ +"""Byte-exact parity for the ACS record fence's chunked scan. + +Invented byte strings only: no archive, no source tree, no engine. ``_records`` +now locates record boundaries with ``bytes.find``/``bytes.count`` over 1 MiB +blocks instead of a Python loop over individual bytes. The sequence of yielded +records and the refusal it raises must be identical to the byte loop's for every +input, so every test here runs both implementations over the same bytes and +compares the whole outcome -- records and refusal together. + +``_reference_records`` below is a verbatim copy of the pre-change body. +""" + +from __future__ import annotations + +import io +import random + +import pytest + +from microcosm.build.us_runtime import acs_person_coverage_authentication as owner + +MAX_RECORD_BYTES = owner.MAX_RECORD_BYTES +MAX_CSV_HEADER_BYTES = owner.MAX_CSV_HEADER_BYTES +MAX_TOKEN_BYTES = owner.MAX_TOKEN_BYTES + + +def _reference_records(stream): + """The exact pre-change body, kept verbatim as the parity reference.""" + + record = bytearray() + quoted, pending_cr, first, token_bytes = False, False, True, 0 + cap = MAX_CSV_HEADER_BYTES + while block := stream.read(4096): + for byte in block: + if pending_cr: + if byte == 10: + owner._require(len(record) < cap, "CSV_RECORD_BYTES") + record.append(byte) + yield bytes(record) + record.clear() + first, pending_cr, token_bytes = False, False, 0 + if byte == 10: + continue + cap = MAX_CSV_HEADER_BYTES if first else MAX_RECORD_BYTES + owner._require(len(record) < cap, "CSV_RECORD_BYTES") + if byte == 44 and not quoted: + token_bytes = 0 + else: + token_bytes += 1 + owner._require(token_bytes <= MAX_TOKEN_BYTES, "CSV_TOKEN_BYTES") + record.append(byte) + if byte == 34: + quoted = not quoted + if not quoted and byte in (10, 13): + if byte == 13: + pending_cr = True + else: + yield bytes(record) + record.clear() + first, token_bytes = False, 0 + if record: + yield bytes(record) + + +def _outcome(fence, data): + """Every record the fence yielded, and the refusal code that stopped it.""" + + records, refusal = [], None + try: + for record in fence(io.BytesIO(data)): + records.append(record) + except owner.ACSCoverageAuthenticationError as error: + refusal = str(error) + return tuple(records), refusal + + +def _assert_identical(data): + reference = _outcome(_reference_records, data) + assert _outcome(owner._records, data) == reference + # The records must also reassemble the input exactly up to a refusal. + if reference[1] is None: + assert b"".join(reference[0]) == data + return reference + + +BOUNDARIES = { + "empty": b"", + "no-terminator": b"a,b,c", + "line-feed": b"a,b\nc,d\n", + "carriage-line-feed": b"a,b\r\nc,d\r\n", + "carriage-only": b"a,b\rc,d\r", + "carriage-then-byte": b"a\rb\n", + "mixed-endings": b"a\r\nb\rc\nd", + "quoted-line-feed": b'"a\nb",c\nd,e\n', + "quoted-carriage": b'"a\rb",c\nd,e\n', + "quoted-carriage-line-feed": b'"a\r\nb",c\nd,e\n', + "odd-quote-runs-on": b'a,"b\nc,d\n', + "parity-carried-across-records": b'a"b\nc"d\ne\n', + "byte-order-mark": b"\xef\xbb\xbfa,b\nc,d\n", + "trailing-carriage-at-eof": b"a,b\r", + "empty-lines": b"\n\n\n", + "carriage-carriage": b"a\r\rb\n", + "quote-inside-unquoted-field": b'a"b",c\n', + "commas-only": b",,,,\n", + "tab-and-space": b"a\t ,b\n", + "nul-byte": b"a\x00b,c\n", +} + + +@pytest.mark.parametrize("name", sorted(BOUNDARIES)) +def test_boundary_cases_fence_identically(name): + _assert_identical(BOUNDARIES[name]) + + +BLOCK = owner._SCAN_BLOCK +SPANNING = { + "record-spans-two-blocks": b"x" * (BLOCK + 500) + b"\n", + "carriage-line-feed-split-across-blocks": b"x" * (BLOCK - 1) + b"\r\ny\n", + "carriage-at-eof-on-a-block-edge": b"x" * (BLOCK - 1) + b"\r", + "carriage-then-byte-across-blocks": b"x" * (BLOCK - 1) + b"\rz\n", + "quoted-region-spans-two-blocks": b'"' + b"x" * BLOCK + b'"\ny\n', + "terminator-exactly-at-block-edge": b"x" * (BLOCK - 1) + b"\ny\n", +} + + +@pytest.mark.parametrize("name", sorted(SPANNING)) +def test_block_boundary_cases_fence_identically(name): + _assert_identical(SPANNING[name]) + + +CEILINGS = { + "first-record-one-over-its-cap": ( + b"x" * MAX_CSV_HEADER_BYTES + b"\n", + "CSV_RECORD_BYTES", + ), + "first-record-exactly-at-its-cap": ( + b"x" * (MAX_CSV_HEADER_BYTES - 1) + b"\n", + None, + ), + "later-record-token-one-over": ( + b"h\n" + b"x" * MAX_TOKEN_BYTES + b"\n", + "CSV_TOKEN_BYTES", + ), + "later-record-token-exactly-at-cap": ( + b"h\n" + b"x" * (MAX_TOKEN_BYTES - 1) + b"\n", + None, + ), + "later-record-many-short-tokens-over-cap": ( + b"h\n" + b",".join([b"y" * 1000] * 500) + b"\n", + "CSV_RECORD_BYTES", + ), + "token-ceiling-fires-before-record-ceiling": ( + b"h\n" + b"a," * 100 + b"w" * (MAX_TOKEN_BYTES + 1) + b"\n", + "CSV_TOKEN_BYTES", + ), + "unterminated-record-past-its-cap": ( + b"h\n" + b"q" * (MAX_RECORD_BYTES + 100), + "CSV_TOKEN_BYTES", + ), + "quoted-terminators-do-not-reset-the-ceiling": ( + b"h\n" + b'"' + b"z" * MAX_TOKEN_BYTES + b"\n", + "CSV_TOKEN_BYTES", + ), +} + + +@pytest.mark.parametrize("name", sorted(CEILINGS)) +def test_ceiling_cases_refuse_identically(name): + data, expected = CEILINGS[name] + assert _assert_identical(data)[1] == expected + + +def test_records_yielded_before_a_refusal_are_the_same_records(): + """A refusal stops the fence; everything it already yielded still matches.""" + + data = b"h\n" + b"good,row\n" + b"x" * (MAX_TOKEN_BYTES + 1) + b"\n" + records, refusal = _assert_identical(data) + assert refusal == "CSV_TOKEN_BYTES" + assert records == (b"h\n", b"good,row\n") + + +def test_randomised_byte_strings_fence_identically(): + """4,200 random strings over the alphabet the fence actually branches on.""" + + alphabet = b'ab,"\r\n\t 0' + generator = random.Random(20260915) + for _ in range(4200): + _assert_identical( + bytes(generator.choice(alphabet) for _ in range(generator.randrange(0, 60))) + ) + + +def test_randomised_record_shaped_inputs_fence_identically(): + """Row-shaped inputs, each ending with one of the three line endings.""" + + generator = random.Random(20260916) + for _ in range(300): + rows = [] + for _ in range(generator.randrange(0, 40)): + fields = [ + bytes( + generator.choice(b"abc0123") + for _ in range(generator.randrange(0, 6)) + ) + for _ in range(generator.randrange(1, 6)) + ] + if generator.random() < 0.2: + fields[0] = b'"' + fields[0] + b'\n"' + rows.append(b",".join(fields)) + ending = generator.choice([b"\n", b"\r\n", b"\r"]) + data = ending.join(rows) + if generator.random() < 0.7: + data += ending + _assert_identical(data) + + +def test_quote_parity_still_persists_across_records(): + """The fence never resets parity at a boundary; both implementations agree.""" + + records, refusal = _assert_identical(b'a"b\nc,d\ne"f\ng,h\n') + assert refusal is None + # The unbalanced quote swallows the next terminator, exactly as before. + assert records == (b'a"b\nc,d\ne"f\n', b"g,h\n") + + +# The suite's other fence tests drive the ceilings by monkeypatching them to +# small values, which is the only practical way to exercise the bounded paths. +# The scan must agree with the byte loop under those ceilings too -- the +# short-circuit that skips the replay is bounded by the smaller of the two live +# ceilings, not by a compile-time relationship between them. +CEILING_SETTINGS = ( + {"MAX_RECORD_BYTES": 8}, + {"MAX_RECORD_BYTES": 12}, + {"MAX_CSV_HEADER_BYTES": 4}, + {"MAX_TOKEN_BYTES": 3}, + {"MAX_TOKEN_BYTES": 2, "MAX_RECORD_BYTES": 9}, + {"MAX_RECORD_BYTES": 5, "MAX_CSV_HEADER_BYTES": 9}, + {"MAX_TOKEN_BYTES": 100_000, "MAX_RECORD_BYTES": 6}, + {"MAX_CSV_HEADER_BYTES": 100_000, "MAX_TOKEN_BYTES": 4}, +) + + +@pytest.mark.parametrize("settings", CEILING_SETTINGS, ids=repr) +def test_small_ceilings_refuse_identically(monkeypatch, settings): + for name, value in settings.items(): + monkeypatch.setattr(owner, name, value) + global MAX_RECORD_BYTES, MAX_CSV_HEADER_BYTES, MAX_TOKEN_BYTES + previous = (MAX_RECORD_BYTES, MAX_CSV_HEADER_BYTES, MAX_TOKEN_BYTES) + MAX_RECORD_BYTES = owner.MAX_RECORD_BYTES + MAX_CSV_HEADER_BYTES = owner.MAX_CSV_HEADER_BYTES + MAX_TOKEN_BYTES = owner.MAX_TOKEN_BYTES + try: + for data in BOUNDARIES.values(): + _assert_identical(data) + generator = random.Random(20260917) + alphabet = b'ab,"\r\n\t 0' + for _ in range(600): + _assert_identical( + bytes( + generator.choice(alphabet) + for _ in range(generator.randrange(0, 40)) + ) + ) + finally: + MAX_RECORD_BYTES, MAX_CSV_HEADER_BYTES, MAX_TOKEN_BYTES = previous + + +def test_the_small_ceiling_case_the_first_scan_got_wrong(monkeypatch): + """A record cap below the token cap must still refuse at the record cap. + + The first version of the scan short-circuited on MAX_TOKEN_BYTES alone, + which silently skipped the record ceiling whenever the record ceiling was + the smaller of the two. This is that case, pinned. + """ + + body = b'"x\r\n\ty",z\n' + monkeypatch.setattr(owner, "MAX_RECORD_BYTES", len(body) - 1) + assert owner.MAX_RECORD_BYTES < owner.MAX_TOKEN_BYTES + with pytest.raises(owner.ACSCoverageAuthenticationError, match="CSV_RECORD_BYTES"): + list(owner._records(io.BytesIO(b"A,B\n" + body))) + + +def _exhaustive(alphabet, length): + yield b"" + current = [b""] + for _ in range(length): + current = [ + prefix + bytes((symbol,)) for prefix in current for symbol in alphabet + ] + yield from current + + +EXHAUSTIVE_SETTINGS = ( + {}, + {"MAX_TOKEN_BYTES": 2}, + {"MAX_RECORD_BYTES": 3, "MAX_CSV_HEADER_BYTES": 3}, + {"MAX_RECORD_BYTES": 4, "MAX_TOKEN_BYTES": 2}, + {"MAX_CSV_HEADER_BYTES": 2, "MAX_RECORD_BYTES": 5, "MAX_TOKEN_BYTES": 4}, +) + + +@pytest.mark.parametrize("settings", EXHAUSTIVE_SETTINGS, ids=repr) +def test_every_short_string_over_the_branching_alphabet(monkeypatch, settings): + """Exhaustive: all 9,331 strings up to five bytes over the fence's alphabet. + + Every byte the fence branches on is in the alphabet -- comma, quote, CR, LF + -- plus one ordinary byte and one whitespace byte. Under each ceiling + setting the scan and the byte loop must agree on every one of them. + """ + + for name, value in settings.items(): + monkeypatch.setattr(owner, name, value) + global MAX_RECORD_BYTES, MAX_CSV_HEADER_BYTES, MAX_TOKEN_BYTES + previous = (MAX_RECORD_BYTES, MAX_CSV_HEADER_BYTES, MAX_TOKEN_BYTES) + MAX_RECORD_BYTES = owner.MAX_RECORD_BYTES + MAX_CSV_HEADER_BYTES = owner.MAX_CSV_HEADER_BYTES + MAX_TOKEN_BYTES = owner.MAX_TOKEN_BYTES + checked = 0 + try: + for data in _exhaustive(b',"\r\n a', 5): + assert _outcome(owner._records, data) == _outcome( + _reference_records, data + ), data + checked += 1 + finally: + MAX_RECORD_BYTES, MAX_CSV_HEADER_BYTES, MAX_TOKEN_BYTES = previous + assert checked == 9331 From 124f158aae61c6d274138d20eca225f26f37cb40 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 15 Sep 2026 23:52:53 -0400 Subject: [PATCH 05/45] Derive each source content key once per run, and prove them at run end executor.py re-derived source_content_key for every source a cold-executed node declared, and keys._directory_identity read_bytes()es every file of the tree: O(executed source-declaring nodes x source bytes), which on the native graph is the 3.48 GiB source tree re-read per node. run_graph now carries a _SourceIdentities cache in its own locals, populated by the run-start pass as it reads. A cached key is reused only while the path's stat signature -- (dev, ino, size, mtime_ns, ctime_ns) for a file, and for a directory its own identity plus the relative name, type and identity of every entry _directory_identity would walk -- is identical to the signature taken both immediately before and immediately after the read that produced the key. A derivation whose two signatures disagree is not cached at all. source_content_key itself stays pure and uncached: three key and codec tests call it twice on one path with the bytes changed in between. Before the manifest is built, every source is re-derived in full with the cache bypassed, written inline rather than through _source_paths_and_keys because a build test counts calls to that code object. That closes the two cases a stat signature cannot decide, and gives the executor a guarantee it never had: a source changed during a node that declares none is now caught. RunManifest gains source_identities, attached exactly like populations and mass_ledgers -- repr=False, compare=False, outside content_addressed, outside to_json, outside every node receipt and cache record. test_graph_executor_source_identity pins the contract that had no test at all: a source rewritten by its own node refuses and writes nothing to the store, a file added to or removed from a directory source refuses, a same-length rewrite with mtime restored refuses, a change during a source-free node refuses at run end, and an unchanged source is read twice per run however many nodes declare it. The graph suite falls from 108 s to 72 s. Also records the real-archive record-fence parity receipt: both ACS person members, 2.4 GB, 3,422,890 records, identical digests, 226.9 s -> 4.6 s. Co-Authored-By: Claude Opus --- .../record-fence-parity.json | 48 ++ .../record_fence_real_archive.py | 162 ++++++ .../src/microcosm/graph/executor.py | 131 ++++- .../src/microcosm/graph/manifest.py | 19 + .../test_graph_executor_source_identity.py | 477 ++++++++++++++++++ 5 files changed, 832 insertions(+), 5 deletions(-) create mode 100644 experiments/native-verify-once/record-fence-parity.json create mode 100644 experiments/native-verify-once/record_fence_real_archive.py create mode 100644 packages/microcosm-graph/tests/test_graph_executor_source_identity.py diff --git a/experiments/native-verify-once/record-fence-parity.json b/experiments/native-verify-once/record-fence-parity.json new file mode 100644 index 000000000..869cca582 --- /dev/null +++ b/experiments/native-verify-once/record-fence-parity.json @@ -0,0 +1,48 @@ +{ + "protocol": "owned.us-native-record-fence-parity.v1", + "archive": "/Users/maxghenis/PolicyEngine/_recovered/pilot-runs/native45-v5/run/sources/acs/csv_pus.zip", + "archive_bytes": 602847146, + "archive_sha256_of_central_directory_names": "c1464ee3afd33e620773581b0feac04c2e87c00018ef2392795a723e658984d4", + "identical": true, + "members": [ + { + "member": "psam_pusa.csv", + "member_bytes": 1226543489, + "byte_limit": 1226543489, + "identical": true, + "byte_loop": { + "records_sha256": "da955ca2c3d53ec608fc4c373ac584e38f50b14732b13420296bb136347fc827", + "records": 1743752, + "bytes": 1226543489, + "cpu_seconds": 116.717 + }, + "chunked_scan": { + "records_sha256": "da955ca2c3d53ec608fc4c373ac584e38f50b14732b13420296bb136347fc827", + "records": 1743752, + "bytes": 1226543489, + "cpu_seconds": 2.379 + }, + "speedup": 49.1 + }, + { + "member": "psam_pusb.csv", + "member_bytes": 1178924624, + "byte_limit": 1178924624, + "identical": true, + "byte_loop": { + "records_sha256": "9bbf6af890dd286c0d76171d72363888218a8bf0395a94eef75a15e490184bba", + "records": 1679138, + "bytes": 1178924624, + "cpu_seconds": 110.14 + }, + "chunked_scan": { + "records_sha256": "9bbf6af890dd286c0d76171d72363888218a8bf0395a94eef75a15e490184bba", + "records": 1679138, + "bytes": 1178924624, + "cpu_seconds": 2.252 + }, + "speedup": 48.9 + } + ], + "note": "Descriptive parity measurement on a read-only staged archive. Not a build, not a certification, not release eligible." +} diff --git a/experiments/native-verify-once/record_fence_real_archive.py b/experiments/native-verify-once/record_fence_real_archive.py new file mode 100644 index 000000000..60188c692 --- /dev/null +++ b/experiments/native-verify-once/record_fence_real_archive.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Prove the ACS record fence rewrite on the real staged archive, read-only. + +The scan in ``acs_person_coverage_authentication._records`` replaced a +byte-at-a-time loop. The unit suite proves equivalence on invented bytes; this +script proves it on the actual source the native build fences, by running both +implementations over every applicable member of a staged ``csv_pus.zip`` and +comparing a rolling digest of the yielded records, the record count and the +total bytes. + +It opens the archive read-only, writes only the receipt path it is given, and +executes no build. Usage: + + .venv/bin/python -I -B experiments/native-verify-once/record_fence_real_archive.py \ + [byte-limit-per-member] +""" + +from __future__ import annotations + +import glob +import hashlib +import io +import json +import os +import sys +import time +import zipfile +from pathlib import Path + +sys.path[:0] = sorted( + glob.glob(str(Path(__file__).resolve().parents[2] / "packages/*/src")) +) + +from microcosm.build.us_runtime import ( # noqa: E402 + acs_person_coverage_authentication as owner, +) + + +def reference_records(stream): + """The exact pre-change body of ``_records``, kept verbatim.""" + + record = bytearray() + quoted, pending_cr, first, token_bytes = False, False, True, 0 + cap = owner.MAX_CSV_HEADER_BYTES + while block := stream.read(4096): + for byte in block: + if pending_cr: + if byte == 10: + owner._require(len(record) < cap, "CSV_RECORD_BYTES") + record.append(byte) + yield bytes(record) + record.clear() + first, pending_cr, token_bytes = False, False, 0 + if byte == 10: + continue + cap = owner.MAX_CSV_HEADER_BYTES if first else owner.MAX_RECORD_BYTES + owner._require(len(record) < cap, "CSV_RECORD_BYTES") + if byte == 44 and not quoted: + token_bytes = 0 + else: + token_bytes += 1 + owner._require(token_bytes <= owner.MAX_TOKEN_BYTES, "CSV_TOKEN_BYTES") + record.append(byte) + if byte == 34: + quoted = not quoted + if not quoted and byte in (10, 13): + if byte == 13: + pending_cr = True + else: + yield bytes(record) + record.clear() + first, token_bytes = False, 0 + if record: + yield bytes(record) + + +class Bounded(io.RawIOBase): + """Read-only truncating wrapper; it never writes to the archive.""" + + def __init__(self, stream, limit): + self._stream, self._left = stream, limit + + def read(self, size=-1): + if self._left <= 0: + return b"" + want = size if size and size > 0 else 65536 + block = self._stream.read(min(want, self._left)) + self._left -= len(block) + return block + + +def fence(archive_path, member, implementation, limit): + digest, count, total = hashlib.sha256(), 0, 0 + with zipfile.ZipFile(archive_path) as archive, archive.open(member) as raw: + started = time.process_time() + for record in implementation(Bounded(raw, limit)): + digest.update(len(record).to_bytes(8, "little")) + digest.update(record) + count += 1 + total += len(record) + elapsed = time.process_time() - started + return { + "records_sha256": digest.hexdigest(), + "records": count, + "bytes": total, + "cpu_seconds": round(elapsed, 3), + } + + +def main(): + archive_path = Path(sys.argv[1]) + receipt_path = Path(sys.argv[2]) + limit = int(sys.argv[3]) if len(sys.argv) > 3 else 1 << 62 + with zipfile.ZipFile(archive_path) as archive: + members = [ + info.filename + for info in archive.infolist() + if info.filename.casefold().startswith("psam_pus") + ] + sizes = {info.filename: info.file_size for info in archive.infolist()} + rows, identical = [], True + for member in sorted(members): + old = fence(archive_path, member, reference_records, limit) + new = fence(archive_path, member, owner._records, limit) + same = {k: v for k, v in old.items() if k != "cpu_seconds"} == { + k: v for k, v in new.items() if k != "cpu_seconds" + } + identical &= same + rows.append( + { + "member": member, + "member_bytes": sizes[member], + "byte_limit": min(limit, sizes[member]), + "identical": same, + "byte_loop": old, + "chunked_scan": new, + "speedup": round(old["cpu_seconds"] / max(new["cpu_seconds"], 1e-9), 1), + } + ) + payload = { + "protocol": "owned.us-native-record-fence-parity.v1", + "archive": str(archive_path), + "archive_bytes": archive_path.stat().st_size, + "archive_sha256_of_central_directory_names": hashlib.sha256( + "\n".join(sorted(members)).encode() + ).hexdigest(), + "identical": identical, + "members": rows, + "note": ( + "Descriptive parity measurement on a read-only staged archive. Not a " + "build, not a certification, not release eligible." + ), + } + receipt_path.parent.mkdir(parents=True, exist_ok=True) + receipt_path.write_text(json.dumps(payload, indent=2) + "\n") + print(json.dumps(payload, indent=2)) + return 0 if identical else 1 + + +if __name__ == "__main__": + os.environ["TZ"] = "UTC" + raise SystemExit(main()) diff --git a/packages/microcosm-graph/src/microcosm/graph/executor.py b/packages/microcosm-graph/src/microcosm/graph/executor.py index 8c0eada62..8f78b6aac 100644 --- a/packages/microcosm-graph/src/microcosm/graph/executor.py +++ b/packages/microcosm-graph/src/microcosm/graph/executor.py @@ -6,6 +6,7 @@ import json import pickle import socket +import stat import struct import time from collections.abc import Callable, Mapping @@ -2058,6 +2059,8 @@ def _source_paths_and_keys( compiled: CompiledGraph, sources: Mapping[str, Path], store: ContentStore, + *, + identities: _SourceIdentities | None = None, ) -> tuple[dict[str, Path], dict[str, str]]: declared = {source.name: source for source in compiled.graph.sources} used = {name for node in compiled.graph.nodes for name in node.sources} @@ -2068,7 +2071,7 @@ def _source_paths_and_keys( if unknown: raise ValueError(f"Source paths supplied for undeclared names {unknown!r}.") resolved: dict[str, Path] = {} - identities: dict[str, str] = {} + derived: dict[str, str] = {} for name in sorted(used): path = Path(sources[name]).resolve(strict=True) # Codec availability is verified before any kernel can execute. The @@ -2086,8 +2089,94 @@ def _source_paths_and_keys( else: # defended by ContentStore.__init__ raise StoreUnavailable("ContentStore has an invalid codec registry.") resolved[name] = path - identities[name] = source_content_key(name, path) - return resolved, identities + derived[name] = ( + source_content_key(name, path) + if identities is None + else identities.key(name, path) + ) + return resolved, derived + + +def _stat_identity(info: object) -> tuple[int, int, int, int, int]: + """The five fields every source owner in this repository binds a file by.""" + + return ( + info.st_dev, + info.st_ino, + info.st_size, + info.st_mtime_ns, + info.st_ctime_ns, + ) + + +def _source_stat_signature(path: Path) -> tuple[object, ...]: + """A read-free signature of everything ``source_content_key`` would read. + + A regular file contributes its own stat identity. A directory contributes + its own, plus the relative name, file type and stat identity of every entry + ``_directory_identity`` would walk -- the roster as well as the members, + because the directory identity hashes relative names and a file added or + removed moves it with no member's stat changing. Anything else contributes + its stat identity alone, so an unexpected node type is never cached past a + change. + """ + + info = path.lstat() + if stat.S_ISREG(info.st_mode): + return ("file", _stat_identity(info)) + if not stat.S_ISDIR(info.st_mode): + return ("other", stat.S_IFMT(info.st_mode), _stat_identity(info)) + entries = [] + for candidate in sorted(path.rglob("*")): + entry = candidate.lstat() + entries.append( + ( + candidate.relative_to(path).as_posix(), + stat.S_IFMT(entry.st_mode), + _stat_identity(entry), + ) + ) + return ("dir", _stat_identity(info), tuple(entries)) + + +class _SourceIdentities: + """One run's source content keys, re-derived only when a source may have moved. + + ``source_content_key`` itself stays pure and uncached: it is the declared + path-independent identity of a source's *content*, and three key and codec + tests call it twice on one path with the bytes changed in between. This + cache is the caller's, lives in one ``run_graph`` call, and never outlives + it. + + A cached key is reused only when the path's stat signature is identical to + the signature taken both immediately before and immediately after the read + that produced the key. A derivation whose two signatures disagree -- the + source moved while it was being read -- is not cached at all, so every + consumer re-derives it exactly as it does today. + """ + + __slots__ = ("_entries",) + + def __init__(self) -> None: + self._entries: dict[str, tuple[tuple[object, ...], str]] = {} + + def key(self, name: str, path: Path) -> str: + signature = _source_stat_signature(path) + entry = self._entries.get(name) + if entry is not None and entry[0] == signature: + return entry[1] + identity = source_content_key(name, path) + if _source_stat_signature(path) == signature: + self._entries[name] = (signature, identity) + else: + self._entries.pop(name, None) + return identity + + def derive(self, name: str, path: Path) -> str: + """Re-derive one key with the cache bypassed, and refresh the cache.""" + + self._entries.pop(name, None) + return self.key(name, path) def _all_node_keys( @@ -2353,7 +2442,13 @@ def run_graph( _preflight_expand_declarations(compiled) started_at = _now() - source_paths, source_keys = _source_paths_and_keys(compiled, sources, store) + # One run's source identities. The run-start pass populates the cache as it + # reads, so the per-node check below is a stat walk rather than a second + # full read; the run-end pass bypasses the cache entirely. + source_identities = _SourceIdentities() + source_paths, source_keys = _source_paths_and_keys( + compiled, sources, store, identities=source_identities + ) keys, implementations = _all_node_keys(compiled, kernels, source_keys) contracts = { node_id: typed_contracts(compiled, compiled.graph.node(node_id), keys, kernels) @@ -2522,7 +2617,11 @@ def run_graph( if before != after: raise NodeRejected(f"Node {node.id!r} mutated its input context.") for name in node.sources: - current = source_content_key(name, source_paths[name]) + # A stat signature that still matches the one taken across this + # source's own read reuses that read's key; anything that moved + # is re-derived in full here, so the refusal still precedes + # every _write_node this run performs. + current = source_identities.key(name, source_paths[name]) if current != source_keys[name]: raise NodeRejected( f"Node {node.id!r} changed source {name!r} while running." @@ -2692,9 +2791,31 @@ def run_graph( ) receipt_payloads[node_id] = receipts[node_id].receipt + # Every source is re-derived in full, cache bypassed, before any caller + # receives a manifest. This closes the two cases a stat signature cannot + # decide -- a rewrite that leaves all five stat fields identical, and a + # change made during a node that declares no source at all, which the + # per-node check has never seen -- and it is written inline rather than + # through _source_paths_and_keys because a build test counts calls to that + # exact code object. + final_identities = {} + moved = [] + for name in sorted(source_paths): + identity = source_identities.derive(name, source_paths[name]) + final_identities[name] = identity + if identity != source_keys[name]: + moved.append(name) + if moved: + raise NodeRejected( + "Run changed source " + + ", ".join(repr(name) for name in moved) + + " while executing." + ) + return RunManifest( country=compiled.graph.country, nodes=MappingProxyType(receipts), + source_identities=MappingProxyType(final_identities), decisions=decisions, started_at=started_at, finished_at=_now(), diff --git a/packages/microcosm-graph/src/microcosm/graph/manifest.py b/packages/microcosm-graph/src/microcosm/graph/manifest.py index 1adb8ded8..6b1a526f3 100644 --- a/packages/microcosm-graph/src/microcosm/graph/manifest.py +++ b/packages/microcosm-graph/src/microcosm/graph/manifest.py @@ -443,6 +443,12 @@ class RunManifest: timestamps, and attached ``Frame`` instances are also outside the content-addressed body. Computational reuse continues to use node keys, not this run-manifest identity. + + :attr:`source_identities` is attached in the same way: the executor's + run-end re-derivation of every declared source's content key, recorded so a + reader can see which bytes the run actually finished against. Like + ``populations`` and ``mass_ledgers`` it is outside :attr:`key`, outside + ``to_json``, and outside every node receipt and cache record. """ country: str @@ -457,6 +463,9 @@ class RunManifest: mass_ledgers: Mapping[str, tuple[MassRecord, ...]] = field( default_factory=dict, repr=False, compare=False ) + source_identities: Mapping[str, str] = field( + default_factory=dict, repr=False, compare=False + ) def __post_init__(self) -> None: if not isinstance(self.country, str): @@ -507,6 +516,16 @@ def __post_init__(self) -> None: ) mass_ledgers[version_id] = frozen_records object.__setattr__(self, "mass_ledgers", MappingProxyType(mass_ledgers)) + source_identities: dict[str, str] = {} + for name, identity in self.source_identities.items(): + if not isinstance(name, str): + raise TypeError("RunManifest.source_identities keys must be strings") + if not isinstance(identity, str): + raise TypeError("RunManifest.source_identities values must be strings") + source_identities[name] = identity + object.__setattr__( + self, "source_identities", MappingProxyType(source_identities) + ) @property def content_addressed(self) -> Mapping[str, object]: diff --git a/packages/microcosm-graph/tests/test_graph_executor_source_identity.py b/packages/microcosm-graph/tests/test_graph_executor_source_identity.py new file mode 100644 index 000000000..1d51ab0ed --- /dev/null +++ b/packages/microcosm-graph/tests/test_graph_executor_source_identity.py @@ -0,0 +1,477 @@ +"""The executor's source-identity contract, and the per-run cache under it. + +Until this file the per-node source re-derivation at the heart of +``run_graph`` had no regression test at all: nothing asserted that a source +mutated while a node ran makes the run refuse, and nothing asserted that the +refusal lands before the node's artifacts reach the store. Both are asserted +here, first, so the cache added underneath them is measured against a pinned +contract rather than against silence. + +The cache reuses a source's content key only while the path's stat signature +is unchanged, so every mutation that moves any stat field -- content, size, +inode, timestamps, or a directory's roster -- still refuses at the same node +it refuses at today. The two cases a stat signature cannot decide are closed +by an unconditional full re-derivation before the manifest is built, which +also gives the executor something it never had: a source changed during a node +that declares no source is now caught. + +Synthetic toy graph only: no data file, no country package, no engine. +""" + +from __future__ import annotations + +import importlib.util +import os +import sys +from collections.abc import Callable +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +import microcosm.graph.executor as graph_executor +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph.decl import ( + Graph, + Node, + Owned, + Slice, + SourceRef, + StructuralDelta, + compile_graph, +) +from microcosm.graph.executor import NodeRejected, run_graph +from microcosm.graph.kernel import ( + Capabilities, + Determinism, + KernelContext, + KernelRegistry, + KernelResult, +) +from microcosm.graph.store import ContentStore + +if "_toy" not in sys.modules: + _SPEC = importlib.util.spec_from_file_location( + "_toy", Path(__file__).with_name("_toy.py") + ) + assert _SPEC is not None and _SPEC.loader is not None + _TOY = importlib.util.module_from_spec(_SPEC) + sys.modules["_toy"] = _TOY + _SPEC.loader.exec_module(_TOY) + +SOURCE = SourceRef("survey", "csv-tables") + + +class _Kernel: + def __init__(self, ref, capabilities, run): + self._ref = ref + self._capabilities = capabilities + self._run = run + + @property + def ref(self): + return self._ref + + @property + def capabilities(self): + return self._capabilities + + def implementation_hash(self): + return f"impl-{self._ref}" + + def run(self, context): + return self._run(context) + + +def _source_path(root: Path, value: int = 0) -> Path: + root.mkdir() + (root / "value.txt").write_text(str(value), encoding="utf-8") + return root + + +def _source_frame(path: Path) -> Frame: + offset = int((path / "value.txt").read_text(encoding="utf-8")) + person = pd.DataFrame( + { + "person_id": np.asarray([1, 2, 3], dtype=np.int64), + "person_household_id": np.asarray([10, 10, 20], dtype=np.int64), + "age": np.asarray([10, 20, 30], dtype=np.int64) + offset, + } + ) + household = pd.DataFrame( + { + "household_id": np.asarray([10, 20], dtype=np.int64), + "size": np.asarray([2, 1], dtype=np.int64), + } + ) + return Frame( + {"person": person, "household": household}, + EntitySchema(group_entities=("household",)), + {"household": Weights(np.asarray([1.0, 2.0]), WeightKind.DESIGN)}, + pd.Series(["a", "a", "b"], name="stratum"), + ) + + +CREATE = Node( + "survey", + "source@1", + structural=StructuralDelta.CREATE, + sources=("survey",), + outputs=(Owned("person", "age", "int64"), Owned("household", "size", "int64")), +) +FOLLOWER = Node( + "follower", + "follower@1", + inputs=(Slice("person", ("age",)),), + outputs=(Owned("person", "doubled", "float64"),), + population="survey", +) + + +def _graph() -> Graph: + return Graph("toy", (SOURCE,), (CREATE, FOLLOWER)) + + +def _registry( + *, + on_source: Callable[[Path], None] | None = None, + on_follower: Callable[[], None] | None = None, +) -> KernelRegistry: + def source(context: KernelContext) -> KernelResult: + frame = _source_frame(context.sources["survey"]) + if on_source is not None: + on_source(context.sources["survey"]) + return KernelResult(frame=frame, receipt={"rows": frame.n("person")}) + + def follower(context: KernelContext) -> KernelResult: + if on_follower is not None: + on_follower() + table = context.tables["person"] + return KernelResult( + columns={ + ("person", "doubled"): pd.Series( + table["age"].to_numpy(dtype=np.float64) * 2.0, + index=pd.Index(table["person_id"], name="person_id"), + dtype="float64", + ) + }, + receipt={"rows": len(table)}, + ) + + registry = KernelRegistry() + registry.register( + _Kernel( + "source@1", + Capabilities(Determinism.DETERMINISTIC, structural=StructuralDelta.CREATE), + source, + ) + ) + registry.register( + _Kernel("follower@1", Capabilities(Determinism.DETERMINISTIC), follower) + ) + return registry + + +def _run(source: Path, store: ContentStore, registry: KernelRegistry): + return run_graph( + compile_graph(_graph()), + sources={"survey": source}, + store=store, + kernels=registry, + ) + + +def _stored_objects(store: ContentStore) -> set[str]: + return { + path.relative_to(store.objects).as_posix() + for path in store.objects.rglob("*") + if path.is_file() + } + + +# -------------------------------------------------------------------------- +# The contract: a source that moves while the run executes makes it refuse. +# -------------------------------------------------------------------------- + + +def test_a_source_rewritten_by_its_own_node_refuses_that_node(tmp_path: Path) -> None: + source = _source_path(tmp_path / "source") + store = ContentStore(tmp_path / "store") + before = _stored_objects(store) + + def rewrite(path: Path) -> None: + (path / "value.txt").write_text("7", encoding="utf-8") + + with pytest.raises(NodeRejected, match="changed source"): + _run(source, store, _registry(on_source=rewrite)) + # The refusal precedes the node's own persistence: nothing new was written. + assert _stored_objects(store) == before + + +def test_a_file_added_to_a_directory_source_refuses(tmp_path: Path) -> None: + """The roster is part of the signature, so an addition is never cached over.""" + + source = _source_path(tmp_path / "source") + store = ContentStore(tmp_path / "store") + + def add(path: Path) -> None: + (path / "extra.txt").write_text("invented", encoding="utf-8") + + with pytest.raises(NodeRejected, match="changed source"): + _run(source, store, _registry(on_source=add)) + + +def test_a_file_removed_from_a_directory_source_refuses(tmp_path: Path) -> None: + source = _source_path(tmp_path / "source") + (source / "extra.txt").write_text("invented", encoding="utf-8") + store = ContentStore(tmp_path / "store") + + def remove(path: Path) -> None: + (path / "extra.txt").unlink() + + with pytest.raises(NodeRejected, match="changed source"): + _run(source, store, _registry(on_source=remove)) + + +def test_a_source_changed_during_a_source_free_node_refuses_at_run_end( + tmp_path: Path, +) -> None: + """The guarantee the per-node check never had: no node declares it here.""" + + source = _source_path(tmp_path / "source") + store = ContentStore(tmp_path / "store") + + def rewrite() -> None: + (source / "value.txt").write_text("9", encoding="utf-8") + + with pytest.raises(NodeRejected, match="Run changed source 'survey'"): + _run(source, store, _registry(on_follower=rewrite)) + + +def test_a_rewritten_source_still_refuses_when_only_its_bytes_moved( + tmp_path: Path, +) -> None: + """Same length, restored mtime: the cache must not answer from memory.""" + + source = _source_path(tmp_path / "source") + store = ContentStore(tmp_path / "store") + target = source / "value.txt" + target.write_text("1", encoding="utf-8") + original = target.stat() + + def rewrite(path: Path) -> None: + (path / "value.txt").write_text("2", encoding="utf-8") + os.utime(path / "value.txt", ns=(original.st_atime_ns, original.st_mtime_ns)) + + with pytest.raises(NodeRejected, match="source"): + _run(source, store, _registry(on_source=rewrite)) + + +# -------------------------------------------------------------------------- +# The cache: fewer reads, identical keys. +# -------------------------------------------------------------------------- + + +def _counting(monkeypatch: pytest.MonkeyPatch) -> list[tuple[str, str]]: + seen: list[tuple[str, str]] = [] + original = graph_executor.source_content_key + + def counted(name, path): + seen.append((name, str(path))) + return original(name, path) + + monkeypatch.setattr(graph_executor, "source_content_key", counted) + return seen + + +def test_each_source_is_read_twice_per_run_not_once_per_node( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = _source_path(tmp_path / "source") + store = ContentStore(tmp_path / "store") + seen = _counting(monkeypatch) + manifest = _run(source, store, _registry()) + # Once when the run's identities are established, once when they are + # re-proved at the end. The node in between reuses the first read. + assert [name for name, _ in seen] == ["survey", "survey"] + assert manifest.nodes["survey"].hit is False + + +def test_an_unchanged_source_is_never_re_read_by_a_node( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A wider graph adds nodes, not reads.""" + + source = _source_path(tmp_path / "source") + store = ContentStore(tmp_path / "store") + wide = Graph( + "toy", + (SOURCE,), + ( + CREATE, + FOLLOWER, + Node( + "second", + "source@1", + structural=StructuralDelta.CREATE, + sources=("survey",), + outputs=( + Owned("person", "age", "int64"), + Owned("household", "size", "int64"), + ), + ), + ), + ) + seen = _counting(monkeypatch) + run_graph( + compile_graph(wide), + sources={"survey": source}, + store=store, + kernels=_registry(), + ) + assert len(seen) == 2 + + +def test_a_touched_but_unchanged_source_re_reads_and_still_matches( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A signature miss is a re-read, not a refusal.""" + + source = _source_path(tmp_path / "source") + store = ContentStore(tmp_path / "store") + seen = _counting(monkeypatch) + + def touch(path: Path) -> None: + os.utime(path / "value.txt", None) + + manifest = _run(source, store, _registry(on_source=touch)) + assert len(seen) == 3 # run start, the node's miss, run end + assert manifest.nodes["survey"].hit is False + + +# -------------------------------------------------------------------------- +# The record, and what it must not move. +# -------------------------------------------------------------------------- + + +def test_the_manifest_records_the_run_end_identities_without_moving_anything( + tmp_path: Path, +) -> None: + source = _source_path(tmp_path / "source") + store = ContentStore(tmp_path / "store") + manifest = _run(source, store, _registry()) + identity = graph_executor.source_content_key("survey", source) + assert dict(manifest.source_identities) == {"survey": identity} + + bare = type(manifest)( + country=manifest.country, + nodes=manifest.nodes, + decisions=manifest.decisions, + started_at=manifest.started_at, + finished_at=manifest.finished_at, + host=manifest.host, + ) + assert bare.key == manifest.key + assert bare.to_json() == manifest.to_json() + assert "source_identities" not in manifest.to_json() + assert dict(manifest.content_addressed) == dict(bare.content_addressed) + + +def test_the_recorded_identities_are_rejected_when_malformed() -> None: + from microcosm.graph.manifest import RunManifest + + with pytest.raises(TypeError, match="source_identities keys"): + RunManifest(country="toy", nodes={}, source_identities={1: "a"}) + with pytest.raises(TypeError, match="source_identities values"): + RunManifest(country="toy", nodes={}, source_identities={"a": 1}) + + +# -------------------------------------------------------------------------- +# The cache object itself. +# -------------------------------------------------------------------------- + + +def test_the_cache_reuses_only_an_unchanged_signature(tmp_path: Path) -> None: + source = _source_path(tmp_path / "source") + identities = graph_executor._SourceIdentities() + first = identities.key("survey", source) + assert identities.key("survey", source) == first + (source / "value.txt").write_text("5", encoding="utf-8") + second = identities.key("survey", source) + assert second != first + assert identities.key("survey", source) == second + + +def test_the_cache_is_bypassed_by_derive(tmp_path: Path) -> None: + source = _source_path(tmp_path / "source") + identities = graph_executor._SourceIdentities() + identities.key("survey", source) + calls: list[str] = [] + original = graph_executor.source_content_key + + def counted(name, path): + calls.append(name) + return original(name, path) + + graph_executor.source_content_key = counted + try: + identities.key("survey", source) + assert calls == [] + identities.derive("survey", source) + assert calls == ["survey"] + finally: + graph_executor.source_content_key = original + + +def test_a_regular_file_source_signature_moves_with_its_bytes(tmp_path: Path) -> None: + path = tmp_path / "table.csv" + path.write_bytes(b"id\n1\n") + before = graph_executor._source_stat_signature(path) + assert before[0] == "file" + path.write_bytes(b"id\n2\n") + assert graph_executor._source_stat_signature(path) != before + + +def test_a_directory_signature_covers_names_as_well_as_contents( + tmp_path: Path, +) -> None: + root = _source_path(tmp_path / "source") + before = graph_executor._source_stat_signature(root) + assert before[0] == "dir" + (root / "value.txt").rename(root / "renamed.txt") + renamed = graph_executor._source_stat_signature(root) + assert renamed != before + nested = root / "inner" + nested.mkdir() + (nested / "deep.txt").write_text("x", encoding="utf-8") + assert graph_executor._source_stat_signature(root) != renamed + + +def test_a_source_that_moves_during_its_own_read_is_not_cached( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A torn read leaves no cache entry, so the next consumer re-derives.""" + + path = tmp_path / "table.csv" + path.write_bytes(b"id\n1\n") + identities = graph_executor._SourceIdentities() + original = graph_executor.source_content_key + + def tearing(name, source_path): + result = original(name, source_path) + Path(source_path).write_bytes(b"id\n22\n") + return result + + monkeypatch.setattr(graph_executor, "source_content_key", tearing) + identities.key("table", path) + monkeypatch.setattr(graph_executor, "source_content_key", original) + calls: list[str] = [] + + def counted(name, source_path): + calls.append(name) + return original(name, source_path) + + monkeypatch.setattr(graph_executor, "source_content_key", counted) + identities.key("table", path) + assert calls == ["table"] From 96423cbb96f2374d846854b600b54fd2477a0a7d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 16 Sep 2026 00:04:50 -0400 Subject: [PATCH 06/45] Add verification epochs to the two native capsules Every borrow of AuthenticatedSurveyPopulationPreparation re-ran the whole authentication: both source catalogues, the ACS native coverage binding (eight full zip reads), the nested ASEC native population (seven files), the ten-file roster re-hash and every pure seal -- once per executed node through the population observer, and again inside every kernel that borrows it. The ASEC native capsule did the same on every .frame access. survey_population_preparation.verification_epoch() makes that happen once. Inside an epoch each borrow still pays a cheap tier in full -- the live authority and attached owner payloads, _encode(_producer()), and _file_stats over the whole roster -- and the expensive tier is skipped only while a signature is unchanged: the stat identity of every path the four foreign verifications and _source_files re-read, the identity and length of every borrowed payload, and a read-free witness of every live Frame's storage (buffer address, shape, strides, dtype and writeable flag per column, index and weight). A signature that moved is a memo miss, not a refusal: the complete validation runs and raises exactly what it would have raised. Leaving the epoch, at every nesting level, re-validates every memoised capsule in full with the memo bypassed, so a change no signature can see -- an in-place write into a live buffer, or a value written into a frozen plan row -- still makes the run refuse before the caller receives anything. Outside an epoch nothing is memoised and every existing borrow, refusal and trace-based mutation test keeps its exact meaning; _validate and _validate_state are untouched. The three foreign owners gain only a _verified_source_stats() helper naming the paths their own verification re-reads. Their inventory contracts do not move. The two capsule modules' unbound_uses digests do, because the new functions use already-imported names in new scopes; both are re-derived through graph_implementation._dependency_contract and recorded in PROGRESS.md. Co-Authored-By: Claude Opus --- PROGRESS.md | 40 ++- .../us_runtime/acs_native_coverage_binding.py | 25 ++ .../us_runtime/acs_population_catalogue.py | 31 ++ .../us_runtime/asec_2024_native_population.py | 118 ++++++- .../us_runtime/asec_population_catalogue.py | 23 ++ .../graph_implementation_inventory.json | 4 +- .../survey_population_preparation.py | 235 ++++++++++++- .../tests/test_us_native_verify_once_epoch.py | 314 ++++++++++++++++++ 8 files changed, 783 insertions(+), 7 deletions(-) create mode 100644 packages/microcosm-build/tests/test_us_native_verify_once_epoch.py diff --git a/PROGRESS.md b/PROGRESS.md index 50dec777d..11357b3c5 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -1422,7 +1422,8 @@ Branch `native-verify-once`, worktree `~/PolicyEngine/_worktrees/microcosm-verif base `f7bb88525a78786f91bc3ebe2083ef4b1c85de18` (PR #893 head). Started 2026-09-15. -**State (2026-09-15, opening):** investigation only. Nothing edited yet. +**State (2026-09-16):** mechanisms 3, 4 and 5 landed with proofs; the +verification epoch (mechanisms 1 and 2) is implemented and under test. **Goal.** The 9/15 pilot v5 measurement (`~/PolicyEngine/_recovered/pilot-runs/native45-v5/out.md` §2) attributes ~79 % @@ -1434,9 +1435,44 @@ without moving any digest value. **Done:** read the evidence base; located all five mechanisms at base HEAD. -**Next:** pin-surface map, memo design note, implementation, tests, measurement, +**Done (2026-09-16):** + +- `docs/us-native-verification-once.md` — the design note, written before any + capsule changed. +- **Mechanism 5**: `_object_stream` builds each plain float/int/bool column's + context-digest bytes with numpy. Byte-identical against a verbatim copy of the + pre-change body over every column kind, the float specials, non-canonical NaN + payloads, both int64 endpoints and a 200-frame random sweep. 0.47 s -> 0.09 s + on a 6,928 x 240 frame. +- **Mechanism 4**: the ACS record fence now uses `bytes.find`/`bytes.count` with + cached terminator cursors. Proven against the byte loop on 46,655 exhaustive + short strings across five ceiling settings, 4,500 random strings, and the real + staged `csv_pus.zip`: 3,422,890 records, identical digests, 226.9 s -> 4.6 s + (49x). Re-pins `_ACCEPTED["acs_person_coverage_authentication.py"]`. +- **Mechanism 3**: `run_graph` carries a `_SourceIdentities` cache keyed on stat + signatures, and re-derives every source in full before building the manifest. + `RunManifest.source_identities` records it without moving any existing value. + The refusal it protects had no test at all; `test_graph_executor_source_identity` + now pins it. Graph suite 108 s -> 72 s. +- **Mechanisms 1 and 2**: `survey_population_preparation.verification_epoch()`, + an opt-in scoped memo. Cheap tier every borrow (live authority, attached + payloads, producer encoding, roster stat identities); expensive tier skipped + only while a signature over every path it reads and every live buffer it + digests is unchanged; unconditional full re-validation on leaving the epoch. + The two moved inventory contracts are re-derived (below). + +**Next:** finish the epoch test run, wire the epoch into the graph entry points, +before/after probe measurement, docs that call the executor check "post-run", draft PR. +**Pins re-derived so far:** + +| pin | old | new | +|---|---|---| +| `acs_native_coverage_binding._ACCEPTED["acs_person_coverage_authentication.py"]` | `475aa795…fe85bcff` | `9ec68721…d88e8e49f` | +| `graph_implementation_inventory.json` `survey_population_preparation.py` `unbound_uses_sha256` | `29c09f6f…d296ef91` | `d114117c…dd4005910` | +| `graph_implementation_inventory.json` `asec_2024_native_population.py` `unbound_uses_sha256` | `71463df4…d7f287608` | `8e15860c…5ca2e902db` | + **Lane notes.** Root `out.md` is a tracked file holding the Amendment 19 lane's committed report; this lane's report goes to `experiments/native-verify-once/out.md`. The v5 cold run (`run_v5.py`, pid diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py index 98e9972ca..bc4ccc291 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_native_coverage_binding.py @@ -402,6 +402,31 @@ def _owned(issuance): return owned +def _verified_source_stats(issuance): + """Stat identities of every path ``_verify_sources`` re-reads, without reading.""" + owned = _owned(issuance) + return tuple( + _path_stat(paths[role]) + for paths in owned.snapshots + for role, _name, _digest, _size in owned.pins + ) + + +def _path_stat(path): + """One path's stat identity, or why it has none. Never raises.""" + try: + info = Path(path).lstat() + except OSError as error: + return ("absent", error.errno) + return ( + info.st_dev, + info.st_ino, + info.st_size, + info.st_mtime_ns, + info.st_ctime_ns, + ) + + def _verify_sources(snapshots, pins): for paths in snapshots: for role, _name, digest, size in pins: diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py index 44d97c96f..50d7f4b7c 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/acs_population_catalogue.py @@ -159,6 +159,37 @@ def _preflight(paths): return inventories +def _verified_source_stats(value): + """Stat identities of every path ``_source_checks`` re-reads, without reading. + + A caller memoising this catalogue's verification uses this as part of its + memo signature: the answer it may reuse depends on exactly these files, so + a change to any of them must move the signature and force the full check. + """ + owned = _lookup(value) + paths = dict(owned.paths) + result = [_path_stat(owned.projection_path)] + for role, name, _digest, _size in owned.pins: + result.append(_path_stat(owned.source_dir / name)) + result.append(_path_stat(paths[role])) + return tuple(result) + + +def _path_stat(path): + """One path's stat identity, or why it has none. Never raises.""" + try: + info = Path(path).lstat() + except OSError as error: + return ("absent", error.errno) + return ( + info.st_dev, + info.st_ino, + info.st_size, + info.st_mtime_ns, + info.st_ctime_ns, + ) + + def _source_checks(source_dir, paths, pins): _require(housing._pins() == pins, "SOURCE_AUTHORITY_CHANGED") for role, name, digest, size in pins: diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py index 7cad3394d..c9a6636b3 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py @@ -440,6 +440,122 @@ def _validate_state(state): ) +# --------------------------------------------------------------------------- +# Verification epoch support. The epoch itself is opened by +# ``survey_population_preparation.verification_epoch``, which imports this +# module; the state lives here so that this capsule's own borrows are memoised +# even when they are reached directly rather than through the preparation. +# Outside an epoch none of this is touched and every borrow re-authenticates in +# full, exactly as before. +# --------------------------------------------------------------------------- + +_EPOCH_DEPTH = [0] +_MEMO: dict[int, tuple] = {} + + +def _path_stat(path): + """One path's stat identity, or why it has none. Never raises.""" + try: + info = Path(path).lstat() + except OSError as error: + return ("absent", error.errno) + return ( + info.st_dev, + info.st_ino, + info.st_size, + info.st_mtime_ns, + info.st_ctime_ns, + ) + + +def _verified_source_stats(value): + """Stat identities of the seven source files, and of their directories. + + ``_file_identity`` opens ``O_NOFOLLOW`` on the final component only, so the + parent directories join the signature here: an added or removed sibling + moves a directory's own mtime and ctime, which is how a roster change + reaches a memo that never lists the roster. + """ + entry = _ISSUED.get(id(value)) + if entry is None or entry[0]() is not value: + return ("unissued",) + paths = [Path(row[0]) for row in entry[2].source_files] + directories = sorted({path.parent for path in paths}) + return ( + tuple(_path_stat(path) for path in paths), + tuple(_path_stat(directory) for directory in directories), + ) + + +def _parent_frame_witness(value): + """A read-free structural identity of this capsule's two live Frames.""" + entry = _ISSUED.get(id(value)) + if entry is None or entry[0]() is not value: + return ("unissued",) + state = entry[2] + return ( + preparation_witness(state.frame), + preparation_witness(state.parent.frame), + ) + + +def preparation_witness(frame): + """Deferred to the preparation owner, which defines the witness shape. + + Imported lazily: the preparation module imports this one, so the dependency + can only run in this direction once both modules exist. + """ + from . import survey_population_preparation as preparation + + return preparation._frame_witness(frame) + + +def _memo_signature(value, state): + borrowed = (state.parent, state.coverage, state.anchors, state.fields) + return ( + _verified_source_stats(value), + tuple(row[1] for row in state.source_files), + tuple((id(item), id(getattr(item, "payload", None))) for item in borrowed), + _parent_frame_witness(value), + ) + + +def _memoized_validate(owner, state): + """Validate ``state`` unless this epoch already validated it unchanged.""" + if not _EPOCH_DEPTH[0]: + _validate_state(state) + return + _require(_encode(_implementation()) == state.producer, "PRODUCER_CHANGED") + signature = _memo_signature(owner, state) + entry = _MEMO.get(id(owner)) + if entry is not None and entry[0]() is owner and entry[1] == signature: + return + _validate_state(state) + _MEMO[id(owner)] = (weakref.ref(owner), signature, state) + + +def _epoch_enter(): + _EPOCH_DEPTH[0] += 1 + + +def _epoch_exit(failed): + """Re-validate every memoised capsule in full, with the memo bypassed.""" + _EPOCH_DEPTH[0] -= 1 + try: + if failed: + return + for key, entry in list(_MEMO.items()): + owner = entry[0]() + if owner is None: + del _MEMO[key] + continue + _validate_state(entry[2]) + _MEMO[key] = (entry[0], _memo_signature(owner, entry[2]), entry[2]) + finally: + if not _EPOCH_DEPTH[0]: + _MEMO.clear() + + @dataclass(frozen=True, slots=True, weakref_slot=True) class AuthenticatedAsec2024NativePopulation: """Process-issued population; every accessor verifies its full live binding.""" @@ -457,7 +573,7 @@ def _checked(self): and self.payload == entry[1], "UNISSUED_OR_CHANGED", ) - _validate_state(entry[2]) + _memoized_validate(self, entry[2]) _require( _ISSUED.get(id(self)) is entry and type(self.payload) is bytes diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py index 03c83d4d7..9c5efae86 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_population_catalogue.py @@ -412,6 +412,29 @@ def _current_records_identity(state): ) +def _verified_source_stats(value): + """Stat identities of every source file this catalogue re-reads per borrow.""" + entry = _ISSUED.get(id(value)) + if entry is None or entry[0]() is not value: + return ("unissued",) + return tuple(_path_stat(row[0]) for row in entry[2].source_files) + + +def _path_stat(path): + """One path's stat identity, or why it has none. Never raises.""" + try: + info = Path(path).lstat() + except OSError as error: + return ("absent", error.errno) + return ( + info.st_dev, + info.st_ino, + info.st_size, + info.st_mtime_ns, + info.st_ctime_ns, + ) + + def _validate_state(state): _require(_source_authority() == state.source_authority, "SOURCE_AUTHORITY_CHANGED") _require(_encode(_implementation()) == state.producer, "PRODUCER_CHANGED") diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json index 72bebd7d7..2fbf137c3 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json @@ -208,7 +208,7 @@ "pandas" ], "resource_accesses_sha256": "466e46803aecf67189f31031cdb4eb9287af37669e5fa0b36610788d62564e46", - "unbound_uses_sha256": "71463df4f645cd2a98f537a212187b4d1e52bffcc4d2317defe6968d7f287608" + "unbound_uses_sha256": "8e15860cf67c6b069bb9f76ca042ce5d15a6b357583dc7847f4fae5ca2e902db" }, "microcosm.build/us_runtime/asec_checkpoint.py": { "imports": [ @@ -947,7 +947,7 @@ "pandas" ], "resource_accesses_sha256": "ccfed1c1acff5a1c50b538424dc129930f69c374d9841e233b3fb593a440c3aa", - "unbound_uses_sha256": "29c09f6fcb25ce8bc6111dd0dc76f3501d9a6b635929e94b80835889d296ef91" + "unbound_uses_sha256": "d114117ca19e9b866982147e5497d601ed91ca603d0568730d4051bdd4005910" }, "microcosm.frame/__init__.py": { "imports": [ diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py index 72428c0d1..f7a3536b9 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py @@ -999,6 +999,237 @@ def _origins(frame, sources, selected, native_receipts): } +# --------------------------------------------------------------------------- +# Verification epochs: validate once per run, and again in full at the end. +# +# Outside an epoch nothing below is reached and every capsule behaves exactly as +# it did before: one complete validation per accessor use. Inside an epoch each +# borrow still pays a cheap tier in full -- the live authority, the attached +# owner payloads, the producer encoding and the whole source roster's stat +# identities -- and the expensive tier is skipped only while a signature over +# every file those checks read, and every live buffer they digest, is identical +# to the signature recorded when that tier last ran in full. A signature that +# moved is a memo miss, not a refusal: the complete validation runs and raises +# whatever it would have raised. Leaving the epoch re-validates every capsule it +# memoised, in full, with the memo bypassed. +# --------------------------------------------------------------------------- + +_EPOCHS = [] +_MEMO = {} +_EPOCH_RECORD = None + + +def _stat_or_absent(path): + """One path's stat identity, or why it has none. Never raises.""" + try: + return _stat_identity(Path(path).lstat()) + except OSError as error: + return ("absent", error.errno) + + +def _array_witness(values): + """A read-free identity of one column's live storage. + + Buffer address, shape, strides, dtype and the writeable flag move whenever + a column is replaced, reindexed, retyped or rebound. An extension array + that exposes no numpy buffer contributes its object identity, which pandas + keeps stable for the arrays that have one. + """ + + data = getattr(values, "_data", None) + mask = getattr(values, "_mask", None) + if isinstance(data, np.ndarray) and isinstance(mask, np.ndarray): + return ("masked", _buffer_witness(data), _buffer_witness(mask)) + array = getattr(values, "_ndarray", None) + if isinstance(array, np.ndarray): + return ("array", _buffer_witness(array)) + if isinstance(values, np.ndarray): + return ("raw", _buffer_witness(values)) + return ("opaque", type(values).__name__, id(values), len(values)) + + +def _buffer_witness(array): + return ( + array.ctypes.data, + array.shape, + array.strides, + array.dtype.str, + array.flags.writeable, + ) + + +def _frame_witness(frame): + """A read-free structural identity of one Frame's live storage.""" + + if not isinstance(frame, Frame): + return ("not-a-frame", type(frame).__name__) + tables = [] + for entity in sorted(frame._tables): + table = frame._tables[entity] + columns = tuple(str(column) for column in table.columns) + tables.append( + ( + entity, + id(table), + len(table), + columns, + tuple(str(table[column].dtype) for column in table.columns), + tuple(_array_witness(table[column].array) for column in table.columns), + _array_witness(table.index), + id(table.index), + ) + ) + weights = tuple( + ( + entity, + frame._weights[entity].kind.value, + _array_witness(frame._weights[entity].values), + ) + for entity in sorted(frame._weights) + ) + strata = frame.strata + return ( + tuple(tables), + weights, + ( + id(strata), + len(strata), + str(strata.dtype), + strata.name, + _array_witness(strata.array), + _array_witness(strata.index), + ), + tuple(sorted(frame._link_tables)), + ) + + +def _verified_source_stats(state): + """Stat identities of every path the memoised tier re-reads, per owner.""" + + return ( + _file_stats(state.root), + acs_catalogue._verified_source_stats(state.catalogues[0]), + asec_catalogue._verified_source_stats(state.catalogues[1]), + acs_native._verified_source_stats(state.native[0]), + asec_native._verified_source_stats(state.native[1]), + ) + + +def _memo_signature(state): + """Everything cheap the memoised tier's answer can depend on.""" + + borrowed = (*state.catalogues, *state.native) + return ( + state.files, + _verified_source_stats(state), + tuple((id(value.payload), len(value.payload)) for value in borrowed), + tuple(id(value) for value in borrowed), + _frame_witness(state.frame), + tuple(_frame_witness(frame) for frame in state.source_frames), + asec_native._parent_frame_witness(state.native[1]), + ( + id(state.plan), + len(state.plan.selected), + len(state.plan.excluded), + len(state.plan.cells), + ), + ) + + +def _cheap_checks(state): + """The tier every borrow pays whether or not the expensive tier is skipped.""" + + _require( + _live() == _LIVE and _authority() == state.authority, "FINAL_AUTHORITY_CHANGED" + ) + _require(_attached(state) == state.attached, "ATTACHED_EVIDENCE_CHANGED") + _require(_encode(_producer()) == state.producer, "PRODUCER_CHANGED") + _require(_file_stats(state.root) == state.file_stats, "SOURCE_STAT_CHANGED") + + +def _memoized_validate(owner, state): + """Validate ``state`` unless this epoch already validated it unchanged.""" + + if not _EPOCHS: + _validate(state) + return + _cheap_checks(state) + signature = _memo_signature(state) + entry = _MEMO.get(id(owner)) + if entry is not None and entry[0]() is owner and entry[1] == signature: + _EPOCH_RECORD["hits"] += 1 + return + _validate(state) + _MEMO[id(owner)] = (weakref.ref(owner), signature, state) + _EPOCH_RECORD["misses"] += 1 + + +def _finalize_epoch(): + """Re-validate every memoised capsule in full, with the memo bypassed.""" + + for key, entry in list(_MEMO.items()): + owner = entry[0]() + if owner is None: + del _MEMO[key] + continue + _validate(entry[2]) + _EPOCH_RECORD["final_validations"] += 1 + _MEMO[key] = (entry[0], _memo_signature(entry[2]), entry[2]) + + +@contextmanager +def verification_epoch(): + """Validate each native capsule once per run, and again in full at the end. + + The memo is opt-in and scoped: nothing outside this context manager changes + behaviour, which is why every existing borrow, refusal and trace-based + mutation test keeps its exact meaning. Leaving the epoch re-validates every + capsule it memoised in full; a capsule whose source or live storage moved + while a memo was warm therefore still makes the run refuse, before the + caller receives anything. + + The record it yields is this epoch's own receipt: how many borrows were + answered from the memo, how many re-ran the complete validation, and how + many unconditional final re-validations closed it. + """ + + global _EPOCH_RECORD + record = { + "protocol": PROTOCOL + "/verification-epoch/1", + "hits": 0, + "misses": 0, + "final_validations": 0, + "capsules": 0, + } + outer = _EPOCH_RECORD + _EPOCH_RECORD = record + _EPOCHS.append(record) + asec_native._epoch_enter() + failed = False + try: + yield record + except BaseException: + failed = True + raise + finally: + _EPOCHS.pop() + try: + if not failed: + record["capsules"] = len(_MEMO) + _finalize_epoch() + finally: + if not _EPOCHS: + _MEMO.clear() + _EPOCH_RECORD = outer + asec_native._epoch_exit(failed) + + +def epoch_record(): + """The innermost open epoch's record, or None outside an epoch.""" + + return _EPOCH_RECORD if _EPOCHS else None + + @dataclass(frozen=True) class _State: root: Path @@ -1146,7 +1377,7 @@ def _checked(self): and self.payload == entry[1], "UNISSUED_OR_CHANGED", ) - _validate(entry[2]) + _memoized_validate(self, entry[2]) _require( _ISSUED.get(id(self)) is entry and type(self.payload) is bytes @@ -1217,7 +1448,7 @@ def verify_materialized_survey_population(preparation, frame): ) entry = preparation._checked() _require(_frame_identity(frame) == entry[2].identity, "MATERIALIZED_FRAME_CHANGED") - _validate(entry[2]) + _memoized_validate(preparation, entry[2]) _require(_frame_identity(frame) == entry[2].identity, "MATERIALIZED_FRAME_CHANGED") _require(preparation.payload == entry[1], "UNISSUED_OR_CHANGED") diff --git a/packages/microcosm-build/tests/test_us_native_verify_once_epoch.py b/packages/microcosm-build/tests/test_us_native_verify_once_epoch.py new file mode 100644 index 000000000..b80fbe867 --- /dev/null +++ b/packages/microcosm-build/tests/test_us_native_verify_once_epoch.py @@ -0,0 +1,314 @@ +"""Verification epochs: validated once, re-used, and re-proved in full at the end. + +The two native capsules re-ran their whole authentication on every borrow -- +both source catalogues, the ACS native coverage binding, the nested ASEC native +population, the ten-file source roster and every pure seal -- which is what made +a native build spend most of its time re-reading source it had already read. + +Inside a ``verification_epoch`` an unchanged capsule is validated once and the +expensive tier is then skipped, while the cheap tier (live authority, attached +owner payloads, the producer encoding and the whole roster's stat identities) +still runs on every borrow. Every test here asserts both halves: that the work +really is skipped, and that the refusal still fires -- either at the borrow, or +at the epoch's unconditional final re-validation. + +Outside an epoch nothing changes, which every other test in this suite already +proves by continuing to pass; the first test here pins it directly. + +Invented fixture only: no staged source tree, no engine. +""" + +from __future__ import annotations + +import os +import sys + +import pytest +from test_us_survey_population_preparation import fixture + +from microcosm.build.us_runtime import asec_2024_native_population as native +from microcosm.build.us_runtime import survey_population_preparation as owner + + +class _Counter: + """Count returns from chosen code objects without rebinding a producer. + + The runtime authenticates its own live callables, so wrapping one refuses + with PRODUCER_CHANGED. A profile hook observes the same calls and changes + nothing the seals can see. + """ + + def __init__(self, *functions): + self._codes = {function.__code__: function.__name__ for function in functions} + self.counts: dict[str, int] = {name: 0 for name in self._codes.values()} + self._previous = None + + def _trace(self, frame, event, arg): + if event == "return": + name = self._codes.get(frame.f_code) + if name is not None: + self.counts[name] += 1 + + def __enter__(self): + self._previous = sys.getprofile() + sys.setprofile(self._trace) + return self + + def __exit__(self, *exception): + sys.setprofile(self._previous) + return False + + +def _borrow(preparation): + return preparation.checked_view() + + +# -------------------------------------------------------------------------- +# Outside an epoch: nothing moves. +# -------------------------------------------------------------------------- + + +def test_without_an_epoch_every_borrow_re_reads_every_source(tmp_path, monkeypatch): + arguments = fixture(tmp_path, monkeypatch) + preparation = owner.prepare_authenticated_survey_population(**arguments) + assert owner.epoch_record() is None + with _Counter(owner._source_files, owner._producer) as counter: + for _ in range(3): + _borrow(preparation) + assert counter.counts["_source_files"] == 3 + + +# -------------------------------------------------------------------------- +# Inside an epoch: validated once, re-used, re-proved at the end. +# -------------------------------------------------------------------------- + + +def test_an_epoch_validates_once_and_reuses_it(tmp_path, monkeypatch): + arguments = fixture(tmp_path, monkeypatch) + preparation = owner.prepare_authenticated_survey_population(**arguments) + with _Counter(owner._source_files, owner._producer) as counter: + with owner.verification_epoch() as record: + for _ in range(6): + _borrow(preparation) + borrows = dict(counter.counts) + closed = dict(counter.counts) + # One full validation for the first borrow; the other five reuse it. + assert borrows["_source_files"] == 1 + assert record["hits"] == 5 + assert record["misses"] == 1 + # The producer encoding is never memoised: it runs on every borrow. + assert borrows["_producer"] >= 6 + # Leaving the epoch re-validated the capsule in full, unconditionally. + assert closed["_source_files"] == 2 + assert record["final_validations"] == 1 + assert record["capsules"] == 1 + + +def test_the_epoch_state_is_gone_afterwards(tmp_path, monkeypatch): + arguments = fixture(tmp_path, monkeypatch) + preparation = owner.prepare_authenticated_survey_population(**arguments) + with owner.verification_epoch(): + _borrow(preparation) + assert owner._MEMO == {} + assert owner._EPOCHS == [] + assert native._MEMO == {} + assert native._EPOCH_DEPTH[0] == 0 + with _Counter(owner._source_files) as counter: + _borrow(preparation) + assert counter.counts["_source_files"] == 1 + + +# -------------------------------------------------------------------------- +# Mutation: every on-disk change still refuses at the borrow that follows it. +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "name", + [ + "selection-request.json", + "acs/csv_pus.zip", + "asec/pppub25.csv", + ], +) +def test_a_source_appended_to_mid_epoch_refuses_at_the_next_borrow( + tmp_path, monkeypatch, name +): + arguments = fixture(tmp_path, monkeypatch) + preparation = owner.prepare_authenticated_survey_population(**arguments) + with owner.verification_epoch(): + _borrow(preparation) + path = arguments["source_dir"] / name + path.write_bytes(path.read_bytes() + b" ") + with pytest.raises(owner.SurveyPopulationPreparationError): + _borrow(preparation) + + +@pytest.mark.parametrize("name", ["selection-request.json", "asec/pppub25.csv"]) +def test_a_source_truncated_mid_epoch_refuses_at_the_next_borrow( + tmp_path, monkeypatch, name +): + arguments = fixture(tmp_path, monkeypatch) + preparation = owner.prepare_authenticated_survey_population(**arguments) + with owner.verification_epoch(): + _borrow(preparation) + path = arguments["source_dir"] / name + path.write_bytes(path.read_bytes()[:-1]) + with pytest.raises(owner.SurveyPopulationPreparationError): + _borrow(preparation) + + +def test_a_source_rewritten_in_place_mid_epoch_refuses_at_the_next_borrow( + tmp_path, monkeypatch +): + """Same length, and the modification time put back: ctime still moves.""" + + arguments = fixture(tmp_path, monkeypatch) + preparation = owner.prepare_authenticated_survey_population(**arguments) + with owner.verification_epoch(): + _borrow(preparation) + path = arguments["source_dir"] / "selection-request.json" + before = path.stat() + raw = bytearray(path.read_bytes()) + raw[-1] = raw[-1] ^ 0x20 + path.write_bytes(bytes(raw)) + os.utime(path, ns=(before.st_atime_ns, before.st_mtime_ns)) + assert path.stat().st_size == before.st_size + assert path.stat().st_mtime_ns == before.st_mtime_ns + with pytest.raises(owner.SurveyPopulationPreparationError): + _borrow(preparation) + + +def test_a_file_added_to_a_source_directory_mid_epoch_refuses(tmp_path, monkeypatch): + arguments = fixture(tmp_path, monkeypatch) + preparation = owner.prepare_authenticated_survey_population(**arguments) + with owner.verification_epoch(): + _borrow(preparation) + (arguments["source_dir"] / "asec" / "extra.txt").write_text("invented") + with pytest.raises(owner.SurveyPopulationPreparationError): + _borrow(preparation) + + +def test_a_touched_but_unchanged_source_re_runs_the_full_validation( + tmp_path, monkeypatch +): + """A signature miss is a re-read, not a refusal.""" + + arguments = fixture(tmp_path, monkeypatch) + preparation = owner.prepare_authenticated_survey_population(**arguments) + with _Counter(owner._source_files) as counter: + with owner.verification_epoch() as record: + _borrow(preparation) + os.utime(arguments["source_dir"] / "selection-request.json", None) + _borrow(preparation) + borrows = dict(counter.counts) + assert borrows["_source_files"] == 2 + assert record["misses"] == 2 + assert record["hits"] == 0 + + +# -------------------------------------------------------------------------- +# The end of the epoch is what closes everything a signature cannot decide. +# -------------------------------------------------------------------------- + + +def test_a_change_a_signature_cannot_see_refuses_when_the_epoch_closes( + tmp_path, monkeypatch +): + """The selection plan's rows are frozen values behind a stable identity. + + The signature records the plan's identity and its three row counts, so a + value written into an existing row with object.__setattr__ is invisible to + it -- the case the unconditional final re-validation exists for. + """ + + from fractions import Fraction + + arguments = fixture(tmp_path, monkeypatch) + preparation = owner.prepare_authenticated_survey_population(**arguments) + state = owner._ISSUED[id(preparation)][2] + with pytest.raises(owner.SurveyPopulationPreparationError, match="PLAN_CHANGED"): + with owner.verification_epoch(): + _borrow(preparation) + object.__setattr__(state.plan.selected[0], "share", Fraction(99)) + _borrow(preparation) # memo hit: the signature did not move + + +def test_a_failing_body_propagates_and_finalizes_nothing(tmp_path, monkeypatch): + arguments = fixture(tmp_path, monkeypatch) + preparation = owner.prepare_authenticated_survey_population(**arguments) + + class _SentinelError(Exception): + pass + + with _Counter(owner._source_files) as counter: + with pytest.raises(_SentinelError): + with owner.verification_epoch(): + _borrow(preparation) + raise _SentinelError + assert counter.counts["_source_files"] == 1 + assert owner._MEMO == {} + assert native._MEMO == {} + + +def test_nested_epochs_finalize_at_every_level(tmp_path, monkeypatch): + arguments = fixture(tmp_path, monkeypatch) + preparation = owner.prepare_authenticated_survey_population(**arguments) + with _Counter(owner._source_files) as counter: + with owner.verification_epoch() as outer: + _borrow(preparation) + with owner.verification_epoch() as inner: + _borrow(preparation) + inner_close = dict(counter.counts) + outer_close = dict(counter.counts) + assert inner["hits"] == 1 + assert inner_close["_source_files"] == 2 # first borrow, then inner's finalize + assert outer_close["_source_files"] == 3 # plus outer's finalize + assert outer["final_validations"] == 1 + assert owner._MEMO == {} + + +# -------------------------------------------------------------------------- +# The ASEC native capsule on its own. +# -------------------------------------------------------------------------- + + +def test_the_native_capsule_validates_once_per_epoch(tmp_path, monkeypatch): + from test_us_asec_2024_native_population import _fixture as asec_fixture + + paths = asec_fixture(tmp_path, monkeypatch) + capsule = native.load_authenticated_asec_2024_native_population(**paths) + with _Counter(native._file_identity) as counter: + for _ in range(3): + capsule.validate() + cold = counter.counts["_file_identity"] + with native_epoch(): + for _ in range(3): + capsule.validate() + warm = counter.counts["_file_identity"] - cold + closed = counter.counts["_file_identity"] - cold - warm + files = cold // 3 + assert files > 0 + assert warm == files # one full pass, then two reuses + assert closed == files # the epoch's unconditional final re-validation + + +def test_the_native_capsule_still_refuses_a_changed_source_inside_an_epoch( + tmp_path, monkeypatch +): + from test_us_asec_2024_native_population import _fixture as asec_fixture + + paths = asec_fixture(tmp_path, monkeypatch) + capsule = native.load_authenticated_asec_2024_native_population(**paths) + with native_epoch(): + capsule.validate() + with paths["parent_path"].open("ab") as handle: + handle.write(b"changed") + with pytest.raises(native.AsecNativePopulationError, match="SOURCE_FILE"): + capsule.validate() + + +def native_epoch(): + """The epoch, opened through the preparation owner that drives both memos.""" + + return owner.verification_epoch() From 3bff62f8575f0b2118fe04266cb1d891c30a363b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 16 Sep 2026 00:16:15 -0400 Subject: [PATCH 07/45] Open a verification epoch around each atomic native run run_atomic_survey_population and run_atomic_survey_financial now wrap their whole body in survey_population_preparation.verification_epoch(). Every native capsule the run borrows is authenticated in full once, reused while its sources and live storage are provably unchanged, and re-authenticated in full before the run returns. The financial run's prefix opens a nested epoch; each closes with its own full re-validation. Both commits are an indentation plus one 'with' line. Proven mechanically: the statement lists of both function bodies parse to identical ASTs before and after (50 statements and 102 statements), the argument lists are unchanged, the docstrings stay outside the wrapper, and both files were ruff-format clean at the base so every hunk lies inside the wrapped function. Keeping the body in its own function matters: a build test asserts caller.f_code is runner.run_atomic_survey_population.__code__. The epoch's exit closes the nested capsule's epoch first, so this owner's final validation reaches the nested population's complete file check rather than its memo, and collects both refusals so neither can be lost to the other in a finally block. A foreign owner's error raised by a deferred borrow is translated exactly as _checked translates it, instead of escaping raw. test_us_native_verify_once_epoch: 17 tests. Validated once and reused (one _source_files pass for six borrows, five memo hits); the producer encoding never memoised; every on-disk mutation -- append, truncate, same-length rewrite with mtime restored, a file added to a source directory, a touch -- refusing at the borrow that follows it AND again when the epoch declines to close over it; a signature miss outside the roster re-running without refusing; a change no signature can see refusing at the close; a failing body finalizing nothing; and nested epochs finalizing at every level. Also adopts the change in docs/validation-cost-and-borrow-boundaries.md, which recorded this optimization as proposed rather than adopted, and corrects the codecs docstring that called the executor's check post-run. Co-Authored-By: Claude Opus --- .gitignore | 3 + docs/validation-cost-and-borrow-boundaries.md | 4 +- experiments/native-verify-once/README.md | 11 + .../us_runtime/asec_2024_native_population.py | 14 +- .../graph_atomic_survey_financial.py | 1224 +++++++++-------- .../graph_atomic_survey_population.py | 358 ++--- .../survey_population_preparation.py | 32 +- .../tests/test_us_native_verify_once_epoch.py | 133 +- .../src/microcosm/graph/codecs.py | 6 +- 9 files changed, 973 insertions(+), 812 deletions(-) create mode 100644 experiments/native-verify-once/README.md diff --git a/.gitignore b/.gitignore index 1e77cefd2..a83de8598 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,6 @@ tools/release.env # Finder metadata — never payload .DS_Store + +# Lane-local before/after measurement staging (see docs/us-native-verification-once.md) +.measure/ diff --git a/docs/validation-cost-and-borrow-boundaries.md b/docs/validation-cost-and-borrow-boundaries.md index f88c2cf4c..0bd28f100 100644 --- a/docs/validation-cost-and-borrow-boundaries.md +++ b/docs/validation-cost-and-borrow-boundaries.md @@ -24,7 +24,9 @@ Three different facts currently appear together in nested validation: - Agreement with files: source bytes, archive snapshots and stored graph artifacts still match their recorded identities. - Retained-object identity: the actual Frame, Population, weights and ancestry still match their issued state. -An independent Fable 5.1 review recommended making these checks explicit and performing expensive file verification primarily at public entry and emission boundaries. That is a proposed contract change, not an adopted optimization. Moving file checks would change when an altered archive is detected. It requires a documented public-entry contract and tests at every relevant callback, file borrow and emission boundary before implementation. +An independent Fable 5.1 review recommended making these checks explicit and performing expensive file verification primarily at public entry and emission boundaries. Moving file checks changes when an altered archive is detected, so it needed a documented entry contract and tests at the boundaries before implementation. + +**Adopted 2026-09-16, as a scoped epoch rather than a boundary rule.** `survey_population_preparation.verification_epoch()` opens a verification epoch around a graph run. Outside one, nothing changes: every borrow re-authenticates in full. Inside one, each borrow still pays the cheap tier — loaded-code identity through `_encode(_producer())`, the attached owner payloads, the live authority, and `_file_stats` over the whole source roster — and the expensive file and retained-object tier is skipped only while a signature is unchanged: the stat identity of every path the four foreign verifications and `_source_files` re-read, the identity and length of every borrowed payload, and a read-free witness of each live Frame's storage. A signature that moved is a memo miss, not a refusal. Leaving the epoch, at every nesting level, re-validates every memoised capsule in full with the memo bypassed, so a change no signature can see still refuses before the run returns anything. The contract, the residuals it accepts and the argument for each are in [verifying native sources once per run](us-native-verification-once.md); the boundary tests are `packages/microcosm-build/tests/test_us_native_verify_once_epoch.py`. Similarly, an AST cache would hold mutable objects, unlike compiled code. It needs its own ownership design and evidence; the compilation-cache result does not establish its safety or benefit. diff --git a/experiments/native-verify-once/README.md b/experiments/native-verify-once/README.md new file mode 100644 index 000000000..af5d15f92 --- /dev/null +++ b/experiments/native-verify-once/README.md @@ -0,0 +1,11 @@ +# Verify-once lane artifacts + +Receipts and reproduction scripts for the change described in +[docs/us-native-verification-once.md](../../docs/us-native-verification-once.md). + +| file | what | +|---|---| +| `record_fence_real_archive.py` | Runs the pre-change ACS record fence and the shipped one over every applicable member of a staged `csv_pus.zip`, read-only, and compares a rolling digest of every yielded record. | +| `record-fence-parity.json` | That comparison's receipt for the 2026-09-15 staged archive. | + +Nothing here is a build, a certification or a release artifact. diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py index c9a6636b3..01ccbd74d 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py @@ -549,7 +549,19 @@ def _epoch_exit(failed): if owner is None: del _MEMO[key] continue - _validate_state(entry[2]) + try: + _validate_state(entry[2]) + except AsecNativePopulationError: + raise + except ( + OSError, + ValueError, + TypeError, + KeyError, + AttributeError, + OverflowError, + ): + raise AsecNativePopulationError("NATIVE_BINDING_REFUSAL") from None _MEMO[key] = (entry[0], _memo_signature(owner, entry[2]), entry[2]) finally: if not _EPOCH_DEPTH[0]: diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py index d0beb224a..e3637dea4 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_financial.py @@ -1291,654 +1291,684 @@ def run_atomic_survey_financial( return_values=False, ): """Verify the base financial graph and its explicitly selected extension.""" - require(type(return_values) is bool, "RETURN_VALUES_FLAG") - require(type(person_status) is bool, "PERSON_STATUS_FLAG") - require(type(rebase_property_taxes) is bool, "PROPERTY_TAX_FLAG") - require( - not rebase_property_taxes or property_income is not None, - "PROPERTY_TAX_REQUIRES_PROPERTY", - ) - require(type(household_roles) is bool, "HOUSEHOLD_ROLES_FLAG") - require(not household_roles or child_property is not None, "ROLES_REQUIRE_CHILD") - require( - child_property is None or property_income is not None, "CHILD_REQUIRES_PROPERTY" - ) - require( - child_property is None or rebase_property_taxes, "CHILD_REQUIRES_TAX_REBASE" - ) - if child_property is not None: - extension = _completion_module() - extension.validate_options(child_property) - base = run_atomic_survey_financial( + # One verification epoch for the whole run; the prefix run below opens + # a nested one, and each closes with a full re-authentication. + with survey._source_owner().verification_epoch(): + require(type(return_values) is bool, "RETURN_VALUES_FLAG") + require(type(person_status) is bool, "PERSON_STATUS_FLAG") + require(type(rebase_property_taxes) is bool, "PROPERTY_TAX_FLAG") + require( + not rebase_property_taxes or property_income is not None, + "PROPERTY_TAX_REQUIRES_PROPERTY", + ) + require(type(household_roles) is bool, "HOUSEHOLD_ROLES_FLAG") + require( + not household_roles or child_property is not None, "ROLES_REQUIRE_CHILD" + ) + require( + child_property is None or property_income is not None, + "CHILD_REQUIRES_PROPERTY", + ) + require( + child_property is None or rebase_property_taxes, "CHILD_REQUIRES_TAX_REBASE" + ) + if child_property is not None: + extension = _completion_module() + extension.validate_options(child_property) + base = run_atomic_survey_financial( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed=seed, + geography_config=geography_config, + demographic_conditioning=demographic_conditioning, + n_estimators=n_estimators, + property_income=property_income, + person_status=person_status, + rebase_property_taxes=False, + resume=resume, + return_values=True, + ) + result = extension.extend( + base, + options=child_property, + household_roles=household_roles, + resume=resume, + ) + return result if return_values else result.manifest + values.feature_columns(demographic_conditioning) + property_graph = None if property_income is None else _property_module() + if property_graph is not None: + require( + type(property_income) is property_graph.PropertyIncomeOptions, + "PROPERTY_OPTIONS_TYPE", + ) + property_income_bytes = property_income.to_bytes() + else: + property_income_bytes = None + live = _live(property_income, rebase_property_taxes, person_status) + config_bytes = values.host.survey_budget._config_payload(geography_config) + require(config_bytes is not None, "ATOMIC_GEOGRAPHY_REQUIRED") + prefix = atomic.run_atomic_survey_population( source_dir, snapshot_root=snapshot_root, store_root=store_root, fraction=fraction, seed=seed, geography_config=geography_config, - demographic_conditioning=demographic_conditioning, - n_estimators=n_estimators, - property_income=property_income, - person_status=person_status, - rebase_property_taxes=False, resume=resume, return_values=True, ) - result = extension.extend( - base, options=child_property, household_roles=household_roles, resume=resume + entry = prefix.preparation._checked() + prefix_objects = ( + prefix.preparation, + prefix.manifest, + prefix.compiled, + prefix.store, + prefix.kernels, + prefix.sources, ) - return result if return_values else result.manifest - values.feature_columns(demographic_conditioning) - property_graph = None if property_income is None else _property_module() - if property_graph is not None: - require( - type(property_income) is property_graph.PropertyIncomeOptions, - "PROPERTY_OPTIONS_TYPE", - ) - property_income_bytes = property_income.to_bytes() - else: - property_income_bytes = None - live = _live(property_income, rebase_property_taxes, person_status) - config_bytes = values.host.survey_budget._config_payload(geography_config) - require(config_bytes is not None, "ATOMIC_GEOGRAPHY_REQUIRED") - prefix = atomic.run_atomic_survey_population( - source_dir, - snapshot_root=snapshot_root, - store_root=store_root, - fraction=fraction, - seed=seed, - geography_config=geography_config, - resume=resume, - return_values=True, - ) - entry = prefix.preparation._checked() - prefix_objects = ( - prefix.preparation, - prefix.manifest, - prefix.compiled, - prefix.store, - prefix.kernels, - prefix.sources, - ) - prefix_manifest_bytes = prefix.manifest.to_json_bytes() - prefix_declaration = graph_to_json(prefix.compiled.graph) - retained = { - name: ( - getattr(prefix, name), - reconstruction._population_stamp(getattr(prefix, name)), - ) - for name in ( - "allocated_population", - "observed_population", - "expanded_population", - "geography_population", - "clone_population", - ) - } - qualified = values.qualify_current_survey_predictors( - prefix.preparation, - prefix.allocated_population, - prefix.clone_population, - demographic_conditioning=demographic_conditioning, - geography_config=geography_config, - ) - projection_bytes, matrix_bytes = qualified.projection, qualified.matrix - property_qualified = ( - None - if property_graph is None - else property_graph.sources.qualify_current_property_income_sources( + prefix_manifest_bytes = prefix.manifest.to_json_bytes() + prefix_declaration = graph_to_json(prefix.compiled.graph) + retained = { + name: ( + getattr(prefix, name), + reconstruction._population_stamp(getattr(prefix, name)), + ) + for name in ( + "allocated_population", + "observed_population", + "expanded_population", + "geography_population", + "clone_population", + ) + } + qualified = values.qualify_current_survey_predictors( prefix.preparation, prefix.allocated_population, prefix.clone_population, demographic_conditioning=demographic_conditioning, geography_config=geography_config, ) - ) - geography = reconstruction.reconstruct_atomic_survey_geography( - prefix.preparation, prefix.allocated_population, geography_config - ) - pins, prefix_artifacts = {}, {} - for node_id, record in prefix.manifest.nodes.items(): - for name, key in record.opaque_artifacts.items(): - prefix_artifacts[node_id, name] = prefix.store.load_bytes(key) - for edge in ( - *financial.host.current_survey_host_edges(), - financial._geography_edge(), - ): - record = prefix.manifest.node(edge.producer) - pins[edge.name] = { - "producer_key": record.key, - "artifact_key": record.opaque_artifacts[edge.artifact], - "payload_sha256": codec.sha(prefix_artifacts[edge.producer, edge.artifact]), - } - status_graph = _person_status_module() if person_status else None - status_boundary, status_nodes = None, () - if status_graph is not None: - status_qualified = status_graph.qualify_current_survey_person_status( - prefix.preparation + projection_bytes, matrix_bytes = qualified.projection, qualified.matrix + property_qualified = ( + None + if property_graph is None + else property_graph.sources.qualify_current_property_income_sources( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + demographic_conditioning=demographic_conditioning, + geography_config=geography_config, + ) + ) + geography = reconstruction.reconstruct_atomic_survey_geography( + prefix.preparation, prefix.allocated_population, geography_config ) - status_edges = ( + pins, prefix_artifacts = {}, {} + for node_id, record in prefix.manifest.nodes.items(): + for name, key in record.opaque_artifacts.items(): + prefix_artifacts[node_id, name] = prefix.store.load_bytes(key) + for edge in ( *financial.host.current_survey_host_edges(), financial._geography_edge(), - ) - status_nodes = status_graph.current_survey_person_status_nodes( - status_qualified, + ): + record = prefix.manifest.node(edge.producer) + pins[edge.name] = { + "producer_key": record.key, + "artifact_key": record.opaque_artifacts[edge.artifact], + "payload_sha256": codec.sha( + prefix_artifacts[edge.producer, edge.artifact] + ), + } + status_graph = _person_status_module() if person_status else None + status_boundary, status_nodes = None, () + if status_graph is not None: + status_qualified = status_graph.qualify_current_survey_person_status( + prefix.preparation + ) + status_edges = ( + *financial.host.current_survey_host_edges(), + financial._geography_edge(), + ) + status_nodes = status_graph.current_survey_person_status_nodes( + status_qualified, + prefix.clone_population.frame, + receiving_version=prefix.clone_population.version, + after=financial._geography_edge(), + host_edges=status_edges, + host_pins=pins, + ) + status_boundary = _PersonStatusBoundary( + prefix.preparation, + entry, + prefix.clone_population, + status_qualified, + status_nodes, + ) + nodes = financial.current_survey_predictor_nodes( + qualified, prefix.clone_population.frame, - receiving_version=prefix.clone_population.version, - after=financial._geography_edge(), - host_edges=status_edges, host_pins=pins, + n_estimators=n_estimators, ) - status_boundary = _PersonStatusBoundary( - prefix.preparation, - entry, - prefix.clone_population, - status_qualified, - status_nodes, - ) - nodes = financial.current_survey_predictor_nodes( - qualified, - prefix.clone_population.frame, - host_pins=pins, - n_estimators=n_estimators, - ) - original_donor = next(n for n in nodes if n.id == financial.DONOR_NODE) - if status_graph is not None: - nodes = tuple( - replace( - node, - artifact_inputs=(*node.artifact_inputs, status_graph.binding_edge()), + original_donor = next(n for n in nodes if n.id == financial.DONOR_NODE) + if status_graph is not None: + nodes = tuple( + replace( + node, + artifact_inputs=( + *node.artifact_inputs, + status_graph.binding_edge(), + ), + ) + if node.id == financial.DONOR_NODE + else node + for node in nodes + ) + property_nodes = ( + () + if property_graph is None + else property_graph.current_survey_property_nodes( + property_qualified, + prefix.clone_population.frame, + host_pins=pins, + options=property_income, ) - if node.id == financial.DONOR_NODE - else node - for node in nodes - ) - property_nodes = ( - () - if property_graph is None - else property_graph.current_survey_property_nodes( - property_qualified, - prefix.clone_population.frame, - host_pins=pins, - options=property_income, ) - ) - tax_nodes = ( - () - if not rebase_property_taxes - else _tax_nodes( - prefix.clone_population.frame, - prefix.clone_population.version, - property_income, - anticipated_outputs=( - *(() if status_graph is None else status_nodes[-1].outputs), - *next(n.outputs for n in nodes if n.id == financial.ATTACH_NODE), - *property_graph.owned_columns(), - ), + tax_nodes = ( + () + if not rebase_property_taxes + else _tax_nodes( + prefix.clone_population.frame, + prefix.clone_population.version, + property_income, + anticipated_outputs=( + *(() if status_graph is None else status_nodes[-1].outputs), + *next(n.outputs for n in nodes if n.id == financial.ATTACH_NODE), + *property_graph.owned_columns(), + ), + ) ) - ) - compiled = compile_graph( - replace( - prefix.compiled.graph, - nodes=( - *prefix.compiled.graph.nodes, - *status_nodes, - *nodes, - *property_nodes, - *tax_nodes, - ), + compiled = compile_graph( + replace( + prefix.compiled.graph, + nodes=( + *prefix.compiled.graph.nodes, + *status_nodes, + *nodes, + *property_nodes, + *tax_nodes, + ), + ) ) - ) - require( - len(prefix.compiled.order) == 9 - and len(property_nodes) == (0 if property_graph is None else 16) - and len(tax_nodes) == (3 if rebase_property_taxes else 0) - and len(status_nodes) == (4 if person_status else 0) - and len(compiled.order) - == 19 + len(status_nodes) + len(property_nodes) + len(tax_nodes), - "ATOMIC_COMPILER_ROSTER", - ) - gate_edge = financial._geography_edge() - require( - gate_edge.producer in compiled.predecessors[financial.DONOR_NODE] - and prefix_artifacts[gate_edge.producer, gate_edge.artifact] - == qualified.geography_validation, - "ATOMIC_GEOGRAPHY_GATE_EDGE", - ) - if status_graph is not None: require( - status_graph.BIND_NODE in compiled.predecessors[financial.DONOR_NODE] - and gate_edge.producer in compiled.predecessors[status_graph.SOURCE_NODE], - "PERSON_STATUS_ORDERING", - ) - declaration = graph_to_json(compiled.graph) - kernels, store, sources = KernelRegistry(), prefix.store, dict(prefix.sources) - for kernel in prefix.kernels.as_mapping().values(): - kernels.register(kernel) - source_items = tuple(sorted(sources.items())) - for cls in ( - financial.CurrentSurveyPredictorProjectionKernel, - financial.CurrentSurveyPredictorDonorFilterKernel, - financial.CurrentSurveyPredictorDonorColumnsKernel, - financial.CurrentSurveyPredictorAttachKernel, - ): - kernel = cls( - prefix.preparation, - prefix.allocated_population, - prefix.clone_population, - host_pins=pins, - n_estimators=n_estimators, - demographic_conditioning=demographic_conditioning, - geography_config=geography_config, + len(prefix.compiled.order) == 9 + and len(property_nodes) == (0 if property_graph is None else 16) + and len(tax_nodes) == (3 if rebase_property_taxes else 0) + and len(status_nodes) == (4 if person_status else 0) + and len(compiled.order) + == 19 + len(status_nodes) + len(property_nodes) + len(tax_nodes), + "ATOMIC_COMPILER_ROSTER", + ) + gate_edge = financial._geography_edge() + require( + gate_edge.producer in compiled.predecessors[financial.DONOR_NODE] + and prefix_artifacts[gate_edge.producer, gate_edge.artifact] + == qualified.geography_validation, + "ATOMIC_GEOGRAPHY_GATE_EDGE", ) - if ( - status_graph is not None - and cls is financial.CurrentSurveyPredictorDonorFilterKernel + if status_graph is not None: + require( + status_graph.BIND_NODE in compiled.predecessors[financial.DONOR_NODE] + and gate_edge.producer + in compiled.predecessors[status_graph.SOURCE_NODE], + "PERSON_STATUS_ORDERING", + ) + declaration = graph_to_json(compiled.graph) + kernels, store, sources = KernelRegistry(), prefix.store, dict(prefix.sources) + for kernel in prefix.kernels.as_mapping().values(): + kernels.register(kernel) + source_items = tuple(sorted(sources.items())) + for cls in ( + financial.CurrentSurveyPredictorProjectionKernel, + financial.CurrentSurveyPredictorDonorFilterKernel, + financial.CurrentSurveyPredictorDonorColumnsKernel, + financial.CurrentSurveyPredictorAttachKernel, ): - kernel = _PersonStatusOrderedDonor( - kernel, - status_boundary, - original_donor, - compiled.graph.node(financial.DONOR_NODE), + kernel = cls( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + host_pins=pins, + n_estimators=n_estimators, + demographic_conditioning=demographic_conditioning, + geography_config=geography_config, ) - kernels.register(kernel) - if status_graph is not None: + if ( + status_graph is not None + and cls is financial.CurrentSurveyPredictorDonorFilterKernel + ): + kernel = _PersonStatusOrderedDonor( + kernel, + status_boundary, + original_donor, + compiled.graph.node(financial.DONOR_NODE), + ) + kernels.register(kernel) + if status_graph is not None: + + def require_status_current(): + status_boundary.pure() + + declared, status_kernels = status_graph.person_status_kernels( + status_qualified, + prefix.clone_population.frame, + receiving_version=prefix.clone_population.version, + after=financial._geography_edge(), + host_edges=status_edges, + host_pins=pins, + require_current=require_status_current, + ) + require(declared == status_nodes, "PERSON_STATUS_DECLARATIONS_CHANGED") + for kernel in status_kernels: + kernels.register(kernel) + kernels.register(LegacyQRFTrainKernel()) + kernels.register(LegacyQRFApplyMatrixKernel()) + if property_graph is not None: + property_graph.register_property_kernels( + kernels, + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + host_pins=pins, + options=property_income, + demographic_conditioning=demographic_conditioning, + geography_config=geography_config, + ) + if rebase_property_taxes: + tax = _tax_module() + for cls in ( + tax.PropertyTaxReceivingKernel, + tax.PropertyTaxLeavesKernel, + tax.PropertyTaxLeafGateKernel, + ): + kernels.register(cls()) + _, source_keys = _source_paths_and_keys(compiled, sources, store) + keys, implementations = _all_node_keys(compiled, kernels, source_keys) + base_expected = { + survey.CREATE_NODE: Population.from_frame( + entry[2].frame, survey.CREATE_NODE + ), + survey.ALLOCATION_NODE: prefix.allocated_population, + **{s.node.id: s.population for s in geography.stages}, + } + receipts = { + **{n: r.receipt for n, r in prefix.manifest.nodes.items()}, + **{s.node.id: json.loads(s.receipt) for s in geography.stages}, + } + donor_node = compiled.graph.node(financial.DONOR_NODE) + donor = population_ops.patch( + base_expected[survey.CREATE_NODE], + donor_node, + KernelResult(frame=qualified.donor_frame), + ) + receipts[donor_node.id] = { + "selection": "ASEC_native_whole_households", + "fit_weight_kind": "design", + "release_eligible": False, + } + columns_node = compiled.graph.node(financial.DONOR_COLUMNS_NODE) + donor_columns = population_ops.patch( + donor, + columns_node, + KernelResult( + columns={ + ("person", c): qualified.donor_columns[c] + for c in ( + *values.feature_columns(demographic_conditioning), + *values.TARGETS, + ) + } + ), + ) + receipts[financial.PROJECTION_NODE] = qualified.evidence + receipts[columns_node.id] = qualified.evidence + observed, observed_stamps = {}, {} - def require_status_current(): - status_boundary.pure() + def observe(node_id, population): + require(node_id not in observed, "ATOMIC_OBSERVER_DUPLICATE") + observed[node_id] = population + observed_stamps[node_id] = reconstruction._population_stamp(population) - declared, status_kernels = status_graph.person_status_kernels( - status_qualified, - prefix.clone_population.frame, - receiving_version=prefix.clone_population.version, - after=financial._geography_edge(), - host_edges=status_edges, - host_pins=pins, - require_current=require_status_current, + manifest = run_graph( + compiled, + sources=sources, + store=store, + kernels=kernels, + resume=resume, + _population_observer=observe, ) - require(declared == status_nodes, "PERSON_STATUS_DECLARATIONS_CHANGED") - for kernel in status_kernels: - kernels.register(kernel) - kernels.register(LegacyQRFTrainKernel()) - kernels.register(LegacyQRFApplyMatrixKernel()) - if property_graph is not None: - property_graph.register_property_kernels( - kernels, - prefix.preparation, - prefix.allocated_population, - prefix.clone_population, - host_pins=pins, - options=property_income, + require(tuple(observed) == compiled.order, "ATOMIC_OBSERVER_ROSTER") + loaded = _artifacts(manifest, compiled, store, kernels, keys, implementations) + require( + all(loaded[k] == v for k, v in prefix_artifacts.items()), + "ATOMIC_PREFIX_ARTIFACTS", + ) + require( + loaded[financial.PROJECTION_NODE, "projection"] == projection_bytes + and loaded[financial.PROJECTION_NODE, "matrix"] == matrix_bytes, + "ATOMIC_SOURCE_ARTIFACTS", + ) + raw = tuple( + loaded[f"{financial.APPLY_PREFIX}.{i:03d}", "raw_draw"] for i in range(3) + ) + applications = tuple( + loaded[f"{financial.APPLY_PREFIX}.{i:03d}", "apply_state"] for i in range(3) + ) + matrix_key = keys[financial.PROJECTION_NODE] + drawn = financial.read_current_survey_draws( + matrix_bytes, + matrix_key, + raw, + applications, demographic_conditioning=demographic_conditioning, - geography_config=geography_config, ) - if rebase_property_taxes: - tax = _tax_module() - for cls in ( - tax.PropertyTaxReceivingKernel, - tax.PropertyTaxLeavesKernel, - tax.PropertyTaxLeafGateKernel, - ): - kernels.register(cls()) - _, source_keys = _source_paths_and_keys(compiled, sources, store) - keys, implementations = _all_node_keys(compiled, kernels, source_keys) - base_expected = { - survey.CREATE_NODE: Population.from_frame(entry[2].frame, survey.CREATE_NODE), - survey.ALLOCATION_NODE: prefix.allocated_population, - **{s.node.id: s.population for s in geography.stages}, - } - receipts = { - **{n: r.receipt for n, r in prefix.manifest.nodes.items()}, - **{s.node.id: json.loads(s.receipt) for s in geography.stages}, - } - donor_node = compiled.graph.node(financial.DONOR_NODE) - donor = population_ops.patch( - base_expected[survey.CREATE_NODE], - donor_node, - KernelResult(frame=qualified.donor_frame), - ) - receipts[donor_node.id] = { - "selection": "ASEC_native_whole_households", - "fit_weight_kind": "design", - "release_eligible": False, - } - columns_node = compiled.graph.node(financial.DONOR_COLUMNS_NODE) - donor_columns = population_ops.patch( - donor, - columns_node, - KernelResult( - columns={ - ("person", c): qualified.donor_columns[c] - for c in ( - *values.feature_columns(demographic_conditioning), - *values.TARGETS, - ) - } - ), - ) - receipts[financial.PROJECTION_NODE] = qualified.evidence - receipts[columns_node.id] = qualified.evidence - observed, observed_stamps = {}, {} - - def observe(node_id, population): - require(node_id not in observed, "ATOMIC_OBSERVER_DUPLICATE") - observed[node_id] = population - observed_stamps[node_id] = reconstruction._population_stamp(population) - - manifest = run_graph( - compiled, - sources=sources, - store=store, - kernels=kernels, - resume=resume, - _population_observer=observe, - ) - require(tuple(observed) == compiled.order, "ATOMIC_OBSERVER_ROSTER") - loaded = _artifacts(manifest, compiled, store, kernels, keys, implementations) - require( - all(loaded[k] == v for k, v in prefix_artifacts.items()), - "ATOMIC_PREFIX_ARTIFACTS", - ) - require( - loaded[financial.PROJECTION_NODE, "projection"] == projection_bytes - and loaded[financial.PROJECTION_NODE, "matrix"] == matrix_bytes, - "ATOMIC_SOURCE_ARTIFACTS", - ) - raw = tuple( - loaded[f"{financial.APPLY_PREFIX}.{i:03d}", "raw_draw"] for i in range(3) - ) - applications = tuple( - loaded[f"{financial.APPLY_PREFIX}.{i:03d}", "apply_state"] for i in range(3) - ) - matrix_key = keys[financial.PROJECTION_NODE] - drawn = financial.read_current_survey_draws( - matrix_bytes, - matrix_key, - raw, - applications, - demographic_conditioning=demographic_conditioning, - ) - receipts.update( - _model_receipts(nodes, donor_columns, qualified, loaded, matrix_key) - ) - receipts[financial.ATTACH_NODE] = { - **qualified.evidence, - "raw_sha256": [codec.sha(r) for r in raw], - "all_output_cells_available": True, - "ACS_financial_origin": "modeled", - "paired_draws": "source_origin_join", - "host_weights_changed": False, - } - property_results = ( - {} - if property_graph is None - else property_graph.reconstruct_property_results( - property_qualified, - prefix.clone_population.frame, - host_pins=pins, - options=property_income, - artifacts=loaded, - legacy_matrix_producer_key=matrix_key, + receipts.update( + _model_receipts(nodes, donor_columns, qualified, loaded, matrix_key) + ) + receipts[financial.ATTACH_NODE] = { + **qualified.evidence, + "raw_sha256": [codec.sha(r) for r in raw], + "all_output_cells_available": True, + "ACS_financial_origin": "modeled", + "paired_draws": "source_origin_join", + "host_weights_changed": False, + } + property_results = ( + {} + if property_graph is None + else property_graph.reconstruct_property_results( + property_qualified, + prefix.clone_population.frame, + host_pins=pins, + options=property_income, + artifacts=loaded, + legacy_matrix_producer_key=matrix_key, + ) ) - ) - receipts.update({name: result.receipt for name, result in property_results.items()}) - expected, current, tax_populations = {}, {}, {} - for node_id in compiled.order: - node, version = compiled.graph.node(node_id), compiled.versions[node_id] - if status_graph is not None and node_id in status_graph.NODE_IDS: - artifacts = { - edge.name: ArtifactValue( - loaded[edge.producer, edge.artifact], - edge.type, - opaque_artifact_key(keys[edge.producer], edge.artifact), - keys[edge.producer], - numeric_scope( - kernels.get( - compiled.graph.node(edge.producer).kernel - ).capabilities + receipts.update( + {name: result.receipt for name, result in property_results.items()} + ) + expected, current, tax_populations = {}, {}, {} + for node_id in compiled.order: + node, version = compiled.graph.node(node_id), compiled.versions[node_id] + if status_graph is not None and node_id in status_graph.NODE_IDS: + artifacts = { + edge.name: ArtifactValue( + loaded[edge.producer, edge.artifact], + edge.type, + opaque_artifact_key(keys[edge.producer], edge.artifact), + keys[edge.producer], + numeric_scope( + kernels.get( + compiled.graph.node(edge.producer).kernel + ).capabilities + ), + ) + for edge in node.artifact_inputs + } + population, status_result = ( + status_graph.expected_person_status_population( + None + if node.structural is StructuralDelta.CREATE + else current[version], + qualified=status_qualified, + node=node, + artifacts=artifacts, + ) + ) + require( + all( + loaded[node_id, name] == payload + for name, payload in status_result.artifacts.items() ), + "PERSON_STATUS_OUTPUT_ARTIFACT", ) - for edge in node.artifact_inputs - } - population, status_result = status_graph.expected_person_status_population( - None if node.structural is StructuralDelta.CREATE else current[version], - qualified=status_qualified, - node=node, - artifacts=artifacts, - ) - require( - all( - loaded[node_id, name] == payload - for name, payload in status_result.artifacts.items() - ), - "PERSON_STATUS_OUTPUT_ARTIFACT", - ) - receipts[node_id] = dict(status_result.receipt) - elif rebase_property_taxes and node_id == _tax_module().RECEIVING_NODE: - tax_populations, tax_results = _reconstruct_tax( - current[node.base], - compiled=compiled, - kernels=kernels, - keys=keys, - loaded=loaded, - options=property_income, - ) - receipts.update( - {name: result.receipt for name, result in tax_results.items()} - ) - population = tax_populations[node_id] - elif node_id in tax_populations: - population = tax_populations[node_id] - elif node_id in property_results: - incumbent = ( - version if node.structural is StructuralDelta.NONE else node.base - ) - property_result = property_results[node_id] - if node.structural is StructuralDelta.FILTER: - frame = current[incumbent].frame - entity = frame.schema.person_entity - id_column = frame.schema.entity_id_column(entity) - ids = pd.Index(frame.table(entity)[id_column], name=id_column) - mask = property_result.keep.reindex(ids).to_numpy( - dtype=np.bool_, copy=True + receipts[node_id] = dict(status_result.receipt) + elif rebase_property_taxes and node_id == _tax_module().RECEIVING_NODE: + tax_populations, tax_results = _reconstruct_tax( + current[node.base], + compiled=compiled, + kernels=kernels, + keys=keys, + loaded=loaded, + options=property_income, + ) + receipts.update( + {name: result.receipt for name, result in tax_results.items()} ) - property_result = replace( - property_result, frame=frame.select(mask), keep=None + population = tax_populations[node_id] + elif node_id in tax_populations: + population = tax_populations[node_id] + elif node_id in property_results: + incumbent = ( + version if node.structural is StructuralDelta.NONE else node.base ) - population = population_ops.patch(current[incumbent], node, property_result) - elif node_id == donor_node.id: - population = donor - elif node_id == columns_node.id: - population = donor_columns - elif node_id == financial.ATTACH_NODE: - population = population_ops.patch( - current[version], - node, - KernelResult( - columns=values.complete_predictor_columns( - qualified, prefix.clone_population.frame, drawn + property_result = property_results[node_id] + if node.structural is StructuralDelta.FILTER: + frame = current[incumbent].frame + entity = frame.schema.person_entity + id_column = frame.schema.entity_id_column(entity) + ids = pd.Index(frame.table(entity)[id_column], name=id_column) + mask = property_result.keep.reindex(ids).to_numpy( + dtype=np.bool_, copy=True ) - ), - ) - elif node_id in base_expected: - population = base_expected[node_id] - else: - population = current[version] - expected[node_id] = current[version] = population - atomic.same_replayed_population(population, observed[node_id]) - if property_graph is not None: - from .graph_property_income_receipts import verify_property_model_receipts + property_result = replace( + property_result, frame=frame.select(mask), keep=None + ) + population = population_ops.patch( + current[incumbent], node, property_result + ) + elif node_id == donor_node.id: + population = donor + elif node_id == columns_node.id: + population = donor_columns + elif node_id == financial.ATTACH_NODE: + population = population_ops.patch( + current[version], + node, + KernelResult( + columns=values.complete_predictor_columns( + qualified, prefix.clone_population.frame, drawn + ) + ), + ) + elif node_id in base_expected: + population = base_expected[node_id] + else: + population = current[version] + expected[node_id] = current[version] = population + atomic.same_replayed_population(population, observed[node_id]) + if property_graph is not None: + from .graph_property_income_receipts import verify_property_model_receipts - receipts.update( - verify_property_model_receipts( - property_nodes, - expected[property_graph.DONOR_COLUMNS_NODE], - expected[property_graph.RECIPIENT_COLUMNS_NODE], - loaded, + receipts.update( + verify_property_model_receipts( + property_nodes, + expected[property_graph.DONOR_COLUMNS_NODE], + expected[property_graph.RECIPIENT_COLUMNS_NODE], + loaded, + ) ) + expected_stamps = { + n: reconstruction._population_stamp(p) for n, p in expected.items() + } + states = atomic._states(compiled, kernels, source_keys, expected, receipts) + survey._check_node_states(manifest, states) + final_node = ( + financial.ATTACH_NODE + if property_graph is None + else property_graph.ATTACH_NODE + ) + property_population = observed[final_node] if rebase_property_taxes else None + if rebase_property_taxes: + final_node = _tax_module().GATE_NODE + legacy_population = observed[financial.ATTACH_NODE] + result = AtomicSurveyFinancialRunValues( + prefix, + observed[final_node], + manifest, + compiled, + store, + kernels, + sources, + projection_bytes, + matrix_bytes, ) - expected_stamps = { - n: reconstruction._population_stamp(p) for n, p in expected.items() - } - states = atomic._states(compiled, kernels, source_keys, expected, receipts) - survey._check_node_states(manifest, states) - final_node = ( - financial.ATTACH_NODE if property_graph is None else property_graph.ATTACH_NODE - ) - property_population = observed[final_node] if rebase_property_taxes else None - if rebase_property_taxes: - final_node = _tax_module().GATE_NODE - legacy_population = observed[financial.ATTACH_NODE] - result = AtomicSurveyFinancialRunValues( - prefix, - observed[final_node], - manifest, - compiled, - store, - kernels, - sources, - projection_bytes, - matrix_bytes, - ) - # Implementation/source/store I/O precedes the last owner/support borrow. - current_keys, current_implementations = _all_node_keys( - compiled, kernels, source_keys - ) - require( - current_keys == keys and current_implementations == implementations, - "ATOMIC_FINAL_IMPLEMENTATIONS", - ) - financial.verify_materialized_current_survey_predictors( - prefix.preparation, - prefix.allocated_population, - prefix.clone_population, - population=_status_complement(status_boundary, legacy_population), - projection=projection_bytes, - matrix=matrix_bytes, - matrix_producer_key=matrix_key, - raw_draws=raw, - apply_states=applications, - host_pins=pins, - n_estimators=n_estimators, - demographic_conditioning=demographic_conditioning, - geography_config=geography_config, - ) - if property_graph is not None: - property_graph.verify_materialized_property_income( + # Implementation/source/store I/O precedes the last owner/support borrow. + current_keys, current_implementations = _all_node_keys( + compiled, kernels, source_keys + ) + require( + current_keys == keys and current_implementations == implementations, + "ATOMIC_FINAL_IMPLEMENTATIONS", + ) + financial.verify_materialized_current_survey_predictors( prefix.preparation, prefix.allocated_population, prefix.clone_population, - legacy_population=_status_complement(status_boundary, legacy_population), - population=_status_complement( - status_boundary, - ( - property_population - if rebase_property_taxes - else result.financial_population - ), - ), + population=_status_complement(status_boundary, legacy_population), + projection=projection_bytes, + matrix=matrix_bytes, + matrix_producer_key=matrix_key, + raw_draws=raw, + apply_states=applications, host_pins=pins, - options=property_income, - artifacts=loaded, - legacy_matrix_producer_key=matrix_key, + n_estimators=n_estimators, demographic_conditioning=demographic_conditioning, geography_config=geography_config, ) - require( - property_income.to_bytes() == property_income_bytes, - "PROPERTY_OPTIONS_CHANGED", - ) - if status_boundary is not None: - status_boundary.requalify() - values.source._pure_final(entry[2]) - require( - values.source._ISSUED.get(id(prefix.preparation)) is entry - and prefix.preparation.payload == entry[1] - and prefix.geography_config is geography_config - and values.host.survey_budget._config_payload(geography_config) == config_bytes - and result.prefix is prefix - and result.manifest is manifest - and result.compiled is compiled - and result.store is store - and result.kernels is kernels - and result.financial_population is observed[final_node] - and result.projection == projection_bytes - and result.matrix == matrix_bytes - and all( - a is b - for a, b in zip( - ( - prefix.preparation, - prefix.manifest, - prefix.compiled, - prefix.store, - prefix.kernels, - prefix.sources, + if property_graph is not None: + property_graph.verify_materialized_property_income( + prefix.preparation, + prefix.allocated_population, + prefix.clone_population, + legacy_population=_status_complement( + status_boundary, legacy_population ), - prefix_objects, - strict=True, + population=_status_complement( + status_boundary, + ( + property_population + if rebase_property_taxes + else result.financial_population + ), + ), + host_pins=pins, + options=property_income, + artifacts=loaded, + legacy_matrix_producer_key=matrix_key, + demographic_conditioning=demographic_conditioning, + geography_config=geography_config, ) - ) - and prefix.manifest.to_json_bytes() == prefix_manifest_bytes - and graph_to_json(prefix.compiled.graph) == prefix_declaration - and prefix.compiled == compile_graph(prefix.compiled.graph) - and tuple(sorted(result.sources.items())) == source_items - and tuple(sorted(prefix.sources.items())) == source_items - and graph_to_json(compiled.graph) == declaration - and compiled == compile_graph(compiled.graph) - and _live(property_income, rebase_property_taxes, person_status) == live, - "ATOMIC_FINAL_BINDINGS", - ) - for name, (population, stamp) in retained.items(): - require( - getattr(prefix, name) is population - and reconstruction._population_stamp(population) == stamp, - "ATOMIC_FINAL_PREFIX_MUTATION", - ) - for version, population in ( - (survey.CREATE_NODE, base_expected[survey.CREATE_NODE]), - (survey.ALLOCATION_NODE, prefix.observed_population), - (atomic.clone.COMBINED_CLONE_NODE, prefix.clone_population), - ): - survey._same_frame(population.frame, prefix.manifest.population(version)) - require( - population.mass_ledger == prefix.manifest.mass_ledger(version), - "ATOMIC_FINAL_PREFIX_LEDGER", - ) - for node_id, population in expected.items(): - require( - reconstruction._population_stamp(population) == expected_stamps[node_id] - and reconstruction._population_stamp(observed[node_id]) - == observed_stamps[node_id], - "ATOMIC_FINAL_POPULATION_MUTATION", - ) - atomic.same_replayed_population(population, observed[node_id]) - for version, population in current.items(): - survey._same_frame(population.frame, manifest.population(version)) + require( + property_income.to_bytes() == property_income_bytes, + "PROPERTY_OPTIONS_CHANGED", + ) + if status_boundary is not None: + status_boundary.requalify() + values.source._pure_final(entry[2]) require( - population.mass_ledger == manifest.mass_ledger(version), - "ATOMIC_FINAL_LEDGER", - ) - survey._check_node_states(manifest, states) - _issue_run( - result, - preparation_entry=entry, - pins=pins, - n_estimators=n_estimators, - demographic_conditioning=demographic_conditioning, - source_keys=source_keys, - keys=keys, - implementations=implementations, - loaded=loaded, - live=live, - property_income=property_income, - legacy_financial_population=None - if property_graph is None - else legacy_population, - rebase_property_taxes=rebase_property_taxes, - property_population=property_population, - person_status_boundary=status_boundary, - node_populations=observed, - node_states=states, - ) - return result if return_values else manifest + values.source._ISSUED.get(id(prefix.preparation)) is entry + and prefix.preparation.payload == entry[1] + and prefix.geography_config is geography_config + and values.host.survey_budget._config_payload(geography_config) + == config_bytes + and result.prefix is prefix + and result.manifest is manifest + and result.compiled is compiled + and result.store is store + and result.kernels is kernels + and result.financial_population is observed[final_node] + and result.projection == projection_bytes + and result.matrix == matrix_bytes + and all( + a is b + for a, b in zip( + ( + prefix.preparation, + prefix.manifest, + prefix.compiled, + prefix.store, + prefix.kernels, + prefix.sources, + ), + prefix_objects, + strict=True, + ) + ) + and prefix.manifest.to_json_bytes() == prefix_manifest_bytes + and graph_to_json(prefix.compiled.graph) == prefix_declaration + and prefix.compiled == compile_graph(prefix.compiled.graph) + and tuple(sorted(result.sources.items())) == source_items + and tuple(sorted(prefix.sources.items())) == source_items + and graph_to_json(compiled.graph) == declaration + and compiled == compile_graph(compiled.graph) + and _live(property_income, rebase_property_taxes, person_status) == live, + "ATOMIC_FINAL_BINDINGS", + ) + for name, (population, stamp) in retained.items(): + require( + getattr(prefix, name) is population + and reconstruction._population_stamp(population) == stamp, + "ATOMIC_FINAL_PREFIX_MUTATION", + ) + for version, population in ( + (survey.CREATE_NODE, base_expected[survey.CREATE_NODE]), + (survey.ALLOCATION_NODE, prefix.observed_population), + (atomic.clone.COMBINED_CLONE_NODE, prefix.clone_population), + ): + survey._same_frame(population.frame, prefix.manifest.population(version)) + require( + population.mass_ledger == prefix.manifest.mass_ledger(version), + "ATOMIC_FINAL_PREFIX_LEDGER", + ) + for node_id, population in expected.items(): + require( + reconstruction._population_stamp(population) == expected_stamps[node_id] + and reconstruction._population_stamp(observed[node_id]) + == observed_stamps[node_id], + "ATOMIC_FINAL_POPULATION_MUTATION", + ) + atomic.same_replayed_population(population, observed[node_id]) + for version, population in current.items(): + survey._same_frame(population.frame, manifest.population(version)) + require( + population.mass_ledger == manifest.mass_ledger(version), + "ATOMIC_FINAL_LEDGER", + ) + survey._check_node_states(manifest, states) + _issue_run( + result, + preparation_entry=entry, + pins=pins, + n_estimators=n_estimators, + demographic_conditioning=demographic_conditioning, + source_keys=source_keys, + keys=keys, + implementations=implementations, + loaded=loaded, + live=live, + property_income=property_income, + legacy_financial_population=None + if property_graph is None + else legacy_population, + rebase_property_taxes=rebase_property_taxes, + property_population=property_population, + person_status_boundary=status_boundary, + node_populations=observed, + node_states=states, + ) + return result if return_values else manifest diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py index 5bd732768..ee7c4481e 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_atomic_survey_population.py @@ -146,189 +146,197 @@ def run_atomic_survey_population( return_values=False, ): """Assign each completed source/clone household and independently verify replay.""" - survey._require(type(return_values) is bool, "RETURN_VALUES_FLAG") - prefix = survey.run_authenticated_survey_population( - source_dir, - snapshot_root=snapshot_root, - store_root=store_root, - fraction=fraction, - seed=seed, - resume=resume, - clones=False, - return_values=True, - ) - source_owner = survey._source_owner() - preparation_entry = prefix.preparation._checked() - geography = reconstruction.reconstruct_atomic_survey_geography( - prefix.preparation, prefix.allocated_population, geography_config - ) - _, view = survey._checked_preparation(prefix.preparation) - instructions = survey.allocation_instructions( - view.selection_plan, view.receipt["origins"]["households"] - ) - _, allocated_context, allocation_payload, _, _ = survey._allocation_output( - view.frame, view.context, instructions, survey._sha(view.payload) - ) - prefix_artifacts = { - (survey.CREATE_NODE, "preparation"): view.payload, - (survey.CREATE_NODE, "frame_context"): view.context, - (survey.ALLOCATION_NODE, "allocation"): allocation_payload, - (survey.ALLOCATION_NODE, "frame_context"): allocated_context, - } - additions = geography.nodes - compiled = compile_graph( - Graph( - "us", - ( - *prefix.compiled.graph.sources, - *(SourceRef(name, "raw-bytes-v1") for name, _ in geography.sources), - ), - (*prefix.compiled.graph.nodes, *additions), + # One verification epoch for the whole run: every native capsule this + # run borrows is authenticated in full once, reused while its sources + # and live storage are provably unchanged, and re-authenticated in + # full when the epoch closes -- before anything is returned. + with survey._source_owner().verification_epoch(): + survey._require(type(return_values) is bool, "RETURN_VALUES_FLAG") + prefix = survey.run_authenticated_survey_population( + source_dir, + snapshot_root=snapshot_root, + store_root=store_root, + fraction=fraction, + seed=seed, + resume=resume, + clones=False, + return_values=True, + ) + source_owner = survey._source_owner() + preparation_entry = prefix.preparation._checked() + geography = reconstruction.reconstruct_atomic_survey_geography( + prefix.preparation, prefix.allocated_population, geography_config + ) + _, view = survey._checked_preparation(prefix.preparation) + instructions = survey.allocation_instructions( + view.selection_plan, view.receipt["origins"]["households"] ) - ) - survey._require(len(compiled.order) == 9, "ATOMIC_COMPILER_ROSTER") - sources = {**prefix.sources, **dict(geography.sources)} - kernels, store = prefix.kernels, prefix.store - store.codecs.register_bytes("raw-bytes-v1", load_raw_bytes) - kernels.register(projection.CurrentSurveyGeographyKernel(prefix.preparation)) - register_atomic_geography_kernels(kernels) - clone.register_us_combined_survey_clone_kernels(kernels) - expected = { - survey.CREATE_NODE: Population.from_frame( - prefix.manifest.population(survey.CREATE_NODE), survey.CREATE_NODE - ), - survey.ALLOCATION_NODE: prefix.allocated_population, - **{stage.node.id: stage.population for stage in geography.stages}, - } - receipts = { - **{name: record.receipt for name, record in prefix.manifest.nodes.items()}, - **{stage.node.id: json.loads(stage.receipt) for stage in geography.stages}, - } - expected_stamps = { - name: reconstruction._population_stamp(value) - for name, value in expected.items() - } - raw_stamp = reconstruction._population_stamp(prefix.allocated_population) - source_items = tuple(sorted(sources.items())) - config_bytes = geography_config.to_bytes() - _, source_keys = _source_paths_and_keys(compiled, sources, store) - states = _states(compiled, kernels, source_keys, expected, receipts) - observed, observed_stamps = {}, {} + _, allocated_context, allocation_payload, _, _ = survey._allocation_output( + view.frame, view.context, instructions, survey._sha(view.payload) + ) + prefix_artifacts = { + (survey.CREATE_NODE, "preparation"): view.payload, + (survey.CREATE_NODE, "frame_context"): view.context, + (survey.ALLOCATION_NODE, "allocation"): allocation_payload, + (survey.ALLOCATION_NODE, "frame_context"): allocated_context, + } + additions = geography.nodes + compiled = compile_graph( + Graph( + "us", + ( + *prefix.compiled.graph.sources, + *(SourceRef(name, "raw-bytes-v1") for name, _ in geography.sources), + ), + (*prefix.compiled.graph.nodes, *additions), + ) + ) + survey._require(len(compiled.order) == 9, "ATOMIC_COMPILER_ROSTER") + sources = {**prefix.sources, **dict(geography.sources)} + kernels, store = prefix.kernels, prefix.store + store.codecs.register_bytes("raw-bytes-v1", load_raw_bytes) + kernels.register(projection.CurrentSurveyGeographyKernel(prefix.preparation)) + register_atomic_geography_kernels(kernels) + clone.register_us_combined_survey_clone_kernels(kernels) + expected = { + survey.CREATE_NODE: Population.from_frame( + prefix.manifest.population(survey.CREATE_NODE), survey.CREATE_NODE + ), + survey.ALLOCATION_NODE: prefix.allocated_population, + **{stage.node.id: stage.population for stage in geography.stages}, + } + receipts = { + **{name: record.receipt for name, record in prefix.manifest.nodes.items()}, + **{stage.node.id: json.loads(stage.receipt) for stage in geography.stages}, + } + expected_stamps = { + name: reconstruction._population_stamp(value) + for name, value in expected.items() + } + raw_stamp = reconstruction._population_stamp(prefix.allocated_population) + source_items = tuple(sorted(sources.items())) + config_bytes = geography_config.to_bytes() + _, source_keys = _source_paths_and_keys(compiled, sources, store) + states = _states(compiled, kernels, source_keys, expected, receipts) + observed, observed_stamps = {}, {} - def observe(node_id, population): - survey._require(node_id not in observed, "DUPLICATE_POPULATION_OBSERVATION") - same_replayed_population(expected[node_id], population) - observed[node_id] = population - observed_stamps[node_id] = reconstruction._population_stamp(population) + def observe(node_id, population): + survey._require(node_id not in observed, "DUPLICATE_POPULATION_OBSERVATION") + same_replayed_population(expected[node_id], population) + observed[node_id] = population + observed_stamps[node_id] = reconstruction._population_stamp(population) - manifest = run_graph( - compiled, - sources=sources, - store=store, - kernels=kernels, - resume=resume, - _population_observer=observe, - ) - survey._require(tuple(observed) == compiled.order, "POPULATION_OBSERVER_COVERAGE") - survey._check_node_states(manifest, states) - for stage in geography.stages: - for name, payload in stage.artifacts: - output = next(o for o in stage.node.artifact_outputs if o.name == name) - survey._final_artifact( - manifest, - store, - node_id=stage.node.id, - name=name, - type_=output.type, - payload=payload, - capabilities=kernels.get(stage.node.kernel).capabilities, - ) - # Validate prefix artifacts separately from column materialization. - for (node_id, name), payload in prefix_artifacts.items(): - key = prefix.manifest.node(node_id).opaque_artifacts[name] + manifest = run_graph( + compiled, + sources=sources, + store=store, + kernels=kernels, + resume=resume, + _population_observer=observe, + ) survey._require( - manifest.node(node_id).opaque_artifacts.get(name) == key, - "ATOMIC_PREFIX_ARTIFACT_KEY", + tuple(observed) == compiled.order, "POPULATION_OBSERVER_COVERAGE" + ) + survey._check_node_states(manifest, states) + for stage in geography.stages: + for name, payload in stage.artifacts: + output = next(o for o in stage.node.artifact_outputs if o.name == name) + survey._final_artifact( + manifest, + store, + node_id=stage.node.id, + name=name, + type_=output.type, + payload=payload, + capabilities=kernels.get(stage.node.kernel).capabilities, + ) + # Validate prefix artifacts separately from column materialization. + for (node_id, name), payload in prefix_artifacts.items(): + key = prefix.manifest.node(node_id).opaque_artifacts[name] + survey._require( + manifest.node(node_id).opaque_artifacts.get(name) == key, + "ATOMIC_PREFIX_ARTIFACT_KEY", + ) + survey._require( + store.load_bytes(key) == payload, "ATOMIC_PREFIX_ARTIFACT_BYTES" + ) + geography_terminal = geography.nodes[-1].id + result = AtomicSurveyPopulationRunValues( + manifest=manifest, + preparation=prefix.preparation, + allocated_population=prefix.allocated_population, + observed_population=observed[projection.NODE], + expanded_population=observed[clone.COMBINED_CLONE_CLAIM_NODE], + geography_population=observed[geography_terminal], + # This historical field is the terminal receiving population. The + # explicit expanded_population is the preassignment clone snapshot. + clone_population=observed[geography_terminal], + geography_config=geography_config, + compiled=compiled, + store=store, + kernels=kernels, + sources=sources, + ) + current_keys, current_implementations = _all_node_keys( + compiled, kernels, source_keys ) survey._require( - store.load_bytes(key) == payload, "ATOMIC_PREFIX_ARTIFACT_BYTES" + current_keys == {name: value["key"] for name, value in states.items()} + and current_implementations + == {name: value["implementation"] for name, value in states.items()}, + "ATOMIC_FINAL_IMPLEMENTATIONS", + ) + # Finish source/support I/O before final comparisons of returned objects. + fresh = reconstruction.reconstruct_atomic_survey_geography( + prefix.preparation, prefix.allocated_population, geography_config ) - geography_terminal = geography.nodes[-1].id - result = AtomicSurveyPopulationRunValues( - manifest=manifest, - preparation=prefix.preparation, - allocated_population=prefix.allocated_population, - observed_population=observed[projection.NODE], - expanded_population=observed[clone.COMBINED_CLONE_CLAIM_NODE], - geography_population=observed[geography_terminal], - # This historical field is the terminal receiving population. The - # explicit expanded_population is the preassignment clone snapshot. - clone_population=observed[geography_terminal], - geography_config=geography_config, - compiled=compiled, - store=store, - kernels=kernels, - sources=sources, - ) - current_keys, current_implementations = _all_node_keys( - compiled, kernels, source_keys - ) - survey._require( - current_keys == {name: value["key"] for name, value in states.items()} - and current_implementations - == {name: value["implementation"] for name, value in states.items()}, - "ATOMIC_FINAL_IMPLEMENTATIONS", - ) - # Finish source/support I/O before final comparisons of returned objects. - fresh = reconstruction.reconstruct_atomic_survey_geography( - prefix.preparation, prefix.allocated_population, geography_config - ) - survey._require( - source_owner._ISSUED.get(id(prefix.preparation)) is preparation_entry - and prefix.preparation.payload == preparation_entry[1], - "ATOMIC_FINAL_PREPARATION", - ) - source_owner._pure_final(preparation_entry[2]) - survey._require( - fresh.config_sha256 == geography.config_sha256 - and fresh.projection_receipt == geography.projection_receipt - and fresh.support_payload == geography.support_payload - and fresh.definition == geography.definition - and geography_config.to_bytes() == config_bytes - and result.geography_config is geography_config - and result.preparation is prefix.preparation - and result.allocated_population is prefix.allocated_population - and result.observed_population is observed[projection.NODE] - and result.expanded_population is observed[clone.COMBINED_CLONE_CLAIM_NODE] - and result.geography_population is observed[geography_terminal] - and result.clone_population is result.geography_population - and result.manifest is manifest - and result.compiled is compiled - and result.store is store - and result.kernels is kernels - and tuple(sorted(result.sources.items())) == source_items - and reconstruction._population_stamp(prefix.allocated_population) == raw_stamp, - "ATOMIC_FINAL_RECONSTRUCTION", - ) - same_replayed_population(fresh.observed_population, result.observed_population) - same_replayed_population(fresh.expanded_population, result.expanded_population) - same_replayed_population(fresh.population, result.geography_population) - for node_id, population in expected.items(): survey._require( - reconstruction._population_stamp(population) == expected_stamps[node_id] - and reconstruction._population_stamp(observed[node_id]) - == observed_stamps[node_id], - "ATOMIC_FINAL_POPULATION_MUTATION", + source_owner._ISSUED.get(id(prefix.preparation)) is preparation_entry + and prefix.preparation.payload == preparation_entry[1], + "ATOMIC_FINAL_PREPARATION", ) - same_replayed_population(population, observed[node_id]) - latest = {} - for node_id in compiled.order: - latest[compiled.versions[node_id]] = node_id - for version, node_id in latest.items(): - survey._same_frame(expected[node_id].frame, manifest.population(version)) + source_owner._pure_final(preparation_entry[2]) survey._require( - manifest.mass_ledger(version) == expected[node_id].mass_ledger, - "ATOMIC_FINAL_MASS_LEDGER", + fresh.config_sha256 == geography.config_sha256 + and fresh.projection_receipt == geography.projection_receipt + and fresh.support_payload == geography.support_payload + and fresh.definition == geography.definition + and geography_config.to_bytes() == config_bytes + and result.geography_config is geography_config + and result.preparation is prefix.preparation + and result.allocated_population is prefix.allocated_population + and result.observed_population is observed[projection.NODE] + and result.expanded_population is observed[clone.COMBINED_CLONE_CLAIM_NODE] + and result.geography_population is observed[geography_terminal] + and result.clone_population is result.geography_population + and result.manifest is manifest + and result.compiled is compiled + and result.store is store + and result.kernels is kernels + and tuple(sorted(result.sources.items())) == source_items + and reconstruction._population_stamp(prefix.allocated_population) + == raw_stamp, + "ATOMIC_FINAL_RECONSTRUCTION", ) - survey._check_node_states(manifest, states) - return result if return_values else manifest + same_replayed_population(fresh.observed_population, result.observed_population) + same_replayed_population(fresh.expanded_population, result.expanded_population) + same_replayed_population(fresh.population, result.geography_population) + for node_id, population in expected.items(): + survey._require( + reconstruction._population_stamp(population) == expected_stamps[node_id] + and reconstruction._population_stamp(observed[node_id]) + == observed_stamps[node_id], + "ATOMIC_FINAL_POPULATION_MUTATION", + ) + same_replayed_population(population, observed[node_id]) + latest = {} + for node_id in compiled.order: + latest[compiled.versions[node_id]] = node_id + for version, node_id in latest.items(): + survey._same_frame(expected[node_id].frame, manifest.population(version)) + survey._require( + manifest.mass_ledger(version) == expected[node_id].mass_ledger, + "ATOMIC_FINAL_MASS_LEDGER", + ) + survey._check_node_states(manifest, states) + return result if return_values else manifest diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py index f7a3536b9..401ccb01b 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/survey_population_preparation.py @@ -1165,14 +1165,26 @@ def _memoized_validate(owner, state): def _finalize_epoch(): - """Re-validate every memoised capsule in full, with the memo bypassed.""" + """Re-validate every memoised capsule in full, with the memo bypassed. + + The refusal class is the one a borrow would have raised: this is the borrow + the epoch deferred, so a foreign owner's error is translated exactly as + ``_checked`` translates it rather than escaping raw to the run. + """ for key, entry in list(_MEMO.items()): owner = entry[0]() if owner is None: del _MEMO[key] continue - _validate(entry[2]) + try: + _validate(entry[2]) + except SurveyPopulationPreparationError: + raise + except Exception: + raise SurveyPopulationPreparationError( + "PREPARATION_VERIFICATION_REFUSED" + ) from None _EPOCH_RECORD["final_validations"] += 1 _MEMO[key] = (entry[0], _memo_signature(entry[2]), entry[2]) @@ -1213,15 +1225,29 @@ def verification_epoch(): raise finally: _EPOCHS.pop() + # The nested capsule's epoch closes first, so this owner's own final + # validation reaches the nested population's complete file check rather + # than its memo. Both refusals are collected and this owner's is + # preferred, because this owner's error class is the one every borrow + # through it raises; a refusal is never dropped, only ordered. + nested = None + try: + asec_native._epoch_exit(failed) + except BaseException as error: # noqa: BLE001 - re-raised below + nested = error + own = None try: if not failed: record["capsules"] = len(_MEMO) _finalize_epoch() + except BaseException as error: # noqa: BLE001 - re-raised below + own = error finally: if not _EPOCHS: _MEMO.clear() _EPOCH_RECORD = outer - asec_native._epoch_exit(failed) + if not failed and (own is not None or nested is not None): + raise own if own is not None else nested def epoch_record(): diff --git a/packages/microcosm-build/tests/test_us_native_verify_once_epoch.py b/packages/microcosm-build/tests/test_us_native_verify_once_epoch.py index b80fbe867..ef87a428e 100644 --- a/packages/microcosm-build/tests/test_us_native_verify_once_epoch.py +++ b/packages/microcosm-build/tests/test_us_native_verify_once_epoch.py @@ -119,10 +119,32 @@ def test_the_epoch_state_is_gone_afterwards(tmp_path, monkeypatch): # -------------------------------------------------------------------------- -# Mutation: every on-disk change still refuses at the borrow that follows it. +# Mutation: every on-disk change still refuses at the borrow that follows it, +# and again when the epoch refuses to close over it. # -------------------------------------------------------------------------- +def _refuses_at_the_borrow_and_at_the_close(preparation, mutate, error): + """The borrow after a mutation refuses, and so does leaving the epoch. + + Both halves matter. The first is the guarantee the memo must not weaken: + the refusal arrives at the same borrow it arrives at today. The second is + the guarantee the memo is allowed to lean on: even a caller that swallows + the first refusal cannot leave the epoch with a changed source behind it. + """ + + borrowed = [] + with pytest.raises(error) as closing: + with owner.verification_epoch(): + _borrow(preparation) + mutate() + with pytest.raises(error) as borrow: + _borrow(preparation) + borrowed.append(str(borrow.value)) + assert borrowed and borrowed[0] + return borrowed[0], str(closing.value) + + @pytest.mark.parametrize( "name", [ @@ -136,12 +158,14 @@ def test_a_source_appended_to_mid_epoch_refuses_at_the_next_borrow( ): arguments = fixture(tmp_path, monkeypatch) preparation = owner.prepare_authenticated_survey_population(**arguments) - with owner.verification_epoch(): - _borrow(preparation) - path = arguments["source_dir"] / name + path = arguments["source_dir"] / name + + def mutate(): path.write_bytes(path.read_bytes() + b" ") - with pytest.raises(owner.SurveyPopulationPreparationError): - _borrow(preparation) + + _refuses_at_the_borrow_and_at_the_close( + preparation, mutate, owner.SurveyPopulationPreparationError + ) @pytest.mark.parametrize("name", ["selection-request.json", "asec/pppub25.csv"]) @@ -150,12 +174,14 @@ def test_a_source_truncated_mid_epoch_refuses_at_the_next_borrow( ): arguments = fixture(tmp_path, monkeypatch) preparation = owner.prepare_authenticated_survey_population(**arguments) - with owner.verification_epoch(): - _borrow(preparation) - path = arguments["source_dir"] / name + path = arguments["source_dir"] / name + + def mutate(): path.write_bytes(path.read_bytes()[:-1]) - with pytest.raises(owner.SurveyPopulationPreparationError): - _borrow(preparation) + + _refuses_at_the_borrow_and_at_the_close( + preparation, mutate, owner.SurveyPopulationPreparationError + ) def test_a_source_rewritten_in_place_mid_epoch_refuses_at_the_next_borrow( @@ -165,46 +191,81 @@ def test_a_source_rewritten_in_place_mid_epoch_refuses_at_the_next_borrow( arguments = fixture(tmp_path, monkeypatch) preparation = owner.prepare_authenticated_survey_population(**arguments) - with owner.verification_epoch(): - _borrow(preparation) - path = arguments["source_dir"] / "selection-request.json" + path = arguments["source_dir"] / "selection-request.json" + + def mutate(): before = path.stat() raw = bytearray(path.read_bytes()) raw[-1] = raw[-1] ^ 0x20 path.write_bytes(bytes(raw)) os.utime(path, ns=(before.st_atime_ns, before.st_mtime_ns)) - assert path.stat().st_size == before.st_size - assert path.stat().st_mtime_ns == before.st_mtime_ns - with pytest.raises(owner.SurveyPopulationPreparationError): - _borrow(preparation) + after = path.stat() + assert after.st_size == before.st_size + assert after.st_mtime_ns == before.st_mtime_ns + assert after.st_ino == before.st_ino + + _refuses_at_the_borrow_and_at_the_close( + preparation, mutate, owner.SurveyPopulationPreparationError + ) def test_a_file_added_to_a_source_directory_mid_epoch_refuses(tmp_path, monkeypatch): arguments = fixture(tmp_path, monkeypatch) preparation = owner.prepare_authenticated_survey_population(**arguments) - with owner.verification_epoch(): - _borrow(preparation) + + def mutate(): (arguments["source_dir"] / "asec" / "extra.txt").write_text("invented") - with pytest.raises(owner.SurveyPopulationPreparationError): - _borrow(preparation) + _refuses_at_the_borrow_and_at_the_close( + preparation, mutate, owner.SurveyPopulationPreparationError + ) + + +def test_a_touched_source_refuses_on_its_stat_identity_alone(tmp_path, monkeypatch): + """Touching a roster file is already a refusal, and stays one.""" + + arguments = fixture(tmp_path, monkeypatch) + preparation = owner.prepare_authenticated_survey_population(**arguments) -def test_a_touched_but_unchanged_source_re_runs_the_full_validation( + def mutate(): + os.utime(arguments["source_dir"] / "selection-request.json", None) + + borrow, closing = _refuses_at_the_borrow_and_at_the_close( + preparation, mutate, owner.SurveyPopulationPreparationError + ) + assert borrow == "SOURCE_STAT_CHANGED" + assert closing == "SOURCE_STAT_CHANGED" + + +def test_a_signature_miss_outside_the_roster_re_runs_without_refusing( tmp_path, monkeypatch ): - """A signature miss is a re-read, not a refusal.""" + """A private snapshot copy is in the signature but not in the roster stats. + + Touching one leaves the cheap tier's ``_file_stats`` comparison untouched + and moves the ACS catalogue's own path signature, so the borrow takes the + full validation again -- and passes, because the bytes are unchanged. That + is the shape of every memo miss: more work, never a refusal by itself. + """ arguments = fixture(tmp_path, monkeypatch) preparation = owner.prepare_authenticated_survey_population(**arguments) + state = owner._ISSUED[id(preparation)][2] + copies = [ + path for _role, path in owner.acs_catalogue._lookup(state.catalogues[0]).paths + ] + assert copies with _Counter(owner._source_files) as counter: with owner.verification_epoch() as record: _borrow(preparation) - os.utime(arguments["source_dir"] / "selection-request.json", None) + first = counter.counts["_source_files"] _borrow(preparation) - borrows = dict(counter.counts) - assert borrows["_source_files"] == 2 + assert counter.counts["_source_files"] == first # a hit + os.utime(copies[0], None) + _borrow(preparation) + assert counter.counts["_source_files"] == first + 1 # a miss, not a refusal + assert record["hits"] == 1 assert record["misses"] == 2 - assert record["hits"] == 0 # -------------------------------------------------------------------------- @@ -300,12 +361,18 @@ def test_the_native_capsule_still_refuses_a_changed_source_inside_an_epoch( paths = asec_fixture(tmp_path, monkeypatch) capsule = native.load_authenticated_asec_2024_native_population(**paths) - with native_epoch(): - capsule.validate() - with paths["parent_path"].open("ab") as handle: - handle.write(b"changed") - with pytest.raises(native.AsecNativePopulationError, match="SOURCE_FILE"): + borrowed = [] + with pytest.raises(native.AsecNativePopulationError, match="SOURCE_FILE"): + with native_epoch(): capsule.validate() + with paths["parent_path"].open("ab") as handle: + handle.write(b"changed") + with pytest.raises( + native.AsecNativePopulationError, match="SOURCE_FILE" + ) as borrow: + capsule.validate() + borrowed.append(str(borrow.value)) + assert borrowed == ["SOURCE_FILE_CHANGED"] def native_epoch(): diff --git a/packages/microcosm-graph/src/microcosm/graph/codecs.py b/packages/microcosm-graph/src/microcosm/graph/codecs.py index 3ebc9d8e9..882633e89 100644 --- a/packages/microcosm-graph/src/microcosm/graph/codecs.py +++ b/packages/microcosm-graph/src/microcosm/graph/codecs.py @@ -541,8 +541,10 @@ def load_raw_bytes(path: Path, *, store: ContentStore | None = None) -> bytes: itself opened, and the open is non-blocking, so the codec cannot be made to wait on a pipe or stream a device. The read is bounded rather than trusted to ``st_size``, because a file may grow after it is measured. - Detecting that a source moved is the executor's post-run content check; - this bound only keeps the codec from reading an unbounded amount first. + Detecting that a source moved is the executor's own content check -- after + every cold node that declares the source, and again over every source in + full before the run manifest is built; this bound only keeps the codec from + reading an unbounded amount first. """ del store # raw bytes are self-describing; no content store is consulted From 7e2f1fbbc004a97f4e9ed761bcd8b0f50131077c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 16 Sep 2026 00:18:01 -0400 Subject: [PATCH 08/45] Start the lane report with the proofs that are already in hand Co-Authored-By: Claude Opus --- experiments/native-verify-once/out.md | 277 ++++++++++++++++++++++++++ 1 file changed, 277 insertions(+) create mode 100644 experiments/native-verify-once/out.md diff --git a/experiments/native-verify-once/out.md b/experiments/native-verify-once/out.md new file mode 100644 index 000000000..f471ac98d --- /dev/null +++ b/experiments/native-verify-once/out.md @@ -0,0 +1,277 @@ +# Build lane: verify native sources once per run + +Branch `native-verify-once`, worktree `~/PolicyEngine/_worktrees/microcosm-verify-once`, +base `f7bb88525a78786f91bc3ebe2083ef4b1c85de18` (PR #893 head — `git rev-parse HEAD` +and `git log -1` checked before any edit). 2026-09-15/16. + +**Where this report is.** The repository's root `out.md` is a tracked file +holding the Amendment 19 lane's committed report, and root `PROGRESS.md` is +cumulative. Writing this lane's report to either would delete another lane's +committed content, so the report is this file and the lane's journal is an +appended section of `PROGRESS.md`. + +**What this is not.** Nothing here is a statement about dataset quality, +calibration or release eligibility. Every measurement is descriptive; none is a +build, a certification or a release artifact. + +## The problem + +`~/PolicyEngine/_recovered/pilot-runs/native45-v5/out.md` §2, measured by stack +sampling on real runs: on the 19-node financial graph at `Fraction(1, 1000)` +(3,168 expanded households, 5,362 s wall), node execution is 1,143 s and the +remaining ~79 % is admission and verification of 3.48 GiB of staged source, +repeated. QRF fits inside that run cost 0.06–0.07 s each. + +The design note is [`docs/us-native-verification-once.md`](../../docs/us-native-verification-once.md): +per mechanism, the guarantee today, the change, why the guarantee survives and +the new cost class. It was written and committed before any capsule changed +(`d9136a20e`). + +## Per mechanism + +### 1. `AuthenticatedSurveyPopulationPreparation._checked()` + +Every borrow re-ran the whole authentication — both source catalogues, the ACS +native coverage binding, the nested ASEC native population, the ten-file roster +re-hash and every pure seal — and the graph registers +`verify_survey_population_preparation` as a **per-node population observer**, so +a run paid one of these per executed node on top of every kernel borrow. + +**Guarantee, in two sentences.** Inside a `verification_epoch()` every borrow +still pays, in full and unmemoised, the live authority, the attached owner +payloads, `_encode(_producer())` and `_file_stats` over the whole source roster, +so every producer change and every on-disk change that moves any of five stat +fields still refuses at the same borrow it refuses at today. What a signature +cannot see — an in-place write into a live buffer, or a value written into a +frozen plan row — is closed by an unconditional full re-validation when the +epoch closes, before the run returns anything; outside an epoch nothing is +memoised at all. + +**Mutation tests.** +`test_a_source_appended_to_mid_epoch_refuses_at_the_next_borrow` +(3 parametrisations), `test_a_source_truncated_mid_epoch_refuses_at_the_next_borrow` +(2), `test_a_source_rewritten_in_place_mid_epoch_refuses_at_the_next_borrow`, +`test_a_file_added_to_a_source_directory_mid_epoch_refuses`, +`test_a_touched_source_refuses_on_its_stat_identity_alone`, +`test_a_change_a_signature_cannot_see_refuses_when_the_epoch_closes`. +Each asserts the refusal **twice**: at the borrow that follows the mutation, and +again when the epoch declines to close over it. + +**Before/after CPU.** Unit level, from +`test_an_epoch_validates_once_and_reuses_it`: six borrows of one preparation +cost six complete validations before (one `_source_files` pass each — the whole +3.48 GiB roster) and one after, plus one unconditional pass when the epoch +closes. Five of six borrows are memo hits; `_producer()` still runs on all six. + +### 2. `AuthenticatedAsec2024NativePopulation.frame` + +Every `.frame`, `.context`, `.receipt` and `.to_bytes()` was a full +re-authentication including a SHA-256 of all seven ASEC source files. + +**Guarantee, in two sentences.** `_encode(_implementation())` still runs on +every borrow, and the signature carries the stat identity of all seven source +files *and of their parent directories*, which `_file_identity` does not cover +because it opens `O_NOFOLLOW` on the final component only — so a roster change +reaches a memo that never lists the roster. The capsule's epoch closes **before** +its owner's, so the owner's final validation reaches the complete file check +rather than the memo. + +**Mutation test.** +`test_the_native_capsule_still_refuses_a_changed_source_inside_an_epoch` +(`SOURCE_FILE_CHANGED` at the borrow, and again at the close). The existing +`test_changed_original_source_refuses_existing_borrow` is unchanged and still +passes: it runs outside an epoch. + +**Before/after CPU.** `test_the_native_capsule_validates_once_per_epoch`: three +`validate()` calls cost three complete seven-file passes before and one after, +plus one at the close. + +### 3. Per-node source content keys in the executor + +`executor.py:2452-2457` re-derived `source_content_key` for every source a +cold-executed node declared; `keys._directory_identity` `read_bytes()`es every +file of the tree. + +**Guarantee, in two sentences.** A cached key is reused only while the path's +stat signature — for a directory, its own identity plus the relative name, type +and identity of every entry `_directory_identity` would walk — is identical to +the signature taken both immediately before and immediately after the read that +produced the key, so every mutation that moves any stat field still refuses at +the same node, before that node's `_write_node`. Every source is then re-derived +in full, cache bypassed, before the manifest is built, which also catches a +change made during a node that declares no source — something the per-node check +has never seen. + +**Regression tests.** The refusal had **no test anywhere in the repository** +before this branch. `test_graph_executor_source_identity` now pins it: +`test_a_source_rewritten_by_its_own_node_refuses_that_node` (and asserts the +store gained no object), `test_a_file_added_to_a_directory_source_refuses`, +`test_a_file_removed_from_a_directory_source_refuses`, +`test_a_rewritten_source_still_refuses_when_only_its_bytes_moved`, +`test_a_source_changed_during_a_source_free_node_refuses_at_run_end`. + +**Before/after CPU.** `test_each_source_is_read_twice_per_run_not_once_per_node` +and `test_an_unchanged_source_is_never_re_read_by_a_node`: two full derivations +per run regardless of how many nodes declare the source. The +`packages/microcosm-graph` suite fell from **108.28 s to 71.81 s** on the same +machine — a synthetic toy source, so the real saving scales with source bytes. + +**The record.** `RunManifest.source_identities`, attached exactly like +`populations` and `mass_ledgers`: `repr=False`, `compare=False`, outside +`content_addressed`, outside `to_json`, outside every node receipt and cache +record. `test_the_manifest_records_the_run_end_identities_without_moving_anything` +asserts the manifest's key, its JSON and its content-addressed body are +identical with and without it. + +### 4. The ACS record fence + +`acs_person_coverage_authentication._records` was a `for byte in block` loop +over 4,096-byte blocks, fencing 2.4 GB of ACS person CSV. + +**Guarantee, in two sentences.** The state machine is unchanged — quote parity +still persists across records, CR/LF/CRLF are still preserved, and both ceilings +are still charged in the same order — only the boundaries are now located with +`bytes.find` and `bytes.count`. A record no longer than the smaller of the two +live ceilings cannot have violated either, because the token counter resets at +every record start; anything longer is replayed byte by byte, so the refusal +code and the byte it fires on stay the fence's own. + +**Equality proof.** `test_us_acs_record_fence_scan` compares the shipped fence +against a verbatim copy of the byte loop on every input, comparing the yielded +record sequence and the refusal together: 20 boundary cases, 6 block-boundary +cases, 8 ceiling cases on both sides of every cap, 4,200 random strings, 300 +row-shaped inputs, and **all 9,331 strings up to five bytes over the branching +alphabet under each of five ceiling settings — 46,655 exhaustive comparisons**. +On the real staged archive +(`experiments/native-verify-once/record-fence-parity.json`, produced by +`record_fence_real_archive.py`): + +| member | bytes | records | digest | byte loop | chunked scan | speedup | +|---|---|---|---|---|---|---| +| `psam_pusa.csv` | 1,226,543,489 | 1,743,752 | `da955ca2…347fc827` identical | 116.72 s | 2.38 s | 49.1× | +| `psam_pusb.csv` | 1,178,924,624 | 1,679,138 | `9bbf6af8…90184bba` identical | 110.14 s | 2.25 s | 48.9× | + +**226.86 s → 4.63 s per pass**, 3,422,890 records, digests identical. + +### 5. The kernel context digest + +`_update_series` boxed every value of a plain float, integer or boolean column +into a Python object and framed them one at a time; on pandas 3.0.3 such a +column is a `NumpyExtensionArray` with no `_data`/`_mask`, so the existing fast +path never applied to it. + +**Guarantee, in two sentences.** `_object_stream` emits the identical framed +bytes — the same `object` dtype header, the same shape header, the same +per-value length prefixes and payloads — and returns `None` for every column +kind it cannot reproduce exactly. Both `_context_digest` calls per node remain, +because the second one is the "did the kernel mutate its context" guard and +reusing its result would delete the guard. + +**Byte-equality proof.** `test_graph_executor_series_stream` compares the live +helper against a verbatim copy of the pre-change body **at the byte level**, not +only at the digest: 19 column kinds, the float specials (signed zero, both +infinities, the subnormal extremes), non-canonical NaN payloads with their sign +bit, both `int64` endpoints, empty/sliced/strided columns, labelled indexes, and +a 200-frame random sweep over ten kinds. + +The first version of the fast path mistook a `Categorical` for an integer column +because `Categorical._ndarray` is its codes array; that is now a named test, +`test_categorical_is_not_mistaken_for_its_integer_codes`, and the shipped guard +keys on `series.dtype` being a plain numpy dtype rather than on any private +attribute. + +**Before/after CPU**, 6,928 × 240 frame: **0.466 s → 0.094 s**. Per value: +float64 182.6 → 17.9 ns, bool 225.6 → 13.9 ns, int64 383.3 → 119 ns. + +## Pins re-derived + +| pin | old | new | command | +|---|---|---|---| +| `acs_native_coverage_binding._ACCEPTED["acs_person_coverage_authentication.py"]` | `475aa795c8a5b49a0dd3405a0877866dddd03012a1f2b9743447fab1fe85bcff` | `9ec68721a4cf480ef412c51ab354db9000eb7a7989e35e6e09b1574d88e8e49f` | `shasum -a 256 packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py` (re-derived after the final formatting pass; a stale pin refuses with `UNREVIEWED_PREPARATION`) | +| `graph_implementation_inventory.json` → `survey_population_preparation.py` → `unbound_uses_sha256` | `29c09f6fcb25ce8bc6111dd0dc76f3501d9a6b635929e94b80835889d296ef91` | `d114117ca19e9b866982147e5497d601ed91ca603d0568730d4051bdd4005910` | `graph_implementation._dependency_contract(payload, name, _covered_imports(name, inventory))` — the module's own generator | +| `graph_implementation_inventory.json` → `asec_2024_native_population.py` → `unbound_uses_sha256` | `71463df4f645cd2a98f537a212187b4d1e52bffcc4d2317defe6968d7f287608` | `8e15860cf67c6b069bb9f76ca042ce5d15a6b357583dc7847f4fae5ca2e902db` | same | + +No other pin moves. Verified by recomputing **every** contract in the inventory +through `graph_implementation._dependency_contract` and building all ten stage +manifests: `moved: 0`, `all stage manifests build`. In particular the three +foreign owners that gained a `_verified_source_stats()` helper +(`acs_population_catalogue`, `acs_native_coverage_binding`, +`asec_population_catalogue`) and both graph-shard modules keep their contracts +unchanged, because the additions introduce no `_RESOURCE_CALLS` name and no +non-stdlib import. + +**One thing that does move, by design and not by this lane's choice.** Each US +stage's `implementation_hash` is `_digest(module bytes)` over its whole module +roster, so editing any inventoried module moves it, and with it every node key. +That is true of any change to these files, including a comment. It is not a +digest this lane computes; the digests this lane computes — frame identities, +source file digests, context digests, source content keys, seals, receipts — all +keep their exact values, and the tests above prove it byte for byte where bytes +are rebuilt. + +## Test summaries, verbatim + +``` +$ .venv/bin/python -I -B -m pytest packages/microcosm-graph/tests -p no:randomly +766 passed, 1 skipped, 1 warning in 71.81s (0:01:11) +``` + +``` +$ .venv/bin/python -I -B -m pytest packages/microcosm-graph/tests/test_graph_executor_series_stream.py +34 passed in 0.75s +``` + +``` +$ .venv/bin/python -I -B -m pytest packages/microcosm-graph/tests/test_graph_executor_source_identity.py -p no:randomly +15 passed in 0.56s +``` + +``` +$ .venv/bin/python -I -B -m pytest packages/microcosm-build/tests/test_us_acs_record_fence_scan.py -p no:randomly +52 passed in 1.08s +``` + +``` +$ .venv/bin/python -I -B -m pytest \ + packages/microcosm-build/tests/test_us_acs_person_coverage_authentication.py \ + packages/microcosm-build/tests/test_us_acs_source_compile_cache.py \ + packages/microcosm-build/tests/test_us_acs_person_coverage_columns.py \ + packages/microcosm-build/tests/test_us_acs_housing_source.py \ + packages/microcosm-build/tests/test_us_acs_record_fence_scan.py -p no:randomly +544 passed, 5 warnings in 14.33s +``` + +``` +$ .venv/bin/python -I -B -m pytest \ + packages/microcosm-build/tests/test_us_survey_population_preparation.py \ + packages/microcosm-build/tests/test_us_asec_2024_native_population.py \ + packages/microcosm-build/tests/test_us_acs_source_compile_cache.py -p no:randomly +112 passed, 2 warnings in 140.94s (0:02:20) +``` + +``` +$ .venv/bin/python -I -B -m pytest packages/microcosm-build/tests/test_us_native_verify_once_epoch.py -p no:randomly +17 passed in 161.69s (0:02:41) +``` + +``` +$ .venv/bin/python -I -B -m ruff check . +All checks passed! +``` + +``` +$ python3 -I -B -S tools/ci_test_groups.py --verify +verification=ok +``` + +``` +$ uv lock --check +Resolved 125 packages in 6ms +``` + +All four new test files land in the groups they should and none under +`[defaulted]`: `test_us_acs_record_fence_scan.py` and both graph files in +`rest` / `us-am` / `wheels`, `test_us_native_verify_once_epoch.py` in `rest` / +`us-not` / `wheels`. + +*(Remaining runs and the probe measurement follow.)* From d9a12e46ce49b112867f9cd060a7d8a8f4b79de6 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 16 Sep 2026 00:19:22 -0400 Subject: [PATCH 09/45] Keep the witness helper private, and re-pin its module contract The frame-witness bridge into the preparation owner is module-internal; naming it without the underscore made it look like an owner API. Renaming it moves the module's unbound_uses digest, because that digest keys each use site by its enclosing scope, so the inventory contract is re-derived through the module's own generator: 71463df4f645cd2a98f537a212187b4d1e52bffcc4d2317defe6968d7f287608 -> 7bb20439da26f75b48551e21e304c4bb7f3983ee07ef78ed804411640d58cdf4 All ten stage manifests still build. Co-Authored-By: Claude Opus --- PROGRESS.md | 2 +- experiments/native-verify-once/out.md | 2 +- .../build/us_runtime/asec_2024_native_population.py | 6 +++--- .../build/us_runtime/graph_implementation_inventory.json | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index 11357b3c5..f040833f9 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -1471,7 +1471,7 @@ draft PR. |---|---|---| | `acs_native_coverage_binding._ACCEPTED["acs_person_coverage_authentication.py"]` | `475aa795…fe85bcff` | `9ec68721…d88e8e49f` | | `graph_implementation_inventory.json` `survey_population_preparation.py` `unbound_uses_sha256` | `29c09f6f…d296ef91` | `d114117c…dd4005910` | -| `graph_implementation_inventory.json` `asec_2024_native_population.py` `unbound_uses_sha256` | `71463df4…d7f287608` | `8e15860c…5ca2e902db` | +| `graph_implementation_inventory.json` `asec_2024_native_population.py` `unbound_uses_sha256` | `71463df4…d7f287608` | `7bb20439…40d58cdf4` | **Lane notes.** Root `out.md` is a tracked file holding the Amendment 19 lane's committed report; this lane's report goes to diff --git a/experiments/native-verify-once/out.md b/experiments/native-verify-once/out.md index f471ac98d..14b62a568 100644 --- a/experiments/native-verify-once/out.md +++ b/experiments/native-verify-once/out.md @@ -189,7 +189,7 @@ float64 182.6 → 17.9 ns, bool 225.6 → 13.9 ns, int64 383.3 → 119 ns. |---|---|---|---| | `acs_native_coverage_binding._ACCEPTED["acs_person_coverage_authentication.py"]` | `475aa795c8a5b49a0dd3405a0877866dddd03012a1f2b9743447fab1fe85bcff` | `9ec68721a4cf480ef412c51ab354db9000eb7a7989e35e6e09b1574d88e8e49f` | `shasum -a 256 packages/microcosm-build/src/microcosm/build/us_runtime/acs_person_coverage_authentication.py` (re-derived after the final formatting pass; a stale pin refuses with `UNREVIEWED_PREPARATION`) | | `graph_implementation_inventory.json` → `survey_population_preparation.py` → `unbound_uses_sha256` | `29c09f6fcb25ce8bc6111dd0dc76f3501d9a6b635929e94b80835889d296ef91` | `d114117ca19e9b866982147e5497d601ed91ca603d0568730d4051bdd4005910` | `graph_implementation._dependency_contract(payload, name, _covered_imports(name, inventory))` — the module's own generator | -| `graph_implementation_inventory.json` → `asec_2024_native_population.py` → `unbound_uses_sha256` | `71463df4f645cd2a98f537a212187b4d1e52bffcc4d2317defe6968d7f287608` | `8e15860cf67c6b069bb9f76ca042ce5d15a6b357583dc7847f4fae5ca2e902db` | same | +| `graph_implementation_inventory.json` → `asec_2024_native_population.py` → `unbound_uses_sha256` | `71463df4f645cd2a98f537a212187b4d1e52bffcc4d2317defe6968d7f287608` | `7bb20439da26f75b48551e21e304c4bb7f3983ee07ef78ed804411640d58cdf4` | same | No other pin moves. Verified by recomputing **every** contract in the inventory through `graph_implementation._dependency_contract` and building all ten stage diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py index 01ccbd74d..f7b76aff7 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/asec_2024_native_population.py @@ -494,12 +494,12 @@ def _parent_frame_witness(value): return ("unissued",) state = entry[2] return ( - preparation_witness(state.frame), - preparation_witness(state.parent.frame), + _preparation_witness(state.frame), + _preparation_witness(state.parent.frame), ) -def preparation_witness(frame): +def _preparation_witness(frame): """Deferred to the preparation owner, which defines the witness shape. Imported lazily: the preparation module imports this one, so the dependency diff --git a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json index 2fbf137c3..cf1c55f03 100644 --- a/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json +++ b/packages/microcosm-build/src/microcosm/build/us_runtime/graph_implementation_inventory.json @@ -208,7 +208,7 @@ "pandas" ], "resource_accesses_sha256": "466e46803aecf67189f31031cdb4eb9287af37669e5fa0b36610788d62564e46", - "unbound_uses_sha256": "8e15860cf67c6b069bb9f76ca042ce5d15a6b357583dc7847f4fae5ca2e902db" + "unbound_uses_sha256": "7bb20439da26f75b48551e21e304c4bb7f3983ee07ef78ed804411640d58cdf4" }, "microcosm.build/us_runtime/asec_checkpoint.py": { "imports": [ From 647063b3c8412f8d72c1cdb97cba62d5c9629ce3 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 16 Sep 2026 00:21:14 -0400 Subject: [PATCH 10/45] Record that this stacked PR's base does not reach CI Co-Authored-By: Claude Opus --- experiments/native-verify-once/out.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/experiments/native-verify-once/out.md b/experiments/native-verify-once/out.md index 14b62a568..1edc6d418 100644 --- a/experiments/native-verify-once/out.md +++ b/experiments/native-verify-once/out.md @@ -269,6 +269,13 @@ $ uv lock --check Resolved 125 packages in 6ms ``` +**CI does not run on this pull request, by design.** `.github/workflows/test.yml` +triggers on `pull_request: branches: [main]`, and this PR is stacked on +`microcosm-us-launch-integration-20260909`. Only that branch's own PR (#893) +reaches CI. Everything below was therefore run locally, against the same +`uv sync --all-packages --locked --extra us` environment CI uses +(Python 3.14.4, pandas 3.0.3, numpy 2.4.6). + All four new test files land in the groups they should and none under `[defaulted]`: `test_us_acs_record_fence_scan.py` and both graph files in `rest` / `us-am` / `wheels`, `test_us_native_verify_once_epoch.py` in `rest` / From bf8712da63f06fd847c3a19406869a9fa65e8fbe Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 16 Sep 2026 00:57:24 -0400 Subject: [PATCH 11/45] Record the lane's second-session state before the measurement runs The split branch, the measurement staging and the remaining work, so the journal is accurate while the probes are running. Co-Authored-By: Claude Opus 5 --- PROGRESS.md | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/PROGRESS.md b/PROGRESS.md index f040833f9..94c887adc 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -1422,8 +1422,10 @@ Branch `native-verify-once`, worktree `~/PolicyEngine/_worktrees/microcosm-verif base `f7bb88525a78786f91bc3ebe2083ef4b1c85de18` (PR #893 head). Started 2026-09-15. -**State (2026-09-16):** mechanisms 3, 4 and 5 landed with proofs; the -verification epoch (mechanisms 1 and 2) is implemented and under test. +**State (2026-09-16, second session):** all five mechanisms landed with +proofs and the epoch wired into both atomic capsules; draft PR #935 open. The +open work is the CI-shaped test sweep, the before/after probe measurement, the +report and the main-only split. **Goal.** The 9/15 pilot v5 measurement (`~/PolicyEngine/_recovered/pilot-runs/native45-v5/out.md` §2) attributes ~79 % @@ -1461,9 +1463,29 @@ without moving any digest value. digests is unchanged; unconditional full re-validation on leaving the epoch. The two moved inventory contracts are re-derived (below). -**Next:** finish the epoch test run, wire the epoch into the graph entry points, -before/after probe measurement, docs that call the executor check "post-run", -draft PR. +**Done (2026-09-16, second session):** + +- Draft PR [#935](https://github.com/PolicyEngine/microcosm/pull/935), base + `microcosm-us-launch-integration-20260909`. +- Main-only split branch `graph-verify-once-main`, worktree + `~/PolicyEngine/_worktrees/microcosm-graph-verify-once-main`, branched from + `origin/main` (51c314382). It carries the graph-shard change only: the three + commits re-applied, plus a changelog fragment. Its one deliberate difference + from this branch is that `_update_scalar` has no exact-float `struct.pack` + shortcut on main -- that belongs to PR #893 -- so the parity test's reference + copy was aligned to main's actual pre-change body, which makes it a literal + verbatim copy there too. +- Before/after measurement staging in `.measure/` (gitignored): a parameterised + copy of the v5 pilot probe that takes the source tree, the staged run inputs + and the output directory from the environment, asserts every imported + `microcosm` module resolves inside the measured tree, and enforces a 16 GiB + resident ceiling alongside the CPU one. Baseline worktree + `~/PolicyEngine/_worktrees/microcosm-verify-once-baseline` is detached at + `f7bb88525`. + +**Next:** finish both probe runs, the CI-shaped test sweep on this branch, the +19-node harness if memory allows, the report, push the split branch and open +its draft PR, and update #935's body with the measurement table. **Pins re-derived so far:** From e32bb46e87a46f33c04bc76c6ae3897880984f4a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 16 Sep 2026 00:59:14 -0400 Subject: [PATCH 12/45] Commit the before/after probe so the measurement can be re-run The measurement staging itself stays out of the tree (.measure/ is ignored): this is the script, parameterised so the identical file measures the base and this branch, with the import-origin assertion that makes that comparison mean something. Co-Authored-By: Claude Opus 5 --- experiments/native-verify-once/README.md | 2 + .../native-verify-once/probe_verify_once.py | 441 ++++++++++++++++++ 2 files changed, 443 insertions(+) create mode 100644 experiments/native-verify-once/probe_verify_once.py diff --git a/experiments/native-verify-once/README.md b/experiments/native-verify-once/README.md index af5d15f92..9c464f51b 100644 --- a/experiments/native-verify-once/README.md +++ b/experiments/native-verify-once/README.md @@ -7,5 +7,7 @@ Receipts and reproduction scripts for the change described in |---|---| | `record_fence_real_archive.py` | Runs the pre-change ACS record fence and the shipped one over every applicable member of a staged `csv_pus.zip`, read-only, and compares a rolling digest of every yielded record. | | `record-fence-parity.json` | That comparison's receipt for the 2026-09-15 staged archive. | +| `probe_verify_once.py` | The before/after nine-node prefix probe: a parameterised copy of the v5 pilot's `probe_repeated_verification.py`, taking the source tree, the staged run inputs and the output directory from the environment so one file measures both trees. | +| `probe-before.json`, `probe-after.json` | Those two runs' measurements, base `f7bb88525` and this branch. | Nothing here is a build, a certification or a release artifact. diff --git a/experiments/native-verify-once/probe_verify_once.py b/experiments/native-verify-once/probe_verify_once.py new file mode 100644 index 000000000..a9f9b8460 --- /dev/null +++ b/experiments/native-verify-once/probe_verify_once.py @@ -0,0 +1,441 @@ +"""Before/after measurement for the verify-once lane: the nine-node prefix. + +A parameterised copy of the v5 pilot probe +(``~/PolicyEngine/_recovered/pilot-runs/native45-v5/probe_repeated_verification.py``). +Everything that decides *what is measured* is verbatim: the same nine-node +population prefix, the same 1/1000 sample, the same seed, the same stack +sampler at the same interval, the same ceilings, the same output shape. The +only changes are the ones a before/after comparison needs: + +* the source tree, the staged run inputs and the output directory come from + the environment, so the identical file measures both trees; +* every imported ``microcosm`` module is asserted to live under that tree, + so a stray editable install cannot silently measure the wrong source; +* a resident-memory ceiling is enforced alongside the CPU ceiling; +* the load average and the interpreter's third-party versions are recorded + with each run, because this machine carries other lanes' work. + +Sources are read by path and never written. Nothing here is a build, a +certification or a release artifact. + + MEASURE_BASE= MEASURE_RUN= \ + MEASURE_PROBE= MEASURE_LABEL=