diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5baf54f3..6b9cf94b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -37,11 +37,30 @@ jobs: - name: Run tests (shard ${{ matrix.shard }}/4) run: pytest -q --splits 4 --group ${{ matrix.shard }} --splitting-algorithm duration_based_chunks --durations-path .test_durations + graph-integration: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.13', '3.14'] + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install the pinned optional graph integration + run: pip install -e ".[graph]" pytest + - name: Require typed graph capabilities + run: python -c "from populace_dynamics.graph._compat import require_graph; require_graph()" + - name: Run graph and existing mortality regressions + run: pytest -q tests/test_graph_mortality.py tests/test_m6_engine_refit.py tests/test_m6_engine_steps.py + # Fan-in jobs keeping the branch-protection context names # ("pytest (3.11)" / "pytest (3.13)") stable across the shard split. pytest: name: pytest (${{ matrix.python-version }}) - needs: pytest-shard + needs: [pytest-shard, graph-integration] if: always() runs-on: ubuntu-latest strategy: @@ -49,4 +68,6 @@ jobs: python-version: ['3.14'] steps: - name: Verify all shards passed - run: test "${{ needs.pytest-shard.result }}" = "success" + run: | + test "${{ needs.pytest-shard.result }}" = "success" + test "${{ needs.graph-integration.result }}" = "success" diff --git a/docs/plans/graph-port-A-shadow-2026-09-03.md b/docs/plans/graph-port-A-shadow-2026-09-03.md index a258db12..fce3d3fd 100644 --- a/docs/plans/graph-port-A-shadow-2026-09-03.md +++ b/docs/plans/graph-port-A-shadow-2026-09-03.md @@ -1,10 +1,10 @@ # Graph port, Plan A: store, ingestion and the Q5 harvest subgraph in shadow -Status: revision 4, 2026-09-04. Gate 20260903-122500-plan-10bdc7ab (Fable main + Sol peer) reached its four-round cap; round 1 AGREED that the shadow scope confers no authority, and rounds 1, 2 and 4 requested changes on the evidence machinery (round 3 could not dispatch for lack of a codex lane). Every replacement from rounds 1–2 is folded; round 4's six findings are folded below as follows: the three plan-level findings (ingestion contract consistency, evidence-complete vs parity-pass states, a frozen H2 stopping rule) are adopted in full; the three authority-grade findings (hermetic execution closure with denied undeclared I/O, an independently trusted attester with tamper tests before any cache read, per-event-type disposition schemas with wrong-disposition fixtures) are adopted as REQUIREMENTS FOR CALLING ANY SHADOW RESULT EVIDENCE and carried into Plan B, with Plan A explicitly producing no certification evidence until they are met. This revision awaits Max's decision on how it is approved (a third gate, or his own approval for shadow-only work with the residual findings logged). Plan A supersedes the scope of `docs/plans/graph-port-2026-09-03.md` revision 3, whose gate (20260903-093201-plan-342fa24d) reached its four-round cap with eight open findings; those findings are carried verbatim into Plan B, which will be gated only once Plan A has produced measured parity evidence. Nothing in Plan A confers authority on any graph artifact. +Status: revision 12, 2026-09-05 (gate 20260905-200417-plan-6deac85c round 1 requested one change, folded: A-5 requires A-1 at LIVE scope). The A-1/A-2 build at exact head 41fe7fe (three commits on `plan-a-shadow-a1a2-20260905`, SHADOW-ONLY) was reviewed clause by clause (run 20260905-190906-plan-a-a1a2-review-r2, REVIEW.md sha256 01c768fb1523b0c7…): A-1 is accepted at FIXTURE scope only and recorded NOT MET at LIVE scope (the build ingests staged reports, tables with sidecars and its own cohort manifest, not the other declared classes), A-2 is accepted for the fixture set, and the totality of `decode_lane`/`reconstruct_T` was then MEASURED by the shadow child at df10c8b (`tests/q5graph/test_kernel_totality.py`, 23 tests): total where the contract holds, an absent source is the declared boundary, and three inputs are neither total nor a typed gate failure — a printed `NaN` in the symbol table (crashes both kernels), a digit run longer than the interpreter's int-conversion limit (raises out of `decode_report`; the limit is an interpreter setting the TCB must pin), and a printed arity above int64 (crashes `decode_lane`; `reconstruct_T` survives the same bytes). This revision makes the typed-failure contract explicit for those inputs (§3 A2a below); this revision makes the two A-1 scopes explicit in A1 and §4, replaces the global path-fragment blacklist with a per-class staged rule in §5 (the blacklist would refuse two declared classes outright), and adds two required fixes for the next slice (the TCB digest in every run artifact's banner; an absent declared digest fails rather than passes). A-3 does not start before this revision is agreed. Revision 11: Gate 20260905-164552-plan-b373bd2b round 1 accepted the revision-10 capacity amendment without finding and requested one A-2 topology change, folded here: at the pinned executor a failed string verdict does not by itself block downstream execution (`_blocked_by` blocks only consumers of unavailable typed artifacts and descendants of execution-unreached nodes), so every gate that must stop execution downstream now emits a typed artifact that its dependents consume, the expected-state vocabulary distinguishes EXECUTED-FAILED from UNREACHED, and fixtures cover both dependency types cold and warm. Revision 9 was AGREED by gate 20260905-162709-plan-c185a30c (round 2 approve, 16:34 EDT). Revision 10 amended ONE resource rule and nothing else, at the Microcosm US session's request under Max's steering to maximise parallel work: A5 and the matching §5 do-not now permit the isolated A-1/A-2 engineering lane to be routed upward to Astra on a Codex home when every Claude lane is capped (measured by return code, never by the status table), subject to CEREMONY PRECEDENCE on that home and a recorded window state at dispatch; the Claude path stays the default and is used whenever it is available first. It is submitted to a fresh gate. Earlier: gate 20260905-162709-plan-c185a30c on revision 8 requested three runtime-contract changes, all folded here against the pinned executor: (i) every owned output of a gate-role kernel is a STRING (executor.py:130 rejects any non-string owned output on the exception path, so a mixed string/boolean gate cannot persist failed-gate evidence), with a mixed-output exception fixture that must fail; (ii) RELEASE's `requires_decisions` names DECISIONS supplied at run time (executor.py:219–249), not ancestral gates, so the required decision tuple is frozen and validated independently of the ancestral gate set, with missing- and wrong-decision checks on warm replay; (iii) A-2's execute-cold / hit-warm assertions are scoped per fixture to the frozen expected state (executed, refused, unreached), because a missing required source is rejected before any kernel runs (executor.py:1922–1924) and a gate exception leaves dependents unexecuted. Revision 7 was AGREED by the third gate 20260905-154850-plan-5cfd8532 (Fable main + Astra peer, round 5 approve, 16:12 EDT; proceed authorized). Revision 8 amends exactly one adopted decision and one rule, on the Microcosm US session's interface-confirmation memo (`~/architecture-reviews/microcosm-20260904/dynamics-q5-current-graph-interface-check.md`, SHA-256 485ea3d327b71d2aa9b7f654f0453cdf65c763821a060dc9415f7be951465d9c; companion receipt SHA-256 8d780c99c401536d52e522dd779e9493a66768a75d341130f2148a4663f7cbed): (A2) the gate-recorded TCB pin moves from microcosm `6022e4f5` to `b55a583f078b3d182c65980f24a4d5a119f169bc`, a NEW TCB, because the A-2 interfaces do not exist at `6022e4f5`; (§1/A-1) the store, codec registry and typed artifact facilities are CONSUMED from the core at that pin, not reimplemented; (§4) gate kernels own a string outcome column. It is submitted to a fresh gate; nothing else moves. Earlier: the third gate's rounds 2, 3 and 4 requested changes, all folded in revisions 5–7: (00, round 4) applicability is declared separately for report-derived and T-derived fields, so an attempt voided after its canonical report but before its T keeps the full report comparison and marks only the T-derived fields ABSENT-BY-DISPOSITION, with a report-bearing-void fixture (§4); (0, round 3) the legacy comparison now carries a per-event-type applicability map with explicit absence semantics — a voided attempt or supersession that never produced a canonical report or T bytes is compared on its disposition schema with those fields marked ABSENT-BY-DISPOSITION, which satisfies A-2 and counts toward EVIDENCE-COMPLETE, while an event whose class requires a payload and lacks one is MISSING REQUIRED EVIDENCE and fails; fixtures cover both (§4); (1) the hermetic-image alternative now binds the complete image identity and the allowed semantic-environment values into the TCB, and any change to either rejects the populated old namespace before any cache read (A2, A2a); (2) the latency population is defined per event class with an explicit timestamp source, and harvest-to-shadow-complete is retained separately for harvest events (§4, H2 stopping rule). Previously: gate 20260903-122500-plan-10bdc7ab (Fable main + Sol peer) reached its four-round cap; round 1 AGREED that the shadow scope confers no authority, and rounds 1, 2 and 4 requested changes on the evidence machinery (round 3 could not dispatch for lack of a codex lane). Every replacement from rounds 1–2 is folded; round 4's six findings are folded below as follows: the three plan-level findings (ingestion contract consistency, evidence-complete vs parity-pass states, a frozen H2 stopping rule) are adopted in full; the three authority-grade findings (hermetic execution closure with denied undeclared I/O, an independently trusted attester with tamper tests before any cache read, per-event-type disposition schemas with wrong-disposition fixtures) are adopted as REQUIREMENTS FOR CALLING ANY SHADOW RESULT EVIDENCE and carried into Plan B, with Plan A explicitly producing no certification evidence until they are met. Revision 4 awaited a decision on how it is approved; under Max's standing instruction to keep independent, gate-respecting work moving, the third gate above is that path, and this revision answers its round 2. Plan A supersedes the scope of `docs/plans/graph-port-2026-09-03.md` revision 3, whose gate (20260903-093201-plan-342fa24d) reached its four-round cap with eight open findings; those findings are carried verbatim into Plan B, which will be gated only once Plan A has produced measured parity evidence. Nothing in Plan A confers authority on any graph artifact. ## 1. Scope -IN: (a) a content-addressed store outside e8-ops with a codec registry; (b) a trusted-computing-base (TCB) recomputed at preflight, covering the runtime and the whole evidence toolchain, bound into every node key, run and report; (c) read-only ingestion of the campaign's stable canonical staged bytes as NONAUTHORITY source caches; (d) the Q5 harvest/transport subgraph run in SHADOW beside the bespoke harvest; (e) a parity harness with frozen legacy schema, graph oracle, mutation manifest, recovery/failure matrix and named negative fixtures; (f) a shadow backfill of every event in a frozen cohort manifest; (g) a parity report per document and a closing census by multiset equality. +IN: (a) a content-addressed store outside e8-ops, bound to the core's `ContentStore`, `SourceCodecRegistry` (`register_bytes` / `raw-bytes-v1`), `SourceRef`, typed `ArtifactType`/`ArtifactInput`/`ArtifactOutput` edges, `Owned` cells, gate roles and RELEASE `requires_decisions` at the A2 pin — Plan A reimplements none of those facilities and adds only the Q5 kernels, codec mappings, source declarations, the occurrence/document Frame (`EntitySchema(person_entity='occurrence', group_entities=('document',))` with unit weights) and the parity harness; (b) a trusted-computing-base (TCB) recomputed at preflight, covering the runtime and the whole evidence toolchain, bound into every node key, run and report; (c) read-only ingestion of the campaign's stable canonical staged bytes as NONAUTHORITY source caches; (d) the Q5 harvest/transport subgraph run in SHADOW beside the bespoke harvest; (e) a parity harness with frozen legacy schema, graph oracle, mutation manifest, recovery/failure matrix and named negative fixtures; (f) a shadow backfill of every event in a frozen cohort manifest; (g) a parity report per document and a closing census by multiset equality. OUT (Plan B, gated separately with Plan A's evidence): any harvest cutover or authority handoff; any board line, seal record or pointer index that cites a graph manifest; the campaign verifier's trust root and profile; the post-ratification successor DAG; the benefit-level gate; record retirement; any change to lane briefs, templates, harvest rules, ledger or library folds, `#405`, `e20-amend20`, `docs/design/` or the registry. @@ -15,51 +15,54 @@ OUT (Plan B, gated separately with Plan A's evidence): any harvest cutover or au ## 3. Decisions adopted by this plan -A1. Source model. Every declared, pinned source class is ingested: stable canonical staged report bytes (the only admissible form of a lane report — never raw lane `.out` or worktree outputs), briefs, ledger and library versions, rubric notes, PDFs, artifact JSONs and commit-pinned board ruling bytes, each hashed by name, bytes and size. The staged-bytes restriction applies to lane-report alternatives only; omitting any other declared source class fails A-1. No kernel calls an engine. Deterministic post-lane transformations are kernels; transport and seal checks are gate-role kernels that own at least one column read by the release. Determinism is a dynamics-side rule enforced by review and by the parity harness, not by the executor. +A1. Source model. Every declared, pinned source class is ingested: stable canonical staged report bytes (the only admissible form of a lane report — never raw lane `.out` or worktree outputs), briefs, ledger and library versions, rubric notes, PDFs, artifact JSONs and commit-pinned board ruling bytes, each hashed by name, bytes and size. The staged-bytes restriction applies to lane-report alternatives only; omitting any other declared source class fails A-1 at LIVE scope. A-1 is accepted in two named scopes and any acceptance states which: FIXTURE scope — every class the frozen cohort manifest declares is ingested and re-hashes to the record (met at 41fe7fe: 11 sources, tri-checked); LIVE scope — every class this paragraph names is ingested from the live cohort (not met at 41fe7fe; the build discloses the exclusion). The ingester must accept every declared class: a class-agnostic path-fragment blacklist that refuses briefs because they live under `thu-briefs/` or `opus-scratch/` is a defect, not a guard. No kernel calls an engine. Deterministic post-lane transformations are kernels; transport and seal checks are gate-role kernels that own at least one column read by the release. Determinism is a dynamics-side rule enforced by review and by the parity harness, not by the executor. -A2. TCB attestation over the runtime and the evidence toolchain. The gate-recorded pin is `microcosm` `origin/main` at commit 6022e4f5c8af31d62fddd1c7dbd1ab9eea81b0f6 (pre-#847). After implementation and before any acceptance run, a reviewed evidence-toolchain bundle is frozen and its digests recorded: the attester itself, the ingestion and H/H2 enumeration code, the parity comparator and report generator, the campaign kernels and codecs, and the frozen mutation manifest, graph oracle, recovery/failure matrix and legacy schema bytes. Before any cache lookup, the attester recomputes the TCB from the actually loaded graph/frame/executor files, the registry name-to-loader mapping, kernel and codec files, the interpreter executable/ABI, installed distribution files and the toolchain bundle; it rejects any mismatch with the recorded commit and digests and uses the recomputed digest as the store namespace. The TCB manifest is a declared source of the CREATE node and of every descendant node, so it enters every key, and its digest is bound into every run and report. Any change to any TCB component invalidates all Plan-A evidence and requires a complete cohort rerun; A-5 rejects mixed-TCB results. Adopting #847 is a new TCB and a full rerun. +A2a. Kernel totality contract. Every dynamics kernel is total over every input the source model admits, or fails through the typed gate failure the plan's blocking topology carries — never through an uncaught interpreter error. Printed `NaN`, digit runs beyond the pinned interpreter limit, and arities beyond int64 are admitted inputs (they occur in lane bytes) and must map to typed failures; the interpreter's int-conversion limit is a TCB component and is pinned in the toolchain bundle. The three counterexamples measured at df10c8b are open A-2 defects until fixed under this contract and re-measured; fixing them changes the toolchain bundle and therefore the TCB namespace, which is expected and must be reported, not hidden. -A2a. Evidence-grade closure (carried to Plan B; a Plan A prerequisite for the word "evidence"). No shadow result is certification evidence until: (i) execution runs in a read-only hermetic image with an environment and input allowlist, or the complete loaded closure (standard library, transitive native libraries, import hooks and resolution, semantic environment) is hashed and traced, undeclared filesystem, clock, randomness and network inputs are denied, and the closure is verified again after execution; (ii) an independently trusted, gate-owned recomputation of raw bundle, run and report bytes exists outside the attester, with tamper tests for every TCB component against a populated old namespace proving rejection before any cache read. Until then Plan A outputs are labelled SHADOW-ONLY, NOT EVIDENCE. +A2. TCB attestation over the runtime and the evidence toolchain. The gate-recorded pin is PolicyEngine/microcosm commit `b55a583f078b3d182c65980f24a4d5a119f169bc` ("Authenticate candidate payload identities and bound local transport reads", 2026-09-05 15:22 EDT), which the Microcosm US session froze for its interface check. Publication state at pinning: that commit is a strict descendant of `origin/main` `c1b8324183819cf91511c002bf724a2676524e87` and was not yet on any remote branch, so build agents read it from the local object store and its first appearance on a remote is a recorded event, not a pin change. The previous pin, `6022e4f5c8af31d62fddd1c7dbd1ab9eea81b0f6` (pre-#847), lacked `SourceCodecRegistry.register_bytes`, the `raw-bytes-v1` reader, and the typed `ArtifactType`/`ArtifactInput`/`ArtifactOutput` edges (all present at `b55a583f` in `packages/microcosm-graph/src/microcosm/graph/codecs.py` and `decl.py`); moving to `b55a583f` is a new TCB under this same paragraph's rule, and no Plan-A result exists yet to rerun. The memo confirms the ENGINEERING interface only: it approves no attestation, hermeticity, or source/scientific authority, and the §5 do-not on declaring `Tolerance` or entrants (#847) is unchanged. After implementation and before any acceptance run, a reviewed evidence-toolchain bundle is frozen and its digests recorded: the attester itself, the ingestion and H/H2 enumeration code, the parity comparator and report generator, the campaign kernels and codecs, and the frozen mutation manifest, graph oracle, recovery/failure matrix and legacy schema bytes. Before any cache lookup, the attester recomputes the TCB from the actually loaded graph/frame/executor files, the registry name-to-loader mapping, kernel and codec files, the interpreter executable/ABI, installed distribution files and the toolchain bundle; it rejects any mismatch with the recorded commit and digests and uses the recomputed digest as the store namespace. When execution runs in a hermetic image (A2a(i), first alternative), the complete image identity (the image digest and the digests of every layer actually loaded) and the exact allowed semantic-environment values (the environment and input allowlist as resolved, including interpreter flags, locale, timezone and hash-seed settings) are TCB components: the attester records them, recomputes them before any cache lookup, and treats a change to any of them exactly as it treats a toolchain change — the populated old namespace is rejected before any cache read and a new namespace is derived. When the alternative closure approach is used instead, the hashed closure (standard library, transitive native libraries, import hooks and resolution, semantic environment) is the TCB component in the same way. In neither case may two images or two environment resolutions share a cache namespace. The TCB manifest is a declared source of the CREATE node and of every descendant node, so it enters every key, and its digest is bound into every run and report. Any change to any TCB component invalidates all Plan-A evidence and requires a complete cohort rerun; A-5 rejects mixed-TCB results. Adopting #847 is a new TCB and a full rerun. + +A2a. Evidence-grade closure (carried to Plan B; a Plan A prerequisite for the word "evidence"). No shadow result is certification evidence until: (i) execution runs in a read-only hermetic image with an environment and input allowlist whose image identity (image and loaded-layer digests) and resolved allowlist values are bound into the TCB per A2, or the complete loaded closure (standard library, transitive native libraries, import hooks and resolution, semantic environment) is hashed and traced, undeclared filesystem, clock, randomness and network inputs are denied, and the closure is verified again after execution; (ii) an independently trusted, gate-owned recomputation of raw bundle, run and report bytes exists outside the attester, with tamper tests for every TCB component against a populated old namespace proving rejection before any cache read. Until then Plan A outputs are labelled SHADOW-ONLY, NOT EVIDENCE. A3. No authority. Every shadow manifest is NONAUTHORITY. Decision records are nonselecting provenance; nothing in Plan A selects an active record, cites a manifest on the board, or retires or rewrites any record. The bespoke harvest remains the sole authority throughout. A4. Storage. Outside e8-ops in a dedicated private location. For every source, before any copy or replica: record access classification, license/replication permission, approved destination and backup owner, repository/binary limits, retention, access controls, and required encryption; do not copy or replicate a source until those checks pass. Copied source payloads are NONAUTHORITY caches preserving canonical paths and storage identities. Inventory source bytes, columns, node-key duplicates, attempts and metadata before choosing Git, Git LFS or private object storage; name the backup owner and off-host replica; clean-store restore-and-rehash drill before acceptance; no capacity ceiling asserted until measured. -A5. Capacity. Build agents run on Claude capacity separate from the campaign's lanes and from the orchestrator's lane, and pause if harvest, A4/C20 or referee capacity is threatened. No codex lane-hours are consumed by Plan A except the gate peer's own review. +A5. Capacity. Build agents run on Claude capacity separate from the campaign's lanes and from the orchestrator's lane, and pause if harvest, A4/C20 or referee capacity is threatened. Codex lane-hours are consumed by Plan A only for (i) the gate peer's own review and (ii) the isolated A-1/A-2 engineering lane when it is routed upward to Astra because every Claude lane is capped — established by a fresh return code from each candidate lane, never by the status table — and then only under CEREMONY PRECEDENCE: no build dispatch on a Codex home while an Amendment-20 ceremony step (A4 freeze or re-freeze, scratch transition, receipt, referee verdict, closure, repin, or the PR #405 gate peer) is running on it, the dispatch records the home, its measured five-hour and weekly window state and the routing decision in the run's ledger entry, and the lane pauses or is not started if a ceremony step needs the home. The Claude path remains the default and is used whenever a Claude lane frees first. The served model and lane are reported in the parity report; a build that ran on Astra is labelled as such and confers nothing. ## 4. Sequence and acceptance -States. Two acceptance states are distinct: EVIDENCE-COMPLETE (every cohort document and event has a shadow result under the A-2 contract) and PARITY-PASS (zero governing discrepancies). Any governing residue makes the parity verdict FAIL for the cohort; A-5 labels such a cohort as failed parity, never as positive evidence. H2 stopping rule (frozen before A-3 starts): A-3 runs for at least 7 calendar days and until at least 12 live events spanning every event class that occurs in that window (harvest, seal, void/repair, supersession or replication if any occur) have been shadowed; H2 may not be chosen retroactively; the latency population is every live event's harvest-to-shadow-complete interval, with failures, retries, residues and censored runs reported separately and never dropped. +States. Two acceptance states are distinct: EVIDENCE-COMPLETE (every cohort document and event has a shadow result under the A-2 contract, where a result for an event whose class carries no canonical report or T bytes is complete when its disposition schema declares those fields ABSENT-BY-DISPOSITION) and PARITY-PASS (zero governing discrepancies). Any governing residue makes the parity verdict FAIL for the cohort; A-5 labels such a cohort as failed parity, never as positive evidence. H2 stopping rule (frozen before A-3 starts): A-3 runs for at least 7 calendar days and until at least 12 live events spanning every event class that occurs in that window (harvest, seal, void/repair, supersession or replication if any occur) have been shadowed; H2 may not be chosen retroactively; the latency population is defined per event class: for harvest events, harvest-to-shadow-complete (start = the canonical staged report's commit-pinned board timestamp); for seal, void, repair, supersession and replication events, event-to-shadow-complete (start = the commit-pinned timestamp of the board ruling or event line that creates the event, never a harvest timestamp of the document concerned, which may predate the event or, for a voided attempt, not exist); shadow-complete is the timestamp of the shadow run's report write. Each class is reported as its own population; harvest-to-shadow-complete is retained separately for harvest events and is never pooled with the other classes; failures, retries, residues and censored runs are reported separately and never dropped. Cohort definition and closing census. At an exact immutable board commit/event H, freeze a content-hashed occurrence manifest listing every document, attempt, canonical staged report, repair, full source digest/path/storage identity, and disposition, each with an immutable coordinate and identity. A-4 covers every event at or before H; A-3 covers every canonical harvest, seal, void, repair, supersession, and replication event after H through a closing coordinate H2, each assigned the same immutable coordinate/identity scheme. At H2, produce a second occurrence manifest in the same schema, re-enumerate the authoritative record, and require multiset equality with explicit missing/duplicate detection. Every post-H source delta passes the same A-1/A4 ingestion checks as the initial cohort. Boundary, duplicate, outage and omitted-event cases are tested against the census. Frozen oracles (recorded in the toolchain bundle before any acceptance run): -- Legacy schema: for every cohort document and event, the closed set of legacy fields compared exactly against the pinned canonical report and T bytes: ordered decoded rows, row counts, per-frame and stream digests, transport outcome and its complete evidence, seal outcome and its complete evidence, reconstructed T bytes, T row count and T digest. Missing or extra fields fail. +- Legacy schema: for every cohort document and event, the closed set of legacy fields compared exactly against the pinned canonical report and T bytes: ordered decoded rows, row counts, per-frame and stream digests, transport outcome and its complete evidence, seal outcome and its complete evidence, reconstructed T bytes, T row count and T digest. Missing or extra fields fail. APPLICABILITY MAP (frozen with the schema): the legacy fields are partitioned into two axes — REPORT-DERIVED (ordered decoded rows, row counts, per-frame and stream digests, transport outcome and its evidence, seal outcome and its evidence) and T-DERIVED (reconstructed T bytes, T row count, T digest) — and each event class declares applicability for each axis separately, from the authoritative disposition of the event, never from what happens to be on disk. Harvest, seal, repair and replication events with both payloads apply both axes. An attempt voided, superseded or otherwise closed AFTER its canonical report exists but BEFORE its T was produced applies the report axis in full and marks every T-derived field ABSENT-BY-DISPOSITION; an event whose canonical record never produced a report applies neither axis and marks both. Each inapplicable field is present with the single marker ABSENT-BY-DISPOSITION and the disposition identity that explains it; the comparison for the inapplicable axis is over the per-event-type disposition schema. EXPECTED ABSENCE (the marker exactly where the map permits it, per axis) satisfies A-2; MISSING REQUIRED EVIDENCE (a field an applicable axis requires that is absent or unchecked, or a marker on an axis the map declares applicable) fails, so a report-bearing void can never be declared evidence-complete with its report-derived evidence unchecked. Fixtures: a voided attempt with neither payload passes with both axes marked; a voided attempt with a canonical report and no T passes only if every report-derived field is compared and only the T fields carry the marker, and fails if any report-derived field is absent or marked; a harvest event lacking its pinned report fails. - Graph oracle: a separately frozen schema for graph-only fields: node keys, exact gate IDs and order, ancestry, tier/outcome mapping and required evidence fields. Missing or extra evidence fields fail. - Mutation manifest: each source digest, exact semantic mutation or omission, and expected row/field/verdict or typed-refusal code. Key changes alone never count. A governing mutant with unchanged semantics fails; a decoy mutation changes identities but leaves semantic rows, verdicts and evidence unchanged apart from provenance. Mutant kernels that ignore a governing source or consume a decoy must make the harness fail. -- Per-event-type schemas (carried to Plan B; required before any void, repair, supersession or replication shadow result is called evidence): for each event type, an exact identity, disposition, ruling digest, supersedes edge and replica source/destination with a derivation projection, plus wrong-disposition, wrong-ruling, wrong-supersedes and wrong-replica fixtures that must fail. +- Per-event-type schemas (carried to Plan B; required before any void, repair, supersession or replication shadow result is called evidence): for each event type, an exact identity, disposition, ruling digest, supersedes edge and replica source/destination with a derivation projection, its entry in the applicability map above for each of the two axes, plus wrong-disposition, wrong-ruling, wrong-supersedes, wrong-replica and wrong-absence (a marker on an applicable axis, a payload on an axis the map declares absent, or a report-bearing void whose report axis is marked instead of compared) fixtures that must fail. +- Gate outcome rule (from the interface check, corrected against the pinned executor): EVERY owned output of a gate-role kernel is a STRING column — the outcome and any check cell alike — because the pinned executor's exception path (executor.py:130) validates every owned output and raises NodeRejected on any non-string one, so a gate that mixes a string outcome with a boolean or numeric cell aborts on an exception without persisting failed-gate evidence. Boolean or numeric owned cells are permitted only on non-gate kernels. Fixture: a gate kernel with a mixed string/boolean output set raising an exception must FAIL the harness (no failed-gate evidence persisted); a gate with all-string outputs raising the same exception must persist the failed-gate outcome. RELEASE owns exactly one string `tier` column whose value must agree with the executor's ancestral gate derivation. `requires_decisions` names DECISIONS supplied at run time (matched by decision name, executor.py:219–249), which is a separate contract from gate ancestry: the plan freezes the required decision tuple (exact names, non-empty) and the expected ancestral gate set independently, and the harness validates both — the tuple against the supplied decisions on cold and on warm replay (a missing or wrongly named decision must leave RELEASE `unreached`, and the fixture decision must yield `pass`), and the gate set against the RELEASE receipt's ancestry; because the runtime permits an empty default tuple and derives ancestry separately, an empty tuple, a tuple of gate IDs, or a wrong gate set each fails the harness explicitly. - Recovery/failure matrix: recovery passes only when commit-pinned ruling bytes and redundant serialization/cross-receipts uniquely authenticate the reconstruction; raw bytes, reconstructed bytes, proof and ruling are persisted. Every historical positive recovery is tested, and for each, removal or substitution of every individual proof element must persist a gate failure. Named negative fixtures with stable typed refusal codes cover malformed, truncated, noncanonical, wrong-sentinel, wrong-count, wrong-digest, stale, swapped, conflicting and ambiguous evidence; none may certify or pass. | Step | What | Owner | Accepted when | |---|---|---|---| -| A-1 | Store, codec registry, TCB attester and toolchain bundle, ingestion of the cohort manifest's sources and of every post-H delta | build agents; this session reviews | every ingested source re-hashes to the SHA cited in the campaign record; canonical paths and storage identities preserved; no Plan-A write to e8-ops, verified by attribution audit; only stable canonical staged bytes ingested; the attester passes against the recorded commit and toolchain digests and its recomputed digest namespaces the store and appears as a source in every node key and in every run and report; restore-and-rehash drill passed from a clean store | -| A-2 | Q5 harvest/transport subgraph (CREATE occurrence universe with `document` as the group entity and unit weights; `q5.decode_lane@1`; `q5.transport_gate@1`; `q5.reconstruct_T@1` owning digest columns; `q5.seal_gate@1`; `q5.harvest_seal@1` RELEASE with real gate ancestry and nonempty `requires_decisions`) against the frozen oracles | build agents; this session reviews | for every cohort document and every mutation and recovery case: cold-A and cold-B executions in fresh interpreters on distinct physical, verified-empty stores execute every kernel with hit=false for every node and agree on canonical stored payload bytes and independently recomputed payload hashes; the unchanged warm run in a fresh interpreter executes zero kernels with hit=true for every node; legacy-schema comparison exact for every document and event; graph-oracle comparison exact; every mutation-manifest entry and every recovery/failure-matrix entry and negative fixture behaves as frozen; `q5.decode_lane@1` and `q5.reconstruct_T@1` are total over declared bytes or all fallible parsing lives inside gate kernels | +| A-1 | Store, codec registry, TCB attester and toolchain bundle, ingestion of the cohort manifest's sources and of every post-H delta | build agents; this session reviews | every ingested source re-hashes to the SHA cited in the campaign record; canonical paths and storage identities preserved; no Plan-A write to e8-ops, verified by attribution audit; only stable canonical staged bytes ingested; the attester passes against the recorded commit and toolchain digests and its recomputed digest namespaces the store and appears as a source in every node key and in every run and report; restore-and-rehash drill passed from a clean store; the attested TCB digest is printed in the banner of every committed run artifact; a source whose declared digest is absent FAILS the re-hash check rather than passing it; the acceptance names its scope (fixture or live) | +| A-2 | Q5 harvest/transport subgraph (CREATE occurrence universe with `document` as the group entity and unit weights; `q5.decode_lane@1`; `q5.transport_gate@1`; `q5.reconstruct_T@1` owning digest columns; `q5.seal_gate@1`; `q5.harvest_seal@1` RELEASE with real gate ancestry and nonempty `requires_decisions`) against the frozen oracles. BLOCKING TOPOLOGY (pinned executor): a failed string verdict does not block downstream execution by itself — `_blocked_by` blocks only consumers of an unavailable typed artifact and descendants of an execution-unreached node — so wherever execution MUST stop after a gate fails, that gate emits a typed `ArtifactOutput` that every kernel required to stop consumes as an `ArtifactInput` (transport gate → reconstruct_T and seal gate; seal gate → RELEASE), in addition to its owned string columns; owned-column dependencies alone never carry a stop. The frozen expected-state vocabulary distinguishes EXECUTED-FAILED (the gate ran and persisted its failed string outcome; its typed artifact is unavailable) from UNREACHED (a node blocked by an unavailable artifact or by an unreached ancestor); RELEASE must be UNREACHED, never executed with an evidence-tier failure, whenever the seal gate failed, even when its required decision is supplied | build agents; this session reviews | for every cohort document and every mutation and recovery case, the frozen fixture declares its EXPECTED STATE per node — executed, refused (typed refusal before any kernel runs, as when a required source is missing: executor.py:1922–1924), or unreached (a dependent of a failed gate) — and the assertions are scoped to it: for fixtures whose expected state is executed, cold-A and cold-B executions in fresh interpreters on distinct physical, verified-empty stores execute every kernel with hit=false for every node and agree on canonical stored payload bytes and independently recomputed payload hashes, and the unchanged warm run in a fresh interpreter executes zero kernels with hit=true for every node; for refusal fixtures, both cold runs and the warm run produce the identical typed refusal code before any kernel executes and store nothing; for executed-failed/unreached fixtures, the gate's failed string outcome is persisted identically on both cold runs, its typed artifact is unavailable, every artifact consumer and every descendant of an unreached node is UNREACHED on both (a RELEASE that executes after a seal-gate failure is a topology defect and fails the harness), and the warm run reproduces the same outcome and unreached set from cache without executing the failed gate's dependents; fixtures cover BOTH dependency types — an artifact-consumer blocked by an unavailable artifact, and a descendant blocked by an unreached ancestor — on cold and warm; any fixture whose observed states differ from its frozen expected states fails; legacy-schema comparison exact for every document and event under that event class's applicability map, with expected absences carried as ABSENT-BY-DISPOSITION markers and never as missing fields; graph-oracle comparison exact; every mutation-manifest entry and every recovery/failure-matrix entry and negative fixture behaves as frozen; `q5.decode_lane@1` and `q5.reconstruct_T@1` are total over declared bytes or all fallible parsing lives inside gate kernels | | A-3 | Shadow every canonical harvest, seal, void, repair, supersession and replication event after H through H2, beside the bespoke harvest, with no effect on it | orchestrator dispatch unchanged; build agents run the shadow | every post-H event is shadowed under the A-2 contract; shadow latency recorded; zero governing discrepancies, or the document is listed as a residue with its discrepancy class; no typed exceptions exist in Plan A | | A-4 | Shadow backfill of every event at or before H, including every void, supersession and replication disposition, each derived from the commit-pinned ruling bytes as a source and reported, not selected | build agents | per-document parity report under the A-2 contract; residue list; for doc_058 the report shows both 38eea5db57118df598e2f2ea1d68466df58d9e1ff1615de38d36c4609da2fef3 (board-ratified) and b3a36967ffdc934521c5459f7bf71b46cb5741f2a089a4a89b5d8d750de89fc8 (preserved superseded) with their board rulings, without choosing | -| A-5 | Closing census, parity report and Plan B proposal | this session | evidence-complete and parity states reported separately; A-1 through A-4 accepted under one TCB; the H2 census shows multiset equality with the authoritative record; the report records both occurrence-manifest digests (H and H2), the TCB and toolchain digests, per-document comparison results, residue classes, measured shadow latency and the store inventory; mixed-TCB results are rejected; Plan B is proposed only with that evidence attached | +| A-5 | Closing census, parity report and Plan B proposal | this session | evidence-complete and parity states reported separately; A-1 accepted at LIVE scope (a fixture-scope acceptance of A-1 does not satisfy A-5: the closing census cannot show multiset equality with the authoritative record while declared source classes are omitted) and A-2 through A-4 accepted, all under one TCB; the H2 census shows multiset equality with the authoritative record; the report records both occurrence-manifest digests (H and H2), the TCB and toolchain digests, per-document comparison results, residue classes, measured shadow latency and the store inventory; mixed-TCB results are rejected; Plan B is proposed only with that evidence attached | ## 5. Do-nots - Do not cut over, cite a manifest as authority, select an active record, append a pointer index, or retire, rewrite or delete any record. - Do not touch lane briefs, templates, harvest rules, ledger or library folds, `#405`, `e20-amend20`, `docs/design/` or the registry; no Plan-A write to e8-ops. - Do not wrap a lane as a kernel or let any kernel call an engine. -- Do not ingest raw lane `.out` or worktree outputs; only stable canonical staged bytes. +- Do not ingest raw lane `.out` or worktree outputs; only stable canonical staged bytes. This rule is PER CLASS and applies to lane-report alternatives: it must be enforced by the source class of the declaration, never by a global path-fragment blacklist, because briefs and rubric notes — declared classes — live under `thu-briefs/` and `opus-scratch/`. - Do not run any cache lookup before the attester passes; do not reuse cache across TCB namespaces; do not use `resume=forbid` on a shared store; do not aggregate results from different TCBs; do not change any toolchain component without a full rerun. - Do not hand-edit or silently repair a shadow object; only matrix-defined deterministic recovery is allowed. - Do not copy or replicate any source before its access, license, destination, backup, retention, access-control and encryption checks pass. - Do not put the store in e8-ops or rely on `/tmp`. - Do not declare `Tolerance` or entrants (#847) in Plan A. -- Do not consume codex lane-hours for build work; do not run build agents on the orchestrator's lane. +- Do not consume codex lane-hours for build work except the single upward-routed A-1/A-2 lane that A5 permits, and never while a ceremony step is running on that home; do not run build agents on the orchestrator's lane. - Do not widen any Opus role beyond Max's cited authorization without a completed digest-bound calibration report. - Do not call any shadow result evidence, or use it in any Plan B argument, before A2a and the per-event-type schemas are met. - Do not choose H2 retroactively or shorten the live window below the frozen rule. diff --git a/docs/population-graph.md b/docs/population-graph.md new file mode 100644 index 00000000..92bc30dd --- /dev/null +++ b/docs/population-graph.md @@ -0,0 +1,139 @@ +# First population graph: mortality and ageing + +The optional `populace_dynamics.graph` package fits the existing M6 mortality +model on a historical synthetic panel, applies the fitted artifact to a +separate starting population, and adds surviving observations for the next +year. It produces an engineering report and a content-addressed execution +manifest. It does not change the existing projection loop, candidate +registries, scientific gates, or committed evidence. + +## Dependencies and execution + +The graph example requires Python **3.13 or 3.14**, NumPy 2+, pandas 2.3+, +and the Microcosm graph/frame revisions containing typed artifact edges and +`microcosm.graph.randomness.keyed_uniform`. The default Dynamics installation +continues to support Python 3.10–3.14 without importing Microcosm. The entry +point checks capabilities and gives installation guidance when they are absent. + +The `graph` extra pins both graph and Frame to core commit +`3ff92b0aea14407d09479bff5623dc7d1a92d008`. An older package merely sharing +the version number `0.1.0` is insufficient. In an isolated Python 3.13 or 3.14 +environment, install with `uv pip install '.[graph]'`. The core change must be +reviewed before this dependent integration is released; replace the Git pins +with a compatible published release when one exists. CI installs this exact +extra, refuses missing capabilities, and runs the integration on both supported +Python versions. Do not modify the existing scientific gate environment. No +rules engine, restricted microdata, or optional forest fitter is needed here. + +Run from that environment, choosing an output directory: + +```sh +python -m populace_dynamics.graph --synthetic --output-dir ./mortality-example +``` + +The command creates small synthetic inputs under `mortality-example/inputs`. +It preserves existing input files so that edits can test cache invalidation. +Repeated execution reuses the verified store under `mortality-example/store`. +The report, manifest, fitted JSON model, entity tables, and next-period slice +are written inside the chosen output directory. A failed engineering or +fixture verdict exits nonzero while retaining the diagnostics. + +If malformed held-out input prevents evaluation, the executor records a failed +`evaluate` gate. The command exports that manifest and a diagnostic report with +the gate's name, outcome and original exception evidence on both cold and warm +runs, then exits nonzero. The report marks `engineering_verdict` and +`fixture_verdict` as `not_evaluated`; it does not invent evaluated deaths or +mass metrics. Successful evaluation retains the existing report fields. + +Four explicit source paths can replace the generated inputs: + +```sh +python -m populace_dynamics.graph \ + --training ./inputs/training.json --rates ./inputs/rates.json \ + --initial ./inputs/initial.json --holdout ./inputs/holdout.json \ + --boundary-year 2014 --external-vintage-year 2014 \ + --experiment-id comparison-a --replicate 0 --base-seed 0 \ + --output-dir ./mortality-example +``` + +These inputs still exercise the synthetic engineering contract. The example +does not confer validity on a real-population projection. Source JSON rejects +duplicate members and nonfinite values. Each source is declared separately; +holdout bytes are available only to evaluation. Domain kernels read the +content-verified JSON directly: the registered source marker deliberately +does not pretend an external rate table or a holdout report is a population. + +## Executable ownership + +The graph has two CREATE roots, each carrying a `person_period` observation +entity and `person` and `period` groups. `person` retains stable identities; +`period.period` is the immutable mass-partition label. The training root +contains exposure records; the initial root contains recipients. Their only +connection is the explicitly typed mortality-model artifact. + +The fit node calls `prepare_mortality_refit_inputs` and +`fit_mortality_model`. Event year, required interview year, and declared +external vintage retain the existing cutoff checks. The JSON model contains +validated contiguous age bands, sex-specific probabilities, fit boundary, +external vintage, and retained row count. The manifest binds its producer +to source identities and implementation digests. The fitter's external-rate +factor cancels in its fitted-window level, so this is not evidence of +independent external calibration. + +Application calls `apply_mortality` with a graph-specific context. Every +uniform is keyed by the original person identity, process, year, and draw +index under the chosen experiment/replicate/seed. It does not use the legacy +ID-sorted ordinal registry. Reordering, splitting, or adding unrelated +people preserves the existing people's draws. Fit and application declare +platform-specific bitwise numeric behavior conservatively; cross-platform +equality is not claimed. + +EXPAND calls `advance_age` on survivors, adds their next-period observations +with lineage to the original observations, and attaches them to one newly +admitted period group. A same-version rewrite node claims the materialized +age values. Historical ages and memberships remain unchanged. The temporary +`year` returned by `advance_age` is never written over a carried observation +column. No new person, birth, or immigrant is implied by admission of the +period group. + +Typed person-period weights are the single authority. Every survivor carries +the same trajectory weight into the new period. The declared mass receipt +shows historical mass unchanged and new-period mass equal to surviving +weight. Total stored observation mass therefore grows by the additional +period. If everyone dies, the graph adds no observations and no orphan period +group; the report explicitly records next-period mass zero. + +## Evaluation and limits + +The report separates `engineering_verdict` (survivor/age parity and population +structure) from `fixture_verdict` (the independently sourced synthetic +death-rate and age expectations). It records weighted expected, observed, +and generated deaths, row counts, period mass, node/model identities, and +cache reuse. The fixture death-rate tolerance is an input named +`fixture_max_abs_death_rate_gap`; it is not a scientific acceptance threshold. +Changing all held-out outcomes to deaths fails that fixture check while +leaving fitting, application, draws, and accounting unchanged. + +Household accounting is explicitly unsupported and refused by the Python +entry point. Household weight sharing, marriage, births, immigration, +alignment replay, repeated years, and the full M6 loop remain later work. +No certified data release or scientific candidate is produced by this graph. + +## Tests + +```sh +python -m pytest -q tests/test_graph_mortality.py \ + tests/test_m6_engine_refit.py tests/test_m6_engine_steps.py +``` + +The integration tests cover direct execution with an independently injected +uniform vector, JSON validation, cutoff and holdout isolation, fitted-artifact +reuse, changed fitting weights, row/chunk/person invariance, cold/warm stores, +and zero/all-survivor expansion. Independent graph-to-direct model parity and +future-event/late-interview mutations protect cutoff mapping. Nondefault +experiment, replicate and seed tests independently derive draw coordinates and +check direct-step parity, fit reuse and changed application identity. They skip +the optional runtime cases when the required core capabilities are unavailable; +the JSON and dependency +boundary tests still run. Importing `populace_dynamics.graph` remains safe +under Python 3.10–3.12. diff --git a/docs/progress.md b/docs/progress.md index 4be0e24e..1cdcb1cc 100644 --- a/docs/progress.md +++ b/docs/progress.md @@ -2,7 +2,7 @@ # Development progress -*Snapshot as of 2026-09-04T18:30:00Z (UTC); updated at least daily while the evidence campaign is active. The data behind this page is committed at [`docs/progress/progress.json`](progress/progress.json).* +*Snapshot as of 2026-09-09T11:51Z (UTC); updated at least daily while the evidence campaign is active. The data behind this page is committed at [`docs/progress/progress.json`](progress/progress.json).* Dynamics builds Social Security earnings histories from the PSID, and the corrected covered-earnings series is only as credible as the reading of the source documentation beneath it. Before that series ships, every piece of documentation the construction relies on is independently verified by the three arms below; the design is pre-registered and ratifies only when they complete. (Internally: the evidence campaign behind Amendment 20 (two-arm evidence charter for the covered-earnings correction design), under review in PR #405, open; draft ratified through round 6.2; next step, the A4 evidence freeze, then the C20 ratification chain toward revision 22.) @@ -27,14 +27,14 @@ The foundations below are complete and in the repository today; the campaign tra align-items: baseline;"> Prompt-purpose census - 18,838 / 20,815 · 90.5% + 20,815 / 20,815 · 100.0%
+ width: 100.0%;">
-
prompts adopted (contiguous R1–R18838). +
prompts adopted (contiguous R1–R20815 — ARM COMPLETE, terminal fold v1.97). Establishing what each of the ~21,000 remaining items in the PSID's 1968–2023 questionnaires and codebooks is actually asking, so the model only uses variables whose meaning has been adjudicated rather than assumed. Each block runs a full pass, an independent dense audit, and a correction sweep before its entries are adopted into the governing ledger.
@@ -61,14 +61,14 @@ The foundations below are complete and in the repository today; the campaign tra align-items: baseline;"> Q5 semantic annotation - 54 / 257 · 21.0% + 63 / 257 · 24.5%
+ width: 24.5%;">
-
documents sealed. +
documents sealed — 63 of the 81 annotated questionnaires (77.8%); the 257 denominator is the fixed PSID source-documentation domain: every survey instrument is annotated, and the other 176 are non-instrument files (setups, codebooks, raw data), 43 of them classed Semantic N/A by the coverage audit (not a law term). Verifying which survey questions connect to which data fields, document by document across the PSID questionnaire corpus: two independent annotators per document, then a reconciliation that seals its table of record.
@@ -129,13 +129,13 @@ The three drill-down tables below decompose each arm's progress bar row by row. | fam2013 complete + q2013 BCDE/G (final-tranche block 1) | R17,889–R18,236 | 348 | adopted | 348/348 regenerated (sweep) | pass 348/348 -> dense audit NOT CALIBRATED both limbs (59-row manifest, seals regenerate) -> regeneration sweep 348/348 byte-exact envelope; folded v1.87 | 2026-09-04 | | q2013 R/P/KL + fam2015 complete (final-tranche block 2) | R18,237–R18,526 | 290 | adopted | APPLY 217 / UNCOVERED 73 | pass recovered byte-exact from durable scratch -> dense audit CALIBRATED 99.37% sample / 99.66% full (one-row manifest R18462); folded v1.88 | 2026-09-04 | | q2015 BCDE/G/R/P (final-tranche block 3) | R18,527–R18,838 | 312 | adopted | — | pass 312/312 -> dense audit CALIBRATED 100% sample and full vector (empty contradiction set; 24/24 receipts byte-exact); folded v1.89 | 2026-09-04 | -| q2015 KL + q2017 complete (final-tranche block 4) | R18,839–R19,111 | 273 | pending | — | — | — | -| q2019 complete + q2021 A (final-tranche block 5) | R19,112–R19,384 | 273 | pending | — | — | — | -| q2021 BC (final-tranche block 6) | R19,385–R19,749 | 365 | pending | — | — | — | -| q2021 G (final-tranche block 7) | R19,750–R20,006 | 257 | pending | — | — | — | -| q2021 P/KL/IMMIG (final-tranche block 8) | R20,007–R20,311 | 305 | pending | — | — | — | -| q2021 ADDRPAYMENT + q2023 BC (final-tranche block 9) | R20,312–R20,618 | 307 | pending | — | — | — | -| q2023 G/P/KL (final-tranche block 10) | R20,619–R20,815 | 197 | pending | — | — | — | +| q2015 KL + q2017 complete (final-tranche block 4) | R18,839–R19,111 | 273 | adopted | — | pass 273/273 -> dense audit CALIBRATED (sampled 144/144; full-vector replay 273/273; row contradictions 0; family concentration none); folded v1.90 | 2026-09-04 | +| q2019 complete + q2021 A (final-tranche block 5) | R19,112–R19,384 | 273 | adopted | — | pass staged -> dense audit CALIBRATED 100% (sample 144/144; 273/273 rows examined); folded v1.91 (terminal identity 317,298 B, SHA 014aeb13…) | 2026-09-04 | +| q2021 BC (final-tranche block 6) | R19,385–R19,749 | 365 | adopted | APPLY 267 / UNCOVERED 98 (12 proposal entries) | pass 365/365 -> dense audit CALIBRATED with a closed three-row manifest (R19623–R19625, BC60A over-fire -> ALL.DIRECT_EXPOSURE; eight of nine charges refuted; EHC recall 25/25); folded v1.92 | 2026-09-05 | +| q2021 G (final-tranche block 7) | R19,750–R20,006 | 257 | adopted | APPLY 203 / UNCOVERED 54 (8 proposal entries) | pass 257/257 -> dense audit CALIBRATED 163/163, zero-row manifest (fifth consecutive calibrated audit); EHC fired 18 times and was upheld 18/18 on precision and recall; folded v1.93 | 2026-09-05 | +| q2021 P/KL/IMMIG (final-tranche block 8) | R20,007–R20,311 | 305 | adopted | 305 ranks in two passes: R20007–R20247 APPLY 153 / UNCOVERED 88; R20248–R20311 APPLY 27 / UNCOVERED 37 | q2021 §P audit (136/241 sampled = 56.4%; manifest rebuilt in both directions with zero discrepancies; era discipline clean) folded v1.94 under a 'block closed' heading that covered only the §P part — an orchestrator scope error (brief cut from the per-section table); the KL+IMMIG remainder (64 ranks) ran as an 8b completion pass, audited CALIBRATED 64/64 = 100.00% with zero corrections, and folded v1.95 | 2026-09-05 | +| q2021 ADDRPAYMENT + q2023 BC (final-tranche block 9) | R20,312–R20,618 | 307 | adopted | APPLY 270 / UNCOVERED 37 (closes q2021 incl. ADDRPAYMENT; opens q2023 §BC) | pass 307/307 -> dense audit CALIBRATED: 161/307 sampled (52.4%), agreement 161/161 = 100.00%, correction manifest empty of rows (three documentary items); adopted as issued; folded v1.96 (99.05%). One 63-character rubric digest in the pass's assertion object traced to the dispatch brief -> standing correction adopted (recompute every emitted digest from the named file) | 2026-09-05 | +| q2023 G/P/KL (final-tranche block 10) | R20,619–R20,815 | 197 | adopted | APPLY 169 / UNCOVERED 28 (G 156, P 17, KL 24; by_construction_exclusions = [Q21822]); the corpus closes at R20815 | pass 197/197 (APPLY 169 / UNCOVERED 28) -> dense audit CALIBRATED, reported as 102/102 = 100% (51.8% sampled) with 101 distinct ranks (one rank counted twice; 51.3% of ranks); DISAGREE set empty; correction manifest empty of rows and items; charge 9 confirmed the Poppler pdftotext 26.04.0 pin and argument equivalence over all 523 pages of q2023; charge 10 upheld the pass's form-feed frame-recipe correction with no row moved; charge 8 swept all 20,815 index lines end to end with zero gaps or duplicates; the ninth consecutive CALIBRATED block; folded v1.97 (379,534 B / ebe3d2b9…) — the corpus closes at R20815 | 2026-09-05 | ### A3 classification, block by block @@ -255,6 +255,15 @@ The three drill-down tables below decompose each arm's progress bar row by row. | 2007 family questionnaire QxQ companion (fam2007_QxQs.pdf) | 952 | RECONCILED — lane B prevails (917/952 as submitted; common-mode 7/389) | 2026-09-04 | | 2007 family questionnaire (q2007.pdf) | 1,605 | lane B adopted unmodified as the table of record (1,605/1,605) | 2026-09-04 | | 2009 family questionnaire QxQ companion (fam2009_QxQs.pdf) | 171 | both lanes rejected; the reconciled table stands (common-mode 5/171 = 2.9%, the highest yet) | 2026-09-04 | +| 2009 family questionnaire (q2009.pdf) | 2,282 | lane A's coordinate frame upheld as the base; lane B refuted on its central doctrine (2,509 minted node ids) | 2026-09-04 | +| 2013 family questionnaire QxQ companion (fam2013_QxQs.pdf) | 1,320 | lane B substantially upheld (551 of 610 contested rows) and adopted as the base with 82 corrections; lane A refuted; both refuted on the aggregate class | 2026-09-04 | +| 2019 family questionnaire (q2019.pdf) | 1,882 | each lane reversed on exactly one axis and upheld on the other (133 + 113 = the 246 contested rows); zero minted identifiers; common-mode 0/1,636 | 2026-09-04 | +| 1968 family questionnaire QxQ companion (fam1968_QxQs.pdf) | 156 | adjudication of the contested set: 38 rows with both lanes upheld, 69 lane A upheld, 19 lane B upheld; 156 rows = seal; zero minted identifiers | 2026-09-04 | +| 1969 family questionnaire QxQ companion (fam1969_QxQs.pdf) | 159 | lane A with exactly 7 row replacements after an 8-block adjudication (confirmed common-mode 0/85 by the per-row method; three exposures recorded on the seal) | 2026-09-05 | +| 1976 family questionnaire QxQ companion (fam1976_QxQs.pdf) | 385 | all 224 agreed rows audited in three strata and upheld (common-mode 0/224, or 5/224 under the stricter reading); reading-invariant, no Q5-status pass asserted | 2026-09-05 | +| 1979 family questionnaire QxQ companion (fam1979_QxQs.pdf) | 473 | revision 2 after two of its four verdicts were adversarially overturned; reading-invariant with three exposures recorded; survived the adversarial seal review | 2026-09-05 | +| 1981 family questionnaire QxQ companion (fam1981_QxQs.pdf) | 449 | revision 2 (six rows and one count corrected under a declared adversarial pass); both lanes and the table write zero component-side sentinels; survived the adversarial seal review | 2026-09-05 | +| 1977 family questionnaire QxQ companion (fam1977_QxQs.pdf) | 432 | revision 2; states the containment ground independently (the strongest of the four seals under adversarial review) | 2026-09-05 | ## Development scorecard @@ -311,6 +320,19 @@ From the pre-registered timeline ledger, entry 22 (registered 2026-09-03). Dates ## Recent milestones +- **2026-09-09** — Morning. Max's routing ruling arrived through his chief-of-staff session at half past four yesterday afternoon and is now in force: no Opus, Sonnet or Haiku turn on any Claude account that still has Fable headroom, because those turns draw the account's shared weekly bucket and four accounts were capped that way before their Fable was touched. The orchestrator switched to Fable 5.1 in the same minute and recorded the ruling on the board; every dispatch since has been Fable on a Claude lane or Astra on a Codex lane. The night was a quiet hold on mains power with nothing in flight and the three ruling packages unchanged. A-3 shadow pass 6 was the day's first decision, and it was run the way the pass-5 adjudication said passes must be run. The seat prepared the worktree and the brief before dawn, with a deltas section carrying the pass-5 acceptance, the pin discipline, the store-write proof, the staging rule, the four failure classes and Max's ruling. At thirteen past six it asked the orchestrator for a Fable slot on the one Claude lane with headroom; the orchestrator read the lane's state at that moment rather than an hour earlier, content-asserted the brief and the worktree from bytes, named a foreign run already on the lane, and granted. The seat dry-ran the dispatch twice first, which surfaced a trap the record now calls D15: the plain Fable form of the dispatcher resolves to a read-only sandbox that would have censored the store write on a host that could reach the store, exactly as the Codex sandbox did in pass 5. Store capability means the host and the sandbox mode, and the dry-run is now a required step. The pass landed in nineteen minutes with four commits, inside its thirty-minute bound, and is accepted. It took its own coordinate from the live board, proved that the seventy-two events shadowed through pass 5b reproduce exactly, and then shadowed forty-one new ones: thirty-four harvests, four repairs and three supersessions, among them the orchestrator's own rotation, the kit's r10 install and the ruling that voided the old standing order. One hundred and twenty-nine A-2 executions ran with no typed exception and no governing discrepancy; custody measured eighty-six source occurrences and copied none, because no new source has a row in the recorded table. The cumulative count is one hundred and thirteen events across four classes, and H2 remains unreachable before the evening of September 12, when the seven-day floor is met. The lane's first report claimed it had overrun the bound; it had estimated elapsed time from its own turns instead of reading the clock, caught the error against the commit timestamps, and corrected the report in a fourth commit that touched nothing else. None of yesterday's four failure classes occurred. The record gained a fifth class an hour later. The farness lane's five-hour window, shared by the pass, the foreign run and the orchestrator's own turns, hit its limit as the orchestrator received the acceptance relay, and its seven o'clock heartbeat failed the same way. It resumes automatically at eleven. A limit modal is not a death, so the seat neither rotated it nor spent credits; it holds, runs the orchestrator's read-only safety instrument on the hour in the meantime, with every axis unchanged, and will relay the acceptance again once the orchestrator is turning. Still with Max: the three ruling packages for gate 3, mortality, and claiming and PIA, and the rotation of the exposed policyengine.org token. +- **2026-09-08** — Afternoon. The claiming and PIA gates closed the lock discipline to a ruling package in one day. The adversarial referee round on the bound thresholds returned two NOT LOCK-READY verdicts with concrete findings: a mislabelled reading of one rule, a single-rule pass under one grammar, an incomplete flip plan, a footer defect, and a run of contract items. One fixes sitting answered every finding without moving a floor number, and an independent refute-first verification confirmed each fix from bytes, corrected one count in the fixes report from five to three, and settled the flip-time disposition. The ruling package went to Max at 11:16 with seven claiming decisions, five PIA decisions, and the reading of the rule-count definition; the three packages now with him, gate 3, mortality, and claiming and PIA, are the campaign's critical path. Flip briefs for all three are templated and take nothing but his written words. The C20 kit's r10 repair passed its review on a scratch clone, with the receipt gate exercised and the fallback unused, and was installed on the orchestrator's concurrence with one standing requirement: renders made under earlier kit versions must be re-rendered, and any clone used for verification must be a same-filesystem wholesale-object clone, because only the S-prime object is unreferenced and a pack-transfer clone drops it. That custody correction reversed a premise both seats had carried for a day. A-3 shadow pass 5 was dispatched to Astra when no Opus lane answered its clock, and the pass was censored: the Codex sandbox cannot reach the shadow store, so the copy step could not run. The plan owner adjudicated the dispatch as its own error and fixed the rule: an A-3 pass runs only on a host whose writable roots include the store, and a clock with no such host slips and records the slip. Pass 5b completed the pass on a Claude host at the same coordinate. Its lane finished the substance, then died on a DNS error before committing, and a continuation verified the salvage byte for byte and committed it. Accepted: all twenty-nine events shadowed, ninety-three A-2 executions, zero governing discrepancies, cumulative seventy-two events complete; H2 stays not reachable before the evening of September 12. The machine spent the afternoon on battery. No adapter was detected until shortly after three, the lid closed at 14:41 with two percent left and the box slept five minutes, and nothing was lost: everything was committed, and the seat and the orchestrator agreed beforehand that wall-clock silence on battery is uninformative, so no stall or death call may rest on time alone. The unpushed A-3 shadow branch now has a second on-disk copy as a git bundle, proved to restore by reconstruction, without breaching the shadow-only rule. The orchestrator wrote a complete handover line at 14:06 at fourteen percent and, once power returned, asked to be rotated deliberately at eighty-five percent of its transcript trigger rather than die mid-task; its successor proved the safety baseline on arrival. The rotation surfaced a hygiene finding: the launcher passed the lane token on the tmux command line, and because tmux keeps the first client's arguments as its server's, the shared server has shown a token in the process list since September 6. Both launchers now fetch the token inside the pane, the successor's launch added nothing to any command line, a sweep of every running process found no other credential, and the rotation of the exposed token is with Max. The shift's failures are filed in four classes, Fable weekly caps, a DNS outage, machine sleep, and a limit modal that read as death, and only the first is capacity. The box went back to battery at 15:57, which is why this refresh ran at four rather than eight. +- **2026-09-08** — Morning. The claiming and PIA gates went through three steps of the lock discipline overnight. Floors v2 answered both adversarial referees in one commit with every alternative emitted rather than chosen: three revision-sd grammars at three K values, a K-by-rounding sweep, settle-age strata with horizon-aware pricing, an OLS window sweep that shows the knife-edge cell is a property of the scanned window grid, and a stated decimal rounding mode with its one exact tie and thirteen-cell knife-edge class. No drafted tolerance or partition count moved; six open decisions are filed for the ruling, and a seventh packet correction records that the structural demotion of the knife-edge cell is unavailable as written because the transition it cites begins a year later. An independent verification then confirmed every fix from bytes under its own code, adjudicated the one referee number that had not reproduced in the statistical referee's favour, and found one new low finding, a fourth committed channel carrying the held-out rows. The threshold-binding sitting followed: two gate-schema artifacts derived from the floors by path with their digests pinned, every filed alternative's partition and tolerances carried beside the draft so a ruling changes a pointer rather than a number, LOCKED-HOT bindings that recompute all forty-two tolerances from the floor artifact, and both DRAFT gate blocks carried inside the artifacts as strings with a placeholder for every ruling. Under the definition's letter the PIA partition awards exactly one rule all three conditions. The seat reproduced the tests. Then the capacity ran out. Every Fable lane hit its weekly limit by three in the morning, and the adversarial referee round on the bound thresholds died on the cap. A retry loop burned eighteen empty dispatches before the orchestrator measured why: a one-line probe answers on a lane whose per-model budget a full brief exhausts. The dispatcher now demotes a lane after a failed dispatch, and the two referees are running on the one Fable lane still open. The seat itself stopped a third time at 03:14: its own scheduled wake did not fire, and the app paused the session two hours later; the cause is undetermined. A one-shot revival at 05:31 repaired the loop and relaunched the seat on Opus as an interim, which found Astra back, dispatched the kit's r9 two-engine gate, and then hit its own weekly cap; Max resumed the Fable seat at 08:12. The r9 gate failed the candidate with concrete repairs, and an r10 repair draft is running on Astra with the orchestrator's concurrence step retained. The orchestrator hit its weekly cap the evening before and was relaunched at midnight on a fresh lane by a daemon armed for it. A-3 shadow pass 5 runs on Astra from a pre-created worktree because no Opus lane answered its clock. Gate 3 and the mortality gate remain LOCK-READY and wait on Max's rulings; H2 stays not yet reachable before September 12. +- **2026-09-07** — Evening. Gate 3 is ratified LOCK-READY. The third independent verification returned VERIFIED with 24,979 non-string leaves identical across the fix and the two ratifying referees, one statistical and one contract, both returned RATIFY LOCK-READY and agreed on all four decisions: cure A for the description, rho at 0.75, lock the 43-pair tranche under exactly its own description, and the candidate-side module change as a registration precondition. The six typed corrections they conditioned on are applied report-only and verified. The one thing left is the ruling itself, which is Max's; the flip commit is drafted so it dispatches the moment he answers, with the pre-lock guards inverted and nothing rebuilt. The mortality gate moved through the whole discipline in one afternoon. Floors v3 answered the pre-registration packet's three data-side blockers from bytes, the weight universe, the death-ascertainment convention and the censoring convention, with the floor rebuilt on the declared universe beside the all-window rebuild. Two adversarial referees followed: the statistical one said LOCK-READY with three findings, the contract one said NOT LOCK-READY narrowly on the record, chiefly that the tests froze the live gates.yaml so the gate block's own commit would have turned them red. Floors v4 fixed both lists in one commit with no floor number moving, an independent verification returned VERIFIED, and the threshold-binding sitting is running now: the gate-schema artifact, the anchor's operating characteristic, the draw stream, the teeth table, and a draft gate block carried inside the artifact by digest. The seat itself stopped for eight hours overnight, and the record says why: a completion notice pushed into an idle seat produced no turn, and the app paused the seat fifteen minutes later. The fix has three layers, a self-wake under the pause, a launchd daemon that resumes the session when no process exists, and lane chains that dispatch their own follow-ons. Two more stop mechanisms were found in daylight: the Mac slept on battery until Max plugged it in, now held awake by caffeinate, and every Claude lane hit its weekly limit by mid-morning, leaving one session window as the day's only capacity. Subfleet's non-build tasks were also found to run without a shell by default; the referees were re-run with a writable sandbox. A-3 shadow pass 4 accepted on recomputed bytes: fifteen new events, forty-three cumulative, zero disagreeing receipts; H2 stays not yet reachable before September 12. +- **2026-09-07** — Morning. The ceremony seat stopped for eight hours overnight, and the record says exactly why: at 21:40 the closing sitting's completion notice was pushed into the seat's inbox and produced no turn, and at 21:49 the desktop app paused the idle seat after its fifteen-minute timer; nothing ran until 05:43. Two mechanisms, both read from the notify code and the app's own log, neither a usage limit. The fix has three layers: the seat now wakes itself every thirteen minutes, under the app's pause; a launchd daemon resumes the session as a one-shot turn whenever no live process exists and hourly as a heartbeat, on a lane it probes first (the top-ranked lane answered 'weekly limit hit' in the test); and the lane chains dispatch their own follow-ons, so a landing no longer waits for the seat. Gate 3. The fourth sitting closed the v3 report (332 KB, 33 of 33 digests matching disk, 52 tests). The second independent verification returned NOT VERIFIED with every computed number reproduced by builder-free code, 5,940 of 5,940 panel deviations, 43 of 43 gated rows and 65 of 65 demotions among them, and three blockers all in typed sentences: two false self-descriptions in the committed artifact, the report printing the superseded fragment texts, and a stale test-tier manifest. Floors v4 answered exactly those in one commit (72069f6, five files); the third verification is running and the two ratifying referees are chained behind it. A-3 shadow pass 4 accepted from recomputed bytes: fifteen new events, forty-three cumulative, sixty-two receipts with zero disagreeing, one recorded retry that exposed a pin discipline (pin your own contribution, never the ledger's totals). H2 stays not yet reachable; the seven-day floor alone binds, earliest 2026-09-12 20:16 EDT. Lane capacity is now the binding constraint rather than the seat: the pass-4 dispatch died on three hard-limited lanes in a row before a fourth carried it, and one non-reserved Opus lane was open at 06:47. +- **2026-09-06** — Evening. The capability track ran all afternoon on separate lanes beside the correction campaign, and eight shadow branches now sit unpushed and PR-ready, every one with the gate contract byte-identical and its own reproduction tests reproduced by the plan owner: gate-3 retrodiction floors (v1, then v2 answering two adversarial referees, then an independent verification that closed the referees' blockers by recomputation and opened seven new ones, with v3 in progress); mortality floors v2 (a hundred seeds, the estimand restated as the 1997-onward window); the claiming publication floor and the benefit-oracle rule-coverage artifact; the three registered battery families published with every gate-1 run (full suite 5,705 passed); the report-only entrant schedule with the Trustees net-immigration table captured and hash-pinned; and revision 5 of the immigration design, folding a sixteen-decision referee round. What the lock discipline found on gate 3 is the point of having it: the claiming module's default reference was the gate's own resolution table, a textbook drift rule passed 63 percent of the draft surface, no pass rule was stated, and, after v2 fixed all three, an unregistered quadratic-trend rule still passed 23 of 88 gated pairs; the gated set shrinks with each cycle and that is the correct direction. The immigration path has a measured fork for Max: the certified-frame donor is 1,155 rows in 562 households with an effective sample of 82, too thin to floor a gate, so a gated module needs the ACS donor or a coarser surface, while the report-only schedule can ship. Two decisions were re-priced by measurement. The Q5 package: an adversarial pass over the revision candidate's fourteen folded ranges found nine defects, three material, so the memo's own rule flips the recommendation to shipping the package of record with the errata and the line-1188 retraction beside it. The ceremony: an attempt candidate without the disputed precommit exception passes the pinned battery at both states, the oracle and the kit's freeze gate, so ruling (i)'s premise does not hold and the fresh round waits on kit r9's gate, the R12/R31 ruling and Astra's return on 12 September. A-3 shadow pass 3 accepted (28 cumulative events; the twelve-event H2 condition met; the seven-day floor alone remains). The seat itself was revived three times today by subfleet onto lane hosts and retired its own duplicates twice; the host ruling stays open. +- **2026-09-06** — Midday. Max asked for the forecast to DynaSim-class capability and what shortens it, and two lane investigations answered from the record rather than from memory. Capability, as the gate ledger states it: eight pre-registered gates, seven locked, every locked gate with a registered first pass (earnings histories 7 July; marital and fertility 9 July; household composition; marriage x earnings; disability; CPS-frame transport; the temporal-holdout projection engine 24 July), 48 one-shot candidate runs on the record. Only gate 3, forward projection, has never been locked and has no thresholds. Against DynaSim the deficits are: no forward mode past 2022; the model's earnings are not covered earnings and there is no SSA calibration or OACT alignment (the covered-earnings correction exists to fix the first); no immigration, education or hours modules; mortality floored but not certifiable on the current surface; the benefit oracle and claiming module built and SSA-exact but ungated. Forecast, measured: capability commits over the last six weeks are zero of 140 (57% evidence and ceremony, 19% dashboard), and scored-run cadence is zero since 25 July after 77 artifacts in the 20-day July sprint; no date is derivable from a rate of zero. The only dated plan, the 16 July claim of DynaSim parity in four to seven weeks, expired between 13 August and 3 September; the full-parity milestone (immigration and alignment) is open and past its p80. The binding mechanism is gate 3's tier-2 arm, which resolves against future annual SSA publications, so its clock cannot start until thresholds are locked; its retrodiction arm needs no publication and is unused. Under redirected capacity the five already-floored links are a 6 to 14 lane-week program; a scored Stage-3 exit is not before Q1 2027. The two top levers cost no lane-weeks because they are decisions: lock gate 3 on the retrodiction arm first, and rule on the graph-port plan's never-ratified D3 and D6 rules that sequence the benefit gate behind the correction's publication (p50 21 October). In motion since this morning: the gate-3 retrodiction floor build, the three registered-but-unpublished battery families, pre-registration packets for the claiming, benefit-oracle and mortality gates, and an immigration/entrant design memo, all on separate lanes beside the correction campaign. Also today: A-3 shadow pass 2 accepted (eleven post-H events, the A-2 contract executed over post-H bytes with the plan owner's per-source custody addendum; H2 not before 12 September); and the decision package's line 1188, which asserts that no report among the 29 declares an agent count, is measured false (at least seven declare, by file and line) and needs a retraction rather than the errata's one-word fix, which re-prices the revise-versus-errata decision. The morning's outage attribution to a usage limit was retracted: the 22:57 kill was the desktop app's own auto-update; the 23:06 'death' was a one-shot revive exiting normally. +- **2026-09-06** — Morning after the halted ceremony. Nothing ceremony-related moved overnight and nothing will until three things are ruled on: how the scratch-state battery requirement and candidate-HEAD authentication should coexist (both referees rejected the re-freeze's reconciliation as an unauthorized precommit exception), whether the two substituted sentences are restored or re-authored, and a kit round that deposits durable evidence of scratch cleanliness; the Codex home that runs the ceremony's Astra steps is at its weekly cap until 12 September, so no re-run is possible before then in any case. Two corrections to what the decision package tells Max were settled from bytes: the seven-sealed-document itemisation omits one document (a defect the errata already records as its first entry and that four instruments had re-derived independently without any of them updating the package), and the errata's own Part 2 summary says four decisions change position where its tables and enumeration show nine and eight — its detail is right, its headline is not. The structural question that follows is Max's: whether the package gets a revision folding the errata's twenty-seven defects, or ships as-is with the errata beside it. Graph side, all shadow-only and never pushed: plan revision 12 (two acceptance scopes for A-1, a per-class staged rule, a kernel totality contract, and a live-scope requirement for the closing census) was agreed by the peer gate with no findings and merged; the cohort was frozen at an immutable board coordinate with a 4,262-entry content-hashed manifest (5.55 GB across the seven declared source classes, every digest recomputed), the A4 storage record was filled from existing local settings with gated PSID documentation ingested by identity only because the host's disk is not encrypted, and the H2 stopping rule was frozen before any A-3 run (earliest H2 the evening of 12 September). A-1 accepted at LIVE scope on run 4's head 3bb53ad: an independent refute-first pass reproduced all eleven clause verdicts with nothing refuted and seven report-level qualifications recorded; run 4's third declaration rule partitioned the 29 staged-report copies 10/11/8 exactly as predicted; the plan owner reproduced its 212-test suite. Run 4's delta was accepted on reading and reproduction, not on a second adversarial pass. A-3 pass 1 landed and was accepted as one bounded pass (run 20260906-062442, 19 minutes): 2 post-H harvest events shadowed identity-only with latency recorded, zero discrepancies against bytes, H2 not yet reachable (earliest 2026-09-12). Nothing was copied into the shadow store: the plan's A4 clause requires the custody checks to be recorded per source before any copy, and the recorded table closed at H, so the pass refused every post-H copy and reported the six sources by identity. The plan owner records the post-H custody rows as a dated addendum before pass 2, which then copies the admissible sources and runs the A-2 contract over them; no plan amendment is needed for custody. Two frozen-enumerator defects (a void pattern without a negation guard; a harvest stated in words that only a creation-receipt sweep caught) are recorded for the next revision. Passes continue daily until H2. Purpose stays at 100.00%; Q5 at 63/257; the twelve decisions remain with Max. +- **2026-09-05** — Late evening: the Amendment 20 closure ceremony ran to its referee step and stopped there. After the re-frozen candidate C′ (820da19) passed the scratch transition S′ (battery 220/220), the executed-transition receipt (kit gate ready 11 of 11) and a first-add on the ceremony branch, both independent closure referees withheld ratification and wrote no verdict. Both rest on the same two blocking grounds: the re-freeze's test change, which made the battery hold at the scratch state, reads as a precommit index-authentication exception the parent design does not authorize; and the repository holds no independent evidence that the scratch worktree was clean at S′, a gap in the ceremony's own evidence design. Both referees also found the candidate's replacements for the parent's status-meaning and exact-failure-control sentences not proposition-equivalent; one treats that as blocking, the other does not. Closure and the revision-22 repin were therefore not attempted, the ratification pull request stays unmerged, and the next candidate needs a design-level decision on how the scratch-state battery and candidate-HEAD authentication coexist, a kit round that deposits durable cleanliness evidence, and a ruling on the two sentences. Every step is recorded with digests in the ceremony record. Graph side: the A-1/A-2 build was reviewed clause by clause (accepted at fixture scope; the plan's live source model not yet met), a shadow child fixed the review's concrete findings and measured the kernels' totality (123 tests; three inputs that crash rather than fail typed), and plan revision 12 is drafted with a kernel totality contract and is in its peer gate. Purpose stays at 100.00%; Q5 at 63/257; the twelve decisions remain with Max. +- **2026-09-05** — Evening: the Amendment 20 A4 freeze was re-frozen after its first candidate failed the pinned battery at the scratch state (a test the freeze itself extended validated the working-tree registry as production whenever ratification was flagged ready, so it could not hold at any scratch state). The re-frozen candidate C′ (820da19, sole parent 86e6994) differs from the first only in that test and its one capture row; the §34 semantic digest is unchanged and was reproduced with the parent commit's own projection code; the candidate battery passed 220/220 and the scratch-state battery is the remaining gate. Scratch transition S′: dispatched to Astra at 18:01 EDT after the re-freeze report showed readiness 31/31 and both batteries 220/220. Read-only review of the candidate's prose found that the freeze removed seven sentences from the parent (the three permitting sentences already known, plus a meaning clause, a blanket prohibition on authority selection before activation, and a rule that a proven residue controls) and added at least twelve law-shaped sentences with no parent counterpart, one of which rules on how the campaign's own board may be read; all reproduced from bytes by a second party and carried to the two closure referees as questions, not rulings. Kit round r8b installed under the two-engine gate: referees now run a whole-document sentence diff in both directions with stated splitter parameters, source-byte probes, and a guard that the withheld-ranks sentence belongs to the missing-reason arm's occurrence census, not the purpose ledger (which stays at 100.00%). Two figures in the decision package were corrected: the MEASURED-family population is 79 occurrence lines, not 78 (one bracket opens with an en dash that a same-line scan cannot see; four parties had run the same scan), and the tag-id unit is 126 under its author's own rule (122 numbered items by five independent methods plus four verified §4/§5 items), not 127. The errata's own 'largest unswept surface', a truncation ending mid-clause somewhere in the decision package's 185 quoted spans, was swept exhaustively: every span ruled (167 complete, 9 truncated at a boundary, 8 altered, 1 not locatable) and zero mid-clause truncations, with the population re-derived blind and all nine control loci recovered. The MEASURED stratum of the decision package was then opened source by source with no wording trigger (78 of 79 loci, 35 of 35 cited files): the source-fidelity defect rate is 11 of 78 (14.1%), so the earlier 13.9% floor is now a bound, and the wording trigger it replaced had found only 3 of those 11; the union of loci carrying a defect on any axis is 15 of 79 (19.0%), stated as a union of two passes rather than one measurement. Both surfaces the errata named as unswept are closed. Graph side: the Plan A shadow plan reached revision 11 through three peer gates and merged to master; its first build slice is running on a Claude lane (Opus) since 17:08 EDT with its first commit (7515f9b, the q5graph package and its test suite) at 17:52, on a Claude lane under the agreed resource policy with no escalation. Q5 stands at 63/257; the twelve decisions remain with Max. +- **2026-09-05** — The purpose census is COMPLETE: 20,815 of 20,815 prompts adopted (100.00%) at the terminal fold, ledger v1.97 (ebe3d2b9…). Blocks 7–10 of the final tranche folded through the day (v1.93–v1.97), every audit CALIBRATED; block 8 was folded in two parts after an orchestrator scope error (the brief was cut from the per-section table, leaving 64 ranks unadjudicated until an 8b completion pass) and block 9 closed the 2021 questionnaire entirely. A standing correction now requires every lane to recompute each digest it emits from the named file and run its own 64-hex self-check, after a 63-character digest was copied forward from a dispatch brief. A compile-only decision package (twelve decisions the package orders by how much work each blocks, against 63 sealed, 17 held and 1 withdrawn of the 81 annotated documents, although an errata compile finds that order does not consistently follow its own criterion) is staged for ruling; it lists five items it says the commissioning brief missed, the sharpest being that the worktree the lanes run in is two ratified design revisions behind master (19 vs 21), although the slot-catalog section the top decision turns on is byte-identical at both. Two figures already in front of the ruling were corrected: the atomisation coverage gap is 331 of 447 enumerated Social Security, pension, annuity and retirement items in no atom (74.05%, not the 72.7% the census printed and its replication reproduced without recomputing), and the corpus's page and frame digests are reproducible under the pinned Poppler pdftotext 26.04.0, which the design, the inventory code and the stage-1 replay artifact all name and the code enforces (cross-host determinism untested) (a Chronicle cross-check with pypdf 6.11.0 reproduced 0 of 101 pages, the pinned recipe 101 of 101); the final-tranche pass reports did not restate the build until the block-10 audit did. Q5 stands at 63/257, which is 77.8% of the 81 annotated questionnaires; the remaining 176 files in the fixed domain are setups, codebooks and raw data, not instruments. The open questions are recorded for ruling; the compiler ruled on none of them. +- **2026-09-05** — Overnight the purpose census reaches 94.88% (final-tranche block 6 folded as ledger v1.92, the fourth consecutive CALIBRATED audit and the first to carry a correction), and the sixteen-document 1968–1982 family back-catalogue is fully annotated and reconciled: six documents sealed, one seal withdrawn on adversarial review, nine held pending rulings (Q5 63/257, which is 77.8% of the 81 annotated questionnaires — the 257 denominator is the fixed source-documentation domain, 43 of which can never seal; a counter reconciliation confirmed the figure across three instruments). A compile-only dossier found the design fixes seven sentinel identifiers (four job-slot, three component-slot) in a section outside every span the briefs had pinned, overturning the campaign's three-sentinel reading and its bar on the seventh literal; a supersession audit reduced whether the tier-1 minting bar is in force to a single unresolved sentence about Amendment 13's recording act; and a full-corpus atomisation census, replicated by an independent method, found that 331 of 447 enumerated questionnaire items naming Social Security, pensions, annuities or retirement lie in no atom of the sealed evidence base (66 of 81 documents). Materialization closed with 83 tables of record on disk and exactly two documents provably unrecoverable. All open questions are recorded for ruling, not resolved locally. +- **2026-09-04** — Final-tranche blocks 4 and 5 close (ledger v1.91; purpose reaches 93.13%), both dense audits CALIBRATED at 100% on full 273-row replays; 1,431 ranks remain across blocks 6–10. Q5 seals q2009, fam2013, q2019 and fam1968 (58/257); fam1968 is the first of a sixteen-document back-catalogue of 1968–1982 family volumes found never briefed and now opened. Five reconciled documents are held and q2023 stays unsealed on the campaign's first 'Q5 status does not pass' verdict, pending five law rulings raised for decision rather than resolved locally: the identifier-minting remedy under §26.5.1, two evidence-integrity classes (digest-only and corrupt sealed payloads), the H-object scope of the pinned law extract, and a minting-clause precondition naming a field that nine artifacts lack. A prepare-only materialization sweep of all 69 reconcile reports writes every reproducible table of record to disk as a first-class artifact with provenance marked decoded or reconstructed (64 tables at last count), and recovery runs prove two of the three corrupt payloads (1995q, 1994f) reproducible from surviving emissions. - **2026-09-04** — Final-tranche blocks 1–3 close (ledger v1.89; purpose reaches 90.50%): block 1 on its regeneration sweep, block 2 CALIBRATED at 99.66% on a one-row manifest, block 3 CALIBRATED at 100% with an empty contradiction set. Q5 seals 2003q, 2005f, 2005q, 2007f, 2007q and 2009f (54/257), the last five on Opus reconciliations; the first Opus/Opus pairs measure common-mode rates of 0.4%, 1.8%, 0.0% and 2.9%. The C20 ceremony-kit correction reaches its fifth round as a Sol-written change that an independent Opus review ACCEPTS, clearing the last precondition on the A4 evidence freeze other than the census itself. Six Claude accounts run Opus lanes in parallel after subfleet's false month-long parks are found and cleared. - **2026-09-03** — The E5 first tranche completes: block 8 closes on its audit's closed 16-row manifest (ledger v1.84; purpose reaches 85.94%). The final tranche R17889–R20815 is planned as the entire remainder — 2,927 ranks across eight documents from 2013 to 2023 in ten blocks (v1.85) — and its first block passes 348/348 and is audited (v1.86) with a 59-row regeneration manifest. Q5 seals 2001f, 2003f and 2001q (48/257), the latter two on Claude Opus reconciliations in 42–43 minutes against a one-to-three-hour Sol baseline. - **2026-09-03** — E5 blocks 6–7 close (ledger v1.83; purpose reaches 84.48%) and the tranche's last block, block 8 to R17888, is in flight. Q5 seals 1999f and 1999q — the latter the campaign's largest table of record at 8,498 rows — reaching 45/257. An output-order mandate (full serialization first, no placeholders) is adopted as standing process law for large deliverables after a truncated sweep had to be re-run. diff --git a/docs/progress/detail.json b/docs/progress/detail.json index 4e145e77..a2aeafff 100644 --- a/docs/progress/detail.json +++ b/docs/progress/detail.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "as_of": "2026-09-04", + "as_of": "2026-09-05", "description": "Row-level drill-down behind the verification arms; extracted from the campaign evidence archive and adversarially verified. Sources name the archive artifact grounding each row.", "purpose": { "unit": "One rank = one row of the pinned 20,815-row residual index (e8-ops/sol-ce-purpose-residual-index-v1.txt, file SHA b35519702e6f...) = one prompt occurrence keyed `psid-questionnaire-occurrence:` — a single printed questionnaire item in a sealed official PSID questionnaire/QxQ-codebook PDF (staged under ~/PolicyEngine/psid-data/documentation/capture1), located by an exact page/character-span receipt, e.g. `q88:p4[2538,2708)#10bc6b307d41`. What is adjudicated per rank is the prompt's \"purpose\": per sol-ce-purpose-evidence-scope-report.md, purpose \"is the closed model-law classification saying which factual dimension a `field_purpose_prompt` establishes for `O_P`\" — not survey designers' motivation. Each rank is disposed either D (determined: an ordered purpose array over the closed 35-literal space, e.g. `[M]`) or SU (source-underdetermined, with a named condition such as U_ONTOLOGY, U_CONFLICT, or U_SCOPE). Example full row from sol-ce-purpose-ledger-pass-R6172-6576-report.md: \"R6176 | Q7316 | psid-questionnaire-occurrence:091095d3... | E3.CALENDAR_EXPOSURE | D | [M] | q88:p4[2538,2708)#10bc6b307d41\". Documents are the per-wave PSID instruments: f-docs are family-interview QxQ codebooks (e.g. f94 = fam1994_QxQs.pdf) and q-docs are questionnaires (e.g. q94 = q94.pdf, \"1994 Questionnaire\", per the e4-era build's source table).", @@ -552,11 +552,11 @@ "rank_start": 18839, "rank_end": 19111, "ranks": 273, - "status": "pending", + "status": "adopted", "dispositions": null, - "audit": null, - "adopted_on": null, - "source": "sol-ce-purpose-ledger-e5final-build-report.md" + "audit": "pass 273/273 -> dense audit CALIBRATED (sampled 144/144; full-vector replay 273/273; row contradictions 0; family concentration none); folded v1.90", + "adopted_on": "2026-09-04", + "source": "sol-ce-purpose-audit-R18839-19111-report.md" }, { "document": "q2019 complete + q2021 A (final-tranche block 5)", @@ -564,11 +564,11 @@ "rank_start": 19112, "rank_end": 19384, "ranks": 273, - "status": "pending", + "status": "adopted", "dispositions": null, - "audit": null, - "adopted_on": null, - "source": "sol-ce-purpose-ledger-e5final-build-report.md" + "audit": "pass staged -> dense audit CALIBRATED 100% (sample 144/144; 273/273 rows examined); folded v1.91 (terminal identity 317,298 B, SHA 014aeb13…)", + "adopted_on": "2026-09-04", + "source": "sol-ce-purpose-audit-R19112-19384-report.md" }, { "document": "q2021 BC (final-tranche block 6)", @@ -576,11 +576,11 @@ "rank_start": 19385, "rank_end": 19749, "ranks": 365, - "status": "pending", - "dispositions": null, - "audit": null, - "adopted_on": null, - "source": "sol-ce-purpose-ledger-e5final-build-report.md" + "status": "adopted", + "dispositions": "APPLY 267 / UNCOVERED 98 (12 proposal entries)", + "audit": "pass 365/365 -> dense audit CALIBRATED with a closed three-row manifest (R19623–R19625, BC60A over-fire -> ALL.DIRECT_EXPOSURE; eight of nine charges refuted; EHC recall 25/25); folded v1.92", + "adopted_on": "2026-09-05", + "source": "sol-ce-purpose-audit-R19385-19749-report.md" }, { "document": "q2021 G (final-tranche block 7)", @@ -588,11 +588,11 @@ "rank_start": 19750, "rank_end": 20006, "ranks": 257, - "status": "pending", - "dispositions": null, - "audit": null, - "adopted_on": null, - "source": "sol-ce-purpose-ledger-e5final-build-report.md" + "status": "adopted", + "dispositions": "APPLY 203 / UNCOVERED 54 (8 proposal entries)", + "audit": "pass 257/257 -> dense audit CALIBRATED 163/163, zero-row manifest (fifth consecutive calibrated audit); EHC fired 18 times and was upheld 18/18 on precision and recall; folded v1.93", + "adopted_on": "2026-09-05", + "source": "sol-ce-purpose-audit-R19750-20006-report.md" }, { "document": "q2021 P/KL/IMMIG (final-tranche block 8)", @@ -600,11 +600,11 @@ "rank_start": 20007, "rank_end": 20311, "ranks": 305, - "status": "pending", - "dispositions": null, - "audit": null, - "adopted_on": null, - "source": "sol-ce-purpose-ledger-e5final-build-report.md" + "status": "adopted", + "dispositions": "305 ranks in two passes: R20007–R20247 APPLY 153 / UNCOVERED 88; R20248–R20311 APPLY 27 / UNCOVERED 37", + "audit": "q2021 §P audit (136/241 sampled = 56.4%; manifest rebuilt in both directions with zero discrepancies; era discipline clean) folded v1.94 under a 'block closed' heading that covered only the §P part — an orchestrator scope error (brief cut from the per-section table); the KL+IMMIG remainder (64 ranks) ran as an 8b completion pass, audited CALIBRATED 64/64 = 100.00% with zero corrections, and folded v1.95", + "adopted_on": "2026-09-05", + "source": "sol-ce-purpose-audit-R20248-20311-report.md" }, { "document": "q2021 ADDRPAYMENT + q2023 BC (final-tranche block 9)", @@ -612,11 +612,11 @@ "rank_start": 20312, "rank_end": 20618, "ranks": 307, - "status": "pending", - "dispositions": null, - "audit": null, - "adopted_on": null, - "source": "sol-ce-purpose-ledger-e5final-build-report.md" + "status": "adopted", + "dispositions": "APPLY 270 / UNCOVERED 37 (closes q2021 incl. ADDRPAYMENT; opens q2023 §BC)", + "audit": "pass 307/307 -> dense audit CALIBRATED: 161/307 sampled (52.4%), agreement 161/161 = 100.00%, correction manifest empty of rows (three documentary items); adopted as issued; folded v1.96 (99.05%). One 63-character rubric digest in the pass's assertion object traced to the dispatch brief -> standing correction adopted (recompute every emitted digest from the named file)", + "adopted_on": "2026-09-05", + "source": "sol-ce-purpose-audit-R20312-20618-report.md" }, { "document": "q2023 G/P/KL (final-tranche block 10)", @@ -624,11 +624,11 @@ "rank_start": 20619, "rank_end": 20815, "ranks": 197, - "status": "pending", - "dispositions": null, - "audit": null, - "adopted_on": null, - "source": "sol-ce-purpose-ledger-e5final-build-report.md" + "status": "adopted", + "dispositions": "APPLY 169 / UNCOVERED 28 (G 156, P 17, KL 24; by_construction_exclusions = [Q21822]); the corpus closes at R20815", + "audit": "pass 197/197 (APPLY 169 / UNCOVERED 28) -> dense audit CALIBRATED, reported as 102/102 = 100% (51.8% sampled) with 101 distinct ranks (one rank counted twice; 51.3% of ranks); DISAGREE set empty; correction manifest empty of rows and items; charge 9 confirmed the Poppler pdftotext 26.04.0 pin and argument equivalence over all 523 pages of q2023; charge 10 upheld the pass's form-feed frame-recipe correction with no row moved; charge 8 swept all 20,815 index lines end to end with zero gaps or duplicates; the ninth consecutive CALIBRATED block; folded v1.97 (379,534 B / ebe3d2b9…) — the corpus closes at R20815", + "adopted_on": "2026-09-05", + "source": "sol-ce-purpose-audit-R20619-20815.md" } ] }, @@ -1481,6 +1481,78 @@ "verdict": "both lanes rejected; the reconciled table stands (common-mode 5/171 = 2.9%, the highest yet)", "sealed_on": "2026-09-04", "source": "sol-ce-q5-doc-2009f-reconciled-report.md" + }, + { + "doc": "document_071", + "label": "2009 family questionnaire (q2009.pdf)", + "rows_sealed": 2282, + "verdict": "lane A's coordinate frame upheld as the base; lane B refuted on its central doctrine (2,509 minted node ids)", + "sealed_on": "2026-09-04", + "source": "sol-ce-q5-doc-2009q-reconciled-report.md" + }, + { + "doc": "document_074", + "label": "2013 family questionnaire QxQ companion (fam2013_QxQs.pdf)", + "rows_sealed": 1320, + "verdict": "lane B substantially upheld (551 of 610 contested rows) and adopted as the base with 82 corrections; lane A refuted; both refuted on the aggregate class", + "sealed_on": "2026-09-04", + "source": "sol-ce-q5-doc-2013f-reconciled-report.md" + }, + { + "doc": "document_079", + "label": "2019 family questionnaire (q2019.pdf)", + "rows_sealed": 1882, + "verdict": "each lane reversed on exactly one axis and upheld on the other (133 + 113 = the 246 contested rows); zero minted identifiers; common-mode 0/1,636", + "sealed_on": "2026-09-04", + "source": "sol-ce-q5-doc-2019q-reconciled-report.md" + }, + { + "doc": "document_001", + "label": "1968 family questionnaire QxQ companion (fam1968_QxQs.pdf)", + "rows_sealed": 156, + "verdict": "adjudication of the contested set: 38 rows with both lanes upheld, 69 lane A upheld, 19 lane B upheld; 156 rows = seal; zero minted identifiers", + "sealed_on": "2026-09-04", + "source": "sol-ce-q5-doc-1968f-reconciled-report.md" + }, + { + "doc": "document_003", + "label": "1969 family questionnaire QxQ companion (fam1969_QxQs.pdf)", + "rows_sealed": 159, + "verdict": "lane A with exactly 7 row replacements after an 8-block adjudication (confirmed common-mode 0/85 by the per-row method; three exposures recorded on the seal)", + "sealed_on": "2026-09-05", + "source": "sol-ce-q5-doc-1969f-reconciled-report.md" + }, + { + "doc": "document_017", + "label": "1976 family questionnaire QxQ companion (fam1976_QxQs.pdf)", + "rows_sealed": 385, + "verdict": "all 224 agreed rows audited in three strata and upheld (common-mode 0/224, or 5/224 under the stricter reading); reading-invariant, no Q5-status pass asserted", + "sealed_on": "2026-09-05", + "source": "sol-ce-q5-doc-1976f-reconciled-report.md" + }, + { + "doc": "document_023", + "label": "1979 family questionnaire QxQ companion (fam1979_QxQs.pdf)", + "rows_sealed": 473, + "verdict": "revision 2 after two of its four verdicts were adversarially overturned; reading-invariant with three exposures recorded; survived the adversarial seal review", + "sealed_on": "2026-09-05", + "source": "sol-ce-q5-doc-1979f-reconciled-report.md" + }, + { + "doc": "document_027", + "label": "1981 family questionnaire QxQ companion (fam1981_QxQs.pdf)", + "rows_sealed": 449, + "verdict": "revision 2 (six rows and one count corrected under a declared adversarial pass); both lanes and the table write zero component-side sentinels; survived the adversarial seal review", + "sealed_on": "2026-09-05", + "source": "sol-ce-q5-doc-1981f-reconciled-report.md" + }, + { + "doc": "document_019", + "label": "1977 family questionnaire QxQ companion (fam1977_QxQs.pdf)", + "rows_sealed": 432, + "verdict": "revision 2; states the containment ground independently (the strongest of the four seals under adversarial review)", + "sealed_on": "2026-09-05", + "source": "sol-ce-q5-doc-1977f-reconciled-report.md" } ] } diff --git a/docs/progress/progress.json b/docs/progress/progress.json index 8f094ddc..09b04767 100644 --- a/docs/progress/progress.json +++ b/docs/progress/progress.json @@ -1,7 +1,7 @@ { "schema_version": 1, "description": "Data behind docs/progress.md (the development-progress dashboard). Update this file and run `python scripts/build_progress_page.py`; never edit docs/progress.md by hand.", - "as_of": "2026-09-04T18:30:00Z", + "as_of": "2026-09-09T11:51Z", "framework": { "amendment": "Amendment 20 (two-arm evidence charter for the covered-earnings correction design)", "pr": 405, @@ -47,9 +47,9 @@ "key": "purpose", "label": "Prompt-purpose census", "description": "Establishing what each of the ~21,000 remaining items in the PSID's 1968–2023 questionnaires and codebooks is actually asking, so the model only uses variables whose meaning has been adjudicated rather than assumed. Each block runs a full pass, an independent dense audit, and a correction sweep before its entries are adopted into the governing ledger.", - "done": 18838, + "done": 20815, "total": 20815, - "unit": "prompts adopted (contiguous R1–R18838)" + "unit": "prompts adopted (contiguous R1–R20815 — ARM COMPLETE, terminal fold v1.97)" }, { "key": "a3", @@ -63,9 +63,9 @@ "key": "q5", "label": "Q5 semantic annotation", "description": "Verifying which survey questions connect to which data fields, document by document across the PSID questionnaire corpus: two independent annotators per document, then a reconciliation that seals its table of record.", - "done": 54, + "done": 63, "total": 257, - "unit": "documents sealed" + "unit": "documents sealed — 63 of the 81 annotated questionnaires (77.8%); the 257 denominator is the fixed PSID source-documentation domain: every survey instrument is annotated, and the other 176 are non-instrument files (setups, codebooks, raw data), 43 of them classed Semantic N/A by the coverage audit (not a law term)" } ], "forecast": { @@ -87,6 +87,58 @@ ] }, "milestones": [ + { + "date": "2026-09-09", + "text": "Morning. Max's routing ruling arrived through his chief-of-staff session at half past four yesterday afternoon and is now in force: no Opus, Sonnet or Haiku turn on any Claude account that still has Fable headroom, because those turns draw the account's shared weekly bucket and four accounts were capped that way before their Fable was touched. The orchestrator switched to Fable 5.1 in the same minute and recorded the ruling on the board; every dispatch since has been Fable on a Claude lane or Astra on a Codex lane. The night was a quiet hold on mains power with nothing in flight and the three ruling packages unchanged. A-3 shadow pass 6 was the day's first decision, and it was run the way the pass-5 adjudication said passes must be run. The seat prepared the worktree and the brief before dawn, with a deltas section carrying the pass-5 acceptance, the pin discipline, the store-write proof, the staging rule, the four failure classes and Max's ruling. At thirteen past six it asked the orchestrator for a Fable slot on the one Claude lane with headroom; the orchestrator read the lane's state at that moment rather than an hour earlier, content-asserted the brief and the worktree from bytes, named a foreign run already on the lane, and granted. The seat dry-ran the dispatch twice first, which surfaced a trap the record now calls D15: the plain Fable form of the dispatcher resolves to a read-only sandbox that would have censored the store write on a host that could reach the store, exactly as the Codex sandbox did in pass 5. Store capability means the host and the sandbox mode, and the dry-run is now a required step. The pass landed in nineteen minutes with four commits, inside its thirty-minute bound, and is accepted. It took its own coordinate from the live board, proved that the seventy-two events shadowed through pass 5b reproduce exactly, and then shadowed forty-one new ones: thirty-four harvests, four repairs and three supersessions, among them the orchestrator's own rotation, the kit's r10 install and the ruling that voided the old standing order. One hundred and twenty-nine A-2 executions ran with no typed exception and no governing discrepancy; custody measured eighty-six source occurrences and copied none, because no new source has a row in the recorded table. The cumulative count is one hundred and thirteen events across four classes, and H2 remains unreachable before the evening of September 12, when the seven-day floor is met. The lane's first report claimed it had overrun the bound; it had estimated elapsed time from its own turns instead of reading the clock, caught the error against the commit timestamps, and corrected the report in a fourth commit that touched nothing else. None of yesterday's four failure classes occurred. The record gained a fifth class an hour later. The farness lane's five-hour window, shared by the pass, the foreign run and the orchestrator's own turns, hit its limit as the orchestrator received the acceptance relay, and its seven o'clock heartbeat failed the same way. It resumes automatically at eleven. A limit modal is not a death, so the seat neither rotated it nor spent credits; it holds, runs the orchestrator's read-only safety instrument on the hour in the meantime, with every axis unchanged, and will relay the acceptance again once the orchestrator is turning. Still with Max: the three ruling packages for gate 3, mortality, and claiming and PIA, and the rotation of the exposed policyengine.org token." + }, + { + "date": "2026-09-08", + "text": "Afternoon. The claiming and PIA gates closed the lock discipline to a ruling package in one day. The adversarial referee round on the bound thresholds returned two NOT LOCK-READY verdicts with concrete findings: a mislabelled reading of one rule, a single-rule pass under one grammar, an incomplete flip plan, a footer defect, and a run of contract items. One fixes sitting answered every finding without moving a floor number, and an independent refute-first verification confirmed each fix from bytes, corrected one count in the fixes report from five to three, and settled the flip-time disposition. The ruling package went to Max at 11:16 with seven claiming decisions, five PIA decisions, and the reading of the rule-count definition; the three packages now with him, gate 3, mortality, and claiming and PIA, are the campaign's critical path. Flip briefs for all three are templated and take nothing but his written words. The C20 kit's r10 repair passed its review on a scratch clone, with the receipt gate exercised and the fallback unused, and was installed on the orchestrator's concurrence with one standing requirement: renders made under earlier kit versions must be re-rendered, and any clone used for verification must be a same-filesystem wholesale-object clone, because only the S-prime object is unreferenced and a pack-transfer clone drops it. That custody correction reversed a premise both seats had carried for a day. A-3 shadow pass 5 was dispatched to Astra when no Opus lane answered its clock, and the pass was censored: the Codex sandbox cannot reach the shadow store, so the copy step could not run. The plan owner adjudicated the dispatch as its own error and fixed the rule: an A-3 pass runs only on a host whose writable roots include the store, and a clock with no such host slips and records the slip. Pass 5b completed the pass on a Claude host at the same coordinate. Its lane finished the substance, then died on a DNS error before committing, and a continuation verified the salvage byte for byte and committed it. Accepted: all twenty-nine events shadowed, ninety-three A-2 executions, zero governing discrepancies, cumulative seventy-two events complete; H2 stays not reachable before the evening of September 12. The machine spent the afternoon on battery. No adapter was detected until shortly after three, the lid closed at 14:41 with two percent left and the box slept five minutes, and nothing was lost: everything was committed, and the seat and the orchestrator agreed beforehand that wall-clock silence on battery is uninformative, so no stall or death call may rest on time alone. The unpushed A-3 shadow branch now has a second on-disk copy as a git bundle, proved to restore by reconstruction, without breaching the shadow-only rule. The orchestrator wrote a complete handover line at 14:06 at fourteen percent and, once power returned, asked to be rotated deliberately at eighty-five percent of its transcript trigger rather than die mid-task; its successor proved the safety baseline on arrival. The rotation surfaced a hygiene finding: the launcher passed the lane token on the tmux command line, and because tmux keeps the first client's arguments as its server's, the shared server has shown a token in the process list since September 6. Both launchers now fetch the token inside the pane, the successor's launch added nothing to any command line, a sweep of every running process found no other credential, and the rotation of the exposed token is with Max. The shift's failures are filed in four classes, Fable weekly caps, a DNS outage, machine sleep, and a limit modal that read as death, and only the first is capacity. The box went back to battery at 15:57, which is why this refresh ran at four rather than eight." + }, + { + "date": "2026-09-08", + "text": "Morning. The claiming and PIA gates went through three steps of the lock discipline overnight. Floors v2 answered both adversarial referees in one commit with every alternative emitted rather than chosen: three revision-sd grammars at three K values, a K-by-rounding sweep, settle-age strata with horizon-aware pricing, an OLS window sweep that shows the knife-edge cell is a property of the scanned window grid, and a stated decimal rounding mode with its one exact tie and thirteen-cell knife-edge class. No drafted tolerance or partition count moved; six open decisions are filed for the ruling, and a seventh packet correction records that the structural demotion of the knife-edge cell is unavailable as written because the transition it cites begins a year later. An independent verification then confirmed every fix from bytes under its own code, adjudicated the one referee number that had not reproduced in the statistical referee's favour, and found one new low finding, a fourth committed channel carrying the held-out rows. The threshold-binding sitting followed: two gate-schema artifacts derived from the floors by path with their digests pinned, every filed alternative's partition and tolerances carried beside the draft so a ruling changes a pointer rather than a number, LOCKED-HOT bindings that recompute all forty-two tolerances from the floor artifact, and both DRAFT gate blocks carried inside the artifacts as strings with a placeholder for every ruling. Under the definition's letter the PIA partition awards exactly one rule all three conditions. The seat reproduced the tests. Then the capacity ran out. Every Fable lane hit its weekly limit by three in the morning, and the adversarial referee round on the bound thresholds died on the cap. A retry loop burned eighteen empty dispatches before the orchestrator measured why: a one-line probe answers on a lane whose per-model budget a full brief exhausts. The dispatcher now demotes a lane after a failed dispatch, and the two referees are running on the one Fable lane still open. The seat itself stopped a third time at 03:14: its own scheduled wake did not fire, and the app paused the session two hours later; the cause is undetermined. A one-shot revival at 05:31 repaired the loop and relaunched the seat on Opus as an interim, which found Astra back, dispatched the kit's r9 two-engine gate, and then hit its own weekly cap; Max resumed the Fable seat at 08:12. The r9 gate failed the candidate with concrete repairs, and an r10 repair draft is running on Astra with the orchestrator's concurrence step retained. The orchestrator hit its weekly cap the evening before and was relaunched at midnight on a fresh lane by a daemon armed for it. A-3 shadow pass 5 runs on Astra from a pre-created worktree because no Opus lane answered its clock. Gate 3 and the mortality gate remain LOCK-READY and wait on Max's rulings; H2 stays not yet reachable before September 12." + }, + { + "date": "2026-09-07", + "text": "Evening. Gate 3 is ratified LOCK-READY. The third independent verification returned VERIFIED with 24,979 non-string leaves identical across the fix and the two ratifying referees, one statistical and one contract, both returned RATIFY LOCK-READY and agreed on all four decisions: cure A for the description, rho at 0.75, lock the 43-pair tranche under exactly its own description, and the candidate-side module change as a registration precondition. The six typed corrections they conditioned on are applied report-only and verified. The one thing left is the ruling itself, which is Max's; the flip commit is drafted so it dispatches the moment he answers, with the pre-lock guards inverted and nothing rebuilt. The mortality gate moved through the whole discipline in one afternoon. Floors v3 answered the pre-registration packet's three data-side blockers from bytes, the weight universe, the death-ascertainment convention and the censoring convention, with the floor rebuilt on the declared universe beside the all-window rebuild. Two adversarial referees followed: the statistical one said LOCK-READY with three findings, the contract one said NOT LOCK-READY narrowly on the record, chiefly that the tests froze the live gates.yaml so the gate block's own commit would have turned them red. Floors v4 fixed both lists in one commit with no floor number moving, an independent verification returned VERIFIED, and the threshold-binding sitting is running now: the gate-schema artifact, the anchor's operating characteristic, the draw stream, the teeth table, and a draft gate block carried inside the artifact by digest. The seat itself stopped for eight hours overnight, and the record says why: a completion notice pushed into an idle seat produced no turn, and the app paused the seat fifteen minutes later. The fix has three layers, a self-wake under the pause, a launchd daemon that resumes the session when no process exists, and lane chains that dispatch their own follow-ons. Two more stop mechanisms were found in daylight: the Mac slept on battery until Max plugged it in, now held awake by caffeinate, and every Claude lane hit its weekly limit by mid-morning, leaving one session window as the day's only capacity. Subfleet's non-build tasks were also found to run without a shell by default; the referees were re-run with a writable sandbox. A-3 shadow pass 4 accepted on recomputed bytes: fifteen new events, forty-three cumulative, zero disagreeing receipts; H2 stays not yet reachable before September 12." + }, + { + "date": "2026-09-07", + "text": "Morning. The ceremony seat stopped for eight hours overnight, and the record says exactly why: at 21:40 the closing sitting's completion notice was pushed into the seat's inbox and produced no turn, and at 21:49 the desktop app paused the idle seat after its fifteen-minute timer; nothing ran until 05:43. Two mechanisms, both read from the notify code and the app's own log, neither a usage limit. The fix has three layers: the seat now wakes itself every thirteen minutes, under the app's pause; a launchd daemon resumes the session as a one-shot turn whenever no live process exists and hourly as a heartbeat, on a lane it probes first (the top-ranked lane answered 'weekly limit hit' in the test); and the lane chains dispatch their own follow-ons, so a landing no longer waits for the seat. Gate 3. The fourth sitting closed the v3 report (332 KB, 33 of 33 digests matching disk, 52 tests). The second independent verification returned NOT VERIFIED with every computed number reproduced by builder-free code, 5,940 of 5,940 panel deviations, 43 of 43 gated rows and 65 of 65 demotions among them, and three blockers all in typed sentences: two false self-descriptions in the committed artifact, the report printing the superseded fragment texts, and a stale test-tier manifest. Floors v4 answered exactly those in one commit (72069f6, five files); the third verification is running and the two ratifying referees are chained behind it. A-3 shadow pass 4 accepted from recomputed bytes: fifteen new events, forty-three cumulative, sixty-two receipts with zero disagreeing, one recorded retry that exposed a pin discipline (pin your own contribution, never the ledger's totals). H2 stays not yet reachable; the seven-day floor alone binds, earliest 2026-09-12 20:16 EDT. Lane capacity is now the binding constraint rather than the seat: the pass-4 dispatch died on three hard-limited lanes in a row before a fourth carried it, and one non-reserved Opus lane was open at 06:47." + }, + { + "date": "2026-09-06", + "text": "Evening. The capability track ran all afternoon on separate lanes beside the correction campaign, and eight shadow branches now sit unpushed and PR-ready, every one with the gate contract byte-identical and its own reproduction tests reproduced by the plan owner: gate-3 retrodiction floors (v1, then v2 answering two adversarial referees, then an independent verification that closed the referees' blockers by recomputation and opened seven new ones, with v3 in progress); mortality floors v2 (a hundred seeds, the estimand restated as the 1997-onward window); the claiming publication floor and the benefit-oracle rule-coverage artifact; the three registered battery families published with every gate-1 run (full suite 5,705 passed); the report-only entrant schedule with the Trustees net-immigration table captured and hash-pinned; and revision 5 of the immigration design, folding a sixteen-decision referee round. What the lock discipline found on gate 3 is the point of having it: the claiming module's default reference was the gate's own resolution table, a textbook drift rule passed 63 percent of the draft surface, no pass rule was stated, and, after v2 fixed all three, an unregistered quadratic-trend rule still passed 23 of 88 gated pairs; the gated set shrinks with each cycle and that is the correct direction. The immigration path has a measured fork for Max: the certified-frame donor is 1,155 rows in 562 households with an effective sample of 82, too thin to floor a gate, so a gated module needs the ACS donor or a coarser surface, while the report-only schedule can ship. Two decisions were re-priced by measurement. The Q5 package: an adversarial pass over the revision candidate's fourteen folded ranges found nine defects, three material, so the memo's own rule flips the recommendation to shipping the package of record with the errata and the line-1188 retraction beside it. The ceremony: an attempt candidate without the disputed precommit exception passes the pinned battery at both states, the oracle and the kit's freeze gate, so ruling (i)'s premise does not hold and the fresh round waits on kit r9's gate, the R12/R31 ruling and Astra's return on 12 September. A-3 shadow pass 3 accepted (28 cumulative events; the twelve-event H2 condition met; the seven-day floor alone remains). The seat itself was revived three times today by subfleet onto lane hosts and retired its own duplicates twice; the host ruling stays open." + }, + { + "date": "2026-09-06", + "text": "Midday. Max asked for the forecast to DynaSim-class capability and what shortens it, and two lane investigations answered from the record rather than from memory. Capability, as the gate ledger states it: eight pre-registered gates, seven locked, every locked gate with a registered first pass (earnings histories 7 July; marital and fertility 9 July; household composition; marriage x earnings; disability; CPS-frame transport; the temporal-holdout projection engine 24 July), 48 one-shot candidate runs on the record. Only gate 3, forward projection, has never been locked and has no thresholds. Against DynaSim the deficits are: no forward mode past 2022; the model's earnings are not covered earnings and there is no SSA calibration or OACT alignment (the covered-earnings correction exists to fix the first); no immigration, education or hours modules; mortality floored but not certifiable on the current surface; the benefit oracle and claiming module built and SSA-exact but ungated. Forecast, measured: capability commits over the last six weeks are zero of 140 (57% evidence and ceremony, 19% dashboard), and scored-run cadence is zero since 25 July after 77 artifacts in the 20-day July sprint; no date is derivable from a rate of zero. The only dated plan, the 16 July claim of DynaSim parity in four to seven weeks, expired between 13 August and 3 September; the full-parity milestone (immigration and alignment) is open and past its p80. The binding mechanism is gate 3's tier-2 arm, which resolves against future annual SSA publications, so its clock cannot start until thresholds are locked; its retrodiction arm needs no publication and is unused. Under redirected capacity the five already-floored links are a 6 to 14 lane-week program; a scored Stage-3 exit is not before Q1 2027. The two top levers cost no lane-weeks because they are decisions: lock gate 3 on the retrodiction arm first, and rule on the graph-port plan's never-ratified D3 and D6 rules that sequence the benefit gate behind the correction's publication (p50 21 October). In motion since this morning: the gate-3 retrodiction floor build, the three registered-but-unpublished battery families, pre-registration packets for the claiming, benefit-oracle and mortality gates, and an immigration/entrant design memo, all on separate lanes beside the correction campaign. Also today: A-3 shadow pass 2 accepted (eleven post-H events, the A-2 contract executed over post-H bytes with the plan owner's per-source custody addendum; H2 not before 12 September); and the decision package's line 1188, which asserts that no report among the 29 declares an agent count, is measured false (at least seven declare, by file and line) and needs a retraction rather than the errata's one-word fix, which re-prices the revise-versus-errata decision. The morning's outage attribution to a usage limit was retracted: the 22:57 kill was the desktop app's own auto-update; the 23:06 'death' was a one-shot revive exiting normally." + }, + { + "date": "2026-09-06", + "text": "Morning after the halted ceremony. Nothing ceremony-related moved overnight and nothing will until three things are ruled on: how the scratch-state battery requirement and candidate-HEAD authentication should coexist (both referees rejected the re-freeze's reconciliation as an unauthorized precommit exception), whether the two substituted sentences are restored or re-authored, and a kit round that deposits durable evidence of scratch cleanliness; the Codex home that runs the ceremony's Astra steps is at its weekly cap until 12 September, so no re-run is possible before then in any case. Two corrections to what the decision package tells Max were settled from bytes: the seven-sealed-document itemisation omits one document (a defect the errata already records as its first entry and that four instruments had re-derived independently without any of them updating the package), and the errata's own Part 2 summary says four decisions change position where its tables and enumeration show nine and eight — its detail is right, its headline is not. The structural question that follows is Max's: whether the package gets a revision folding the errata's twenty-seven defects, or ships as-is with the errata beside it. Graph side, all shadow-only and never pushed: plan revision 12 (two acceptance scopes for A-1, a per-class staged rule, a kernel totality contract, and a live-scope requirement for the closing census) was agreed by the peer gate with no findings and merged; the cohort was frozen at an immutable board coordinate with a 4,262-entry content-hashed manifest (5.55 GB across the seven declared source classes, every digest recomputed), the A4 storage record was filled from existing local settings with gated PSID documentation ingested by identity only because the host's disk is not encrypted, and the H2 stopping rule was frozen before any A-3 run (earliest H2 the evening of 12 September). A-1 accepted at LIVE scope on run 4's head 3bb53ad: an independent refute-first pass reproduced all eleven clause verdicts with nothing refuted and seven report-level qualifications recorded; run 4's third declaration rule partitioned the 29 staged-report copies 10/11/8 exactly as predicted; the plan owner reproduced its 212-test suite. Run 4's delta was accepted on reading and reproduction, not on a second adversarial pass. A-3 pass 1 landed and was accepted as one bounded pass (run 20260906-062442, 19 minutes): 2 post-H harvest events shadowed identity-only with latency recorded, zero discrepancies against bytes, H2 not yet reachable (earliest 2026-09-12). Nothing was copied into the shadow store: the plan's A4 clause requires the custody checks to be recorded per source before any copy, and the recorded table closed at H, so the pass refused every post-H copy and reported the six sources by identity. The plan owner records the post-H custody rows as a dated addendum before pass 2, which then copies the admissible sources and runs the A-2 contract over them; no plan amendment is needed for custody. Two frozen-enumerator defects (a void pattern without a negation guard; a harvest stated in words that only a creation-receipt sweep caught) are recorded for the next revision. Passes continue daily until H2. Purpose stays at 100.00%; Q5 at 63/257; the twelve decisions remain with Max." + }, + { + "date": "2026-09-05", + "text": "Late evening: the Amendment 20 closure ceremony ran to its referee step and stopped there. After the re-frozen candidate C′ (820da19) passed the scratch transition S′ (battery 220/220), the executed-transition receipt (kit gate ready 11 of 11) and a first-add on the ceremony branch, both independent closure referees withheld ratification and wrote no verdict. Both rest on the same two blocking grounds: the re-freeze's test change, which made the battery hold at the scratch state, reads as a precommit index-authentication exception the parent design does not authorize; and the repository holds no independent evidence that the scratch worktree was clean at S′, a gap in the ceremony's own evidence design. Both referees also found the candidate's replacements for the parent's status-meaning and exact-failure-control sentences not proposition-equivalent; one treats that as blocking, the other does not. Closure and the revision-22 repin were therefore not attempted, the ratification pull request stays unmerged, and the next candidate needs a design-level decision on how the scratch-state battery and candidate-HEAD authentication coexist, a kit round that deposits durable cleanliness evidence, and a ruling on the two sentences. Every step is recorded with digests in the ceremony record. Graph side: the A-1/A-2 build was reviewed clause by clause (accepted at fixture scope; the plan's live source model not yet met), a shadow child fixed the review's concrete findings and measured the kernels' totality (123 tests; three inputs that crash rather than fail typed), and plan revision 12 is drafted with a kernel totality contract and is in its peer gate. Purpose stays at 100.00%; Q5 at 63/257; the twelve decisions remain with Max." + }, + { + "date": "2026-09-05", + "text": "Evening: the Amendment 20 A4 freeze was re-frozen after its first candidate failed the pinned battery at the scratch state (a test the freeze itself extended validated the working-tree registry as production whenever ratification was flagged ready, so it could not hold at any scratch state). The re-frozen candidate C′ (820da19, sole parent 86e6994) differs from the first only in that test and its one capture row; the §34 semantic digest is unchanged and was reproduced with the parent commit's own projection code; the candidate battery passed 220/220 and the scratch-state battery is the remaining gate. Scratch transition S′: dispatched to Astra at 18:01 EDT after the re-freeze report showed readiness 31/31 and both batteries 220/220. Read-only review of the candidate's prose found that the freeze removed seven sentences from the parent (the three permitting sentences already known, plus a meaning clause, a blanket prohibition on authority selection before activation, and a rule that a proven residue controls) and added at least twelve law-shaped sentences with no parent counterpart, one of which rules on how the campaign's own board may be read; all reproduced from bytes by a second party and carried to the two closure referees as questions, not rulings. Kit round r8b installed under the two-engine gate: referees now run a whole-document sentence diff in both directions with stated splitter parameters, source-byte probes, and a guard that the withheld-ranks sentence belongs to the missing-reason arm's occurrence census, not the purpose ledger (which stays at 100.00%). Two figures in the decision package were corrected: the MEASURED-family population is 79 occurrence lines, not 78 (one bracket opens with an en dash that a same-line scan cannot see; four parties had run the same scan), and the tag-id unit is 126 under its author's own rule (122 numbered items by five independent methods plus four verified §4/§5 items), not 127. The errata's own 'largest unswept surface', a truncation ending mid-clause somewhere in the decision package's 185 quoted spans, was swept exhaustively: every span ruled (167 complete, 9 truncated at a boundary, 8 altered, 1 not locatable) and zero mid-clause truncations, with the population re-derived blind and all nine control loci recovered. The MEASURED stratum of the decision package was then opened source by source with no wording trigger (78 of 79 loci, 35 of 35 cited files): the source-fidelity defect rate is 11 of 78 (14.1%), so the earlier 13.9% floor is now a bound, and the wording trigger it replaced had found only 3 of those 11; the union of loci carrying a defect on any axis is 15 of 79 (19.0%), stated as a union of two passes rather than one measurement. Both surfaces the errata named as unswept are closed. Graph side: the Plan A shadow plan reached revision 11 through three peer gates and merged to master; its first build slice is running on a Claude lane (Opus) since 17:08 EDT with its first commit (7515f9b, the q5graph package and its test suite) at 17:52, on a Claude lane under the agreed resource policy with no escalation. Q5 stands at 63/257; the twelve decisions remain with Max." + }, + { + "date": "2026-09-05", + "text": "The purpose census is COMPLETE: 20,815 of 20,815 prompts adopted (100.00%) at the terminal fold, ledger v1.97 (ebe3d2b9…). Blocks 7–10 of the final tranche folded through the day (v1.93–v1.97), every audit CALIBRATED; block 8 was folded in two parts after an orchestrator scope error (the brief was cut from the per-section table, leaving 64 ranks unadjudicated until an 8b completion pass) and block 9 closed the 2021 questionnaire entirely. A standing correction now requires every lane to recompute each digest it emits from the named file and run its own 64-hex self-check, after a 63-character digest was copied forward from a dispatch brief. A compile-only decision package (twelve decisions the package orders by how much work each blocks, against 63 sealed, 17 held and 1 withdrawn of the 81 annotated documents, although an errata compile finds that order does not consistently follow its own criterion) is staged for ruling; it lists five items it says the commissioning brief missed, the sharpest being that the worktree the lanes run in is two ratified design revisions behind master (19 vs 21), although the slot-catalog section the top decision turns on is byte-identical at both. Two figures already in front of the ruling were corrected: the atomisation coverage gap is 331 of 447 enumerated Social Security, pension, annuity and retirement items in no atom (74.05%, not the 72.7% the census printed and its replication reproduced without recomputing), and the corpus's page and frame digests are reproducible under the pinned Poppler pdftotext 26.04.0, which the design, the inventory code and the stage-1 replay artifact all name and the code enforces (cross-host determinism untested) (a Chronicle cross-check with pypdf 6.11.0 reproduced 0 of 101 pages, the pinned recipe 101 of 101); the final-tranche pass reports did not restate the build until the block-10 audit did. Q5 stands at 63/257, which is 77.8% of the 81 annotated questionnaires; the remaining 176 files in the fixed domain are setups, codebooks and raw data, not instruments. The open questions are recorded for ruling; the compiler ruled on none of them." + }, + { + "date": "2026-09-05", + "text": "Overnight the purpose census reaches 94.88% (final-tranche block 6 folded as ledger v1.92, the fourth consecutive CALIBRATED audit and the first to carry a correction), and the sixteen-document 1968–1982 family back-catalogue is fully annotated and reconciled: six documents sealed, one seal withdrawn on adversarial review, nine held pending rulings (Q5 63/257, which is 77.8% of the 81 annotated questionnaires — the 257 denominator is the fixed source-documentation domain, 43 of which can never seal; a counter reconciliation confirmed the figure across three instruments). A compile-only dossier found the design fixes seven sentinel identifiers (four job-slot, three component-slot) in a section outside every span the briefs had pinned, overturning the campaign's three-sentinel reading and its bar on the seventh literal; a supersession audit reduced whether the tier-1 minting bar is in force to a single unresolved sentence about Amendment 13's recording act; and a full-corpus atomisation census, replicated by an independent method, found that 331 of 447 enumerated questionnaire items naming Social Security, pensions, annuities or retirement lie in no atom of the sealed evidence base (66 of 81 documents). Materialization closed with 83 tables of record on disk and exactly two documents provably unrecoverable. All open questions are recorded for ruling, not resolved locally." + }, + { + "date": "2026-09-04", + "text": "Final-tranche blocks 4 and 5 close (ledger v1.91; purpose reaches 93.13%), both dense audits CALIBRATED at 100% on full 273-row replays; 1,431 ranks remain across blocks 6–10. Q5 seals q2009, fam2013, q2019 and fam1968 (58/257); fam1968 is the first of a sixteen-document back-catalogue of 1968–1982 family volumes found never briefed and now opened. Five reconciled documents are held and q2023 stays unsealed on the campaign's first 'Q5 status does not pass' verdict, pending five law rulings raised for decision rather than resolved locally: the identifier-minting remedy under §26.5.1, two evidence-integrity classes (digest-only and corrupt sealed payloads), the H-object scope of the pinned law extract, and a minting-clause precondition naming a field that nine artifacts lack. A prepare-only materialization sweep of all 69 reconcile reports writes every reproducible table of record to disk as a first-class artifact with provenance marked decoded or reconstructed (64 tables at last count), and recovery runs prove two of the three corrupt payloads (1995q, 1994f) reproducible from surviving emissions." + }, { "date": "2026-09-04", "text": "Final-tranche blocks 1–3 close (ledger v1.89; purpose reaches 90.50%): block 1 on its regeneration sweep, block 2 CALIBRATED at 99.66% on a one-row manifest, block 3 CALIBRATED at 100% with an empty contradiction set. Q5 seals 2003q, 2005f, 2005q, 2007f, 2007q and 2009f (54/257), the last five on Opus reconciliations; the first Opus/Opus pairs measure common-mode rates of 0.4%, 1.8%, 0.0% and 2.9%. The C20 ceremony-kit correction reaches its fifth round as a Sol-written change that an independent Opus review ACCEPTS, clearing the last precondition on the A4 evidence freeze other than the census itself. Six Claude accounts run Opus lanes in parallel after subfleet's false month-long parks are found and cleared." diff --git a/pyproject.toml b/pyproject.toml index 370d161a..3bdf706c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,12 @@ dependencies = [ ] [project.optional-dependencies] +# Typed artifacts are not yet in the published 0.1.0 packages. Pin both +# graph and Frame to the same reviewed core revision; this extra needs 3.13+. +graph = [ + "microcosm-frame @ git+https://github.com/PolicyEngine/microcosm.git@3ff92b0aea14407d09479bff5623dc7d1a92d008#subdirectory=packages/microcosm-frame", + "microcosm-graph @ git+https://github.com/PolicyEngine/microcosm.git@3ff92b0aea14407d09479bff5623dc7d1a92d008#subdirectory=packages/microcosm-graph", +] dev = [ "pytest>=7.4.0", "black>=23.7.0", diff --git a/scripts/first_estimates_birth_evidence.py b/scripts/first_estimates_birth_evidence.py index efae0662..911ae21f 100644 --- a/scripts/first_estimates_birth_evidence.py +++ b/scripts/first_estimates_birth_evidence.py @@ -156,6 +156,15 @@ Path("src/populace_dynamics/estimates/anchor_context_registry.py"), Path("src/populace_dynamics/estimates/anchor_context_rehearsal.py"), Path("src/populace_dynamics/estimates/anchor_context_report.py"), + # The opt-in graph integration is outside the historical reducer and + # registered production call paths. Keep exact file exclusions, with + # import-reachability coverage, rather than changing any evidence pin. + Path("src/populace_dynamics/graph/__init__.py"), + Path("src/populace_dynamics/graph/__main__.py"), + Path("src/populace_dynamics/graph/_compat.py"), + Path("src/populace_dynamics/graph/model.py"), + Path("src/populace_dynamics/graph/runtime.py"), + Path("src/populace_dynamics/graph/synthetic.py"), ) POST_REVIEW_SHARED_SOURCE_BLOBS = { Path( diff --git a/src/populace_dynamics/graph/__init__.py b/src/populace_dynamics/graph/__init__.py new file mode 100644 index 00000000..03c8c7ab --- /dev/null +++ b/src/populace_dynamics/graph/__init__.py @@ -0,0 +1,18 @@ +"""Optional synthetic population-graph integration. + +Importing this package does not import Microcosm or change legacy execution. +The graph entry point checks Python and the installed graph capabilities. +""" + + +def run_mortality_graph(**kwargs): + """Run the existing mortality/ageing operations through Microcosm.""" + from ._compat import require_graph + + require_graph() + from .runtime import run_mortality_graph as run + + return run(**kwargs) + + +__all__ = ["run_mortality_graph"] diff --git a/src/populace_dynamics/graph/__main__.py b/src/populace_dynamics/graph/__main__.py new file mode 100644 index 00000000..fb502f35 --- /dev/null +++ b/src/populace_dynamics/graph/__main__.py @@ -0,0 +1,66 @@ +"""Run the mortality graph with explicit inputs and output placement.""" + +import argparse +from pathlib import Path + +from . import run_mortality_graph +from .synthetic import write_synthetic_inputs + + +def parser(): + result = argparse.ArgumentParser(description=__doc__) + result.add_argument("--output-dir", type=Path, required=True) + result.add_argument("--synthetic", action="store_true") + for name in ("training", "rates", "initial", "holdout"): + result.add_argument(f"--{name}", type=Path) + result.add_argument("--boundary-year", type=int, default=2014) + result.add_argument("--external-vintage-year", type=int, default=2014) + result.add_argument("--experiment-id", default="mortality") + result.add_argument("--replicate", type=int, default=0) + result.add_argument("--base-seed", type=int, default=0) + return result + + +def main(argv=None): + arg_parser = parser() + args = arg_parser.parse_args(argv) + sources = { + name: getattr(args, name) + for name in ("training", "rates", "initial", "holdout") + } + if args.synthetic: + if any(sources.values()): + arg_parser.error("--synthetic cannot be combined with input paths") + if args.boundary_year != 2014: + arg_parser.error( + "the supplied synthetic fixture has boundary year 2014" + ) + sources = write_synthetic_inputs(args.output_dir / "inputs") + elif not all(sources.values()): + arg_parser.error("supply all four input paths or --synthetic") + try: + run = run_mortality_graph( + **sources, + output_dir=args.output_dir, + boundary_year=args.boundary_year, + external_vintage_year=args.external_vintage_year, + experiment_id=args.experiment_id, + replicate=args.replicate, + base_seed=args.base_seed, + ) + except (ImportError, ValueError) as error: + arg_parser.exit(2, f"{error}\n") + print( + f"{args.output_dir / 'report.json'}: engineering={run.report['engineering_verdict']}, fixture={run.report['fixture_verdict']}" + ) + return ( + 0 + if run.report["engineering_verdict"] + == run.report["fixture_verdict"] + == "pass" + else 1 + ) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/populace_dynamics/graph/_compat.py b/src/populace_dynamics/graph/_compat.py new file mode 100644 index 00000000..86ffff07 --- /dev/null +++ b/src/populace_dynamics/graph/_compat.py @@ -0,0 +1,36 @@ +"""An explicit dependency boundary for the optional graph example.""" + +import importlib +import sys + +_GUIDANCE = ( + "The population graph needs Microcosm's typed model-artifact and keyed " + "randomness interfaces. Install the reviewed microcosm-graph and " + "microcosm-frame revisions together; see docs/population-graph.md. " + "Legacy Dynamics does not require these packages." +) + + +def _python_version(): + return sys.version_info[:2] + + +def require_graph(): + """Refuse unsupported Python or a graph lacking the required interfaces.""" + if _python_version() < (3, 13): + raise ImportError( + "The optional population graph requires Python >=3.13." + ) + try: + decl = importlib.import_module("microcosm.graph.decl") + kernel = importlib.import_module("microcosm.graph.kernel") + randomness = importlib.import_module("microcosm.graph.randomness") + except (ImportError, SyntaxError) as error: + raise ImportError(_GUIDANCE) from error + for name in ("ArtifactType", "ArtifactInput", "ArtifactOutput"): + if not hasattr(decl, name): + raise ImportError(_GUIDANCE) + if not hasattr(kernel.SeedSource, "KEYED") or not hasattr( + randomness, "keyed_uniform" + ): + raise ImportError(_GUIDANCE) diff --git a/src/populace_dynamics/graph/model.py b/src/populace_dynamics/graph/model.py new file mode 100644 index 00000000..0f2da4be --- /dev/null +++ b/src/populace_dynamics/graph/model.py @@ -0,0 +1,179 @@ +"""Validated JSON for the existing fitted M6 mortality law; no pickle.""" + +from __future__ import annotations + +import json +import math +from dataclasses import dataclass + +_FORMAT = "populace-dynamics.mortality" + + +def _unique_object(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON member {key!r}") + result[key] = value + return result + + +def read_json(path): + """Read JSON without accepting duplicate fields or nonfinite numbers.""" + return parse_json(path.read_bytes()) + + +def parse_json(payload): + def invalid(value): + raise ValueError(f"nonfinite JSON number {value}") + + try: + return json.loads( + payload, object_pairs_hook=_unique_object, parse_constant=invalid + ) + except (UnicodeError, json.JSONDecodeError) as error: + raise ValueError("invalid mortality JSON") from error + + +def json_bytes(value): + return json.dumps( + value, sort_keys=True, separators=(",", ":"), allow_nan=False + ).encode("utf-8") + + +def _integer(value, label): + if type(value) is not int: + raise ValueError(f"{label} must be an integer") + return value + + +@dataclass(frozen=True) +class MortalityArtifact: + """A compact model with an explicit fit boundary and source vintage.""" + + bands: tuple[tuple[int, int], ...] + probabilities: tuple[tuple[str, str, float], ...] + boundary_year: int + external_vintage_year: int + fit_rows: int + + def __post_init__(self): + _integer(self.boundary_year, "boundary_year") + _integer(self.external_vintage_year, "external_vintage_year") + if _integer(self.fit_rows, "fit_rows") <= 0: + raise ValueError("fit_rows must be positive") + if self.external_vintage_year > self.boundary_year: + raise ValueError("external vintage is later than the fit boundary") + seen = set() + for label, sex, probability in self.probabilities: + if not isinstance(label, str) or sex not in ("female", "male"): + raise ValueError("invalid mortality probability cell") + if (label, sex) in seen: + raise ValueError("duplicate mortality probability cell") + seen.add((label, sex)) + if isinstance(probability, bool) or not isinstance( + probability, (int, float) + ): + raise ValueError("mortality probability must be numeric") + if not math.isfinite(probability) or not 0 <= probability <= 1: + raise ValueError("mortality probability must lie in [0, 1]") + for band in self.bands: + if len(band) != 2 or any(type(age) is not int for age in band): + raise ValueError("mortality bands must contain integer bounds") + # The real model validates complete, contiguous bands and sex cells. + _ = self.model + + @property + def model(self): + from populace_dynamics.engine.steps import AgeSexMortalityModel + + return AgeSexMortalityModel( + self.bands, + {(band, sex): p for band, sex, p in self.probabilities}, + ) + + def to_bytes(self): + return json_bytes( + { + "format": _FORMAT, + "schema_version": 1, + "boundary_year": self.boundary_year, + "external_vintage_year": self.external_vintage_year, + "fit_rows": self.fit_rows, + "bands": self.bands, + "probabilities": [ + {"age_band": band, "sex": sex, "probability": p} + for band, sex, p in sorted(self.probabilities) + ], + } + ) + + @classmethod + def from_bytes(cls, payload): + raw = parse_json(payload) + expected = { + "format", + "schema_version", + "boundary_year", + "external_vintage_year", + "fit_rows", + "bands", + "probabilities", + } + if not isinstance(raw, dict) or set(raw) != expected: + raise ValueError("mortality model has an invalid field set") + if raw["format"] != _FORMAT or type(raw["schema_version"]) is not int: + raise ValueError("invalid mortality format or schema version") + if raw["schema_version"] != 1: + raise ValueError("unsupported mortality schema version") + if not isinstance(raw["bands"], list) or not all( + isinstance(band, list) for band in raw["bands"] + ): + raise ValueError("mortality bands must be arrays") + if not isinstance(raw["probabilities"], list): + raise ValueError("mortality probabilities must be an array") + for row in raw["probabilities"]: + if not isinstance(row, dict) or set(row) != { + "age_band", + "sex", + "probability", + }: + raise ValueError("invalid mortality probability fields") + return cls( + bands=tuple(tuple(band) for band in raw["bands"]), + probabilities=tuple( + (row["age_band"], row["sex"], row["probability"]) + for row in raw["probabilities"] + ), + boundary_year=raw["boundary_year"], + external_vintage_year=raw["external_vintage_year"], + fit_rows=raw["fit_rows"], + ) + + +def fit_mortality( + exposure, external_rates, *, boundary_year, external_vintage_year +): + """Use the existing cutoff-safe fitter and retain its compact results.""" + from populace_dynamics.engine.refit import ( + fit_mortality_model, + prepare_mortality_refit_inputs, + ) + + prepared = prepare_mortality_refit_inputs( + exposure, + external_rates, + boundary_year=boundary_year, + external_vintage_year=external_vintage_year, + ) + model = fit_mortality_model(prepared) + return MortalityArtifact( + bands=model.bands, + probabilities=tuple( + (band, sex, p) + for (band, sex), p in sorted(model.probability.items()) + ), + boundary_year=boundary_year, + external_vintage_year=external_vintage_year, + fit_rows=len(prepared.exposure), + ) diff --git a/src/populace_dynamics/graph/runtime.py b/src/populace_dynamics/graph/runtime.py new file mode 100644 index 00000000..4d1181fc --- /dev/null +++ b/src/populace_dynamics/graph/runtime.py @@ -0,0 +1,847 @@ +"""Fit the existing mortality law, transport it, and retain two periods. + +This module is imported only after the optional capability check. Its graph +is a synthetic engineering integration, separate from the locked M6 loop. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +from microcosm.frame import EntitySchema, Frame, WeightKind, Weights +from microcosm.graph.canonical import canonical_json +from microcosm.graph.codecs import SOURCE_CODECS +from microcosm.graph.decl import ( + ArtifactInput, + ArtifactOutput, + ArtifactType, + Graph, + Node, + Owned, + Slice, + SourceRef, + StructuralDelta, + compile_graph, +) +from microcosm.graph.executor import run_graph +from microcosm.graph.kernel import ( + Capabilities, + Determinism, + KernelRegistry, + KernelResult, + KernelRole, + Numeric, + SeedSource, + source_hash, +) +from microcosm.graph.randomness import keyed_uniform +from microcosm.graph.store import ContentStore + +from populace_dynamics.engine.refit import ( + fit_mortality_model, + prepare_mortality_refit_inputs, +) +from populace_dynamics.engine.steps import advance_age, apply_mortality + +from . import model as model_module +from .model import ( + MortalityArtifact, + fit_mortality, + json_bytes, + parse_json, + read_json, +) + +OBS = "person_period" +PID = "person_period_person_id" +PERIOD_ID = "person_period_period_id" +OID = "person_period_id" +MODEL_TYPE = ArtifactType("populace-dynamics.mortality", 1) +CODEC = "dynamics-mortality-json-v1" +DEPENDENCIES = ("numpy", "pandas") +TRAIN_COLUMNS = ( + ("age_band", "string"), + ("sex", "string"), + ("required_interview_year", "int64"), + ("exposure", "float64"), + ("death", "float64"), +) + + +def _json_source_marker(path, *, store=None): + """Raw declared sources are read by domain kernels, never as Frames.""" + del path, store + raise ValueError("Dynamics JSON sources need their declared domain kernel") + + +def _series(values, ids, entity=OBS, dtype="int64"): + return pd.Series( + values, + index=pd.Index(ids, name=f"{entity}_id", dtype="int64"), + dtype=dtype, + ) + + +def _records(path, expected): + raw = read_json(path) + if not isinstance(raw, list) or not raw: + raise ValueError(f"{path.name} must contain nonempty record inputs") + if any( + not isinstance(row, dict) or set(row) != set(expected) for row in raw + ): + raise ValueError( + f"{path.name} has unsupported fields; expected {expected}" + ) + return pd.DataFrame(raw) + + +def _integer_column(frame, column): + if any(type(v) is not int for v in frame[column].tolist()): + raise ValueError(f"{column} must contain integer identifiers/years") + frame[column] = frame[column].astype("int64") + + +def _weights(values): + result = np.asarray(values, dtype=np.float64) + if not np.isfinite(result).all() or (result <= 0).any(): + raise ValueError("source weights must be positive and finite") + return Weights(result, WeightKind.DESIGN) + + +def _frame(records, periods, weights, columns): + """Give observations an explicit persistent-person and period partition.""" + observations = pd.DataFrame( + { + OID: np.arange(1, len(records) + 1, dtype=np.int64), + PID: records["person_id"].to_numpy(dtype=np.int64), + PERIOD_ID: np.asarray(periods, dtype=np.int64), + } + ) + for column, dtype in columns: + observations[column] = records[column].astype(dtype).array + person_ids = np.sort(observations[PID].unique()) + period_ids = np.sort(observations[PERIOD_ID].unique()) + return Frame( + { + OBS: observations, + "person": pd.DataFrame({"person_id": person_ids}), + "period": pd.DataFrame( + {"period_id": period_ids, "period": period_ids} + ), + }, + EntitySchema(person_entity=OBS, group_entities=("person", "period")), + {OBS: weights}, + pd.Series(["synthetic"] * len(observations), dtype=object), + ) + + +def _create_training(context): + fields = ["person_id", "event_year", "start_weight", *dict(TRAIN_COLUMNS)] + data = _records(context.sources["training"], fields) + for column in ("person_id", "event_year", "required_interview_year"): + _integer_column(data, column) + if data.duplicated(["person_id", "event_year"]).any(): + raise ValueError("training source repeats a person-period") + if not data.sex.isin(["female", "male"]).all(): + raise ValueError("training sex must be female or male") + for column in ("exposure", "death"): + numeric = data[column].to_numpy(dtype=np.float64) + if not np.isfinite(numeric).all() or (numeric < 0).any(): + raise ValueError( + f"training {column} must be finite and nonnegative" + ) + if (data.death > 1).any(): + raise ValueError("training death must lie in [0, 1]") + data = data.sort_values(["person_id", "event_year"]).reset_index(drop=True) + return KernelResult( + frame=_frame( + data, data.event_year, _weights(data.start_weight), TRAIN_COLUMNS + ) + ) + + +def _create_initial(context): + data = _records( + context.sources["initial"], ("person_id", "age", "sex", "weight") + ) + for column in ("person_id", "age"): + _integer_column(data, column) + if data.person_id.duplicated().any(): + raise ValueError("initial population repeats a person_id") + if not data.age.between(0, 120).all(): + raise ValueError("initial ages must lie in [0, 120]") + if not data.sex.isin(["female", "male"]).all(): + raise ValueError("initial sex must be female or male") + data = data.sort_values("person_id").reset_index(drop=True) + return KernelResult( + frame=_frame( + data, + [context.params["boundary_year"]] * len(data), + _weights(data.weight), + (("age", "int64"), ("sex", "string")), + ) + ) + + +def _periods(context, observations): + periods = context.tables["period"].set_index("period_id")["period"] + return observations[PERIOD_ID].map(periods).astype("int64") + + +def _fit(context): + observations = context.tables[OBS] + exposure = observations[[column for column, _ in TRAIN_COLUMNS]].copy() + exposure["person_id"] = observations[PID].to_numpy() + exposure["event_year"] = _periods(context, observations).to_numpy() + exposure["start_weight"] = context.weights[OBS].values + rates = pd.DataFrame(read_json(context.sources["rates"])) + artifact = fit_mortality( + exposure, + rates, + boundary_year=context.params["boundary_year"], + external_vintage_year=context.params["external_vintage_year"], + ) + return KernelResult( + artifacts={"model": artifact.to_bytes()}, + receipt={ + "fit_rows": artifact.fit_rows, + "boundary_year": artifact.boundary_year, + }, + ) + + +def mortality_uniforms( + person_ids, + *, + experiment_id="mortality", + replicate=0, + base_seed=0, + period=2015, + draw_index=0, +): + """Stable original-person draws, independent of observation row ordinals.""" + if any( + isinstance(pid, (bool, np.bool_)) + or not isinstance(pid, (int, np.integer)) + for pid in person_ids + ): + raise ValueError("person identities must be integers") + if ( + type(period) is not int + or type(draw_index) is not int + or draw_index < 0 + ): + raise ValueError("period and nonnegative draw index must be integers") + return keyed_uniform( + stream=("sha256-u53-v1", experiment_id, replicate, base_seed), + keys=[ + (int(pid), "mortality", int(period), int(draw_index)) + for pid in person_ids + ], + ) + + +class _PersonGenerator: + def __init__(self, person_id, module, period, stream): + self.person_id = person_id + self.module = getattr(module, "value", str(module)) + self.period = period + self.stream = stream + self.draw_index = 0 + + def random(self): + value = keyed_uniform( + stream=self.stream, + keys=[ + ( + int(self.person_id), + self.module, + self.period, + self.draw_index, + ) + ], + )[0] + self.draw_index += 1 + return float(value) + + +class _GraphPeriodContext: + """Adapter for existing steps, intentionally bypassing ordinal mapping.""" + + def __init__(self, params): + self.year = int(params["boundary_year"]) + 1 + self.metadata = {} + self.rng_registry = self # Existing apply_mortality tests for None. + self.stream = tuple(params["stream"]) + + def person_generator(self, module, person_id): + return _PersonGenerator(person_id, module, self.year, self.stream) + + +def _slice(context, observations=None): + observations = ( + context.tables[OBS] if observations is None else observations + ) + return pd.DataFrame( + { + "person_id": observations[PID].to_numpy(dtype=np.int64), + "age": observations.age.to_numpy(dtype=np.int64), + "sex": observations.sex.astype(str).to_numpy(), + "year": _periods(context, observations).to_numpy(), + } + ) + + +def _apply(context): + artifact = MortalityArtifact.from_bytes(context.artifacts["model"].payload) + if artifact.boundary_year != context.params["boundary_year"]: + raise ValueError("mortality fit and application boundary years differ") + model = artifact.model + initial = _slice(context) + survived = apply_mortality( + initial, _GraphPeriodContext(context.params), context.rng, model=model + ) + ids = context.tables[OBS][OID].tolist() + return KernelResult( + columns={ + (OBS, "death_probability"): _series( + model.probabilities(initial), ids, dtype="float64" + ), + (OBS, "survives"): _series( + initial.person_id.isin(survived.person_id).to_numpy(), + ids, + dtype="bool", + ), + } + ) + + +def _mass(weights, survivor_weights, strata, survivor_strata, boundary_year): + def totals(values, labels): + frame = pd.DataFrame({"weight": values, "stratum": list(labels)}) + return { + str(label): float(value) + for label, value in frame.groupby("stratum", observed=True) + .weight.sum() + .items() + } + + before = totals(weights, strata) + future = totals(survivor_weights, survivor_strata) + after = totals( + np.concatenate([weights, survivor_weights]), + [*strata, *survivor_strata], + ) + partition_after = {str(boundary_year): before} + if len(survivor_weights): + partition_after[str(boundary_year + 1)] = future + return { + "policy": "declared", + "before": float(np.sum(weights)), + "after": float(np.sum(np.concatenate([weights, survivor_weights]))), + "stratum_before": before, + "stratum_after": after, + "partition": { + "entity": "period", + "column": "period", + "stratum_before": {str(boundary_year): before}, + "stratum_after": partition_after, + }, + } + + +def _advance(context): + observations = context.tables[OBS] + mask = observations.survives.to_numpy(dtype=bool) + surviving = observations.loc[mask] + boundary = int(context.params["boundary_year"]) + # The existing adapter's year is used for the entrant period group only; + # it is never written back onto an incumbent observation. + aged = advance_age( + _slice(context, surviving), + SimpleNamespace(year=boundary + 1, metadata={}), + context.rng, + ) + old_ids = observations[OID].tolist() + new_ids = list(range(max(old_ids) + 1, max(old_ids) + 1 + len(surviving))) + target_ids = old_ids + new_ids + period_ids = context.tables["period"].period_id.tolist() + period_values = context.tables["period"].period.tolist() + next_period = [boundary + 1] if len(new_ids) else [] + weights = context.weights[OBS].values + survivor_weights = weights[mask] + return KernelResult( + expand={ + OBS: _series(surviving[OID].tolist(), new_ids), + "person": _series([], [], "person"), + "period": _series( + [pd.NA] * len(next_period), next_period, "period", "Int64" + ), + }, + columns={ + (OBS, PERIOD_ID): _series( + observations[PERIOD_ID].tolist() + + [boundary + 1] * len(new_ids), + target_ids, + ), + (OBS, "age"): _series( + observations.age.tolist() + aged.age.tolist(), target_ids + ), + ("period", "period"): _series( + period_values + next_period, period_ids + next_period, "period" + ), + }, + weights=Weights( + np.concatenate([weights, survivor_weights]), + context.weights[OBS].kind, + ), + receipt={ + "mass": _mass( + weights, + survivor_weights, + context.strata.tolist(), + context.strata.to_numpy()[mask], + boundary, + ) + }, + ) + + +def _age_claim(context): + observations = context.tables[OBS] + return KernelResult( + columns={ + (OBS, "age"): _series( + observations.age.tolist(), observations[OID].tolist() + ) + } + ) + + +def _evaluate(context): + observations = context.tables[OBS] + boundary = int(context.params["boundary_year"]) + periods = _periods(context, observations) + initial = observations.loc[periods == boundary].copy() + future = observations.loc[periods == boundary + 1].copy() + truth_document = read_json(context.sources["holdout"]) + if ( + not isinstance(truth_document, dict) + or set(truth_document) + != {"scope", "fixture_max_abs_death_rate_gap", "outcomes"} + or truth_document["scope"] != "synthetic_engineering" + ): + raise ValueError( + "holdout requires an explicit synthetic engineering contract" + ) + truth = pd.DataFrame(truth_document["outcomes"]) + if set(truth.columns) != {"person_id", "year", "age", "death"}: + raise ValueError("invalid held-out outcome columns") + for column in truth.columns: + _integer_column(truth, column) + if truth.person_id.duplicated().any() or set(truth.person_id) != set( + initial[PID] + ): + raise ValueError( + "held-out identities must match the initial population exactly" + ) + if ( + not (truth.year == boundary + 1).all() + or not truth.death.isin([0, 1]).all() + ): + raise ValueError( + "held-out outcomes must be binary deaths in the next period" + ) + truth = truth.set_index("person_id").loc[initial[PID]] + weights = context.weights[OBS].values + start_weights = weights[(periods == boundary).to_numpy()] + next_weights = weights[(periods == boundary + 1).to_numpy()] + mass = _mass( + start_weights, + next_weights, + context.strata[(periods == boundary).to_numpy()].tolist(), + context.strata[(periods == boundary + 1).to_numpy()].tolist(), + boundary, + ) + mass["next_period"] = float(next_weights.sum()) + probability = initial.death_probability.to_numpy() + expected_deaths = float(np.dot(start_weights, probability)) + observed_deaths = float(np.dot(start_weights, truth.death.to_numpy())) + generated_deaths = float( + np.dot(start_weights, ~initial.survives.to_numpy(dtype=bool)) + ) + discrepancy = abs(expected_deaths - observed_deaths) / float( + start_weights.sum() + ) + threshold = truth_document["fixture_max_abs_death_rate_gap"] + if ( + isinstance(threshold, bool) + or not isinstance(threshold, (int, float)) + or not np.isfinite(threshold) + or not 0 <= threshold <= 1 + ): + raise ValueError("fixture death-rate gap must be finite and in [0, 1]") + artifact = MortalityArtifact.from_bytes(context.artifacts["model"].payload) + original = _slice(context, initial).sort_values("person_id") + stream = tuple(context.params["stream"]) + uniforms = mortality_uniforms( + original.person_id.tolist(), + experiment_id=stream[1], + replicate=stream[2], + base_seed=stream[3], + period=boundary + 1, + ) + expected_ids = original.loc[ + uniforms >= artifact.model.probabilities(original), "person_id" + ].tolist() + actual_ids = sorted(future[PID].tolist()) + expected_ages = original.set_index("person_id").age + 1 + age_parity = all( + int(row.age) == int(expected_ages.loc[getattr(row, PID)]) + for row in future.itertuples(index=False) + ) + engineering_pass = ( + expected_ids == actual_ids + and age_parity + and not observations[OID].duplicated().any() + ) + heldout_age_error = float( + np.abs(truth.age.to_numpy() - (initial.age.to_numpy() + 1)).mean() + ) + fixture_pass = discrepancy <= threshold and heldout_age_error == 0 + report = { + "scope": "synthetic_engineering", + "boundary_year": boundary, + "next_period": boundary + 1, + "fit_rows": artifact.fit_rows, + "initial_records": len(initial), + "survivor_records": len(future), + "heldout_records": len(truth), + "expected_deaths": expected_deaths, + "observed_deaths": observed_deaths, + "generated_deaths": generated_deaths, + "absolute_death_rate_gap": discrepancy, + "fixture_max_abs_death_rate_gap": float(threshold), + "heldout_mean_absolute_age_error": heldout_age_error, + "fixture_verdict": "pass" if fixture_pass else "fail", + "engineering_verdict": "pass" if engineering_pass else "fail", + "mass": mass, + "limitations": [ + "Synthetic engineering fixture; no scientific or national-population certification.", + "The mortality fit's external-rate factor cancels in the fitted-window level.", + "Household accounting, births, immigration, and the full M6 loop are outside this slice.", + ], + } + return KernelResult( + artifacts={"report": json_bytes(report)}, + receipt={ + "outcome": "pass" if engineering_pass and fixture_pass else "fail", + "evidence": report, + }, + ) + + +class _Kernel: + def __init__( + self, + ref, + function, + *, + structural=StructuralDelta.NONE, + numeric=Numeric.PLATFORM_BITWISE, + seeded=False, + gate=False, + ): + self.ref = ref + self.function = function + self.capabilities = Capabilities( + Determinism.SEEDED if seeded else Determinism.DETERMINISTIC, + numeric=numeric, + structural=structural, + seed_source=SeedSource.KEYED if seeded else SeedSource.NONE, + role=KernelRole.GATE if gate else KernelRole.COMPUTE, + dependencies=DEPENDENCIES, + ) + + def implementation_hash(self): + return source_hash( + self.function, + model_module, + fit_mortality_model, + prepare_mortality_refit_inputs, + apply_mortality, + advance_age, + keyed_uniform, + canonical_json, + dependencies=self.capabilities.dependencies, + ) + + def run(self, context): + return self.function(context) + + +def build_graph( + *, + boundary_year=2014, + external_vintage_year=2014, + experiment_id="mortality", + replicate=0, + base_seed=0, +): + """Return the declared graph and registered existing-operation wrappers.""" + stream = ("sha256-u53-v1", experiment_id, replicate, base_seed) + keyed_uniform(stream=stream, keys=[]) # Validate even an empty population. + if ( + type(boundary_year) is not int + or type(external_vintage_year) is not int + ): + raise ValueError( + "fit boundary and external vintage must be integer years" + ) + params = {"boundary_year": boundary_year, "stream": stream} + binding = (ArtifactInput("model", "fit", "model", MODEL_TYPE),) + roots = ( + Node( + "training", + "dynamics.training@1", + sources=("training",), + structural=StructuralDelta.CREATE, + outputs=tuple( + Owned(OBS, column, dtype) for column, dtype in TRAIN_COLUMNS + ) + + (Owned("period", "period", "int64"),), + ), + Node( + "fit", + "dynamics.mortality.fit@1", + population="training", + sources=("rates",), + inputs=( + Slice(OBS, tuple(dict(TRAIN_COLUMNS))), + Slice("period", ("period",)), + ), + params={ + "boundary_year": boundary_year, + "external_vintage_year": external_vintage_year, + }, + artifact_outputs=(ArtifactOutput("model", MODEL_TYPE),), + ), + Node( + "initial", + "dynamics.initial@1", + sources=("initial",), + structural=StructuralDelta.CREATE, + params={"boundary_year": boundary_year}, + outputs=( + Owned(OBS, "age", "int64"), + Owned(OBS, "sex", "string"), + Owned("period", "period", "int64"), + ), + ), + Node( + "apply", + "dynamics.mortality.apply@1", + population="initial", + artifact_inputs=binding, + inputs=(Slice(OBS, ("age", "sex")), Slice("period", ("period",))), + params=params, + outputs=( + Owned(OBS, "death_probability", "float64"), + Owned(OBS, "survives", "bool"), + ), + ), + Node( + "advance", + "dynamics.advance@1", + base="initial", + structural=StructuralDelta.EXPAND, + entrants=True, + mass="declared", + inputs=( + Slice(OBS, ("age", "sex", "survives")), + Slice("period", ("period",)), + ), + params={ + "boundary_year": boundary_year, + "expand_cells": ( + (OBS, PERIOD_ID, "int64"), + (OBS, "age", "int64"), + ("period", "period", "int64"), + ), + "expand_weight_entity": OBS, + "expand_weight_kind": "design", + }, + ), + Node( + "age", + "dynamics.age-claim@1", + population="advance", + inputs=(Slice(OBS, ("age",)),), + outputs=(Owned(OBS, "age", "int64", rewrite=True),), + ), + Node( + "evaluate", + "dynamics.mortality.evaluate@1", + population="advance", + sources=("holdout",), + artifact_inputs=binding, + params=params, + inputs=( + Slice(OBS, ("age", "sex", "survives", "death_probability")), + Slice("period", ("period",)), + ), + ), + ) + registry = KernelRegistry() + for kernel in ( + _Kernel( + "dynamics.training@1", + _create_training, + structural=StructuralDelta.CREATE, + numeric=Numeric.BITWISE, + ), + _Kernel("dynamics.mortality.fit@1", _fit), + _Kernel( + "dynamics.initial@1", + _create_initial, + structural=StructuralDelta.CREATE, + numeric=Numeric.BITWISE, + ), + _Kernel("dynamics.mortality.apply@1", _apply, seeded=True), + _Kernel( + "dynamics.advance@1", _advance, structural=StructuralDelta.EXPAND + ), + _Kernel("dynamics.age-claim@1", _age_claim), + _Kernel( + "dynamics.mortality.evaluate@1", _evaluate, seeded=True, gate=True + ), + ): + registry.register(kernel) + SOURCE_CODECS.register(CODEC, _json_source_marker) + return ( + Graph( + "dynamics-mortality", + tuple( + SourceRef(name, CODEC) + for name in ("training", "rates", "initial", "holdout") + ), + roots, + mass_partition=("period", "period"), + ), + registry, + ) + + +@dataclass(frozen=True) +class MortalityGraphRun: + manifest: object + report: dict + model_payload: bytes + next_slice: pd.DataFrame + + +def run_mortality_graph( + *, + training, + rates, + initial, + holdout, + output_dir, + boundary_year=2014, + external_vintage_year=2014, + experiment_id="mortality", + replicate=0, + base_seed=0, + household_accounting=False, +): + """Run/reuse the graph and write artifacts only in the explicit directory.""" + if household_accounting: + raise ValueError( + "household accounting is unsupported by this person-period slice" + ) + output = Path(output_dir).resolve() + sources = { + name: Path(path).resolve() + for name, path in ( + ("training", training), + ("rates", rates), + ("initial", initial), + ("holdout", holdout), + ) + } + graph, registry = build_graph( + boundary_year=boundary_year, + external_vintage_year=external_vintage_year, + experiment_id=experiment_id, + replicate=replicate, + base_seed=base_seed, + ) + output.mkdir(parents=True, exist_ok=True) + store = ContentStore(output / "store") + manifest = run_graph( + compile_graph(graph), sources=sources, store=store, kernels=registry + ) + # Gate exceptions are evidenced failures in the executor's contract and + # may deliberately produce no report artifact. Retain that receipt before + # reading optional diagnostics, including on a cached failed-gate hit. + (output / "manifest.json").write_text(manifest.to_json(), encoding="utf-8") + model_payload = store.load_bytes( + manifest.nodes["fit"].opaque_artifacts["model"] + ) + evaluation = manifest.nodes["evaluate"] + report_key = evaluation.opaque_artifacts.get("report") + if report_key is not None: + report = parse_json(store.load_bytes(report_key)) + elif evaluation.receipt.get("outcome") == "fail": + report = { + "scope": "synthetic_engineering", + "boundary_year": boundary_year, + "next_period": boundary_year + 1, + "engineering_verdict": "not_evaluated", + "fixture_verdict": "not_evaluated", + "evaluation_gate": { + "node_id": "evaluate", + "kernel_ref": evaluation.kernel_ref, + "outcome": "fail", + "evidence": dict(evaluation.receipt.get("evidence", {})), + }, + } + else: + raise ValueError( + "evaluation report is absent without a failed gate receipt" + ) + report["node_keys"] = { + name: node.key for name, node in manifest.nodes.items() + } + report["cache_hits"] = { + name: node.hit for name, node in manifest.nodes.items() + } + report["model_artifact_key"] = manifest.nodes["fit"].opaque_artifacts[ + "model" + ] + population = manifest.population("advance") + observations = population.table(OBS) + next_rows = observations.loc[observations[PERIOD_ID] == boundary_year + 1] + next_slice = ( + pd.DataFrame( + { + "person_id": next_rows[PID].to_numpy(dtype=np.int64), + "age": next_rows.age.to_numpy(dtype=np.int64), + "year": np.full( + len(next_rows), boundary_year + 1, dtype=np.int64 + ), + } + ) + .sort_values("person_id") + .reset_index(drop=True) + ) + (output / "report.json").write_bytes(json_bytes(report)) + (output / "model.json").write_bytes(model_payload) + for entity in (OBS, "person", "period"): + population.table(entity).to_csv(output / f"{entity}.csv", index=False) + next_slice.to_csv(output / "next_period.csv", index=False) + return MortalityGraphRun(manifest, report, model_payload, next_slice) diff --git a/src/populace_dynamics/graph/synthetic.py b/src/populace_dynamics/graph/synthetic.py new file mode 100644 index 00000000..24808993 --- /dev/null +++ b/src/populace_dynamics/graph/synthetic.py @@ -0,0 +1,78 @@ +"""Small hand-specified engineering inputs, independent of generated draws.""" + +from pathlib import Path + +from .model import json_bytes + + +def write_synthetic_inputs(directory): + """Write synthetic sources to an explicit directory; preserve edits.""" + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + training = [ + { + "person_id": 1001 + i, + "event_year": 2013, + "required_interview_year": 2013, + "age_band": "0+", + "sex": "female" if i < 4 else "male", + "start_weight": 1.0, + "exposure": 1.0, + "death": float(i % 4 == 0), + } + for i in range(8) + ] + training.extend( + [ + {**training[0], "person_id": 1009, "event_year": 2015}, + { + **training[0], + "person_id": 1010, + "required_interview_year": 2015, + }, + ] + ) + rates = [ + { + "lower_age": 0, + "upper_age": 120, + "age_band": "0+", + "sex": sex, + "central_rate": rate, + } + for sex, rate in (("female", 0.005), ("male", 0.006)) + ] + initial = [ + { + "person_id": 100 + i, + "age": 30 + 2 * i, + "sex": "female" if i % 2 == 0 else "male", + "weight": float(1 + i % 3), + } + for i in range(20) + ] + holdout = { + "scope": "synthetic_engineering", + "fixture_max_abs_death_rate_gap": 0.25, + "outcomes": [ + { + "person_id": row["person_id"], + "year": 2015, + "age": row["age"] + 1, + "death": int(i % 5 == 0), + } + for i, row in enumerate(initial) + ], + } + result = {} + for name, value in ( + ("training", training), + ("rates", rates), + ("initial", initial), + ("holdout", holdout), + ): + path = directory / f"{name}.json" + if not path.exists(): + path.write_bytes(json_bytes(value)) + result[name] = path + return result diff --git a/tests/README-tiers.md b/tests/README-tiers.md index a5ef9187..19b4bcf2 100644 --- a/tests/README-tiers.md +++ b/tests/README-tiers.md @@ -38,9 +38,9 @@ pytest --collect-only -q -m oracle_policyengine | tail -1 | Tier | Tests at HEAD | |---|---:| -| `unit` | 1,563 | +| `unit` | 1,596 | | `artifact` | 2,668 | | `integration_psid` | 848 | | `reproduction_legacy` | 520 | | `oracle_policyengine` | 159 | -| **Total** | **5,758** | +| **Total** | **5,791** | diff --git a/tests/estimates/test_birth_evidence_artifact.py b/tests/estimates/test_birth_evidence_artifact.py index d4e838a1..2f4c176c 100644 --- a/tests/estimates/test_birth_evidence_artifact.py +++ b/tests/estimates/test_birth_evidence_artifact.py @@ -88,6 +88,12 @@ def test_post_review_sources_are_outside_historical_reducer_identity(): Path("src/populace_dynamics/estimates/anchor_context_registry.py"), Path("src/populace_dynamics/estimates/anchor_context_rehearsal.py"), Path("src/populace_dynamics/estimates/anchor_context_report.py"), + Path("src/populace_dynamics/graph/__init__.py"), + Path("src/populace_dynamics/graph/__main__.py"), + Path("src/populace_dynamics/graph/_compat.py"), + Path("src/populace_dynamics/graph/model.py"), + Path("src/populace_dynamics/graph/runtime.py"), + Path("src/populace_dynamics/graph/synthetic.py"), ) assert reducer.POST_REVIEW_SHARED_SOURCE_BLOBS == { Path( @@ -157,7 +163,7 @@ def _internal_imports( return imports -def test_psid_identity_exclusions_are_unreachable_from_birth_evidence(): +def test_psid_and_graph_exclusions_are_unreachable_from_birth_evidence(): module_paths = _repository_module_paths() root_module = "scripts.first_estimates_birth_evidence" psid_exclusions = { @@ -170,6 +176,13 @@ def test_psid_identity_exclusions_are_unreachable_from_birth_evidence(): } assert root_module in module_paths assert psid_exclusions.issubset(module_paths) + graph_exclusions = { + name + for name in module_paths + if name == "populace_dynamics.graph" + or name.startswith("populace_dynamics.graph.") + } + assert graph_exclusions module_by_path = { path.resolve(): module_name for module_name, path in module_paths.items() @@ -205,6 +218,10 @@ def test_psid_identity_exclusions_are_unreachable_from_birth_evidence(): "historically excluded PSID modules became reachable from the " f"birth-evidence reducer: {sorted(psid_exclusions & reachable)}" ) + assert graph_exclusions.isdisjoint(reachable), ( + "opt-in graph modules became reachable from the birth-evidence " + f"reducer: {sorted(graph_exclusions & reachable)}" + ) def test_reducer_accepts_explicit_unresolved_upstream_boundary(): diff --git a/tests/test_graph_mortality.py b/tests/test_graph_mortality.py new file mode 100644 index 00000000..ab14240d --- /dev/null +++ b/tests/test_graph_mortality.py @@ -0,0 +1,478 @@ +"""Synthetic engineering tests for the optional population graph.""" + +import copy +import json +from pathlib import Path +from types import SimpleNamespace + +import numpy as np +import pandas as pd +import pytest + +from populace_dynamics.graph.model import MortalityArtifact, fit_mortality +from populace_dynamics.graph.synthetic import write_synthetic_inputs + + +@pytest.fixture +def inputs(tmp_path): + return write_synthetic_inputs(tmp_path / "inputs") + + +@pytest.fixture +def runtime(): + from populace_dynamics.graph._compat import require_graph + + try: + require_graph() + except ImportError as error: + pytest.skip(str(error)) + from populace_dynamics.graph import runtime + + return runtime + + +def _read(path): + return json.loads(path.read_text()) + + +def _write(path, value): + path.write_text(json.dumps(value)) + + +def _fit(inputs): + return fit_mortality( + pd.DataFrame(_read(inputs["training"])), + pd.DataFrame(_read(inputs["rates"])), + boundary_year=2014, + external_vintage_year=2014, + ) + + +def _run(runtime, inputs, tmp_path, **kwargs): + return runtime.run_mortality_graph( + **inputs, output_dir=tmp_path / "output", **kwargs + ) + + +def test_existing_mortality_fit_and_json_roundtrip(inputs): + artifact = _fit(inputs) + assert artifact.fit_rows == 8 + assert artifact.model.probability[("0+", "female")] == pytest.approx( + -np.expm1(-0.25) + ) + restored = MortalityArtifact.from_bytes(artifact.to_bytes()) + assert restored.to_bytes() == artifact.to_bytes() + assert restored.model == artifact.model + + +def test_future_exposure_and_interview_information_are_excluded(inputs): + baseline = _fit(inputs) + training = _read(inputs["training"]) + training[-1]["death"] = 0.0 + _write(inputs["training"], training) + assert _fit(inputs).to_bytes() == baseline.to_bytes() + with pytest.raises(ValueError, match="vintage"): + fit_mortality( + pd.DataFrame(training), + pd.DataFrame(_read(inputs["rates"])), + boundary_year=2013, + external_vintage_year=2014, + ) + + +@pytest.mark.parametrize("mutation", ["schema", "duplicate", "missing", "nan"]) +def test_model_payload_fails_closed(inputs, mutation): + raw = json.loads(_fit(inputs).to_bytes()) + if mutation == "schema": + raw["schema_version"] = 99 + elif mutation == "duplicate": + raw["probabilities"].append(copy.deepcopy(raw["probabilities"][0])) + elif mutation == "missing": + raw["probabilities"].pop() + else: + raw["probabilities"][0]["probability"] = float("nan") + with pytest.raises(ValueError): + MortalityArtifact.from_bytes(json.dumps(raw).encode()) + + +def test_model_rejects_duplicate_json_members(inputs): + payload = ( + _fit(inputs) + .to_bytes() + .replace( + b'"schema_version":1', b'"schema_version":1,"schema_version":1' + ) + ) + with pytest.raises(ValueError, match="duplicate"): + MortalityArtifact.from_bytes(payload) + + +def test_graph_reuses_fit_and_reports_period_mass(runtime, inputs, tmp_path): + cold = _run(runtime, inputs, tmp_path) + warm = _run(runtime, inputs, tmp_path) + assert cold.report["scope"] == "synthetic_engineering" + assert cold.report["fixture_verdict"] == "pass" + assert cold.report["engineering_verdict"] == "pass" + assert all(node.hit for node in warm.manifest.nodes.values()) + assert cold.model_payload == warm.model_payload + assert cold.report["mass"] == warm.report["mass"] + mass = cold.report["mass"] + assert mass["after"] == mass["before"] + mass["next_period"] + assert mass["partition"]["stratum_before"]["2014"] == ( + mass["partition"]["stratum_after"]["2014"] + ) + assert (tmp_path / "output" / "report.json").is_file() + assert (tmp_path / "output" / "manifest.json").is_file() + + +def test_graph_matches_existing_steps_with_explicit_uniforms( + runtime, inputs, tmp_path +): + from populace_dynamics.engine.steps import ( + advance_age, + apply_mortality, + ) + + result = _run(runtime, inputs, tmp_path) + initial = pd.DataFrame(_read(inputs["initial"])).sort_values("person_id") + initial["year"] = 2014 + uniforms = runtime.mortality_uniforms(initial.person_id.tolist()) + + class FixedUniforms: + def random(self, n): + assert n == len(uniforms) + return uniforms.copy() + + context = SimpleNamespace(rng_registry=None, year=2015, metadata={}) + model = MortalityArtifact.from_bytes(result.model_payload).model + survived = apply_mortality(initial, context, FixedUniforms(), model=model) + assert ( + survived.person_id.tolist() + == initial.loc[ + uniforms >= model.probabilities(initial), "person_id" + ].tolist() + ) + expected = advance_age(survived, context, np.random.default_rng(0)) + actual = result.next_slice.sort_values("person_id") + pd.testing.assert_frame_equal( + actual[["person_id", "age", "year"]].reset_index(drop=True), + expected[["person_id", "age", "year"]].reset_index(drop=True), + ) + + +def test_holdout_changes_only_evaluation_and_can_fail_fixture( + runtime, inputs, tmp_path +): + baseline = _run(runtime, inputs, tmp_path) + holdout = _read(inputs["holdout"]) + for row in holdout["outcomes"]: + row["death"] = 1 + _write(inputs["holdout"], holdout) + changed = _run(runtime, inputs, tmp_path) + assert changed.report["fixture_verdict"] == "fail" + assert changed.report["engineering_verdict"] == "pass" + assert changed.model_payload == baseline.model_payload + for node_id in ("training", "fit", "initial", "apply", "advance", "age"): + assert changed.manifest.nodes[node_id].hit + assert not changed.manifest.nodes["evaluate"].hit + pd.testing.assert_frame_equal(baseline.next_slice, changed.next_slice) + + +def test_recipient_edit_keeps_model_fit(runtime, inputs, tmp_path): + baseline = _run(runtime, inputs, tmp_path) + initial = _read(inputs["initial"]) + initial[0]["age"] += 1 + _write(inputs["initial"], initial) + changed = _run(runtime, inputs, tmp_path) + assert changed.manifest.nodes["fit"].hit + assert changed.model_payload == baseline.model_payload + assert not changed.manifest.nodes["apply"].hit + + +def test_training_weight_edit_invalidates_fit_and_application( + runtime, inputs, tmp_path +): + baseline = _run(runtime, inputs, tmp_path) + training = _read(inputs["training"]) + training[0]["start_weight"] = 8.0 + _write(inputs["training"], training) + changed = _run(runtime, inputs, tmp_path) + assert not changed.manifest.nodes["fit"].hit + assert not changed.manifest.nodes["apply"].hit + assert changed.model_payload != baseline.model_payload + + +def test_stable_draws_are_row_chunk_and_unrelated_person_invariant(runtime): + first = runtime.mortality_uniforms([11, 21, 31]) + np.testing.assert_array_equal( + runtime.mortality_uniforms([31, 11, 21]), first[[2, 0, 1]] + ) + np.testing.assert_array_equal( + runtime.mortality_uniforms([1, 11, 21, 31])[1:], first + ) + np.testing.assert_array_equal( + np.concatenate( + [ + runtime.mortality_uniforms([11]), + runtime.mortality_uniforms([21, 31]), + ] + ), + first, + ) + + +@pytest.mark.parametrize("identity", [None, 1.5, "1", True]) +def test_mortality_draws_reject_noninteger_person_ids(runtime, identity): + with pytest.raises(ValueError, match="identities"): + runtime.mortality_uniforms([identity]) + + +def test_population_reorder_and_unrelated_person_preserve_survivors( + runtime, inputs, tmp_path +): + baseline = _run(runtime, inputs, tmp_path) + initial = _read(inputs["initial"]) + _write(inputs["initial"], initial[::-1]) + reordered = _run(runtime, inputs, tmp_path) + pd.testing.assert_frame_equal(baseline.next_slice, reordered.next_slice) + initial.append({"person_id": 1, "age": 44, "sex": "male", "weight": 2.0}) + _write(inputs["initial"], initial) + holdout = _read(inputs["holdout"]) + holdout["outcomes"].append( + {"person_id": 1, "year": 2015, "age": 45, "death": 0} + ) + _write(inputs["holdout"], holdout) + extended = _run(runtime, inputs, tmp_path) + pd.testing.assert_frame_equal( + baseline.next_slice, + extended.next_slice.query("person_id != 1").reset_index(drop=True), + ) + + +@pytest.mark.parametrize("all_die", [False, True]) +def test_empty_or_complete_survivor_expansion( + runtime, inputs, tmp_path, all_die +): + training = _read(inputs["training"]) + for row in training: + row["death"] = 1.0 if all_die else 0.0 + row["exposure"] = 1e-9 if all_die else 1.0 + _write(inputs["training"], training) + result = _run(runtime, inputs, tmp_path) + warm = _run(runtime, inputs, tmp_path) + assert warm.manifest.nodes["advance"].hit + assert len(result.next_slice) == (0 if all_die else 20) + assert result.report["mass"]["next_period"] == ( + 0.0 if all_die else result.report["mass"]["before"] + ) + if all_die: + assert ( + "2015" not in result.report["mass"]["partition"]["stratum_after"] + ) + + +def test_household_accounting_is_explicitly_unsupported( + runtime, inputs, tmp_path +): + with pytest.raises(ValueError, match="household"): + _run(runtime, inputs, tmp_path, household_accounting=True) + + +def test_optional_entrypoint_has_actionable_missing_capability(monkeypatch): + from populace_dynamics.graph import _compat + + monkeypatch.setattr(_compat, "_python_version", lambda: (3, 12)) + with pytest.raises(ImportError, match="Python >=3.13"): + _compat.require_graph() + + +def test_optional_entrypoint_reports_old_core_capabilities(monkeypatch): + from populace_dynamics.graph import _compat + + monkeypatch.setattr(_compat, "_python_version", lambda: (3, 14)) + monkeypatch.setattr( + _compat.importlib, "import_module", lambda name: SimpleNamespace() + ) + with pytest.raises(ImportError, match="typed model-artifact"): + _compat.require_graph() + + +def test_cli_requires_output_directory(): + from populace_dynamics.graph.__main__ import parser + + with pytest.raises(SystemExit): + parser().parse_args(["--synthetic"]) + + +def test_keyed_kernel_hashes_random_coordinate_encoding(runtime, monkeypatch): + from microcosm.graph import canonical + + _, registry = runtime.build_graph() + kernel = registry.get("dynamics.mortality.apply@1") + assert kernel.capabilities.dependencies == ("numpy", "pandas") + before = kernel.implementation_hash() + encoding_source = Path(canonical.__file__).resolve() + original = Path.read_bytes + + def changed_source(path): + payload = original(path) + if path.resolve() == encoding_source: + payload += b"\n# coordinate encoding change\n" + return payload + + monkeypatch.setattr(Path, "read_bytes", changed_source) + assert kernel.implementation_hash() != before + + +@pytest.mark.parametrize( + ("mutation", "message"), + [ + ("missing_person", "held-out identities must match"), + ("wrong_period", "held-out outcomes must be binary deaths"), + ("invalid_json", "invalid mortality JSON"), + ], +) +def test_failed_holdout_exports_named_gate_evidence_cold_and_warm( + runtime, inputs, tmp_path, mutation, message, capsys +): + from populace_dynamics.graph.__main__ import main + + holdout = _read(inputs["holdout"]) + if mutation == "missing_person": + holdout["outcomes"].pop() + elif mutation == "wrong_period": + holdout["outcomes"][0]["year"] = 2014 + _write(inputs["holdout"], holdout) + if mutation == "invalid_json": + inputs["holdout"].write_text("{invalid JSON") + + for warm in (False, True): + result = _run(runtime, inputs, tmp_path) + gate = result.manifest.nodes["evaluate"] + assert gate.hit is warm + assert gate.receipt["outcome"] == "fail" + assert "report" not in gate.opaque_artifacts + diagnostic = result.report["evaluation_gate"] + assert diagnostic["node_id"] == "evaluate" + assert diagnostic["kernel_ref"] == "dynamics.mortality.evaluate@1" + assert diagnostic["outcome"] == "fail" + assert diagnostic["evidence"]["exception_type"] == "ValueError" + assert message in diagnostic["evidence"]["message"] + assert result.report["engineering_verdict"] == "not_evaluated" + assert result.report["fixture_verdict"] == "not_evaluated" + assert "expected_deaths" not in result.report + assert "mass" not in result.report + assert _read(tmp_path / "output" / "report.json") == result.report + assert _read(tmp_path / "output" / "manifest.json") == json.loads( + result.manifest.to_json() + ) + + arguments = ["--output-dir", str(tmp_path / "output")] + for name, path in inputs.items(): + arguments.extend([f"--{name}", str(path)]) + assert main(arguments) == 1 + assert "not_evaluated" in capsys.readouterr().out + assert ( + message + in _read(tmp_path / "output" / "report.json")["evaluation_gate"][ + "evidence" + ]["message"] + ) + + +@pytest.mark.parametrize("excluded_person_id", [1009, 1010]) +def test_graph_fit_matches_direct_cutoff_and_ignores_excluded_outcomes( + runtime, inputs, tmp_path, excluded_person_id +): + """Future events and late interviews cannot leak through graph mapping.""" + baseline = _run(runtime, inputs, tmp_path) + direct = _fit(inputs) + assert baseline.model_payload == direct.to_bytes() + assert direct.fit_rows == 8 + training = _read(inputs["training"]) + excluded = next( + row for row in training if row["person_id"] == excluded_person_id + ) + assert ( + excluded["event_year"] > 2014 + or excluded["required_interview_year"] > 2014 + ) + excluded["death"] = 1.0 - excluded["death"] + excluded["start_weight"] = 50.0 + excluded["exposure"] = 0.25 + _write(inputs["training"], training) + changed = _run(runtime, inputs, tmp_path) + assert not changed.manifest.nodes["training"].hit + assert not changed.manifest.nodes["fit"].hit + assert changed.model_payload == _fit(inputs).to_bytes() + assert changed.model_payload == baseline.model_payload + pd.testing.assert_frame_equal(changed.next_slice, baseline.next_slice) + + +@pytest.mark.parametrize( + "coordinates", + [ + {"experiment_id": "independent-stream"}, + {"replicate": 7}, + {"base_seed": 831}, + ], +) +def test_nondefault_stream_matches_direct_steps_and_reuses_fit( + runtime, inputs, tmp_path, coordinates +): + """An accidentally hard-coded default cannot pass both sides of parity.""" + from microcosm.graph.randomness import keyed_uniform + + from populace_dynamics.engine.steps import advance_age, apply_mortality + + baseline = _run(runtime, inputs, tmp_path) + changed = _run(runtime, inputs, tmp_path, **coordinates) + assert changed.manifest.nodes["fit"].hit + assert ( + changed.manifest.nodes["fit"].key == baseline.manifest.nodes["fit"].key + ) + assert changed.model_payload == baseline.model_payload + assert not changed.manifest.nodes["apply"].hit + assert ( + changed.manifest.nodes["apply"].key + != baseline.manifest.nodes["apply"].key + ) + assert changed.report["engineering_verdict"] == "pass" + + initial = pd.DataFrame(_read(inputs["initial"])).sort_values("person_id") + initial["year"] = 2014 + # Derive the oracle directly from the registered coordinate contract, + # independently of both mortality_uniforms and _GraphPeriodContext. + stream = ( + "sha256-u53-v1", + coordinates.get("experiment_id", "mortality"), + coordinates.get("replicate", 0), + coordinates.get("base_seed", 0), + ) + keys = [(int(pid), "mortality", 2015, 0) for pid in initial.person_id] + uniforms = keyed_uniform(stream=stream, keys=keys) + default = keyed_uniform( + stream=("sha256-u53-v1", "mortality", 0, 0), keys=keys + ) + assert not np.array_equal(uniforms, default) + + class FixedUniforms: + def random(self, n): + assert n == len(uniforms) + return uniforms.copy() + + context = SimpleNamespace(rng_registry=None, year=2015, metadata={}) + # Use the independent direct model, not the graph's returned artifact. + model = _fit(inputs).model + survivors = apply_mortality(initial, context, FixedUniforms(), model=model) + expected = advance_age(survivors, context, np.random.default_rng(0)) + pd.testing.assert_frame_equal( + changed.next_slice[["person_id", "age", "year"]].reset_index( + drop=True + ), + expected[["person_id", "age", "year"]].reset_index(drop=True), + ) + assert ( + changed.next_slice.person_id.tolist() + != baseline.next_slice.person_id.tolist() + ) diff --git a/tests/tier_counts.json b/tests/tier_counts.json index 71769153..e7b10bcb 100644 --- a/tests/tier_counts.json +++ b/tests/tier_counts.json @@ -1,7 +1,7 @@ { "schema_version": 1, "counts": { - "unit": 1563, + "unit": 1596, "artifact": 2668, "integration_psid": 848, "reproduction_legacy": 520,