Handle the capital-gains tail clone in release stages on a PUF-support base

docs/evidence/spec-engine/us-f0-coverage.json

@@ -1656,13 +1656,13 @@
           "compiler_ir.node_slices"
         ],
         "expected": {
-          "map_sha256": "a6feafaf62ed99b4f3fa9fee32e986f83919aed7335848df6470db6ecd17987d",
-          "protocol_sha256": "45f82554869c7a966b3c12b71edab43a6c887806b0fc6e70cd7598e62f375922"
+          "map_sha256": "18594e59d7b060265ac55afa41314adab640579e6ccef929123f39bd0bad5ee0",
+          "protocol_sha256": "56f832aae97f7b8c938f332a2ea52f1a3b067e31dc504d4d60717c970ef615c8"
         },
         "failures": [],
         "observed": {
-          "map_sha256": "a6feafaf62ed99b4f3fa9fee32e986f83919aed7335848df6470db6ecd17987d",
-          "protocol_sha256": "45f82554869c7a966b3c12b71edab43a6c887806b0fc6e70cd7598e62f375922"
+          "map_sha256": "18594e59d7b060265ac55afa41314adab640579e6ccef929123f39bd0bad5ee0",
+          "protocol_sha256": "56f832aae97f7b8c938f332a2ea52f1a3b067e31dc504d4d60717c970ef615c8"
         },
         "status": "covered"
       },
@@ -1677,7 +1677,7 @@
           "compiler_ir.seed_stream_map"
         ],
         "expected": {
-          "implementation_sha256": "45f82554869c7a966b3c12b71edab43a6c887806b0fc6e70cd7598e62f375922",
+          "implementation_sha256": "56f832aae97f7b8c938f332a2ea52f1a3b067e31dc504d4d60717c970ef615c8",
           "protocol": "legacy-v1",
           "streams": [
             "build_model",
@@ -1698,7 +1698,7 @@
         },
         "failures": [],
         "observed": {
-          "implementation_sha256": "45f82554869c7a966b3c12b71edab43a6c887806b0fc6e70cd7598e62f375922",
+          "implementation_sha256": "56f832aae97f7b8c938f332a2ea52f1a3b067e31dc504d4d60717c970ef615c8",
           "protocol": "legacy-v1",
           "streams": [
             "build_model",
@@ -2599,7 +2599,7 @@
       "country": "us",
       "schema_id": "country_spec",
       "schema_version": 1,
-      "spec_sha256": "1c46dda9cff4e53e5a44c6502a46fa695d02d92731f23d618616a892071d4841"
+      "spec_sha256": "3667e30ebfdac4d893e3a111bfba998dc7acf29b0da05360eee521d3e144d08a"
     }
   },
   "report_schema_version": 3,
@@ -2609,7 +2609,7 @@
     "country": "us",
     "schema_id": "country_spec",
     "schema_version": 1,
-    "spec_sha256": "1c46dda9cff4e53e5a44c6502a46fa695d02d92731f23d618616a892071d4841"
+    "spec_sha256": "3667e30ebfdac4d893e3a111bfba998dc7acf29b0da05360eee521d3e144d08a"
   },
   "status": "pass"
 }

docs/us-release-build-rule.md

@@ -145,6 +145,37 @@ on every arm after loading the base. It runs the other checks unchanged on the
 whole base. With `--release-manifest`, it also requires that release to record
 `build.selection_source` as `{"enabled": false}`.
 
+**23 September 2026:** without a selection source the release runs its stages
+on every support copy the base carries, including the capital-gains own-tail
+copy (clone index 2) of each tail household. That copy keeps its source IDs and
+the `puf_tax_detail` channel, so the base from `build_us_puf_support_base.py`
+holds two PUF-role rows for each tail source person (42,336 on base
+`cb1bd1e6…`). SIPP Head Start and voluntary filing identified a copy by source
+ID and role on bases without a raw spine ID, so they refused that base. Release
+stages now identify a copy by source ID and clone index
+(`support_copy_rank_series`), whether or not the base carries a raw spine ID.
+A repeated pair is still refused. Source-level decisions fan out to the tail
+copy from the lowest surviving clone index. Row-level imputations treat the
+tail copy as a PUF-role row, as the base's own post-transfer stages do.
+Only a base without a raw spine ID may lack clone indices. An assembled table
+(one with a raw spine ID) must carry both its support channel and its clone
+index. `require_assembled_support_provenance` refuses one that lacks either.
+The metadata-presence check includes the raw spine ID, so losing both columns
+cannot make an assembled table look historical. `support_role_series` calls
+the validator before resolving roles; this covers the prior-year and SSI
+summaries before occurrence pairing while preserving their attested source
+bytes and the bundle digest. Head Start, voluntary filing and copy ranking
+also validate explicitly before their fallbacks. Missing provenance refuses
+imputation and fails or raises from the signal gates. With clone indices the
+gates compare every copy of a source unit, including repeated
+`(source ID, clone index)` copies; imputations refuse repeated pairs.
+
+Temporary source-kernel projections validate assembled provenance before
+removing the channel, clone index and raw spine ID together. They retain the
+stable source IDs, and their outputs are compared with or merged into the
+original assembled table. This lets historical source kernels consume the
+projection without treating intentionally removed provenance as corruption.
+
 `--dense-default-dataset` is diagnostic only. A release build leaves it unset,
 so the default is the sparse dataset that runs on standard machines.
 

packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py

@@ -359,8 +359,8 @@
     "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f",
     "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356",
     "primary_tuples": "987b501c695e31f45521c4a178528f75ab3df22c09bc407b182213b2de99ee57",
-    "seed_map": "a6feafaf62ed99b4f3fa9fee32e986f83919aed7335848df6470db6ecd17987d",
-    "seed_protocol": "45f82554869c7a966b3c12b71edab43a6c887806b0fc6e70cd7598e62f375922",
+    "seed_map": "18594e59d7b060265ac55afa41314adab640579e6ccef929123f39bd0bad5ee0",
+    "seed_protocol": "56f832aae97f7b8c938f332a2ea52f1a3b067e31dc504d4d60717c970ef615c8",
     "source_manifest": "a87278445d928ea20e15935a8cc07312d6e819e34409b04e8c814f17910d828f",
     "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35",
     "tail": "ac92829c88a1a4fb6460d61190918d5d99c6c377fc8dd8f62f02b332d09bf59c",

packages/microcosm-build/src/microcosm/build/us_runtime/prior_year_income.py

@@ -40,8 +40,8 @@
     run_source_stage,
 )
 from microcosm.build.us_runtime.support_provenance import (
-    has_assembled_support_metadata,
     has_support_role_metadata,
+    support_clone_index_column,
     support_role_series,
     without_support_role_metadata,
 )
@@ -793,9 +793,12 @@ def _share(mask: np.ndarray) -> float:
             }
         )
         group_columns = ["_source_id"]
-        if has_support_role_metadata(
-            person, entity="person"
-        ) and not has_assembled_support_metadata(person, entity="person"):
+        # With clone provenance every copy of a source person, the
+        # capital-gains own-tail copy included, must agree. Only channel-only
+        # fixtures, which predate clone indices, pair copies by occurrence.
+        if has_support_role_metadata(person, entity="person") and (
+            support_clone_index_column("person") not in person
+        ):
             clone_work["_role"] = support_role_series(
                 person, entity="person"
             ).to_numpy()

packages/microcosm-build/src/microcosm/build/us_runtime/sipp_head_start.py

@@ -35,9 +35,9 @@
 from microcosm.build.gates import GateResult
 from microcosm.build.source_manifest import SourceStageSpec, load_source_manifest
 from microcosm.build.us_runtime.support_provenance import (
-    has_assembled_support_metadata,
     has_support_role_metadata,
-    support_clone_index_column,
+    require_assembled_support_provenance,
+    support_copy_rank_series,
     support_role_series,
 )
 from microcosm.build.us_runtime.voluntary_filing import (
@@ -103,8 +103,6 @@
 _OUTPUT = US_SIPP_HEAD_START_OUTPUT_COLUMNS[0]
 _DONOR_WEIGHT_COLUMN = "sipp_weight"
 _PERSON_SOURCE_ID_COLUMN = "person_source_id"
-_ASEC_CHANNEL = "asec"
-_PUF_CHANNEL = "puf_tax_detail"
 _DEFAULT_N_ESTIMATORS = 100
 _ELIGIBLE_MIN_AGE = 3
 _ELIGIBLE_MAX_AGE = 5
@@ -483,25 +481,37 @@ def _support_group_keys(
     person: pd.DataFrame,
     source_id: pd.Series,
 ) -> tuple[pd.Series, pd.Series]:
-    """Return source-person keys and validated legacy-compatible roles."""
+    """Return source-person keys and each row's validated support-copy rank.
+
+    Every support copy of a source person shares its key. The rank orders the
+    copies (native first) and, with the source ID, identifies exactly one row:
+    the capital-gains own-tail copy is a second PUF-role row of its source
+    person, distinguished from the primary PUF-detail copy by clone index.
+
+    Only a historical table without support metadata ranks every row 0. An
+    assembled table that lost its provenance columns is refused before that
+    early return, so a stripped assembled copy cannot become canonical.
+    """
 
-    if not has_support_role_metadata(person, entity="person"):
-        return (
-            source_id.astype(object),
-            pd.Series(_ASEC_CHANNEL, index=person.index),
-        )
     try:
-        roles = support_role_series(person, entity="person")
+        require_assembled_support_provenance(person, entity="person")
+        if not has_support_role_metadata(person, entity="person"):
+            return (
+                source_id.astype(object),
+                pd.Series(0, index=person.index, dtype="int64"),
+            )
+        copy_rank = support_copy_rank_series(person, entity="person")
     except ValueError as exc:
         raise ValueError(
             "US SIPP Head Start found unsupported support channel or "
             f"clone-role metadata: {exc}"
         ) from exc
-    return source_id.astype(object), roles
+    return source_id.astype(object), copy_rank
 
 
 def _recipient_predictors(frame: Frame) -> tuple[pd.DataFrame, pd.Series, np.ndarray]:
     person = frame.table("person")
+    require_assembled_support_provenance(person, entity="person")
     missing = [
         column
         for column in US_SIPP_HEAD_START_REQUIRED_SOURCE_COLUMNS
@@ -563,7 +573,7 @@ def _recipient_predictors(frame: Frame) -> tuple[pd.DataFrame, pd.Series, np.nda
     )
 
     source_id = _decoded_strings(person[_PERSON_SOURCE_ID_COLUMN])
-    source_key, roles = _support_group_keys(person, source_id)
+    source_key, copy_rank = _support_group_keys(person, source_id)
     age_unique = (
         pd.Series(age, index=person.index).groupby(source_key, sort=False).nunique()
     )
@@ -576,55 +586,22 @@ def _recipient_predictors(frame: Frame) -> tuple[pd.DataFrame, pd.Series, np.nda
     order = pd.DataFrame(index=person.index)
     order["source_id"] = source_id
     order["source_key"] = source_key
-    if has_assembled_support_metadata(person, entity="person"):
-        clone_column = support_clone_index_column("person")
-        clone_indices = pd.to_numeric(person[clone_column], errors="raise").astype(
-            "int64"
+    order["copy_rank"] = copy_rank
+    duplicate_copies = order.duplicated(["source_id", "copy_rank"], keep=False)
+    if duplicate_copies.any():
+        bad = (
+            order.loc[duplicate_copies, ["source_id", "copy_rank"]]
+            .drop_duplicates()
+            .itertuples(index=False, name=None)
         )
-        clone_rows = pd.DataFrame(
-            {"source_id": source_id, "clone_index": clone_indices}
-        )
-        duplicate_clones = clone_rows.duplicated(
-            ["source_id", "clone_index"],
-            keep=False,
-        )
-        if duplicate_clones.any():
-            bad = (
-                clone_rows.loc[
-                    duplicate_clones,
-                    ["source_id", "clone_index"],
-                ]
-                .drop_duplicates()
-                .itertuples(index=False, name=None)
-            )
-            raise ValueError(
-                "US SIPP Head Start assembled source units carry duplicated "
-                f"clone-index rows; invalid source clone(s): {list(bad)[:5]}."
-            )
-        order["clone_priority"] = clone_indices
-        priority_column = "clone_priority"
-    else:
-        role_rows = pd.DataFrame({"source_id": source_id, "role": roles})
-        duplicate_roles = role_rows.duplicated(
-            ["source_id", "role"],
-            keep=False,
+        raise ValueError(
+            "US SIPP Head Start source units carry duplicated support copies; "
+            f"invalid (source, copy rank) pair(s): {list(bad)[:5]}."
         )
-        if duplicate_roles.any():
-            bad = (
-                role_rows.loc[duplicate_roles, ["source_id", "role"]]
-                .drop_duplicates()
-                .itertuples(index=False, name=None)
-            )
-            raise ValueError(
-                "US SIPP Head Start source units carry duplicated same-role "
-                f"rows; invalid source role(s): {list(bad)[:5]}."
-            )
-        order["role_priority"] = roles.map({_ASEC_CHANNEL: 0, _PUF_CHANNEL: 1})
-        priority_column = "role_priority"
     order["person_key"] = person["person_id"].astype(str)
     canonical_index = (
         order.sort_values(
-            ["source_id", priority_column, "person_key"],
+            ["source_id", "copy_rank", "person_key"],
             kind="mergesort",
         )
         .drop_duplicates("source_key", keep="first")
@@ -670,6 +647,7 @@ def impute_us_sipp_head_start(
 
     if frame.schema != US_SCHEMA:
         raise ValueError("US SIPP Head Start imputation requires the US schema.")
+    require_assembled_support_provenance(frame.table("person"), entity="person")
     if n_estimators < 1:
         raise ValueError("n_estimators must be positive")
     required = [
@@ -805,11 +783,15 @@ def us_sipp_head_start_summary(frame: Frame) -> dict[str, object]:
     )
     role_invalid = False
     roles: pd.Series | None = None
-    if has_support_role_metadata(person, entity="person"):
-        try:
+    try:
+        # Flag an assembled table missing a provenance column before the
+        # metadata check could read it as a historical table without support
+        # roles. Its copies are still grouped by source ID below.
+        require_assembled_support_provenance(person, entity="person")
+        if has_support_role_metadata(person, entity="person"):
             roles = support_role_series(person, entity="person")
-        except ValueError:
-            role_invalid = True
+    except ValueError:
+        role_invalid = True
     clone_groups = clone_mismatches = 0
     if not provenance_missing:
         source_ids = _decoded_strings(person[_PERSON_SOURCE_ID_COLUMN])
@@ -861,12 +843,22 @@ def us_sipp_head_start_signal_gate(frame: Frame) -> GateResult:
     """Require a nondefault, in-domain, source-clone-consistent take-up flag."""
 
     person = frame.table("person")
+    provenance_failures: tuple[str, ...] = ()
+    try:
+        require_assembled_support_provenance(person, entity="person")
+    except ValueError as exc:
+        # Retain the summary's clone diagnostics when output is available,
+        # and report invalid provenance even on the missing-output path.
+        provenance_failures = (str(exc),)
     if _OUTPUT not in person:
         return GateResult(
             name="sipp_head_start_signal",
             passed=False,
-            failures=(f"person.{_OUTPUT}: missing",),
-            details={"missing": [_OUTPUT]},
+            failures=(f"person.{_OUTPUT}: missing", *provenance_failures),
+            details={
+                "missing": [_OUTPUT],
+                "support_channel_invalid": bool(provenance_failures),
+            },
         )
     try:
         summary = us_sipp_head_start_summary(frame)

packages/microcosm-build/src/microcosm/build/us_runtime/spm_universe_source.py

@@ -93,6 +93,8 @@
 
 from .support_provenance import (
     BASE_ASEC_SUPPORT_CHANNEL,
+    _validated_clone_indices,
+    require_assembled_support_provenance,
     support_channel_column,
     support_clone_index_column,
 )
@@ -400,34 +402,23 @@ def _assert_one_kind_per_household(
 def _person_clone_copies(person: pd.DataFrame) -> np.ndarray:
     """Each person's support-clone copy; refuses an undecidable clone index.
 
-    A support clone deep-copies every column of a table and remaps only the
-    structural id and membership columns
-    (``puf_support._clone_entity_table``), so clone copy ``k`` carries the
-    native row's source fields under new ids. With no clone-index column the
-    frame was never support-cloned and every person is native. When the column
-    is present, a missing, non-finite, negative or non-integral value is
-    undecidable provenance, and the module refuses it rather than guess a copy:
-    reading it as native could merge a genuine cross-copy collision into an
-    apparent bijection in the degraded-partition check. The maintained readers
-    of this column refuse the same values (``acs_transfer`` raises on an
-    invalid clone index; ``stacked_spine`` and ``puf_support`` parse it with
-    ``errors="raise"`` or refuse a missing one).
+    Historical input without a clone-index column retains the native-copy
+    fallback. Assembled input must carry complete support provenance before
+    that fallback is considered. Shared int64 validation rejects invalid
+    values and preserves distinct large integer copy indices, so the native
+    partition check does not merge copies through a float conversion.
     """
-    if _PERSON_CLONE_INDEX_COLUMN not in person.columns:
-        return np.full(len(person), _NATIVE_CLONE_INDEX, dtype=float)
-    values = pd.to_numeric(
-        person[_PERSON_CLONE_INDEX_COLUMN], errors="coerce"
-    ).to_numpy(dtype=float)
-    invalid = ~np.isfinite(values) | (values < 0) | (values != np.floor(values))
-    _require(
-        not invalid.any(),
-        "SPM_UNIVERSE_INVALID_CLONE_INDEX",
-        f"{int(invalid.sum())} person row(s) carry a missing, non-finite, "
-        f"negative or non-integral {_PERSON_CLONE_INDEX_COLUMN}; the row's "
-        "support-clone copy is undecidable, so neither the group-quarters nor "
-        "the degraded-partition check can be judged.",
-    )
-    return values
+    try:
+        require_assembled_support_provenance(person, entity="person")
+        if _PERSON_CLONE_INDEX_COLUMN not in person.columns:
+            return np.full(len(person), _NATIVE_CLONE_INDEX, dtype=float)
+        return _validated_clone_indices(
+            person,
+            _PERSON_CLONE_INDEX_COLUMN,
+            owner="SPM universe support provenance",
+        )
+    except ValueError as exc:
+        _require(False, "SPM_UNIVERSE_INVALID_CLONE_INDEX", str(exc))
 
 
 def _assert_group_quarters_are_single_native_records(