diff --git a/.gitignore b/.gitignore index 1e77cefd2..506298ae6 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ out/ # Release alert webhooks (real file is machine-local; .example is tracked) tools/release.env +tools/route_a/route_a.env /inputs/ # Finder metadata — never payload diff --git a/changelog.d/route-a-driver-in-repo.added.md b/changelog.d/route-a-driver-in-repo.added.md new file mode 100644 index 000000000..20944dfc2 --- /dev/null +++ b/changelog.d/route-a-driver-in-repo.added.md @@ -0,0 +1 @@ +The Route A release driver now lives in `tools/route_a/` (driver, supervisor, sampler, static release-flag check, runbook) with every machine path in a git-ignored `route_a.env`; it previously existed only on the build machine. Its release stage stages telemetry by default: `with_hf_token.sh` fetches the Hugging Face token with `agent-secret` at exec time, so the token never reaches the stage config, argv or logs, and `ROUTE_A_STAGING=0` restores `--no-staging`. Every gate, the publisher's offline preflight and the `published: false` hand-off are unchanged; the published `4b57d15a2` release is unaffected. diff --git a/docs/agent-guide.md b/docs/agent-guide.md index f006103a3..7e5a4d96b 100644 --- a/docs/agent-guide.md +++ b/docs/agent-guide.md @@ -171,6 +171,13 @@ to the private repository; when you run `microcosm-build-uk` yourself, pass `--staging-local-only` unless the operator asked for a staged upload. +The versioned Route A driver is `tools/route_a/route_a.sh`; see its +[runbook](../tools/route_a/README.md) for configuration, preserved admission +and release gates, and the Modal-base hand-off. Its release stage enables +staging telemetry by default, obtains the HF credential only in a runtime +wrapper, and accepts `ROUTE_A_STAGING=0` as the opt-out. It runs only the +publisher's offline `--preflight-only` check and leaves publication to Max. + The US fiscal-refresh builder scores its written H5 in household batches. Before a release rerun, run the small-H5 guard sweep described in [the release build rule](us-release-build-rule.md#post-export-scoring). diff --git a/packages/microcosm-build/tests/engine_free/us/test_route_a_driver.py b/packages/microcosm-build/tests/engine_free/us/test_route_a_driver.py new file mode 100644 index 000000000..1f4a45ada --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/us/test_route_a_driver.py @@ -0,0 +1,390 @@ +"""Exercise Route A orchestration without loading a country engine.""" + +from __future__ import annotations + +import importlib.util +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest + +from test_support.microcosm_build.route_a_driver import ( + ROUTE_A_TOOLS, + SECRET_ALIASES, + write_executable, +) +from test_support.paths import paths_for + +_TEST_PATHS = paths_for("microcosm-build") + + +def _function(name: str) -> str: + source = (ROUTE_A_TOOLS / "route_a.sh").read_text(encoding="utf-8") + match = re.search(rf"(?ms)^{name}\(\)\s*\{{.*?^\}}", source) + assert match is not None, f"driver has no {name} function" + return match.group(0) + + +def _release_result( + staging: str | None, *, tail: str = "", extras: str = "" +) -> subprocess.CompletedProcess[bytes]: + settings = { + "W": "/fixture/worktree", + "PY": sys.executable, + "BASE_H5": "/fixture/base with spaces.h5", + "FEED": "/fixture/feed.jsonl", + "FEED_SHA": "a" * 64, + "TAIL": tail, + "SSI": "/fixture/ssi.json", + "SSI_SHA": "b" * 64, + "SCF": "/fixture/scf.dta", + "REL_OUT": "/fixture/release-out", + "RID": "fixture-release", + "REL_CKPT": "/fixture/checkpoints", + "RELEASE_EXTRA_ARGS": extras, + "TOKEN_WRAPPER": str(ROUTE_A_TOOLS / "with_hf_token.sh"), + "AGENT_SECRET": "/fixture/agent-secret", + } + script = "set -eu\nfail() { printf '%s\\n' \"$*\" >&2; exit 1; }\n" + script += "\n".join( + f"{key}={shlex.quote(value)}" for key, value in settings.items() + ) + script += "\nunset ROUTE_A_STAGING\n" + if staging is not None: + script += f"ROUTE_A_STAGING={shlex.quote(staging)}\n" + script += _function("release_command") + script += '\nrelease_command\nprintf "%s\\0" "${RELEASE_ARGV[@]}"\n' + return subprocess.run( + ["bash", "-c", script], capture_output=True, check=False, timeout=20 + ) + + +def _release_argv(staging: str | None, **kwargs: str) -> list[str]: + result = _release_result(staging, **kwargs) + assert result.returncode == 0, result.stderr.decode() + return result.stdout.decode().rstrip("\0").split("\0") + + +def _flag_checker(): + spec = importlib.util.spec_from_file_location( + "route_a_check_flags", ROUTE_A_TOOLS / "check_flags.py" + ) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _shell_sources() -> list[Path]: + return sorted(ROUTE_A_TOOLS.glob("*.sh")) + [ROUTE_A_TOOLS / "route_a.env.example"] + + +def _example_extra_args() -> str: + source = shlex.quote(str(ROUTE_A_TOOLS / "route_a.env.example")) + result = subprocess.run( + [ + "bash", + "-c", + "PE=/fixture; EXPORT_MASS_REF_H5=/fixture/mass.h5; unset RELEASE_EXTRA_ARGS; " + f". {source}; " + 'printf "%s" "$RELEASE_EXTRA_ARGS"', + ], + capture_output=True, + check=False, + timeout=20, + ) + assert result.returncode == 0, result.stderr.decode() + return result.stdout.decode() + + +def test_shell_scripts_are_valid_bash() -> None: + scripts = _shell_sources() + assert scripts + for script in scripts: + result = subprocess.run( + ["bash", "-n", str(script)], capture_output=True, check=False, timeout=20 + ) + assert result.returncode == 0, result.stderr.decode() + + +def test_shell_scripts_pass_shellcheck() -> None: + shellcheck = shutil.which("shellcheck") + if shellcheck is None: + pytest.skip("shellcheck is not installed") + scripts = _shell_sources() + assert scripts + result = subprocess.run( + [shellcheck, *map(str, scripts)], capture_output=True, check=False, timeout=120 + ) + assert result.returncode == 0, result.stdout.decode() + result.stderr.decode() + + +@pytest.mark.parametrize("staging", [None, "1", "0"]) +def test_every_release_flag_is_declared(staging: str | None) -> None: + checker = _flag_checker() + source = ( + _TEST_PATHS.repository / "tools" / "build_us_fiscal_refresh_release.py" + ).read_text(encoding="utf-8") + argv = _release_argv( + staging, + tail="/fixture/tail register.json", + extras=_example_extra_args() + " --epochs 6000", + ) + flags = [item.partition("=")[0] for item in argv if item.startswith("--")] + assert flags + assert not checker.missing_flags(source, flags) + assert checker.missing_flags(source, ["--route-a-undeclared-flag"]) + + +@pytest.mark.parametrize("staging", [None, "1", "0"]) +def test_staging_controls_only_release_wrapper_and_no_staging_flag( + staging: str | None, +) -> None: + argv = _release_argv(staging) + enabled = staging != "0" + assert ("--no-staging" in argv) is not enabled + assert argv.count("--no-staging") == (0 if enabled else 1) + if enabled: + assert argv[:2] == [ + str(ROUTE_A_TOOLS / "with_hf_token.sh"), + "/fixture/agent-secret", + ] + else: + assert argv[:9] == [ + "/usr/bin/env", + "-u", + "HF_TOKEN", + "-u", + "HUGGING_FACE_HUB_TOKEN", + "-u", + "HUGGINGFACE_HUB_TOKEN", + "-u", + "HUGGING_FACE_TOKEN_MAX", + ] + assert argv[9] == sys.executable + assert str(ROUTE_A_TOOLS / "with_hf_token.sh") not in argv + assert argv[argv.index("--base-h5") + 1] == "/fixture/base with spaces.h5" + + +@pytest.mark.parametrize("staging", ["", "2", "false", "yes"]) +def test_release_rejects_invalid_staging_setting(tmp_path: Path, staging: str) -> None: + source = (ROUTE_A_TOOLS / "route_a.sh").read_text(encoding="utf-8") + prefix, delimiter, _ = source.partition('mkdir -p "$RUN_ROOT"') + assert delimiter + # Execute only settings validation. The remainder can fetch refs and run + # stages, so it must never be executed by an engine-free test. + script = tmp_path / "settings-check.sh" + script.write_text(prefix, encoding="utf-8") + required = ( + "PE MAIN WT_ROOT RUN_ROOT CHAIN_ROOT CHAIN_LOG DISK_PATH UV PYTHON " + "STORAGE EDU FEED LADDER SSI SCF AGENT_SECRET" + ).split() + settings = tmp_path / "route_a.env" + settings.write_text( + "\n".join(f"{key}=/fixture/{key.lower()}" for key in required) + + f"\nROUTE_A_STAGING={shlex.quote(staging)}\n", + encoding="utf-8", + ) + result = subprocess.run( + ["bash", str(script)], + env={**os.environ, "ROUTE_A_ENV": str(settings)}, + capture_output=True, + check=False, + timeout=20, + ) + assert result.returncode != 0 + assert "ROUTE_A_STAGING" in result.stderr.decode() + + +@pytest.mark.parametrize( + "extra", + [ + "--no-staging", + "--no-staging=0", + "--out=/fixture", + "--base-h5=/fixture", + "--allow-unpinned-feed", + "--skip-reform-validation", + "--dense-default-dataset", + ], +) +def test_release_extra_arguments_preserve_refusals(extra: str) -> None: + script = "set -eu\nfail() { printf '%s\\n' \"$*\" >&2; exit 1; }\n" + script += f"RELEASE_EXTRA_ARGS={shlex.quote(extra)}\nDENSE_RELEASE_D122=0\n" + script += _function("refuse_bad_release_args") + "\nrefuse_bad_release_args\n" + result = subprocess.run( + ["bash", "-c", script], capture_output=True, check=False, timeout=20 + ) + assert result.returncode != 0 + assert extra in result.stderr.decode() + + +def test_flag_checker_reads_argument_declarations() -> None: + checker = _flag_checker() + source = """ +import argparse +parser = argparse.ArgumentParser() +parser.add_argument("--real", "-r") +group = parser.add_mutually_exclusive_group() +group.add_argument("--other") +unrelated = "--not-declared" +# parser.add_argument("--comment") +""" + assert checker.declared_flags(source) == {"--real", "-r", "--other"} + assert not checker.missing_flags(source, ["--real", "--other"]) + assert set(checker.missing_flags(source, ["--comment", "--not-declared"])) == { + "--comment", + "--not-declared", + } + + +def test_wrapper_secret_reaches_only_child_environment(tmp_path: Path) -> None: + """The wrapper hands the credential to the exec'd child's environment only. + + Engine-free and psutil-free: it runs the wrapper directly under ``bash -x``. + The same check through the real supervisor, which needs psutil, lives in + ``engine_workflow/us/test_route_a_driver.py``. + """ + + wrapper_source = (ROUTE_A_TOOLS / "with_hf_token.sh").read_text(encoding="utf-8") + assert re.search(r"(?m)^export HF_TOKEN\s*$", wrapper_source) + assert re.search(r'(?m)^exec "\$@"\s*$', wrapper_source) + # An intermediate `env HF_TOKEN=...` launcher would expose the credential + # briefly even when the final child's argv is clean. + assert not re.search(r"\b(?:exec|env)\b[^\n]*\bHF_TOKEN\b", wrapper_source) + # This deliberately does not resemble a Hub credential and never invokes + # the actual agent-secret executable. + marker = "route-a-dummy-environment-value" + secret = write_executable( + tmp_path / "secret-stub", + """#!/bin/bash +set -eu +[ "$#" = 2 ] && [ "$1" = get ] && [ "$2" = HUGGING_FACE_TOKEN_MAX ] +printf '%s\\n' "$ROUTE_A_TEST_SECRET" +printf '%s\\n' "$ROUTE_A_TEST_SECRET" >&2 +""", + ) + report = tmp_path / "child-report.json" + child = tmp_path / "child.py" + child.write_text( + """import json, os, subprocess, sys +marker = os.environ["ROUTE_A_TEST_SECRET"] +argv = subprocess.run( + ["ps", "-ww", "-o", "args=", "-p", str(os.getpid())], + capture_output=True, text=True, check=True, +).stdout +payload = { + "token_present": os.environ.get("HF_TOKEN") == marker, + "argv_read": bool(argv.strip()), + "argv_contains_token": marker in argv, + "aliases_present": [key for key in ( + "HUGGING_FACE_HUB_TOKEN", "HUGGINGFACE_HUB_TOKEN", "HUGGING_FACE_TOKEN_MAX" + ) if key in os.environ], + "pid": os.getpid(), +} +with open(sys.argv[1], "w") as stream: + json.dump(payload, stream) +print("dummy release child finished") +""", + encoding="utf-8", + ) + env = {**os.environ, "ROUTE_A_TEST_SECRET": marker, "HF_TOKEN": "stale-value"} + env.update(dict.fromkeys(SECRET_ALIASES, "stale-value")) + process = subprocess.Popen( + [ + "bash", + "-x", + str(ROUTE_A_TOOLS / "with_hf_token.sh"), + str(secret), + sys.executable, + str(child), + str(report), + ], + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = process.communicate(timeout=60) + assert process.returncode == 0, stderr.decode() + payload = json.loads(report.read_text(encoding="utf-8")) + assert payload["token_present"] is True + assert payload["argv_read"] is True + assert payload["argv_contains_token"] is False + assert payload["aliases_present"] == [] + # exec, not a fork: the release runs as the wrapper's own process. + assert payload["pid"] == process.pid + assert marker.encode() not in stdout + stderr + for path in tmp_path.rglob("*"): + if path.is_file(): + assert marker.encode() not in path.read_bytes(), path.name + + +@pytest.mark.parametrize("secret_result", ["empty", "failed"]) +def test_wrapper_refuses_missing_secret(tmp_path: Path, secret_result: str) -> None: + secret = write_executable( + tmp_path / "secret-stub", + "#!/bin/bash\n" + + ( + "exit 0\n" + if secret_result == "empty" + else "printf 'dummy-lookup-output\\n'\nexit 9\n" + ), + ) + sentinel = tmp_path / "child-started" + child = write_executable(tmp_path / "child-stub", '#!/bin/bash\ntouch "$1"\n') + result = subprocess.run( + [ + "bash", + str(ROUTE_A_TOOLS / "with_hf_token.sh"), + str(secret), + str(child), + str(sentinel), + ], + env={**os.environ, "HF_TOKEN": "inherited-value-must-not-be-used"}, + capture_output=True, + check=False, + timeout=20, + ) + assert result.returncode != 0 + assert not sentinel.exists() + + +def test_route_a_sources_contain_no_machine_paths_or_credentials() -> None: + result = subprocess.run( + [ + "git", + "-C", + str(_TEST_PATHS.repository), + "ls-files", + "--cached", + "--others", + "--exclude-standard", + "-z", + "--", + "tools/route_a/", + ], + capture_output=True, + check=False, + timeout=20, + ) + assert result.returncode == 0, result.stderr.decode() + source_paths = sorted( + ROUTE_A_TOOLS / Path(path).relative_to("tools/route_a") + for path in result.stdout.decode().rstrip("\0").split("\0") + if path + ) + assert source_paths + credential = re.compile( + r"hf_[A-Za-z0-9]{20,}|sk-[A-Za-z0-9]{20,}|" + r"gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}" + ) + for path in source_paths: + source = path.read_text(encoding="utf-8") + assert "/Users/" not in source, path.name + assert not credential.search(source), path.name diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_route_a_driver.py b/packages/microcosm-build/tests/engine_workflow/us/test_route_a_driver.py new file mode 100644 index 000000000..03409a1bd --- /dev/null +++ b/packages/microcosm-build/tests/engine_workflow/us/test_route_a_driver.py @@ -0,0 +1,111 @@ +"""Route A's release credential through the real supervisor. + +The supervisor imports psutil, which the engine-free job does not install, so +this runs in the US engine job. The engine-free twin in +``engine_free/us/test_route_a_driver.py`` (same basename, per the agent guide's split rule) checks the wrapper on its own. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import psutil # noqa: F401 - the supervisor needs it; fail here, not in the child + +from test_support.microcosm_build.route_a_driver import ( + ROUTE_A_TOOLS, + SECRET_ALIASES, + write_executable, +) + + +def test_supervised_release_gets_the_credential_only_in_its_environment( + tmp_path: Path, +) -> None: + # This deliberately does not resemble a Hub credential and never invokes + # the actual agent-secret executable. + marker = "route-a-dummy-environment-value" + secret = write_executable( + tmp_path / "secret-stub", + """#!/bin/bash +set -eu +[ "$#" = 2 ] && [ "$1" = get ] && [ "$2" = HUGGING_FACE_TOKEN_MAX ] +printf '%s\\n' "$ROUTE_A_TEST_SECRET" +printf '%s\\n' "$ROUTE_A_TEST_SECRET" >&2 +""", + ) + report = tmp_path / "child-report.json" + child = tmp_path / "child.py" + child.write_text( + """import json, os, sys +import psutil +marker = os.environ["ROUTE_A_TEST_SECRET"] +payload = { + "token_present": os.environ.get("HF_TOKEN") == marker, + "argv_contains_token": any(marker in a for a in psutil.Process().cmdline()), + "aliases_present": [key for key in ( + "HUGGING_FACE_HUB_TOKEN", "HUGGINGFACE_HUB_TOKEN", "HUGGING_FACE_TOKEN_MAX" + ) if key in os.environ], + "pid": os.getpid(), +} +with open(sys.argv[1], "w") as stream: + json.dump(payload, stream) +print("dummy release child finished") +""", + encoding="utf-8", + ) + config = tmp_path / "release-config.json" + config.write_text( + json.dumps( + { + "argv": [ + "bash", + "-x", + str(ROUTE_A_TOOLS / "with_hf_token.sh"), + str(secret), + sys.executable, + str(child), + str(report), + ], + "cwd": str(tmp_path), + "env": {"PYTHONUNBUFFERED": "1"}, + "limits": { + "wall_seconds": 60, + "cpu_seconds": 10, + "rss_bytes": 512 * 1024**2, + "output_bytes": 1024**2, + "log_bytes": 1024**2, + "disk_floor_bytes": 0, + "disk_admission_bytes": 0, + "available_ram_admission_bytes": 0, + }, + } + ), + encoding="utf-8", + ) + env = {**os.environ, "ROUTE_A_TEST_SECRET": marker, "HF_TOKEN": "stale-value"} + env.update(dict.fromkeys(SECRET_ALIASES, "stale-value")) + out = tmp_path / "release-sup" + result = subprocess.run( + [sys.executable, str(ROUTE_A_TOOLS / "supervise.py"), str(out), str(config)], + env=env, + capture_output=True, + check=False, + timeout=90, + ) + assert result.returncode == 0, result.stderr.decode() + payload = json.loads(report.read_text(encoding="utf-8")) + assert payload["token_present"] is True + assert payload["argv_contains_token"] is False + assert payload["aliases_present"] == [] + # exec, not a fork: the supervisor's recorded pid is the release itself, + # so its process-group kill reaches the release. + assert payload["pid"] == json.loads((out / "PID.json").read_text())["pid"] + assert json.loads((out / "RESULT.json").read_text())["status"] == "COMPLETED" + assert marker.encode() not in result.stdout + result.stderr + for path in tmp_path.rglob("*"): + if path.is_file(): + assert marker.encode() not in path.read_bytes(), path.name diff --git a/test_support/microcosm_build/route_a_driver.py b/test_support/microcosm_build/route_a_driver.py new file mode 100644 index 000000000..e2633cdc0 --- /dev/null +++ b/test_support/microcosm_build/route_a_driver.py @@ -0,0 +1,24 @@ +"""Shared helpers for the Route A driver tests (engine-free and US engine).""" + +from __future__ import annotations + +from pathlib import Path + +from test_support.paths import paths_for + +#: ``tools/route_a``: the driver, supervisor, sampler and token wrapper. +ROUTE_A_TOOLS = paths_for("microcosm-build").repository / "tools" / "route_a" +#: Hub credential spellings the wrapper must strip before exporting HF_TOKEN. +SECRET_ALIASES = ( + "HUGGING_FACE_HUB_TOKEN", + "HUGGINGFACE_HUB_TOKEN", + "HUGGING_FACE_TOKEN_MAX", +) + + +def write_executable(path: Path, source: str) -> Path: + """Write ``source`` to ``path`` and make it executable.""" + + path.write_text(source, encoding="utf-8") + path.chmod(0o755) + return path diff --git a/tools/route_a/README.md b/tools/route_a/README.md new file mode 100644 index 000000000..4d6645541 --- /dev/null +++ b/tools/route_a/README.md @@ -0,0 +1,226 @@ +# Route A release driver + +`route_a.sh` builds a US PUF-support base, builds a fiscal-refresh release +candidate from that base, runs two release-gate preflights and runs the +publisher with `--preflight-only`. It leaves a hand-off with `published: false`. +Publication remains a separate manual step that Max authorizes. + +The driver, `supervise.py`, `sample_series.py`, `check_flags.py` and +`with_hf_token.sh` live together here. The driver, supervisor and sampler +originated in the off-repository Route A tooling. The release stage now enables +staging telemetry by default; `ROUTE_A_STAGING=0` restores `--no-staging`. + +## Configure and run + +Copy the example and edit its paths and commit before running: + +```bash +cp tools/route_a/route_a.env.example tools/route_a/route_a.env +# Edit tools/route_a/route_a.env, then resolve without launching a stage: +bash tools/route_a/route_a.sh --resolve +# After reviewing inputs.resolved.json and every BLOCKER in resolve.log: +bash tools/route_a/route_a.sh +``` + +`ROUTE_A_ENV=` selects another settings file. The local +`route_a.env` is ignored by git. The example uses shell defaults, so a +nonempty environment override wins. `ROUTE_A_STAGING` accepts only `0` or `1`; +an explicitly empty value is refused. Do not put credentials in either file. +`RELEASE_EXTRA_ARGS` and `PREFLIGHT_NEW_LINEAGE_ARGS` retain shell word splitting; +embedded quotes do not quote paths, and paths used there must have no spaces. + +All machine locations come from the settings file: + +| Setting | Purpose | +| --- | --- | +| `PE`, `MAIN`, `WT_ROOT` | PolicyEngine root, git repository used for fetching refs and creating worktrees, and detached build/preflight worktree parent | +| `RUN_ROOT` | Driver lock, logs, input resolution, terminal hand-offs and `run-<12-character-commit>` directories | +| `CHAIN_ROOT`, `CHAIN_LOG` | Previous overnight chain's supervisor run parent and chain log | +| `DISK_PATH` | Filesystem checked by the driver's free-space wait | +| `UV`, `PYTHON` | uv executable and engine-free interpreter for the static parser check | +| `STORAGE`, `EDU` | Processed ASEC/PUF/ACS inputs and the three ASEC education person archives | +| `FEED`, `LADDER`, `SSI`, `SCF` | Pinned Ledger feed, block ladder, SSI prior basis and SCF summary extract | +| `AGENT_SECRET` | Credential helper executable used by the release wrapper | +| `EXPORT_MASS_REF_H5`, `QRF_TAIL_EXCLUSIONS` | Export-mass reference and optional tail exclusion register | + +Choose a pushed `COMMIT`; the example deliberately leaves it empty. By default +the build commit must be reachable from `origin/main`, carry the SPM role stage, +pin the expected feed and ASEC inputs, carry the expected district crosswalk, +and declare the driver's base and release flags. `PREFLIGHT_COMMIT` defaults to +the build commit; with gate preflights enabled it must be on a remote branch. +The static release flag check +reads `add_argument` declarations without importing the release tool, including +flags in `RELEASE_EXTRA_ARGS`. + +`--resolve` (also `--dry-run`) fetches refs, hashes registered inputs, writes +`inputs.resolved.json`, reports commit blockers and launches no stages. It exits +nonzero for failed fetches or input problems; commit blockers are reported in +`resolve.log` and do not alone change that mode's exit status. A real run refuses +both input problems and commit blockers. + +The example retains the historical d122 dense whole-base national/state +settings, batch size 2000, export-mass reference and d490 tail register from the +published run. Those are operator settings, not a new waiver. The driver permits +`--dense-default-dataset` only with `DENSE_RELEASE_D122=1`, and refuses the +release-defeating, selection, evidence, exact-k, staging and driver-owned +overrides listed in `refuse_bad_release_args`, including `--flag=value` forms. +Use `ROUTE_A_STAGING=0` to opt out; `--no-staging` in extra arguments is refused. + +The next base uses the d713 CT-fixed ladder at +`$PE/_build_artifacts/us-ct-cbsa/us_block_ladder_2020.npz`, verified against +sha256 `6840b990acdfa2003d7723da5595e3cfff7205a4fa3d8d6206ed835c85c3f233` +and 18,991,218 bytes. Use a fresh `RUN_ROOT` for that attempt: an existing +`base-sup/ACCEPTED` skips the base stage. Merely pointing `LADDER` at new bytes +does not rebuild a previously accepted base. + +The driver uses macOS `stat -f` and requires bash, git, openssl, uv and the +configured paths. It creates clean detached worktrees at the chosen commits, +runs `uv sync --all-packages --locked --extra us`, and checks that their Python +can import psutil and policyengine-us. The supervised stages use that worktree's +interpreter. + +## Stages and gates + +Before launching, the driver verifies every registered input digest and recorded byte +size, checks commit pins, and takes a driver lock. It waits while the old +overnight chain is alive, unless `EARLY_START_AFTER_HEAVY=1` and its log records +`local release preflight passed`. It verifies inputs again after that wait. + +| Stage | Command and acceptance | +| --- | --- | +| `prefetch` | Fetches the ASEC unemployment archive, SCF summary/full extracts, SIPP financial-assets/tips donors and ORG donor through the release's fetch helpers. Exit 0 is required before release. An initial failure lets the base proceed; release retries it. | +| `base` | Runs `build_us_puf_support_base.py --stage all` on the pinned raw inputs, with seed 0, 32 estimators, target year 2024, district assignment/crosswalk and block ladder. Requires exit 0 and the expected H5, then hashes it. | +| `preflight-base` | Runs `preflight_us_release_gates.py` on that H5 and the pinned feed, with the configured new-lineage mode and optional export-mass reference. Accepts 0 (clean) or 2 (AT-RISK); `RELEASE_DESPITE_PREFLIGHT_FAIL=1` also accepts 1 here. | +| `release` | Runs `build_us_fiscal_refresh_release.py --base-h5` with the feed, SSI basis, SCF extract and configured release arguments. Requires exit 0 and `release_manifest.json`. Staging telemetry is enabled for this stage. | +| `preflight-release` | Repeats the release-gate preflight with the candidate's release manifest. Accepts 0 or 2. | +| `publisher-preflight` | Runs `microcosm-publish-release --preflight-only` against the local release directory and artifacts. Requires exit 0. The publisher calls `prepare_release` and returns before its publishing call. | + +Both gate preflights and the publisher preflight remove the HF credential +variables and Slack webhook variables, set `POPULACE_RELEASE_ENV=/dev/null` and +`HF_HUB_OFFLINE=1`. The base and prefetch retain their original command and +`PYTHONUNBUFFERED=1` configuration. An empty `PREFLIGHT_NEW_LINEAGE_ARGS` records +both gate preflights as `BLOCKED`, permits the build, and ends with +`ROUTE_A_BLOCKED.json` and exit 4 instead of `ROUTE_A_DONE.json`. + +Every stage runs through the supervisor's admission check and limits: + +| Stage | RSS cap (GiB) | RAM admission (GiB) | Disk admission (GiB) | Wall limit (seconds) | CPU limit (seconds) | +| --- | ---: | ---: | ---: | ---: | ---: | +| prefetch | 16 | 12 | 26 | 10800 | 20000 | +| base | 100 | 80 | max(26, 70 − checkpoint GiB) | 21600 | 64800 | +| gate preflights | 24 | 16 | 22 | 3600 | 7200 | +| release | 110 | `RELEASE_RAM_GIB` (example: 50) | 45 | `RELEASE_WALL_SECONDS` (example: 345600) | 2000000 | +| publisher preflight | 24 | 16 | 22 | 7200 | 14400 | + +The driver waits up to `RESOURCE_WAIT_HOURS` (example: 12) for admission and +retries a supervisor admission race every five minutes. Every config includes +a 20 GiB disk floor that must persist for 180 seconds before `DISK_FLOOR` kills +the process group. The supervisor also enforces RSS, wall, CPU, a 2 GiB size +limit for its own output directory and a 512 MiB child-log limit. The output +directory limit does not count the build's separate artifacts/checkpoints. + +For each local attempt, the driver writes `-config.json` and the +supervisor records admission. Admitted launches also record `COMMAND.json`, +`PID.json`, heartbeat, result and child-log files under `-sup/`; +refused admission writes a result without launching the command. +The sampler appends process-tree RSS/CPU, +available RAM, swap and disk to `.series.csv` every 30 seconds until a +result appears or the supervisor disappears. These resource records stay local. + +On restart, accepted stages are skipped; a result with `refusal: null` and an +allowed return code can be accepted without rerunning. A live prior supervisor +is waited for, and failed/refused directories are moved aside. Base stages +resume by completed checkpoint prefix. Failed release attempts get a new +release ID while retaining the shared checkpoint root. With +`PRUNE_BASE_CHECKPOINTS=1`, base `*.frame.h5` files are removed after hashing the +finished H5. Terminal hand-offs record the candidate paths and preflight +verdicts, plus a suggested manual publication command; they never execute it. + +## Published run's release argv + +The following preserves the argv order and values in the historical +`run-4b57d15a287c/release-config.json`, replacing only machine paths with +variables. `W` is that run's build worktree, `RUN` its run directory, +`QRF_TAIL_EXCLUSIONS` its `qrf_tail_exclusions_routea_d490_20260930.json`, and +`EXPORT_MASS_REF_H5` its `populace_us_2024.h5` reference. Its config environment +was `PYTHONUNBUFFERED=1`. + +```bash +RID=populace-us-2024-0fb05b6-4b57d15a287c-20260930T150401Z +"$W/.venv/bin/python" -B tools/build_us_fiscal_refresh_release.py \ + --base-h5 "$RUN/base-out/base_populace_us_2024_puf_support.h5" \ + --ledger-facts "$FEED" \ + --ledger-facts-sha256 b85437390021777e746f507c5890305496baf5fc7f2c78ba08ddb090f4839801 \ + --qrf-tail-concentration-exclusions "$QRF_TAIL_EXCLUSIONS" \ + --ssi-take-up-prior-weight-basis "$SSI" \ + --ssi-take-up-prior-weight-basis-sha256 25fe8af50a99d717f3408b2de7f0849d2307d4f05b1a7d55d2703999002fff0a \ + --scf-summary-extract "$SCF" \ + --out "$RUN/release-out/$RID" \ + --release-id "$RID" \ + --checkpoint-root "$RUN/release-checkpoints" \ + --seed 0 --no-staging \ + --maximum-microsim-batch-size 2000 \ + --target-surface national_state \ + --dense-default-dataset \ + --export-input-mass-reference-h5 "$EXPORT_MASS_REF_H5" +``` + +That config capped RSS at 110 GiB, admitted at 50 GiB available RAM/45 GiB free +disk, and used wall/CPU limits of 345600/2000000 seconds. This is a historical +receipt of the published `4b57d15a2` run, not the command to launch the next +attempt. This migration does not modify its files or its publication. + +## Modal base hand-off + +Follow [the Modal runbook](../../docs/us-modal-stage-runbook.md#route-as-base-stage) +to verify a completed base receipt and compare its lineage before copying any +base into the Route A run directory. Its hand-off block refuses existing +`base-out` or `base-sup`, runs `compare-lineage`, copies `base-out`, verifies the +copies with `verify-receipt --prefix base-out --strict`, writes `base.sha256`, +moves old local checkpoints aside, stores the receipt/comparison/run-context +evidence in `base-sup`, and writes `ACCEPTED` last. Set that block's `RUN` to +`$RUN_ROOT/run-<12-character-build-commit>` before rerunning this driver. + +The committed Modal example plan and local reference target the historical +`4b57d15a2` base and its `7ba39b95…` ladder. They do not qualify a new CT-fixed +base: its plan and comparison evidence must match the chosen commit and CT +ladder. The existing comparison covers only the first two of 24 outer stages; +it does not prove Mac/Linux equality for the remaining stages. The receipt +identifies produced bytes; release gates and certification still run locally. + +## Staging and the Build progress tab + +For a new release with staging enabled, the config contains the wrapper path +and credential-helper path followed by the release argv. The wrapper disables +shell tracing, removes inherited HF credential aliases, calls the configured +`agent-secret get HUGGING_FACE_TOKEN_MAX` at run time, refuses a failed/empty +lookup, exports `HF_TOKEN` and immediately `exec`s the release. It does not put +the returned value into argv, a config or a log. Do not fetch a token before +starting the driver. With `ROUTE_A_STAGING=0`, the release gets `--no-staging`, +inherited HF credential variables are removed and the wrapper is not called. + +The release parser defaults to `policyengine/populace-us-staging` and reads +`POPULACE_STAGING_REPO_ID` from its environment. Export that variable in the +settings file to override the destination; a blank value uses the default. +The upload storage constructs `HfApi()` with ambient credentials, so the +wrapper's `HF_TOKEN` supplies authentication. Uploads are best-effort and stop +after three consecutive failures; local telemetry continues to be written. + +Telemetry starts inside the release after compiling its target registry; +starting the driver or waiting for admission creates no staging run. Once +uploads reach the dashboard's staging repository, the records for its Build +progress tab include the release's run ID and candidate release ID, current status/stage, +timestamps, messages and stage details from `progress.json`, the event history +from `events.ndjson`, and calibration epoch/loss/budget-search information from +`calibration_progress.json`. The builder emits stages for loading the base, +source enrichments, target compilation, calibration, export/gates, diagnostics +and manifests, and uploads attached diagnostics as they are written. Completion +records status `passed`; reported failures record status `failed`. + +Telemetry is written locally under +`/staging/runs//` and uploaded under +`runs//`, with `run_manifest.json`, `latest_staging.json` and +`runs.json` allowing discovery. The base, donor prefetch, offline preflights +and supervisor resource series do not emit these staging records. If uploads +fail, dashboard visibility is not guaranteed. A staging upload does not publish +the release or update production `latest.json`. diff --git a/tools/route_a/check_flags.py b/tools/route_a/check_flags.py new file mode 100644 index 000000000..3823a5e32 --- /dev/null +++ b/tools/route_a/check_flags.py @@ -0,0 +1,39 @@ +"""Check CLI flags against add_argument declarations without importing an engine.""" + +from __future__ import annotations + +import ast +import sys +from collections.abc import Iterable + + +def declared_flags(source: str) -> set[str]: + """Read literal option names from the tool's argument parser.""" + return { + argument.value + for node in ast.walk(ast.parse(source)) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "add_argument" + for argument in node.args + if isinstance(argument, ast.Constant) + and isinstance(argument.value, str) + and argument.value.startswith("-") + } + + +def missing_flags(source: str, flags: Iterable[str]) -> list[str]: + """Return undeclared options, including options supplied as --flag=value.""" + return sorted({flag.split("=", 1)[0] for flag in flags} - declared_flags(source)) + + +def main() -> int: + missing = missing_flags(sys.stdin.read(), sys.argv[1:]) + if missing: + print("release parser does not declare: " + " ".join(missing), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/route_a/route_a.env.example b/tools/route_a/route_a.env.example new file mode 100644 index 000000000..e9b325045 --- /dev/null +++ b/tools/route_a/route_a.env.example @@ -0,0 +1,46 @@ +# shellcheck shell=bash +# Copy to route_a.env, or point ROUTE_A_ENV at a machine-local settings file. +# No credentials belong here. Environment overrides win over these defaults. +# All machine locations are configured here; the driver uses colocated helpers. +: "${PE:=$HOME/PolicyEngine}" +: "${MAIN:=$PE/microcosm}" +: "${WT_ROOT:=$PE/_worktrees}" +# Use a fresh root for a new attempt. Do not reuse a pre-CT base's ACCEPTED marker. +: "${RUN_ROOT:=$PE/_route-a-runs}" +: "${CHAIN_ROOT:=$PE/_recovered/scratch-backup/893/overnight-20260923}" +: "${CHAIN_LOG:=$CHAIN_ROOT/chain.log}" +: "${DISK_PATH:=/System/Volumes/Data}" +: "${UV:=$HOME/.local/bin/uv}" +: "${PYTHON:=python3}" +: "${AGENT_SECRET:=$HOME/bin/agent-secret}" +: "${STORAGE:=$PE/policyengine-us-data/policyengine_us_data/storage}" +: "${EDU:=$PE/_buildm-runtime/inputs/asec_education}" +: "${FEED:=$PE/_buildh-runtime/inputs/consumer_facts_us_c5e5bf8.jsonl}" +# d713: CT-fixed ladder; route_a.sh verifies its digest and size. +: "${LADDER:=$PE/_build_artifacts/us-ct-cbsa/us_block_ladder_2020.npz}" +: "${SSI:=$PE/_buildo-runtime/inputs/attempt6_basis_schema3_seed.json}" +: "${SCF:=$PE/_buildm-runtime/inputs/scf_cache/rscfp2022.dta}" + +# Choose a pushed build commit. Empty COMMIT refuses a real run. +: "${COMMIT:=}" +: "${REQUIRE_ON_MAIN:=1}" +: "${REQUIRE_SPM_ROLE:=1}" +: "${PREFLIGHT_COMMIT:=}" +: "${PREFLIGHT_NEW_LINEAGE_ARGS:=--new-lineage}" +: "${EARLY_START_AFTER_HEAVY:=0}" +: "${PRUNE_BASE_CHECKPOINTS:=1}" +: "${RELEASE_DESPITE_PREFLIGHT_FAIL:=0}" +: "${RESOURCE_WAIT_HOURS:=12}" +: "${RELEASE_RAM_GIB:=50}" +: "${RELEASE_WALL_SECONDS:=345600}" +# 0 is the sole opt-out and restores --no-staging without a credential lookup. +: "${ROUTE_A_STAGING=1}" + +# Retain d122 whole-base dense national/state settings and the published run's +# export-mass reference. Review the historical tail waiver against the new export. +: "${DENSE_RELEASE_D122:=1}" +: "${EXPORT_MASS_REF_H5:=$PE/_buildg-runtime/forensics/populace_us_2024.h5}" +: "${RELEASE_EXTRA_ARGS:=--maximum-microsim-batch-size 2000 --target-surface national_state --dense-default-dataset --export-input-mass-reference-h5 $EXPORT_MASS_REF_H5}" +: "${QRF_TAIL_EXCLUSIONS:=$CHAIN_ROOT/route-a/qrf_tail_exclusions_routea_d490_20260930.json}" +: "${QRF_TAIL_EXCLUSIONS_SHA:=fa5760993ebcd1386f02c0b1c4c4e867a0c4b1e689a33774a41f623f95132d69}" +# Optional base feed override: BASE_LEDGER_FACTS and BASE_LEDGER_FACTS_SHA. diff --git a/tools/route_a/route_a.sh b/tools/route_a/route_a.sh new file mode 100755 index 000000000..79dc61c70 --- /dev/null +++ b/tools/route_a/route_a.sh @@ -0,0 +1,652 @@ +#!/bin/bash +# Route A (microcosm epic #956, acceleration A): a from-scratch US base, then a +# --base-h5 release candidate on the committed labelled Ledger feed, then the +# release-gate preflight and the publisher's offline --preflight-only check. +# +# Builds a release candidate, runs offline preflights, and leaves publication +# to Max. Release staging telemetry is enabled unless ROUTE_A_STAGING=0. +# +# Usage: ./route_a.sh [--resolve] +# --resolve fetches refs, hashes inputs, and checks commit pins; no stages run. +# Settings come from ROUTE_A_ENV, defaulting to route_a.env beside this script. +# Accepted stages are skipped on restart. Live supervisors are waited for; +# failed attempts are moved aside. Releases use fresh IDs and shared caches. +set +x +set -u + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +ROUTE_A_ENV=${ROUTE_A_ENV:-$SCRIPT_DIR/route_a.env} +SUP=$SCRIPT_DIR/supervise.py +SAMPLER=$SCRIPT_DIR/sample_series.py +TOKEN_WRAPPER=$SCRIPT_DIR/with_hf_token.sh +GIB=1073741824 + +MODE=run +case "${1:-}" in + --resolve|--dry-run) MODE=resolve ;; + "") ;; + -h|--help) sed -n '2,/^set +x$/p' "$0"; exit 0 ;; + *) echo "usage: $0 [--resolve]" >&2; exit 64 ;; +esac + +[ -f "$ROUTE_A_ENV" ] || { echo "missing settings: $ROUTE_A_ENV (copy route_a.env.example)" >&2; exit 64; } +# shellcheck source=/dev/null +. "$ROUTE_A_ENV" +: "${PE:?set PE in route_a.env}" "${MAIN:?set MAIN}" "${WT_ROOT:?set WT_ROOT}" +: "${RUN_ROOT:?set RUN_ROOT}" "${CHAIN_ROOT:?set CHAIN_ROOT}" "${CHAIN_LOG:?set CHAIN_LOG}" +: "${DISK_PATH:?set DISK_PATH}" "${UV:?set UV}" "${PYTHON:?set PYTHON}" +: "${STORAGE:?set STORAGE}" "${EDU:?set EDU}" "${FEED:?set FEED}" +: "${LADDER:?set LADDER}" "${SSI:?set SSI}" "${SCF:?set SCF}" +: "${AGENT_SECRET:?set AGENT_SECRET}" "${ROUTE_A_STAGING=1}" "${RELEASE_RAM_GIB:=50}" +case "$ROUTE_A_STAGING" in 0|1) ;; *) echo "ROUTE_A_STAGING must be 0 or 1" >&2; exit 64 ;; esac +mkdir -p "$RUN_ROOT" +LOG=$RUN_ROOT/route_a.log +: "${COMMIT:=}" "${REQUIRE_ON_MAIN:=1}" "${REQUIRE_SPM_ROLE:=1}" "${PREFLIGHT_COMMIT:=}" +: "${PREFLIGHT_NEW_LINEAGE_ARGS:=}" "${EARLY_START_AFTER_HEAVY:=0}" "${PRUNE_BASE_CHECKPOINTS:=0}" +: "${RELEASE_DESPITE_PREFLIGHT_FAIL:=0}" "${RELEASE_EXTRA_ARGS:=}" "${EXPORT_MASS_REF_H5:=}" +: "${RESOURCE_WAIT_HOURS:=12}" + +# --------------------------------------------------------------------------- +# Inputs, resolved and hashed on this machine on 2026-09-23 (see status.md). +# Format: role|path|sha256|bytes|where main pins it +# --------------------------------------------------------------------------- +FEED_SHA=b85437390021777e746f507c5890305496baf5fc7f2c78ba08ddb090f4839801 +# The base stage reads the feed only for the SOI congressional-district return +# counts. On 2026-09-23 the pinned feed and the 2026-09-16 base feed +# (consumer_facts_builde_aging_v5.jsonl, a5d34d4a...) gave the same 436-row +# distribution through main's congressional_district_distribution_from_ledger_facts; +# after the vintage crosswalk, 73 weights differ by at most 2.2e-16 relative. +# The pinned feed is used for both stages so the lineage has one feed identity. +BASE_LEDGER_FACTS=${BASE_LEDGER_FACTS:-$FEED} +BASE_LEDGER_FACTS_SHA=${BASE_LEDGER_FACTS_SHA:-$FEED_SHA} +# d713 (Max 2026-10-03): the next certified build uses the Connecticut-fixed ladder from microcosm#1072 +# (CT blocks get planning-region CBSAs; only cbsa_code differs from 7ba39b95). The published 4b57d15a2 release used 7ba39b95. +# QRF tail-concentration exclusion register. Empty by default since 2026-09-23: +# the Build P register was measured on another lineage and cannot match a +# whole-base export (route A pre-mortem); a route-A register must be measured +# on the export that ships. Set QRF_TAIL_EXCLUSIONS to pass one. +TAIL=${QRF_TAIL_EXCLUSIONS-} +SSI_SHA=25fe8af50a99d717f3408b2de7f0849d2307d4f05b1a7d55d2703999002fff0a +ASEC_2024_SHA=ec36604cb735a660b51b0b2f90be27d803b5878f3464fb30d0eacead59c1260d +ASEC_2023_SHA=cb57817327799f42b741caed5f9be94d04021c2e6809c1ad7bd0686da5428d88 +ASEC_2022_SHA=7ccca976284bb47815d84460cc4f75a0a65d26d7754ab0a0f417de351b3d474e +CROSSWALK_REL=packages/microcosm-build/src/microcosm/build/us_runtime/data/congressional_district_vintage_crosswalk.csv +CROSSWALK_SHA=c7cb040b1f57ca2ea2adcbfe60cc2b250ca23acbc4b640cd421e766fa54c1aec +FEED_PIN_REL=packages/microcosm-build/src/microcosm/build/us/chronicle_feed.json +ASEC_PIN_REL=packages/microcosm-build/src/microcosm/build/us_runtime/asec_sources.py +SPM_ROLE_REL=packages/microcosm-build/src/microcosm/build/us_runtime/spm_independence_role.py +BASE_FLAGS="--stage --checkpoint-dir --asec-h5 --asec-h5-sha256 --puf-h5 --puf-source-year-csv --acs-h5 --asec-education-source --target-year --seed --n-estimators --ledger-facts --assign-congressional-districts --congressional-district-vintage-crosswalk --congressional-district-seed --block-ladder-artifact --out" +RELEASE_FLAGS="--base-h5 --ledger-facts --ledger-facts-sha256 --qrf-tail-concentration-exclusions --ssi-take-up-prior-weight-basis --ssi-take-up-prior-weight-basis-sha256 --scf-summary-extract --out --release-id --checkpoint-root --seed" +[ "$ROUTE_A_STAGING" = 0 ] && RELEASE_FLAGS="$RELEASE_FLAGS --no-staging" +for extra in $RELEASE_EXTRA_ARGS; do + case "$extra" in --*) RELEASE_FLAGS="$RELEASE_FLAGS ${extra%%=*}" ;; esac +done +PREFLIGHT_FLAGS="--base-h5 --ledger-facts --ledger-facts-sha256 --release-manifest --json-out" + +input_rows() { + cat < "$RUN_ROOT/ROUTE_A_FAILED" + exit 1 +} +sha_of() { openssl dgst -sha256 -r "$1" 2>/dev/null | cut -c1-64; } +free_bytes() { df -k "$DISK_PATH" | awk 'NR==2 {print $4*1024}'; } + +INPUT_PROBLEMS=() +verify_inputs() { # fills INPUT_PROBLEMS; writes $RUN_ROOT/inputs.resolved.json + INPUT_PROBLEMS=() + local json=$RUN_ROOT/inputs.resolved.json.tmp first=1 role path sha size pin actual asize status + printf '{\n "resolved_at": "%s",\n "host": "%s",\n "inputs": [\n' "$(date '+%F %T %Z')" "$(hostname -s)" > "$json" + while IFS='|' read -r role path sha size pin; do + [ -n "$role" ] || continue + status=ok; actual=; asize= + if [ ! -f "$path" ]; then + status=missing; INPUT_PROBLEMS+=("$role missing: $path") + else + asize=$(stat -f %z "$path") + if [ "$size" != "-" ] && [ "$asize" != "$size" ]; then + status=size_mismatch; INPUT_PROBLEMS+=("$role size $asize != $size: $path") + fi + actual=$(sha_of "$path") + if [ "$actual" != "$sha" ]; then + status=sha_mismatch; INPUT_PROBLEMS+=("$role sha256 $actual != $sha: $path") + fi + fi + [ $first = 1 ] || printf ',\n' >> "$json" + first=0 + printf ' {"role": "%s", "path": "%s", "expected_sha256": "%s", "observed_sha256": "%s", "bytes": "%s", "status": "%s", "pinned_by": "%s"}' \ + "$role" "$path" "$sha" "$actual" "$asize" "$status" "$pin" >> "$json" + done < <(input_rows) + printf '\n ]\n}\n' >> "$json" + mv "$json" "$RUN_ROOT/inputs.resolved.json" +} + +COMMIT_BLOCKERS=() +COMMIT_NOTES=() +check_commit() { # $1 = build ref, $2 = preflight ref; fills COMMIT_BLOCKERS/COMMIT_NOTES + local ref=$1 pref=$2 full branches tok missing= + COMMIT_BLOCKERS=(); COMMIT_NOTES=() + full=$(git -C "$MAIN" rev-parse --verify --quiet "$ref^{commit}") || { + COMMIT_BLOCKERS+=("build commit $ref does not exist locally after git fetch origin"); return; } + branches=$(git -C "$MAIN" branch -r --contains "$full" 2>/dev/null | grep -v -- '->' | tr -d ' ' | tr '\n' ' ') + if [ -z "$branches" ]; then + COMMIT_BLOCKERS+=("build commit $full is on no origin branch (not pushed)") + else + COMMIT_NOTES+=("build commit $full is on: $branches") + fi + if ! git -C "$MAIN" merge-base --is-ancestor "$full" origin/main; then + if [ "$REQUIRE_ON_MAIN" = 1 ]; then + COMMIT_BLOCKERS+=("build commit $full is not reachable from origin/main (set REQUIRE_ON_MAIN=0 only for a PR-tree receipt run)") + else + COMMIT_NOTES+=("build commit $full is not on origin/main (REQUIRE_ON_MAIN=0: PR-tree receipt run)") + fi + fi + if git -C "$MAIN" show "$full:$FEED_PIN_REL" 2>/dev/null | grep -q "\"facts_sha256\": \"$FEED_SHA\""; then + COMMIT_NOTES+=("us/chronicle_feed.json at $full pins facts_sha256 $FEED_SHA") + else + COMMIT_BLOCKERS+=("us/chronicle_feed.json at $full does not pin facts_sha256 $FEED_SHA (the release tool would refuse the feed)") + fi + local s + for s in $ASEC_2024_SHA $ASEC_2023_SHA $ASEC_2022_SHA; do + git -C "$MAIN" show "$full:$ASEC_PIN_REL" 2>/dev/null | grep -q "$s" \ + || COMMIT_BLOCKERS+=("asec_sources.py at $full lacks canonical ASEC pin $s (the base would refuse the CLI pin)") + done + if [ "$(git -C "$MAIN" show "$full:$CROSSWALK_REL" 2>/dev/null | openssl dgst -sha256 -r | cut -c1-64)" = "$CROSSWALK_SHA" ]; then + COMMIT_NOTES+=("packaged CD vintage crosswalk at $full = $CROSSWALK_SHA") + else + COMMIT_BLOCKERS+=("packaged CD vintage crosswalk at $full is not $CROSSWALK_SHA") + fi + if git -C "$MAIN" show "$full:tools/build_us_fiscal_refresh_release.py" 2>/dev/null | grep -q '_check_committed_us_ledger_feed_pin'; then + COMMIT_NOTES+=("release tool at $full holds the feed to the committed pin") + else + COMMIT_BLOCKERS+=("release tool at $full has no committed-feed-pin check (#955 missing)") + fi + # Every flag the driver passes must be declared by the tool at that commit. + local tool flag missing + tool="tools/build_us_puf_support_base.py:$BASE_FLAGS" + missing= + for flag in ${tool#*:}; do + git -C "$MAIN" show "$full:${tool%%:*}" 2>/dev/null | grep -q -- "\"$flag\"" || missing="$missing $flag" + done + if [ -n "$missing" ]; then COMMIT_BLOCKERS+=("${tool%%:*} at $full does not declare:$missing") + else COMMIT_NOTES+=("${tool%%:*} at $full declares every flag the driver passes"); fi + # shellcheck disable=SC2086 # flags are a whitespace-separated option list + if git -C "$MAIN" show "$full:tools/build_us_fiscal_refresh_release.py" | \ + "$PYTHON" "$SCRIPT_DIR/check_flags.py" $RELEASE_FLAGS; then + COMMIT_NOTES+=("release parser at $full declares every flag the driver passes") + else + COMMIT_BLOCKERS+=("release parser at $full does not declare every driver flag") + fi + if git -C "$MAIN" cat-file -e "$full:$SPM_ROLE_REL" 2>/dev/null; then + COMMIT_NOTES+=("SPM independence role stage (#959) present at $full") + elif [ "$REQUIRE_SPM_ROLE" = 1 ]; then + COMMIT_BLOCKERS+=("#959 SPM independence role stage absent at $full ($SPM_ROLE_REL); without it the release's reform-coverage smoke refuses (docs/us-release-build-rule.md section 2)") + else + COMMIT_NOTES+=("#959 absent at $full; REQUIRE_SPM_ROLE=0") + fi + if [ -z "$PREFLIGHT_NEW_LINEAGE_ARGS" ]; then + COMMIT_NOTES+=("PREFLIGHT_NEW_LINEAGE_ARGS empty: both release-gate preflights will be BLOCKED (tools/preflight_us_release_gates.py requires --selection-source-manifest)") + else + local pfull + pfull=$(git -C "$MAIN" rev-parse --verify --quiet "$pref^{commit}") || { + COMMIT_BLOCKERS+=("preflight commit $pref does not exist locally"); return; } + missing= + for flag in $PREFLIGHT_FLAGS; do + git -C "$MAIN" show "$pfull:tools/preflight_us_release_gates.py" 2>/dev/null | grep -q -- "\"$flag\"" || missing="$missing $flag" + done + [ -z "$missing" ] || COMMIT_BLOCKERS+=("tools/preflight_us_release_gates.py at $pfull does not declare:$missing") + tok=${PREFLIGHT_NEW_LINEAGE_ARGS%% *} + if git -C "$MAIN" show "$pfull:tools/preflight_us_release_gates.py" 2>/dev/null | grep -q -- "\"$tok\""; then + COMMIT_NOTES+=("preflight tool at $pfull declares $tok") + else + COMMIT_BLOCKERS+=("preflight tool at $pfull does not declare \"$tok\" (new-lineage PR not in that commit?)") + fi + git -C "$MAIN" branch -r --contains "$pfull" 2>/dev/null | grep -qv -- '->' \ + || COMMIT_BLOCKERS+=("preflight commit $pfull is on no origin branch") + fi +} + +refuse_bad_release_args() { + local a + for a in $RELEASE_EXTRA_ARGS; do + case "${a%%=*}" in + --dense-default-dataset) + # Max's ruling d122 (2026-09-23): route A's first certified release is + # calibrated whole-base dense. Allowed only with this explicit opt-in. + [ "${DENSE_RELEASE_D122:-0}" = 1 ] || fail "RELEASE_EXTRA_ARGS may carry --dense-default-dataset only with DENSE_RELEASE_D122=1 (Max's d122 ruling)" ;; + --skip-reform-validation|--allow-unpinned-feed|--selection-source-*|\ + --selection-mass-protection|--evidence-release|--evidence-failure-owners|--allow-*|\ + --skip-reform-coverage-smoke|--skip-out-of-sample-reforms|--no-target-materialization-cache|\ + --exact-k*|--pool-manifest*|--staging-*|--no-staging|--release-id|--out|--checkpoint-root|--base-h5|--ledger-facts*) + fail "RELEASE_EXTRA_ARGS may not carry $a (it would make the run not a release, or it is set by the driver)" ;; + esac + done +} + +report_resolution() { # $1 = label of the ref checked + local x + log "inputs: ${#INPUT_PROBLEMS[@]} problem(s); see $RUN_ROOT/inputs.resolved.json" + for x in ${INPUT_PROBLEMS[@]+"${INPUT_PROBLEMS[@]}"}; do log " INPUT PROBLEM: $x"; done + for x in ${COMMIT_NOTES[@]+"${COMMIT_NOTES[@]}"}; do log " commit ($1): $x"; done + for x in ${COMMIT_BLOCKERS[@]+"${COMMIT_BLOCKERS[@]}"}; do log " BLOCKER ($1): $x"; done +} + +release_command() { + local -a extra=() staging=() launcher=() + # shellcheck disable=SC2206 # retain the driver's word-split extra-args contract + extra=($RELEASE_EXTRA_ARGS) + case "${ROUTE_A_STAGING-1}" in + 0) + staging=(--no-staging) + launcher=(/usr/bin/env -u HF_TOKEN -u HUGGING_FACE_HUB_TOKEN -u HUGGINGFACE_HUB_TOKEN -u HUGGING_FACE_TOKEN_MAX) + ;; + 1) + launcher=("$TOKEN_WRAPPER" "$AGENT_SECRET") + ;; + *) fail "ROUTE_A_STAGING must be 0 or 1" ;; + esac + RELEASE_ARGV=("${launcher[@]}" "$PY" -B tools/build_us_fiscal_refresh_release.py + --base-h5 "$BASE_H5" + --ledger-facts "$FEED" --ledger-facts-sha256 "$FEED_SHA" + ${TAIL:+--qrf-tail-concentration-exclusions "$TAIL"} + --ssi-take-up-prior-weight-basis "$SSI" --ssi-take-up-prior-weight-basis-sha256 "$SSI_SHA" + --scf-summary-extract "$SCF" + --out "$REL_OUT" --release-id "$RID" --checkpoint-root "$REL_CKPT" --seed 0 + ${staging[@]+"${staging[@]}"} ${extra[@]+"${extra[@]}"}) +} + +# --------------------------------------------------------------------------- +# Dry run. +# --------------------------------------------------------------------------- +FETCH_OK=1 +git -C "$MAIN" fetch origin --quiet || FETCH_OK=0 +if [ "$MODE" = resolve ]; then + LOG=$RUN_ROOT/resolve.log + REF=${COMMIT:-origin/main} + PREF=${PREFLIGHT_COMMIT:-$REF} + [ "$FETCH_OK" = 1 ] || { echo "git fetch origin failed" >&2; exit 1; } + log "resolve: build ref $REF ($(git -C "$MAIN" rev-parse --short=12 "$REF" 2>/dev/null)), preflight ref $PREF" + verify_inputs + check_commit "$REF" "$PREF" + report_resolution "$REF" + log "free disk $(( $(free_bytes) / GIB )) GiB; chain: $(grep -E '^[0-9-]{10} [0-9:]{8} ' "$CHAIN_LOG" | tail -1)" + [ ${#INPUT_PROBLEMS[@]} = 0 ] || exit 2 + exit 0 +fi + +# --------------------------------------------------------------------------- +# Real run. +# --------------------------------------------------------------------------- +LOCK=$RUN_ROOT/.route_a.lock +if ! mkdir "$LOCK" 2>/dev/null; then + other=$(cat "$LOCK/pid" 2>/dev/null) + if [ -n "$other" ] && kill -0 "$other" 2>/dev/null; then + echo "another route_a.sh (pid $other) holds $LOCK" >&2; exit 75 + fi + rm -f "$LOCK/pid"; rmdir "$LOCK" 2>/dev/null + mkdir "$LOCK" || { echo "cannot take $LOCK" >&2; exit 75; } +fi +echo $$ > "$LOCK/pid" +trap 'rm -f "$LOCK/pid"; rmdir "$LOCK" 2>/dev/null' EXIT +rm -f "$RUN_ROOT/ROUTE_A_FAILED" + +log "route A driver start (pid $$)" +[ "$FETCH_OK" = 1 ] || log "warning: git fetch origin failed; checking pins against local refs" +[ -n "$COMMIT" ] || fail "COMMIT is unset in route_a.env: choose a pushed commit carrying #959 (and the new-lineage preflight PR)" +C=$(git -C "$MAIN" rev-parse --verify --quiet "$COMMIT^{commit}") || fail "COMMIT $COMMIT not found after git fetch origin" +PC=$(git -C "$MAIN" rev-parse --verify --quiet "${PREFLIGHT_COMMIT:-$C}^{commit}") || fail "PREFLIGHT_COMMIT $PREFLIGHT_COMMIT not found" +refuse_bad_release_args +verify_inputs +check_commit "$C" "$PC" +report_resolution "$C" +[ ${#INPUT_PROBLEMS[@]} = 0 ] || fail "input problems (see above)" +[ ${#COMMIT_BLOCKERS[@]} = 0 ] || fail "commit blockers (see above)" + +RUN=$RUN_ROOT/run-${C:0:12} +mkdir -p "$RUN" + +# --- 1. Clean worktree(s) at pushed commits, with a locked environment. Light +# (git worktree add + uv sync), so it runs before the wait and fails early. --- +ensure_worktree() { # $1 = full sha; sets WT + local c=$1 + WT=$WT_ROOT/microcosm-route-a-${c:0:12} + if [ -d "$WT" ]; then + [ "$(git -C "$WT" rev-parse HEAD)" = "$c" ] || fail "$WT exists but is not at $c" + else + git -C "$MAIN" worktree add --detach "$WT" "$c" >>"$LOG" 2>&1 || fail "git worktree add $WT $c" + log "created worktree $WT at $c" + fi + [ -z "$(git -C "$WT" status --porcelain)" ] || fail "worktree $WT is dirty" + if [ ! -f "$WT/.venv/.route_a_synced" ]; then + log "uv sync --all-packages --locked --extra us in $WT" + (cd "$WT" && "$UV" sync --all-packages --locked --extra us) >>"$LOG" 2>&1 || fail "uv sync in $WT" + "$WT/.venv/bin/python" -c 'import psutil, policyengine_us' >>"$LOG" 2>&1 || fail "venv in $WT lacks psutil/policyengine_us" + date '+%F %T' > "$WT/.venv/.route_a_synced" + fi + [ -z "$(git -C "$WT" status --porcelain)" ] || fail "worktree $WT dirty after uv sync" +} +ensure_worktree "$C"; W=$WT +ensure_worktree "$PC"; PW=$WT +PY=$W/.venv/bin/python +[ "$(sha_of "$W/$CROSSWALK_REL")" = "$CROSSWALK_SHA" ] || fail "packaged crosswalk digest in $W" +PEUS=$("$PY" -c 'import importlib.metadata as m; print(m.version("policyengine-us"))') +[ -x "$W/.venv/bin/microcosm-publish-release" ] || fail "no microcosm-publish-release in $W/.venv" +log "build worktree $W (policyengine-us $PEUS); preflight worktree $PW" + +# --- 2. Wait for the overnight chain to leave the machine. --- +chain_alive() { + pgrep -f 'overnight_chain\.sh' >/dev/null 2>&1 && return 0 + pgrep -f "supervise\.py $CHAIN_ROOT/run/" >/dev/null 2>&1 && return 0 + return 1 +} +chain_terminal_line() { + grep -E '^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} (CHAIN DONE|FAIL: )' "$CHAIN_LOG" 2>/dev/null | tail -1 +} +waited=0 +while chain_alive; do + if [ "$EARLY_START_AFTER_HEAVY" = 1 ] && grep -q 'local release preflight passed' "$CHAIN_LOG" 2>/dev/null; then + log "overnight chain still running its certification/publication tail; heavy stages are over (EARLY_START_AFTER_HEAVY=1)" + break + fi + [ $(( waited % 1800 )) = 0 ] && log "waiting for the overnight chain: $(grep -E '^[0-9-]{10} [0-9:]{8} ' "$CHAIN_LOG" | tail -1 | cut -c1-160)" + sleep 300; waited=$(( waited + 300 )) +done +end_line=$(chain_terminal_line) +if chain_alive; then + log "overnight chain still alive in its tail; starting early (last terminal line: ${end_line:-none})" +else + case "$end_line" in + *"CHAIN DONE"*) log "overnight chain finished: $end_line" ;; + *"FAIL: "*) log "overnight chain ended in failure; the machine is free, proceeding: $end_line" ;; + *) log "overnight chain process gone without CHAIN DONE or FAIL; proceeding" ;; + esac +fi +verify_inputs +[ ${#INPUT_PROBLEMS[@]} = 0 ] || { report_resolution "$C"; fail "inputs changed while waiting"; } + +# --- helpers for supervised stages --- +avail_ram_bytes() { "$PY" -c 'import psutil; print(psutil.virtual_memory().available)'; } +wait_resources() { # $1 RAM GiB, $2 disk GiB, $3 label, $4 disk hint + local ram=$(( $1 * GIB )) disk=$(( $2 * GIB )) deadline=$(( $(date +%s) + RESOURCE_WAIT_HOURS * 3600 )) n=0 + while [ "$(avail_ram_bytes)" -lt "$ram" ] || [ "$(free_bytes)" -lt "$disk" ]; do + [ "$(date +%s)" -lt "$deadline" ] || fail "$3: needs $1 GiB available RAM and $2 GiB free disk; have $(( $(avail_ram_bytes) / GIB )) / $(( $(free_bytes) / GIB )) GiB after ${RESOURCE_WAIT_HOURS} h. ${4:-}" + [ $(( n % 6 )) = 0 ] && log "$3 waiting: RAM $(( $(avail_ram_bytes) / GIB ))/$1 GiB, disk $(( $(free_bytes) / GIB ))/$2 GiB. ${4:-}" + n=$(( n + 1 )); sleep 300 + done +} + +AUTH="Route A for microcosm epic #956 acceleration A, staged 2026-09-23 by an Opus 5.5 subagent; builds a local release candidate only; publication is Max's call. Build commit $C." + +# write_config -- argv... +write_config() { + local name=$1 rss=$2 ram=$3 adm=$4 wall=$5 cpu=$6 cwd=$7 envj=$8; shift 9 + "$PY" - "$RUN/$name-config.json" "$cwd" "$C" "$rss" "$ram" "$adm" "$wall" "$cpu" "$envj" "$AUTH" "$@" <<'EOF' +import json, subprocess, sys +path, cwd, commit, rss, ram, adm, wall, cpu, envj, auth, *argv = sys.argv[1:] +G = 1024 ** 3 +head = subprocess.check_output(["git", "-C", cwd, "rev-parse", "HEAD"], text=True).strip() +json.dump({ + "argv": argv, + "cwd": cwd, + "env": json.loads(envj), + "authorization": auth, + "source_identity": {"worktree": cwd, "head": head, "build_commit": commit}, + "limits": { + "wall_seconds": int(wall), "cpu_seconds": int(cpu), + "rss_bytes": int(float(rss) * G), + # supervise.py counts only its own directory (child log + JSON) here. + "output_bytes": 2 * G, "log_bytes": 512 * 1024 * 1024, + "disk_floor_bytes": 20 * G, "disk_floor_persist_seconds": 180, + "disk_admission_bytes": int(float(adm) * G), + "available_ram_admission_bytes": int(float(ram) * G), + }, +}, open(path, "w"), indent=1) +EOF + [ -s "$RUN/$name-config.json" ] || fail "could not write $name config" +} + +stage_done() { [ -f "$RUN/$1-sup/ACCEPTED" ]; } +result_accepted() { # $1 dir, $2 accepted return codes ("0" or "0 2") + local rc + [ -f "$1/RESULT.json" ] || return 1 + grep -q '"refusal": null' "$1/RESULT.json" || return 1 + rc=$(sed -n 's/^ *"returncode": \(-\{0,1\}[0-9]*\),*$/\1/p' "$1/RESULT.json") + case " $2 " in *" $rc "*) echo "$rc"; return 0 ;; esac + return 1 +} +wait_prior_supervisor() { # $1 name: a killed driver can leave its supervisor running + while pgrep -f "supervise\.py $RUN/$1-sup " >/dev/null 2>&1; do + log "$1: a supervisor from an earlier driver is still running it; waiting"; sleep 300 + done +} +run_stage() { # $1 name, $2 accepted return codes, $3 RAM GiB, $4 disk GiB, $5 disk hint + local name=$1 accept=$2 dir=$RUN/$1-sup sp smp rc + local deadline=$(( $(date +%s) + RESOURCE_WAIT_HOURS * 3600 )) + wait_prior_supervisor "$name" + if rc=$(result_accepted "$dir" "$accept"); then + echo "returncode=$rc $(date '+%F %T')" > "$dir/ACCEPTED"; log "$name finished earlier (rc $rc); accepted"; return 0 + fi + [ -d "$dir" ] && mv "$dir" "$dir.failed-$(date +%s)" + while :; do + wait_resources "$3" "$4" "$name" "${5:-}" + log "$name start" + "$PY" "$SUP" "$dir" "$RUN/$name-config.json" >>"$LOG" 2>&1 & + sp=$! + "$PY" "$SAMPLER" "$dir" "$RUN/$name.series.csv" "$sp" 30 >/dev/null 2>&1 & + smp=$! + wait "$sp" + kill "$smp" 2>/dev/null; wait "$smp" 2>/dev/null + if grep -q '"status": "REFUSED_ADMISSION"' "$dir/RESULT.json" 2>/dev/null; then + [ "$(date +%s)" -lt "$deadline" ] || fail "$name refused admission until the deadline: $(tr -d '\n ' < "$dir/ADMISSION.json" | cut -c1-200)" + log "$name refused admission (a race with another process); retrying in 5 min" + mv "$dir" "$dir.refused-$(date +%s)"; sleep 300; continue + fi + break + done + if rc=$(result_accepted "$dir" "$accept"); then + echo "returncode=$rc $(date '+%F %T')" > "$dir/ACCEPTED" + log "$name ACCEPTED (rc $rc) $(tr -d '\n ' < "$dir/RESULT.json" | cut -c1-220)" + return 0 + fi + fail "$name $(tr -d '\n' < "$dir/RESULT.json" 2>/dev/null | cut -c1-300); log $dir/run.log" +} + +PF_ENV='{"HF_HUB_OFFLINE": "1", "PYTHONUNBUFFERED": "1"}' +BUILD_ENV='{"PYTHONUNBUFFERED": "1"}' +NOSECRETS=(/usr/bin/env -u HF_TOKEN -u HUGGING_FACE_HUB_TOKEN -u HUGGINGFACE_HUB_TOKEN -u HUGGING_FACE_TOKEN_MAX + -u SLACK_WEBHOOK_POPULACE_US -u SLACK_WEBHOOK_POPULACE_UK POPULACE_RELEASE_ENV=/dev/null HF_HUB_OFFLINE=1) + +# --- 3. Prefetch the donors the release tool fetches itself when no flag names +# them (build_us_fiscal_refresh_release.py :9444, :10393, :10401, :10595, +# :10708, :10749 on origin/main), so a network failure surfaces in minutes, +# not hours into the release. Each fetch helper verifies its pinned digest per +# its module docstring; results land in ~/.cache/microcosm. --- +if ! stage_done prefetch; then + write_config prefetch 16 12 26 10800 20000 "$W" "$BUILD_ENV" -- "$PY" -B -c ' +import json +from microcosm.build.us_runtime import ( + fetch_asec_2023_weeks_unemployed_source, fetch_org_2024_donor, + fetch_scf_2022_full_extract, fetch_scf_2022_summary_extract, + fetch_sipp_2023_financial_asset_donor, fetch_sipp_2023_tip_donor) +paths = {} +for fn in (fetch_asec_2023_weeks_unemployed_source, fetch_scf_2022_summary_extract, + fetch_sipp_2023_financial_asset_donor, fetch_scf_2022_full_extract, + fetch_sipp_2023_tip_donor, fetch_org_2024_donor): + paths[fn.__name__] = str(fn()) + print(fn.__name__, paths[fn.__name__], flush=True) +print(json.dumps(paths, indent=1)) +' + # Non-fatal here: the base needs none of these, so a fetch problem should not + # hold it back. The release stage below requires the prefetch to pass. + if ! ( run_stage prefetch 0 12 26 ); then + rm -f "$RUN_ROOT/ROUTE_A_FAILED" + log "prefetch failed (see $RUN/prefetch-sup/run.log); the base proceeds and the release retries the prefetch" + fi +fi + +# --- 4. Base stage from raw sources (main + #959). Measured 2026-09-16 under +# policyengine-us 1.819.0: 2,787.90 s wall, 6,238 CPU-s, 72.47 GB (67.5 GiB) +# peak, 44 GB of frame checkpoints + a 2.35 GB H5; under contention 5,058 s, +# 8,560 CPU-s, 65.10 GB. Not re-measured on policyengine-us 2.2.1. --- +BASE_CKPT=$RUN/base-checkpoints +BASE_OUT=$RUN/base-out +BASE_H5=$BASE_OUT/base_populace_us_2024_puf_support.h5 +if ! stage_done base; then + mkdir -p "$BASE_CKPT" + have_ckpt=$(( $(du -sk "$BASE_CKPT" | cut -f1) / 1048576 )) + base_disk=$(( 20 + 50 - have_ckpt )); [ $base_disk -lt 26 ] && base_disk=26 + write_config base 100 80 "$base_disk" 21600 64800 "$W" "$BUILD_ENV" -- "$PY" -B tools/build_us_puf_support_base.py \ + --stage all --checkpoint-dir "$BASE_CKPT" \ + --asec-h5 "2024=$STORAGE/census_cps_2024.h5" --asec-h5 "2023=$STORAGE/census_cps_2023.h5" --asec-h5 "2022=$STORAGE/census_cps_2022.h5" \ + --asec-h5-sha256 "2024=$ASEC_2024_SHA" --asec-h5-sha256 "2023=$ASEC_2023_SHA" --asec-h5-sha256 "2022=$ASEC_2022_SHA" \ + --puf-h5 "$STORAGE/puf_2024.h5" --puf-source-year-csv "$STORAGE/puf_2015.csv" --acs-h5 "$STORAGE/acs_2022.h5" \ + --asec-education-source "2022=$EDU/asecpub23csv.zip" --asec-education-source "2023=$EDU/asecpub24csv.zip" \ + --asec-education-source "2024=$EDU/asecpub25csv.zip" \ + --target-year 2024 --seed 0 --n-estimators 32 \ + --ledger-facts "$BASE_LEDGER_FACTS" \ + --assign-congressional-districts --congressional-district-vintage-crosswalk "$W/$CROSSWALK_REL" \ + --congressional-district-seed 0 --block-ladder-artifact "$LADDER" \ + --out "$BASE_OUT" + run_stage base 0 80 "$base_disk" +fi +[ -f "$BASE_H5" ] || fail "base stage accepted but $BASE_H5 is missing" +if [ ! -f "$RUN/base.sha256" ] || [ "$BASE_H5" -nt "$RUN/base.sha256" ]; then + echo "$(sha_of "$BASE_H5") $BASE_H5" > "$RUN/base.sha256" +fi +BASE_SHA=$(cut -c1-64 "$RUN/base.sha256") +log "base H5 $BASE_H5 sha256 $BASE_SHA ($(stat -f %z "$BASE_H5") bytes)" +if [ "$PRUNE_BASE_CHECKPOINTS" = 1 ] && ls "$BASE_CKPT"/*.frame.h5 >/dev/null 2>&1; then + log "PRUNE_BASE_CHECKPOINTS=1: removing base frame checkpoints ($(du -sh "$BASE_CKPT" | cut -f1)) after the base H5 was hashed" + rm -f "$BASE_CKPT"/*.frame.h5 +fi + +# --- 5. Release-gate preflight on the fresh base (32.58 s / 9.76 GB on 09-16). +# Needs the new-lineage mode: on origin/main --selection-source-manifest is +# required (tools/preflight_us_release_gates.py:273-277). --- +PF_BLOCKED=0 +PF_COMMON=(--base-h5 "$BASE_H5" --ledger-facts "$FEED" --ledger-facts-sha256 "$FEED_SHA") +[ -n "$EXPORT_MASS_REF_H5" ] && PF_COMMON+=(--export-input-mass-reference-h5 "$EXPORT_MASS_REF_H5") +if [ -z "$PREFLIGHT_NEW_LINEAGE_ARGS" ]; then + PF_BLOCKED=1 + log "preflight-base BLOCKED: PREFLIGHT_NEW_LINEAGE_ARGS is empty (new-lineage preflight PR not landed); the release still runs" + echo "BLOCKED $(date '+%F %T'): no new-lineage preflight mode" > "$RUN/preflight-base.BLOCKED" +else + # shellcheck disable=SC2206 + PF_MODE=($PREFLIGHT_NEW_LINEAGE_ARGS) + if ! stage_done preflight-base; then + write_config preflight-base 24 16 22 3600 7200 "$PW" "$PF_ENV" -- "${NOSECRETS[@]}" "$PW/.venv/bin/python" -B \ + tools/preflight_us_release_gates.py "${PF_COMMON[@]}" "${PF_MODE[@]}" --json-out "$RUN/preflight-base.json" + if [ "$RELEASE_DESPITE_PREFLIGHT_FAIL" = 1 ]; then run_stage preflight-base "0 1 2" 16 22 + else run_stage preflight-base "0 2" 16 22; fi + fi + log "preflight-base: $(cat "$RUN/preflight-base-sup/ACCEPTED") (0 clean, 2 AT-RISK only, 1 FAIL); report $RUN/preflight-base.json" +fi + +# --- 6. Release tool, --base-h5 arm, committed feed, no +# selection source. Unmeasured at this size: without a selection the tool +# materializes PolicyEngine over the whole ~353k-household base +# (build_us_fiscal_refresh_release.py:9500-9506 on origin/main); July's 57k +# household run took 2 h 42 min at 85 GB. Limits are sized to the machine: +# 110 GiB RSS, configurable wall and available-RAM admission. --- +refuse_bad_release_args +stage_done prefetch || run_stage prefetch 0 12 26 +REL_CKPT=$RUN/release-checkpoints +wait_prior_supervisor release +if ! stage_done release && ! result_accepted "$RUN/release-sup" 0 >/dev/null; then + RID=populace-us-2024-${BASE_SHA:0:7}-${C:0:12}-$(date -u +%Y%m%dT%H%M%SZ) + REL_OUT=$RUN/release-out/$RID + release_command + write_config release 110 "$RELEASE_RAM_GIB" 45 "${RELEASE_WALL_SECONDS:-345600}" 2000000 "$W" "$BUILD_ENV" -- "${RELEASE_ARGV[@]}" +fi +if ! stage_done release; then + run_stage release 0 "$RELEASE_RAM_GIB" 45 "Set PRUNE_BASE_CHECKPOINTS=1 in route_a.env to reclaim the base's frame checkpoints." +fi +RID=$("$PY" -c 'import json,sys; a=json.load(open(sys.argv[1]))["argv"]; print(a[a.index("--release-id")+1])' "$RUN/release-sup/COMMAND.json") \ + || fail "cannot read the release id from release-sup/COMMAND.json" +REL_OUT=$RUN/release-out/$RID +REL_DIR=$REL_OUT/releases/$RID +[ -f "$REL_DIR/release_manifest.json" ] || fail "release accepted but $REL_DIR/release_manifest.json is missing" +log "release candidate $RID at $REL_DIR" + +# --- 7. Release-gate preflight against the built release manifest. --- +if [ "$PF_BLOCKED" = 1 ]; then + log "preflight-release BLOCKED: no new-lineage preflight mode" + echo "BLOCKED $(date '+%F %T'): no new-lineage preflight mode" > "$RUN/preflight-release.BLOCKED" +else + if ! stage_done preflight-release; then + write_config preflight-release 24 16 22 3600 7200 "$PW" "$PF_ENV" -- "${NOSECRETS[@]}" "$PW/.venv/bin/python" -B \ + tools/preflight_us_release_gates.py "${PF_COMMON[@]}" "${PF_MODE[@]}" \ + --release-manifest "$REL_DIR/release_manifest.json" --json-out "$RUN/preflight-release.json" + run_stage preflight-release "0 2" 16 22 + fi + log "preflight-release: $(cat "$RUN/preflight-release-sup/ACCEPTED"); report $RUN/preflight-release.json" +fi + +# --- 8. Publisher preflight, offline: publish_cli.py returns after +# prepare_release when --preflight-only is set (:325-331 on origin/main), and +# builds no Hub client. Secrets are stripped as overnight_chain.sh does. --- +if ! stage_done publisher-preflight; then + write_config publisher-preflight 24 16 22 7200 14400 "$W" "$PF_ENV" -- "${NOSECRETS[@]}" \ + "PATH=$W/.venv/bin:/usr/bin:/bin:/usr/sbin:/sbin" "$W/.venv/bin/microcosm-publish-release" \ + "$REL_DIR" --repo-id policyengine/populace-us --artifact-root "$REL_OUT/artifacts" --preflight-only + run_stage publisher-preflight 0 16 22 +fi +PUB_VERDICT=$(grep -E '^\{"valid"' "$RUN/publisher-preflight-sup/run.log" | tail -1) +log "publisher --preflight-only: $PUB_VERDICT" + +# --- 9. Hand-off. Nothing was published. --- +STATE="done"; [ "$PF_BLOCKED" = 1 ] && STATE=blocked +OUTF=$RUN_ROOT/ROUTE_A_DONE.json; [ "$STATE" = blocked ] && OUTF=$RUN_ROOT/ROUTE_A_BLOCKED.json +PFB=BLOCKED; PFR=BLOCKED +if [ "$PF_BLOCKED" = 0 ]; then + PFB=$(cat "$RUN/preflight-base-sup/ACCEPTED"); PFR=$(cat "$RUN/preflight-release-sup/ACCEPTED") +fi +"$PY" - "$OUTF" "$STATE" "$C" "$PC" "$PEUS" "$BASE_H5" "$BASE_SHA" "$RID" "$REL_DIR" "$REL_OUT/artifacts" \ + "$PFB" "$PFR" "$PUB_VERDICT" <<'EOF' +import json, sys +(out, state, commit, pcommit, peus, base_h5, base_sha, rid, rel_dir, art, pfb, pfr, pub) = sys.argv[1:] +json.dump({ + "state": state, + "published": False, + "build_commit": commit, + "preflight_commit": pcommit, + "policyengine_us": peus, + "base_h5": base_h5, + "base_sha256": base_sha, + "release_id": rid, + "release_dir": rel_dir, + "artifact_root": art, + "preflight_base": pfb, + "preflight_release": pfr, + "publisher_preflight_only": pub, + "next": ( + "Publication is Max's call: tools/publish_release.sh " + f"{rel_dir} --repo-id policyengine/populace-us --artifact-root {art}" + ), +}, open(out, "w"), indent=1) +EOF +if [ "$STATE" = blocked ]; then + log "ROUTE A BLOCKED: base and release candidate built, release-gate preflight needs the new-lineage mode; nothing published" + exit 4 +fi +log "ROUTE A DONE: release candidate $RID passed the release tool's gates and both preflights; nothing published (Max's call)" diff --git a/tools/route_a/sample_series.py b/tools/route_a/sample_series.py new file mode 100644 index 000000000..4e36e7da6 --- /dev/null +++ b/tools/route_a/sample_series.py @@ -0,0 +1,97 @@ +"""Record a resource time series for one supervise.py stage (route A). + + python sample_series.py [interval-seconds] + +supervise.py keeps only the latest HEARTBEAT.json (peak RSS, overwritten every +15 s), so it cannot answer "what did RSS do over the run". This sampler appends +one row per interval with the stage's current process-tree RSS (the child named +in PID.json plus its descendants, the same tree supervise.py sums), machine +available RAM, swap in use and free disk. It stops when RESULT.json appears or +the supervisor process is gone. It records no row values of any dataset. +""" + +from __future__ import annotations + +import csv +import json +import shutil +import sys +import time +from datetime import UTC, datetime +from pathlib import Path + +import psutil + +FIELDS = ( + "utc", + "stage_dir", + "child_pid", + "tree_rss_bytes", + "tree_cpu_seconds", + "available_ram_bytes", + "swap_used_bytes", + "free_disk_bytes", +) + + +def _tree(pid: int) -> tuple[int, float]: + try: + root = psutil.Process(pid) + members = [root, *root.children(recursive=True)] + except psutil.Error: + return 0, 0.0 + rss = 0 + cpu = 0.0 + for member in members: + try: + rss += member.memory_info().rss + times = member.cpu_times() + cpu += times.user + times.system + except psutil.Error: + pass + return rss, cpu + + +def main() -> int: + stage_dir = Path(sys.argv[1]) + series = Path(sys.argv[2]) + supervisor_pid = int(sys.argv[3]) + interval = float(sys.argv[4]) if len(sys.argv) > 4 else 30.0 + new_file = not series.exists() + with series.open("a", newline="") as stream: + writer = csv.writer(stream) + if new_file: + writer.writerow(FIELDS) + while True: + if (stage_dir / "RESULT.json").exists(): + return 0 + if not psutil.pid_exists(supervisor_pid): + return 0 + child_pid = 0 + try: + child_pid = int(json.loads((stage_dir / "PID.json").read_text())["pid"]) + except (OSError, ValueError, KeyError): + pass + rss, cpu = _tree(child_pid) if child_pid else (0, 0.0) + try: + free = shutil.disk_usage(stage_dir).free + except OSError: + free = -1 + writer.writerow( + ( + datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"), + stage_dir.name, + child_pid, + rss, + round(cpu, 1), + psutil.virtual_memory().available, + psutil.swap_memory().used, + free, + ) + ) + stream.flush() + time.sleep(interval) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/route_a/supervise.py b/tools/route_a/supervise.py new file mode 100644 index 000000000..1ce812c4c --- /dev/null +++ b/tools/route_a/supervise.py @@ -0,0 +1,191 @@ +"""Bounded supervisor for local hours-rebuild runs. Metadata-only progress. + + python supervise.py + +The config names ``argv``, ``cwd``, ``env`` and ``limits`` (wall_seconds, +cpu_seconds, rss_bytes, output_bytes, log_bytes, disk_floor_bytes, +disk_admission_bytes, available_ram_admission_bytes). The output directory must +not exist. The child runs in its own process group; on a breached limit the +supervisor kills exactly that group. It writes ADMISSION.json, COMMAND.json, a +HEARTBEAT.json every 15 seconds and RESULT.json; it never prints or records +row values, and the child's log is capped, not parsed. +""" + +from __future__ import annotations + +import json +import os +import shutil +import signal +import subprocess +import sys +import time +from pathlib import Path + +import psutil + + +def _tree_size(root: Path) -> int: + total = 0 + for path in root.rglob("*"): + try: + if path.is_file(): + total += path.stat().st_size + except OSError: + pass + return total + + +def main() -> int: + out = Path(sys.argv[1]) + config = json.loads(Path(sys.argv[2]).read_text()) + limits = config["limits"] + out.mkdir(parents=True, exist_ok=False) + + available = psutil.virtual_memory().available + free = shutil.disk_usage(out).free + admitted = ( + available >= limits["available_ram_admission_bytes"] + and free >= limits["disk_admission_bytes"] + ) + (out / "ADMISSION.json").write_text( + json.dumps( + { + "admitted": admitted, + "observed_available_ram_bytes": available, + "observed_free_disk_bytes": free, + "limits": limits, + "authorization": config.get("authorization"), + "automatic_retry": False, + }, + indent=1, + ) + + "\n" + ) + if not admitted: + (out / "RESULT.json").write_text( + json.dumps({"status": "REFUSED_ADMISSION"}, indent=1) + "\n" + ) + return 3 + + (out / "COMMAND.json").write_text( + json.dumps( + { + "argv": config["argv"], + "cwd": config["cwd"], + "env": config.get("env", {}), + "source_identity": config.get("source_identity"), + "started_at_unix": time.time(), + }, + indent=1, + ) + + "\n" + ) + env = {**os.environ, **config.get("env", {})} + env.pop("UV_FROZEN", None) + start = time.monotonic() + peak_rss = 0 + cpu = 0.0 + reason = None + last_beat = 0.0 + # A disk-floor breach must persist before it kills the child: local + # snapshot churn on this Mac can swing free space by tens of GB within + # seconds, and the staging child writes nothing until its export. + floor_breach_seconds = float(limits.get("disk_floor_persist_seconds", 0)) + floor_breached_since = None + with (out / "run.log").open("wb") as log: + child = subprocess.Popen( + config["argv"], + cwd=config["cwd"], + env=env, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + (out / "PID.json").write_text(json.dumps({"pid": child.pid}) + "\n") + try: + while child.poll() is None: + try: + root = psutil.Process(child.pid) + members = [root, *root.children(recursive=True)] + rss = 0 + cpu_now = 0.0 + for member in members: + try: + rss += member.memory_info().rss + times = member.cpu_times() + cpu_now += times.user + times.system + except psutil.Error: + pass + peak_rss = max(peak_rss, rss) + cpu = max(cpu, cpu_now) + except psutil.Error: + pass + wall = time.monotonic() - start + output_bytes = _tree_size(out) + log_bytes = os.fstat(log.fileno()).st_size + free_now = shutil.disk_usage(out).free + if wall > limits["wall_seconds"]: + reason = "WALL" + elif cpu > limits["cpu_seconds"]: + reason = "CPU" + elif peak_rss > limits["rss_bytes"]: + reason = "RSS" + elif output_bytes > limits["output_bytes"]: + reason = "OUTPUT" + elif log_bytes > limits["log_bytes"]: + reason = "LOG" + elif free_now < limits["disk_floor_bytes"]: + if floor_breached_since is None: + floor_breached_since = wall + if wall - floor_breached_since >= floor_breach_seconds: + reason = "DISK_FLOOR" + else: + floor_breached_since = None + if reason: + break + if wall - last_beat >= 15: + last_beat = wall + (out / "HEARTBEAT.json").write_text( + json.dumps( + { + "pid": child.pid, + "wall_seconds": wall, + "cpu_seconds": cpu, + "peak_rss_bytes": peak_rss, + "output_bytes": output_bytes, + "log_bytes": log_bytes, + "free_disk_bytes": free_now, + } + ) + + "\n" + ) + time.sleep(1.0) + finally: + if child.poll() is None: + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + returncode = child.wait() + (out / "RESULT.json").write_text( + json.dumps( + { + "status": "COMPLETED" if reason is None and returncode == 0 else "FAIL", + "returncode": returncode, + "refusal": reason, + "wall_seconds": time.monotonic() - start, + "cpu_seconds": cpu, + "peak_rss_bytes": peak_rss, + "output_bytes": _tree_size(out), + "release_acceptance": False, + }, + indent=1, + ) + + "\n" + ) + return 0 if reason is None and returncode == 0 else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/route_a/with_hf_token.sh b/tools/route_a/with_hf_token.sh new file mode 100755 index 000000000..e304ab958 --- /dev/null +++ b/tools/route_a/with_hf_token.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Only the exec'd release process receives the runtime credential. +set +x +set -euo pipefail + +[ "$#" -ge 2 ] || { echo "usage: with_hf_token.sh [args...]" >&2; exit 64; } +secret_helper=$1 +shift +unset HF_TOKEN HUGGING_FACE_HUB_TOKEN HUGGINGFACE_HUB_TOKEN HUGGING_FACE_TOKEN_MAX +HF_TOKEN=$("$secret_helper" get HUGGING_FACE_TOKEN_MAX 2>/dev/null) || { + echo "FAIL: HF credential lookup failed" >&2 + exit 1 +} +[ -n "$HF_TOKEN" ] || { echo "FAIL: empty HF credential" >&2; exit 1; } +export HF_TOKEN +exec "$@"