diff --git a/.autoskillit/test-filter-manifest.yaml b/.autoskillit/test-filter-manifest.yaml index 52def2ee5f..b95a8c0c18 100644 --- a/.autoskillit/test-filter-manifest.yaml +++ b/.autoskillit/test-filter-manifest.yaml @@ -156,6 +156,12 @@ tests/fixtures/codex/*.ndjson: tests/fixtures/codex/*.jsonl: - hooks/ +tests/fixtures/codex_recipe_diagnostic/*: + - execution/ + +tests/fixtures/codex_recipe_protected/*: + - execution/ + tests/execution/backends/fixtures/codex_ndjson/*.json: - execution/ diff --git a/.github/workflows/conformance-probes.yml b/.github/workflows/conformance-probes.yml index 478af9ae1f..247670df86 100644 --- a/.github/workflows/conformance-probes.yml +++ b/.github/workflows/conformance-probes.yml @@ -98,6 +98,7 @@ jobs: .venv/bin/python -m pytest \ tests/execution/test_smoke_codex.py \ tests/execution/backends/test_cli_conformance_probes.py \ + tests/execution/backends/test_codex_recipe_delivery_conformance.py \ tests/server/test_output_budget_e2e.py \ -m smoke -v --tb=short -o "addopts=" \ --basetemp="$TMPDIR/basetemp" \ diff --git a/Taskfile.yml b/Taskfile.yml index 37f7c922fc..91f58693d5 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -246,7 +246,7 @@ tasks: set -o pipefail set +e - $PYTEST_CMD tests/execution/test_smoke_codex.py tests/execution/backends/test_cli_conformance_probes.py -m smoke -v --tb=short -o "addopts=" --basetemp={{.PYTEST_TMPDIR}} -o "cache_dir={{.PYTEST_CACHEDIR}}" 2>&1 | tee "$TEST_OUTPUT" + $PYTEST_CMD tests/execution/test_smoke_codex.py tests/execution/backends/test_cli_conformance_probes.py tests/execution/backends/test_codex_recipe_delivery_conformance.py -m smoke -v --tb=short -o "addopts=" --basetemp={{.PYTEST_TMPDIR}} -o "cache_dir={{.PYTEST_CACHEDIR}}" 2>&1 | tee "$TEST_OUTPUT" PYTEST_EXIT=$? set +o pipefail set -e diff --git a/docs/README.md b/docs/README.md index 1907f9b9a0..dc5130b195 100644 --- a/docs/README.md +++ b/docs/README.md @@ -33,5 +33,6 @@ multi-level orchestrator. The bundled recipes implement issue → plan → workt - [research/experiment-type-rationale.md](research/experiment-type-rationale.md) — dimension-weight rationale for experiment types - [research/silent-type-convention.md](research/silent-type-convention.md) — silent type detection and advisory convention - [research/audit-trail-format.md](research/audit-trail-format.md) — audit/ artifact structure and lifecycle +- [research/codex-delivery-conformance.md](research/codex-delivery-conformance.md) — Codex recipe envelope/pull conformance and protected-host blocker - [audit/surface-freeze-checklist.md](audit/surface-freeze-checklist.md) — commands.py public import surface freeze checklist - [phoropter/](phoropter/README.md) — phoropter lens framework: execution contracts, recipe blocks, synthesis strategies, and authoring guide diff --git a/docs/configuration.md b/docs/configuration.md index 8941d4cb36..37d3f078f2 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -200,25 +200,26 @@ run_skill: stale_threshold: 1200 # 20 minutes of no output before declaring stale ``` -### Per-repo Codex output ceilings - -The global `~/.codex/config.toml` sets `tool_output_token_limit` to 54,500 (written -by `autoskillit init` via `ensure_codex_mcp_registered`). This value is sized for -autoskillit kitchen sessions — see [ADR-0005](decisions/0005-output-budget-protocol.md) -for the derivation. - -For non-autoskillit repos, cap casual reads at ~40 KB (10,000 tokens × 4 bytes/token) -using either approach: - -1. **Per-invocation**: `codex -c tool_output_token_limit=10000` -2. **Per-project**: set `CODEX_HOME=/.codex` with a dedicated `config.toml` - containing `tool_output_token_limit = 10000` - -**Scope caveat**: the ceiling clamps regular-path exec/tool output -(`min(model request, tool_output_token_limit)`), but code-mode models (gpt-5.6-sol) -honor model-declared `max_output_tokens` unclamped — for those sessions the ceiling -is not a hard cap and the intake digest's `max_output_tokens` <= 10000 rule is the -operative bound. +### Codex result and history budgets + +The global `~/.codex/config.toml` sets `tool_output_token_limit` to 56,750 (written +by `autoskillit init` via `ensure_codex_mcp_registered`). The setting governs how +much tool output Codex retains in later conversation history. It does not select the +outer result limit of the current `functions.exec` call. + +Ordinary calls use Codex's 10,000-token omitted outer default and must not request a +larger result. A narrow exception exists for full `open_kitchen` and `load_recipe` +delivery: the outer cell starts with +`// @exec: {"max_output_tokens": 56750}` and passes the protected host-provided +`delivery_request` unchanged. AutoSkillit accepts that exception only for a supported, +unforgeable host evidence identity and only for the thread's first oversized insertion. +Without that evidence—including direct MCP calls and current writable rollout files—the +server returns the bounded, content-addressed `recipe_pull` envelope. + +Changing `tool_output_token_limit` is a history-retention choice, not a per-call output +selector. Bound ordinary producers directly and use `max_output_tokens` only within the +generated recipe-delivery contract. See +[ADR-0005](decisions/0005-output-budget-protocol.md) for the separate authority domains. ## MCP Response Tracking diff --git a/docs/decisions/0005-output-budget-protocol.md b/docs/decisions/0005-output-budget-protocol.md index 6942f4636a..4ddaed9ef5 100644 --- a/docs/decisions/0005-output-budget-protocol.md +++ b/docs/decisions/0005-output-budget-protocol.md @@ -39,43 +39,33 @@ containment; downstream layers are independent backstops rather than substitutes classifier and its `output_budget_guard` are deleted. Codex native shell is now bounded by a PreToolUse input-rewrite hook (`shell_capture_hook`) that captures complete output to a mechanism-owned artifact and emits only a bounded inline - slice. The transport ceiling (CODEX_TOOL_OUTPUT_TOKEN_LIMIT) remains as the - backstop for hook-failure paths.* + slice. The ordinary outer-result limit remains the backstop for hook-failure paths.* 4. **Producer-aware discipline and derived transport ceiling:** one evidence-output policy is delivered on backend surfaces that support it, while Codex's stored tool/function output receives a derived damage ceiling. The policy is advisory; the ceiling is downstream containment. Neither is cumulative-context accounting. -The raw-text `open_kitchen` and `load_recipe` responses are measured exemptions from the -universal backstop. `RESPONSE_BACKSTOP_EXEMPTION_REGISTRY` is the closed authority for -their independent character ceiling, UTF-8 byte ceiling, and measurement identity. Its -canonical digest is carried in tool metadata and probe-cache identity. Adding or relaxing -an exemption requires re-measurement and a deliberate registry-digest change. The raw-text -exemption now applies only to the fits-within-bound fast path; when a recipe payload exceeds -the delivery bound, `maybe_envelope_recipe_response` replaces it with a bounded envelope -(via `build_recipe_envelope`) that fits every backend by construction, with full step -content available on demand via `get_recipe_section`. +The four recipe-bearing routes are owned by `RECIPE_DELIVERY_SURFACE_REGISTRY`. Its +canonical digest, together with `RESPONSE_BACKSTOP_EXEMPTION_REGISTRY`, participates in +the delivery contract and probe-cache identity. `finalize_recipe_delivery` persists every +canonical payload as an immutable content-addressed generation, then selects one explicit +mode: ordinary inline, host-attested inline, or a bounded `recipe_pull` envelope. The +registered FastMCP handlers still return exact strings, and the response decorator consumes +the selected decision without applying a second static shaping pass. ## Numeric Limits and Rationale | Limit | Decision and rationale | |---|---| -| `load_recipe`: `max_chars = 188_000`, `max_utf8_bytes = 188_000` | The 2026-07-21 independent all-recipe/all-mode pre-backstop measurement reached 186,621 characters and UTF-8 bytes for `remediation` with all truthy ingredients. The 1,379-unit margin makes serving growth explicit. Measurement identity: `bundled-recipes-all-modes-2026-07-21/load-recipe`. For payloads that exceed the delivery bound, see envelope note below. | -| `open_kitchen`: `max_chars = 188_000`, `max_utf8_bytes = 188_000` | The 2026-07-21 independent all-recipe/all-mode pre-backstop measurement reached 186,680 characters and UTF-8 bytes for `remediation` with all truthy ingredients. The 1,320-unit margin covers the open-kitchen routing fields without conflating this handler ceiling with the smaller formatted presentation. Measurement identity: `bundled-recipes-all-modes-2026-07-21/open-kitchen`. For payloads that exceed the delivery bound, see envelope note below. | - -> **Envelope note (oversized recipes, issue #4304 Part B):** When a recipe's serialized payload -> exceeds a backend's effective delivery bound, `maybe_envelope_recipe_response` replaces the -> raw text with a bounded envelope built via `build_recipe_envelope`. The envelope carries a -> step-flow skeleton plus a `recipe_pull` reference (`pull_tool: get_recipe_section`, -> `artifact_path`, `artifact_sha256`) so the agent re-acquires each step body on demand via -> `get_recipe_section(section=)` (chunked via `part` / `has_more` / `next_part` -> for oversized sections). The raw-text ceilings above continue to govern the fits-within-bound -> fast path; the envelope introduces a separate, fit-by-construction delivery channel. -| `CODEX_TOOL_OUTPUT_TOKEN_LIMIT = 55_000` | Derive it from the largest registered exemption as `((188_000 + 3) // 4) + 8_000`: 47,000 tokens under the current client's four-byte heuristic, plus 8,000 tokens for serialized-payload headroom. It is a blast-radius damage bound, not the mechanism that makes evidence lossless. | +| `load_recipe`: `max_chars = 195_000`, `max_utf8_bytes = 195_000` | The registered 2026-07-22 all-recipe/all-mode measurement identity is `bundled-recipes-all-modes-2026-07-22/load-recipe`. Growth beyond the measured ceiling fails closed to an immutable pull generation. | +| `open_kitchen`: `max_chars = 195_000`, `max_utf8_bytes = 195_000` | The independent registered identity is `bundled-recipes-all-modes-2026-07-22/open-kitchen`; the deferred branch shares the producer but remains a separate delivery surface. | +| `ordinary_omitted_result_token_limit = 10_000` | Conservative outer result for ordinary Codex calls and every untrusted or unsupported recipe request. | +| `authoritative_attested_recipe_result_token_limit = 56_750` | Derived as `((195_000 + 3) // 4) + 8_000`. It is selectable only from protected host evidence for the current call, never from nested arguments or rollout files. | +| `CODEX_HISTORY_RETENTION_TOKEN_LIMIT = 56_750` | Written to upstream `tool_output_token_limit`; controls later stored history and does not select the current outer result. Equality with the attested result limit is intentional but does not merge the authority domains. | | `CODEX_AUTO_COMPACT_LIMIT = 999_999_999` | Retain the unreachable sentinel and the recovery obligation accepted in [ADR-0004](0004-recipe-redelivery.md). This protocol does not relax recipe-preservation policy. | | `inline_max_chars = 5_000` | Preserve the previous truncation threshold while changing the representation from destructive clipping to an artifact-backed preview. The configured 2,500-character head and 2,500-character tail retain both diagnostic setup and terminal status; a spill marker is added outside those source slices. | | `response_max_bytes = 90_000` | Bound the exact compact serialized handler payload before a coarser transport can clip it. Bytes are authoritative here; this is not a token or full JSON-RPC-envelope estimate. | -| `MAX_MCP_OUTPUT_TOKENS = 50_000` | Keep Claude's independently defined setting separate. It has no shared source of truth with `CODEX_TOOL_OUTPUT_TOKEN_LIMIT` and does not control Claude Code's observed disk-persistence gate. Claude's native Bash spill behavior covers shell output on that backend. | +| `MAX_MCP_OUTPUT_TOKENS = 50_000` | Keep Claude's independently defined setting separate. It has no shared source of truth with Codex result or history limits and does not control Claude Code's observed disk-persistence gate. Claude's native Bash spill behavior covers shell output on that backend. | The shell capture hook uses `shell_max_inline_bytes = 12_000` as the inline threshold: commands whose combined output fits within that budget are inlined in full (artifact @@ -94,47 +84,33 @@ tokenization and provides neither a tokenizer guarantee nor a cumulative-context estimate. The project therefore requires the stricter relationship -`response_max_bytes // 3 < CODEX_TOOL_OUTPUT_TOKEN_LIMIT`. The three-byte divisor is -deliberate margin: the 90,000-byte response backstop must fire before Codex's 54,500-token -transport ceiling can clip a producer-blind response. A static test pins the relationship, -and the live large-output probe must pass before either side is retuned. - -The measured raw-text exemptions, `open_kitchen` and `load_recipe`, must each remain below -their own registered character and UTF-8 byte ceilings, which in turn remain below the -218,000-byte budget implied by the current 54,500-token, four-byte heuristic. Their -measurements are independent release gates; the heuristic is not permission to omit those -tests or reuse one surface's observed maximum as the other's authority. - -### Per-Repo Ceiling Guidance - -The global `~/.codex/config.toml` `tool_output_token_limit` (54,500, written by -`autoskillit init` via `ensure_codex_mcp_registered`) is intentionally sized for -autoskillit kitchen sessions (the `open_kitchen` exemption ceiling above). For -non-autoskillit repos, two lower-ceiling launch paths cap casual reads at ~40 KB -(10,000 tokens × 4 bytes/token): - -1. **Per-invocation**: `codex -c tool_output_token_limit=10000` -2. **Per-project `CODEX_HOME`**: `export CODEX_HOME=/.codex` with its own - `config.toml` containing `tool_output_token_limit = 10000` - -The ceiling clamps regular-path exec/tool output via -`min(model request, tool_output_token_limit)`, but code-mode models (gpt-5.6-sol) -honor model-declared `max_output_tokens` unclamped — for those sessions the ceiling -is not a hard cap and the operative bound is enforced server-side by -`BackendCapabilities.effective_delivery_token_limit` (Codex: 10,000 tokens). The -response backstop reads that capability at spill time and spills any payload whose -estimated token count exceeds it, with the artifact path surfaced via the existing -`_autoskillit_response_spill` envelope (`reason="delivery_bound"`). The intake -discipline digest's numeric rule (`max_output_tokens` <= 10000) remains as the -prompt-level reinforcement. +`response_max_bytes // 3 < ordinary_omitted_result_token_limit`. The three-byte divisor is +deliberate margin: the 90,000-byte response backstop must fire before an ordinary 10,000-token +outer result can clip a producer-blind response. A static test pins the relationship, and the +live large-output probe must pass before either side is retuned. + +The measured recipe surfaces must each remain below their registered character and UTF-8 +byte ceilings. Their measurements are independent release gates; the heuristic is not +permission to omit those tests or reuse one surface's observed maximum as another domain's +authority. + +### Codex Authority Domains + +The generated calling contract keeps five values distinct: caller-requested outer tokens, +host-observed requested tokens, derived selected result tokens, required serialized tokens, +and history-retained tokens. Ordinary calls retain the 10,000-token rule. A full recipe call +may use the exact 56,750-token pragma only when a protected host channel supplies the +immutable `RecipeDeliveryRequest`; otherwise the request is omitted and the response uses +the bounded pull path. The `recipe://{name}` resource is never negotiation-eligible. ## Corrections of Record Commit `6b421e38e` introduced the `_codex_config.py` comment framing `tool_output_token_limit` as a per-MCP-tool response budget sized for `open_kitchen`. -That framing was incorrect. The Codex setting governs tool/function output stored in -context, including native shell, `unified_exec`, and MCP output. It is a global damage -ceiling and does not make `open_kitchen` lossless. +That framing was incorrect. The Codex setting governs tool/function output retained in +later context, including native shell, `unified_exec`, and MCP output. It is a history +damage bound, not the current call's outer result selector, and does not make +`open_kitchen` lossless. [PR #4259](https://github.com/TalonT-Org/AutoSkillit/pull/4259) included `Closes #4253`, but GitHub did not auto-close the issue because the PR merged into @@ -158,21 +134,19 @@ recorded explicitly in the issue body. 6. Routing- and shape-preserving projections retain control-plane keys and value types and place complete domain data in the artifact. A caller needing pruned collection members must retrieve bounded slices from that artifact. -7. Repeated individually bounded calls can still exhaust cumulative context. No pre-call - component owns current context usage, so reserve instructions remain advisory. +7. A durable per-thread receipt limits cumulative context insertion by preventing more than + one oversized attested insertion across normal, deferred, load, and resource routes. + Repeated or changed recipes use pull. 8. Closed for the `run_cmd` channel by #4286 (capture files promoted in place; only bounded slices enter worker memory). Still open for `run_skill` and `test_check`, whose adjudication requires the full text. ## Resolved -- **Code-mode bypass (`gpt-5.6-sol` honoring `max_output_tokens` unclamped)**: prior to - this revision, the `tool_output_token_limit` ceiling did not constrain code-mode - responses and the intake discipline digest's numeric rule was the only operative - bound — prompt-level and unenforced. The server-side response backstop now reads - `BackendCapabilities.effective_delivery_token_limit` (Codex: 10,000 tokens; - Claude Code: 46,500 tokens) and spills payloads that exceed it. This makes the - delivery bound authoritative on every backend, not advisory. +- **Scalar budget coupling:** general responses now use + `BackendCapabilities.unnegotiated_tool_result_token_limit`; recipe responses carry an + explicit decision through final enforcement. History retention is never read as the + selected outer result. ## Operational Signals @@ -181,11 +155,14 @@ Output-budget instrumentation uses low-cardinality structured counters or events - spill count by producer/tool class; - original and artifact UTF-8 byte totals; - measured exemption use; and -- spill failures grouped by bounded cause code. +- spill failures grouped by bounded cause code; +- recipe delivery decision mode and bounded reason; and +- receipt reservation outcomes without thread, call, path, or payload identities. Signals must never contain artifact paths, hashes, or output content. This decision does -not introduce artifact quota, artifact cleanup, cumulative reserve accounting, or -reserve-trigger metrics, so instrumentation must not claim those mechanisms exist. +not introduce an artifact quota or current-context token accounting, so instrumentation +must not claim those mechanisms exist. The one-high-insertion receipt is cumulative +insertion control, not a measurement of remaining context. ## Forward Obligations @@ -200,8 +177,8 @@ reserve-trigger metrics, so instrumentation must not claim those mechanisms exis pulling each step body via `get_recipe_section(section=)`, chunked via `part` / `has_more` / `next_part` for oversized sections — not as a replay of the full raw payload. Reconciles with the ADR-0004 cross-reference amendment. -- After each codex-cli upgrade, re-verify the truncation heuristic - (`CODEX_TOOL_OUTPUT_TOKEN_LIMIT`) and auto-compact sentinel +- After each codex-cli upgrade, re-verify the result-limit parser, history-retention setting + (`CODEX_HISTORY_RETENTION_TOKEN_LIMIT`), and auto-compact sentinel (`CODEX_AUTO_COMPACT_LIMIT`) against the upstream registry AND observed session windows, then bump `CODEX_LIMITS_LAST_VERIFIED_VERSION`. Doctor Check 39 (`codex_limits_verified`) mechanizes the reminder. Issue #4280's investigation @@ -225,7 +202,6 @@ reserve-trigger metrics, so instrumentation must not claim those mechanisms exis - Ordinary large responses become lossless artifacts with bounded inline evidence. - High-confidence unbounded shell calls are refused before their output is produced. -- Transport limits are derived from a measured control-plane payload instead of an - unrelated generous constant. +- Recipe and ordinary result decisions are explicit and independent from history retention. - Artifact/invariant failures become explicit bounded errors instead of context floods. - The accepted gaps above remain visible work rather than implicit guarantees. diff --git a/docs/decisions/0006-output-containment.md b/docs/decisions/0006-output-containment.md index 711f40db41..8331c55900 100644 --- a/docs/decisions/0006-output-containment.md +++ b/docs/decisions/0006-output-containment.md @@ -28,8 +28,8 @@ output-boundary bounding on measured bytes: 3. **Codex native shell** — PreToolUse input-rewrite hook wraps every shell command in a capture harness: complete output to a mechanism-owned artifact, bounded inline slice with provenance marker, exit code preserved. The - transport ceiling (CODEX_TOOL_OUTPUT_TOKEN_LIMIT) remains as the backstop for - hook-failure paths. + ordinary outer-result limit remains the backstop for hook-failure paths. The + separately configured `CODEX_HISTORY_RETENTION_TOKEN_LIMIT` governs later history. ### Sequencing Rule diff --git a/docs/research/codex-delivery-conformance.md b/docs/research/codex-delivery-conformance.md new file mode 100644 index 0000000000..e8ba355789 --- /dev/null +++ b/docs/research/codex-delivery-conformance.md @@ -0,0 +1,62 @@ +# Codex Recipe Delivery Conformance + +Status: blocked + +**Probe contract:** `codex-recipe-delivery-v1` +**Decision date:** 2026-07-22 +**Model identity:** `gpt-5.6-sol` + +## Decision + +`SUPPORTED_CODEX_RECIPE_EVIDENCE_REGISTRY` remains empty. Current Codex rollout and +rollout-trace files are writable by the same OS identity as model-invoked commands. They +record requested input but do not expose an independently observed resolved outer limit or +raw outer pre-truncation bytes before the nested MCP call. They therefore cannot authorize +the attested-inline mode. + +The production resolver fails closed to the bounded, content-addressed `recipe_pull` path. +A positive identity requires a protected pre-call host channel that the model and direct MCP +callers cannot mint, alter, replace, or replay. + +## Evidence Domains + +| Domain | Current observation | +|---|---| +| Caller pragma/request | The generated contract names the exact 56,750-token pragma, but no protected request values are available to the current runtime. | +| Host thread/turn/call and selected limit | Rollout `thread_id` is diagnostic. Protected turn/call identity and a resolved selected limit are unavailable. | +| History configuration | `tool_output_token_limit` is 56,750 and is recorded only as later-history retention. | +| Canonical payload and persisted blob | Envelope/pull tests independently validate `payload_sha256`, `artifact_blob_sha256`, and `body_sha256`. | +| Compact MCP wire and nested result | The live probe hashes its transcript. A distinct nested JavaScript result is unavailable without Code Mode host instrumentation. | +| Raw outer result | Raw outer pre-truncation bytes are unavailable in the current host schema. | +| Model-visible retention | The credentialed probe pulls a bounded content section and rejects transport truncation markers. | +| Next request | The next model request must reproduce the envelope's `body_sha256` and the probe completion marker. | + +## Negative Matrix + +| Case | Required outcome | +|---|---| +| Direct MCP or forged nested fields | Envelope/pull | +| Explicit high without protected selected-limit evidence | Envelope/pull | +| Omitted, lower, malformed, or over-ceiling maximum | Envelope/pull | +| Writable rollout or unsigned trace | Envelope/pull | +| Replayed call, repeated recipe, or changed recipe | Envelope/pull | +| `recipe://{name}` resource | Envelope/pull; negotiation ineligible | + +## Live Probe + +**Envelope/pull oracle:** PASS (2026-07-22) + +The dedicated smoke test initializes an isolated Codex home against the worktree's MCP +server, opens `remediation` ingredients-only, loads it without a forged delivery request, +pulls the immutable `content` generation, and requires the following model request to echo +the retained body digest. With the isolated MCP process explicitly identified as Codex, the +probe retained the bounded `recipe_pull`, pulled content with the same `body_sha256`, found +no transport-truncation marker, and echoed the digest in the terminal next-request canary. +`task test-smoke-codex` passed the dedicated oracle on `gpt-5.6-sol`. + +This pass proves the fail-closed envelope/pull recovery path. It does not prove an +authoritative selected outer limit, raw pre-truncation bytes, or a protected pre-call event, +so it cannot enable `ATTESTED_INLINE`. + +Until the protected-host prerequisite exists and this report contains a passing +authoritative-high/next-request oracle, no supported evidence identity may be added. diff --git a/src/autoskillit/AGENTS.md b/src/autoskillit/AGENTS.md index e886396afc..016fa867a3 100644 --- a/src/autoskillit/AGENTS.md +++ b/src/autoskillit/AGENTS.md @@ -9,6 +9,7 @@ Package root — entry points, hook registry, and cross-cutting utilities. | `__init__.py` | Package exports | | `__main__.py` | `python -m autoskillit` entry point | | `_llm_triage.py` | Contract staleness triage (Haiku subprocess) | +| `_recipe_delivery_framing.py` | Stdlib-only attested recipe wire markers and recognition predicate shared with formatter hooks | | `smoke_utils/` | Callables for smoke-test pipeline `run_python` steps (package) | | `hook_registry.py` | `HookDef`, `HOOK_REGISTRY`, `generate_hooks_json` | | `_probe_canary.py` | Canary state machine and GitHub issue updater for live probes (IL-1) | diff --git a/src/autoskillit/_recipe_delivery_framing.py b/src/autoskillit/_recipe_delivery_framing.py new file mode 100644 index 0000000000..14f3a5841d --- /dev/null +++ b/src/autoskillit/_recipe_delivery_framing.py @@ -0,0 +1,17 @@ +"""Stdlib-only recipe-delivery wire framing shared with hook subprocesses.""" + +from __future__ import annotations + +RECIPE_BODY_START = "--- AUTOSKILLIT RECIPE BODY START ---" +RECIPE_BODY_END = "--- AUTOSKILLIT RECIPE BODY END ---" +RECIPE_COMPLETION_SENTINEL = "AUTOSKILLIT_RECIPE_DELIVERY_COMPLETE" + + +def is_attested_recipe_delivery(text: str) -> bool: + """Return whether text contains the complete attested recipe wire frame.""" + return ( + text.startswith('{"recipe_delivery":') + and RECIPE_BODY_START in text + and RECIPE_BODY_END in text + and RECIPE_COMPLETION_SENTINEL in text + ) diff --git a/src/autoskillit/cli/_prompts_kitchen.py b/src/autoskillit/cli/_prompts_kitchen.py index 8e535cae57..6512a48c8c 100755 --- a/src/autoskillit/cli/_prompts_kitchen.py +++ b/src/autoskillit/cli/_prompts_kitchen.py @@ -10,6 +10,7 @@ _read_full_sous_chef, ) from autoskillit.core import PIPELINE_FORBIDDEN_TOOLS, ROUTING_AUTHORITY_CLAUSE +from autoskillit.execution import codex_recipe_delivery_calling_contract __all__ = [ "_build_open_kitchen_prompt", @@ -67,7 +68,13 @@ def _build_open_kitchen_prompt( f"call {mcp_prefix}get_recipe_section(section='', " "recipe_name=recipe_pull.recipe_name, " "producer_tool=recipe_pull.producer_tool, " - "artifact_sha256=recipe_pull.sha256). " + "descriptor_version=recipe_pull.descriptor_version, " + "schema_version=recipe_pull.schema_version, " + "payload_sha256=recipe_pull.payload_sha256, " + "artifact_blob_sha256=recipe_pull.artifact_blob_sha256, " + "artifact_blob_size_bytes=recipe_pull.artifact_blob_size_bytes, " + "body_sha256=recipe_pull.body_sha256, " + "body_size_bytes=recipe_pull.body_size_bytes). " "Do not read recipe YAML files directly.\n\n" "OPTIONAL STEP SEMANTICS:\n" "- optional: true means the step is SKIPPED when its skip_when_false ingredient\n" @@ -95,7 +102,12 @@ def _build_open_kitchen_prompt( "to migrate automatically, or ask me to do it for you." ) - return text + _backend_supplement(has_unguarded_filesystem_access) + return ( + text + + _backend_supplement(has_unguarded_filesystem_access) + + "\n\n" + + codex_recipe_delivery_calling_contract(mcp_prefix=mcp_prefix) + ) def _build_fleet_dispatch_prompt( diff --git a/src/autoskillit/cli/_prompts_orchestrator.py b/src/autoskillit/cli/_prompts_orchestrator.py index 2586eac266..5cd3116d76 100644 --- a/src/autoskillit/cli/_prompts_orchestrator.py +++ b/src/autoskillit/cli/_prompts_orchestrator.py @@ -17,6 +17,7 @@ ROUTING_AUTHORITY_CLAUSE, get_logger, ) +from autoskillit.execution import codex_recipe_delivery_calling_contract logger = get_logger(__name__) @@ -106,7 +107,8 @@ def _build_orchestrator_prompt( "(between --- INGREDIENTS TABLE --- and --- END TABLE --- markers)" ) - return f"""\ + return ( + f"""\ You are a pipeline orchestrator. Execute the recipe '{recipe_name}' step-by-step. {_ing_section}FIRST ACTION — before prompting for any inputs: @@ -375,4 +377,8 @@ def _build_orchestrator_prompt( - The string "None" is NOT the same as null. If the captured value is the Python None object, do not pass the literal string "None". {sous_chef_content} -""" + _backend_supplement(has_unguarded_filesystem_access) +""" + + _backend_supplement(has_unguarded_filesystem_access) + + "\n\n" + + codex_recipe_delivery_calling_contract(mcp_prefix=mcp_prefix) + ) diff --git a/src/autoskillit/cli/doctor/_doctor_runtime.py b/src/autoskillit/cli/doctor/_doctor_runtime.py index ab2ef73aa3..43ba6931e0 100644 --- a/src/autoskillit/cli/doctor/_doctor_runtime.py +++ b/src/autoskillit/cli/doctor/_doctor_runtime.py @@ -105,8 +105,10 @@ def _check_codex_limits_verified(*, backend: CodingAgentBackend | None = None) - Severity.WARNING, check_name, f"Codex CLI {cur_str} is newer than verified pin {pin_str}; " - f"re-verify CODEX_TOOL_OUTPUT_TOKEN_LIMIT and CODEX_AUTO_COMPACT_LIMIT " - f"against upstream registry, then bump CODEX_LIMITS_LAST_VERIFIED_VERSION", + "re-verify the recipe outer-result parser/protected evidence contract, " + "CODEX_HISTORY_RETENTION_TOKEN_LIMIT (later history only), and " + "CODEX_AUTO_COMPACT_LIMIT against upstream behavior, then bump " + "CODEX_LIMITS_LAST_VERIFIED_VERSION", ) cur_str = ".".join(str(v) for v in ver.parsed) return DoctorResult(Severity.OK, check_name, f"Codex CLI {cur_str} at or below verified pin") diff --git a/src/autoskillit/core/AGENTS.md b/src/autoskillit/core/AGENTS.md index 8d02c3426c..8769110473 100644 --- a/src/autoskillit/core/AGENTS.md +++ b/src/autoskillit/core/AGENTS.md @@ -14,7 +14,7 @@ Sub-packages: types/ (see types/AGENTS.md) and runtime/ (see runtime/AGENTS.md). | `logging.py` | Logging configuration | | `paths.py` | `pkg_root()`, `is_git_worktree()`, `is_git_main_checkout()`, `is_in_git_repo()` | | `_claude_env.py` | IDE-scrubbing canonical env builder for agent subprocesses | -| `_delivery_bounds.py` | `resolve_effective_delivery_bound()` — canonical accessor for per-backend effective delivery token limits | +| `_delivery_bounds.py` | General-output limit accessor plus typed recipe-delivery decision resolver | | `_terminal_table.py` | IL-0 color-agnostic terminal table primitive | | `_version_snapshot.py` | Process-scoped version snapshot for session telemetry (`lru_cache`'d) | | `branch_guard.py` | Branch protection helpers | diff --git a/src/autoskillit/core/__init__.pyi b/src/autoskillit/core/__init__.pyi index 2e7ae85c37..f10c7002f3 100644 --- a/src/autoskillit/core/__init__.pyi +++ b/src/autoskillit/core/__init__.pyi @@ -4,7 +4,11 @@ from ._cmd_runner import CmdRunner as CmdRunner from ._cmd_runner import default_cmd_runner as default_cmd_runner from ._cmd_runner import run_gh as run_gh from ._cmd_runner import run_git as run_git -from ._delivery_bounds import resolve_effective_delivery_bound as resolve_effective_delivery_bound +from ._delivery_bounds import recipe_delivery_request_digest as recipe_delivery_request_digest +from ._delivery_bounds import ( + resolve_general_output_token_limit as resolve_general_output_token_limit, +) +from ._delivery_bounds import resolve_recipe_delivery_decision as resolve_recipe_delivery_decision from ._execution_marker import execution_marker as execution_marker from ._install_detect import DirectUrlInfo as DirectUrlInfo from ._install_detect import _is_release_tag as _is_release_tag @@ -209,6 +213,11 @@ from .types import QUOTA_GUARD_DENY_TRIGGER as QUOTA_GUARD_DENY_TRIGGER from .types import QUOTA_POST_BUDGET_EXCEEDED_TRIGGER as QUOTA_POST_BUDGET_EXCEEDED_TRIGGER from .types import QUOTA_POST_WARNING_TRIGGER as QUOTA_POST_WARNING_TRIGGER from .types import READING_TOKEN_PATTERN as READING_TOKEN_PATTERN +from .types import RECIPE_DELIVERY_ATTESTATION_AUDIENCE as RECIPE_DELIVERY_ATTESTATION_AUDIENCE +from .types import RECIPE_DELIVERY_SURFACE_REGISTRY as RECIPE_DELIVERY_SURFACE_REGISTRY +from .types import ( + RECIPE_DELIVERY_SURFACE_REGISTRY_DIGEST as RECIPE_DELIVERY_SURFACE_REGISTRY_DIGEST, +) from .types import RECIPE_PACK_REGISTRY as RECIPE_PACK_REGISTRY from .types import RECIPE_PACK_TAGS as RECIPE_PACK_TAGS from .types import REQUIRED_CONSUMER_FIELDS as REQUIRED_CONSUMER_FIELDS @@ -226,7 +235,6 @@ from .types import REVIEW_APPROACH_MARKER as REVIEW_APPROACH_MARKER from .types import ROUTING_AUTHORITY_CLAUSE as ROUTING_AUTHORITY_CLAUSE from .types import RUN_PYTHON_SENTINEL_KEYS as RUN_PYTHON_SENTINEL_KEYS from .types import SCOPE_DIRECTION_SOURCE_TYPES as SCOPE_DIRECTION_SOURCE_TYPES -from .types import SERVE_SURFACES as SERVE_SURFACES from .types import SESSION_TYPE_ENV_VAR as SESSION_TYPE_ENV_VAR from .types import SESSION_TYPE_FLEET as SESSION_TYPE_FLEET from .types import SESSION_TYPE_ORCHESTRATOR as SESSION_TYPE_ORCHESTRATOR @@ -347,6 +355,13 @@ from .types import QuotaPolicy as QuotaPolicy from .types import QuotaRefreshTask as QuotaRefreshTask from .types import ReadingToken as ReadingToken from .types import ReadOnlyResolver as ReadOnlyResolver +from .types import RecipeDeliveryAttestation as RecipeDeliveryAttestation +from .types import RecipeDeliveryBudgetDef as RecipeDeliveryBudgetDef +from .types import RecipeDeliveryDecision as RecipeDeliveryDecision +from .types import RecipeDeliveryEvidenceDef as RecipeDeliveryEvidenceDef +from .types import RecipeDeliveryMode as RecipeDeliveryMode +from .types import RecipeDeliveryRequest as RecipeDeliveryRequest +from .types import RecipeDeliverySurfaceDef as RecipeDeliverySurfaceDef from .types import RecipeIdentity as RecipeIdentity from .types import RecipeLoadError as RecipeLoadError from .types import RecipeNotFoundError as RecipeNotFoundError diff --git a/src/autoskillit/core/_delivery_bounds.py b/src/autoskillit/core/_delivery_bounds.py index ca7eb9c7c1..51eeff90f7 100644 --- a/src/autoskillit/core/_delivery_bounds.py +++ b/src/autoskillit/core/_delivery_bounds.py @@ -1,19 +1,200 @@ -"""Per-backend effective delivery-bound resolution. - -The configured ``tool_output_token_limit`` written to Codex ``config.toml`` -is a config-file ceiling. Code-mode models may bypass that ceiling when -they emit ``max_output_tokens`` without an upper bound, so the operative -bound for those paths is the harness default (~10K tokens). The -``BackendCapabilities.effective_delivery_token_limit`` field encodes this -worst-case operative bound; ``resolve_effective_delivery_bound`` is the -canonical accessor. -""" +"""Static general-output and recipe-delivery decision resolution.""" from __future__ import annotations +import hashlib +import json +import time + from .types._type_backend import BackendCapabilities +from .types._type_recipe_delivery import ( + RECIPE_DELIVERY_ATTESTATION_AUDIENCE, + RecipeDeliveryAttestation, + RecipeDeliveryBudgetDef, + RecipeDeliveryDecision, + RecipeDeliveryEvidenceDef, + RecipeDeliveryMode, + RecipeDeliveryRequest, +) + + +def _is_sha256_identity(value: str) -> bool: + prefix = "sha256:" + if not value.startswith(prefix) or len(value) != len(prefix) + 64: + return False + return all(character in "0123456789abcdef" for character in value[len(prefix) :]) + + +def recipe_delivery_request_digest(request: RecipeDeliveryRequest) -> str: + """Return the domain-labelled digest of the exact nested delivery request.""" + canonical = json.dumps( + { + "audience": request.audience, + "caller_requested_outer_tokens": request.caller_requested_outer_tokens, + "code_digest": request.code_digest, + "contract_digest": request.contract_digest, + "contract_version": request.contract_version, + "delivery_call_id": request.delivery_call_id, + }, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + return f"sha256:{hashlib.sha256(canonical).hexdigest()}" + + +def resolve_general_output_token_limit(caps: BackendCapabilities) -> int: + """Return the backend's conservative limit for an unnegotiated tool result.""" + return caps.unnegotiated_tool_result_token_limit + + +def resolve_recipe_delivery_decision( + *, + capabilities: BackendCapabilities, + required_serialized_tokens: int, + budget: RecipeDeliveryBudgetDef | None, + producer: str, + payload_sha256: str, + request: RecipeDeliveryRequest | None = None, + attestation: RecipeDeliveryAttestation | None = None, + supported_evidence: RecipeDeliveryEvidenceDef | None = None, + now_unix: int | None = None, +) -> RecipeDeliveryDecision: + """Resolve one recipe response without treating caller claims as authority.""" + ordinary_limit = resolve_general_output_token_limit(capabilities) + requested = request.caller_requested_outer_tokens if request is not None else None + observed = ( + attestation.host_observed_requested_outer_tokens if attestation is not None else None + ) + + def _decision( + mode: RecipeDeliveryMode, + *, + selected_limit: int, + reason: str, + receipt_status: str, + ) -> RecipeDeliveryDecision: + return RecipeDeliveryDecision( + mode=mode, + caller_requested_outer_tokens=requested, + host_observed_requested_outer_tokens=observed, + required_outer_tokens=required_serialized_tokens, + unnegotiated_tool_result_token_limit=ordinary_limit, + selected_result_token_limit=selected_limit, + contract_digest=budget.contract_digest if budget is not None else "", + evidence_identity=(attestation.evidence_identity if attestation is not None else None), + reason=reason, + producer=producer, + payload_sha256=payload_sha256, + receipt_status=receipt_status, + ) + + def _envelope(reason: str) -> RecipeDeliveryDecision: + return _decision( + RecipeDeliveryMode.ENVELOPE, + selected_limit=ordinary_limit, + reason=reason, + receipt_status="not_reserved", + ) + if required_serialized_tokens < 0: + return _envelope("invalid_required_token_count") + if not producer or not _is_sha256_identity(payload_sha256): + return _envelope("invalid_payload_identity") + if required_serialized_tokens <= ordinary_limit: + return _decision( + RecipeDeliveryMode.ORDINARY_INLINE, + selected_limit=ordinary_limit, + reason="fits_unnegotiated_result_limit", + receipt_status="not_required", + ) + if not capabilities.protected_recipe_delivery_capable: + return _envelope("protected_host_delivery_unavailable") + if budget is None: + return _envelope("protected_delivery_budget_unavailable") + if not _is_sha256_identity(budget.contract_digest): + return _envelope("invalid_contract_digest") + if required_serialized_tokens > budget.authoritative_attested_recipe_result_token_limit: + return _envelope("authoritative_result_limit_exceeded") + if request is None: + return _envelope("delivery_request_missing") + if attestation is None: + return _envelope("host_attestation_missing") + if supported_evidence is None: + return _envelope("supported_evidence_missing") + if request.contract_version != budget.contract_version: + return _envelope("request_contract_version_mismatch") + if not request.delivery_call_id or not _is_sha256_identity(request.code_digest): + return _envelope("request_identity_invalid") + if request.audience != RECIPE_DELIVERY_ATTESTATION_AUDIENCE: + return _envelope("request_audience_mismatch") + if request.contract_digest != budget.contract_digest: + return _envelope("request_contract_digest_mismatch") + if request.caller_requested_outer_tokens != ( + budget.authoritative_attested_recipe_result_token_limit + ): + return _envelope("requested_result_limit_mismatch") + if attestation.delivery_call_id != request.delivery_call_id: + return _envelope("delivery_call_id_mismatch") + if attestation.audience != request.audience: + return _envelope("attestation_audience_mismatch") + if not all( + ( + attestation.thread_id, + attestation.turn_id, + attestation.outer_call_id, + attestation.code_mode_cell_id, + attestation.delivery_call_id, + attestation.nonce, + ) + ): + return _envelope("attestation_identity_incomplete") + if attestation.contract_version != request.contract_version: + return _envelope("attestation_contract_version_mismatch") + if attestation.contract_digest != request.contract_digest: + return _envelope("attestation_contract_digest_mismatch") + if attestation.host_observed_requested_outer_tokens != (request.caller_requested_outer_tokens): + return _envelope("host_observation_mismatch") + if attestation.selected_result_token_limit != ( + budget.authoritative_attested_recipe_result_token_limit + ): + return _envelope("host_selected_limit_mismatch") + if attestation.code_digest != request.code_digest: + return _envelope("code_digest_mismatch") + if attestation.request_digest != recipe_delivery_request_digest(request): + return _envelope("request_digest_mismatch") + effective_now = int(time.time()) if now_unix is None else now_unix + if attestation.expires_at_unix <= effective_now: + return _envelope("attestation_expired") + if attestation.parser_version != budget.parser_version: + return _envelope("attestation_parser_version_mismatch") + if attestation.evidence_version != budget.evidence_version: + return _envelope("attestation_evidence_version_mismatch") + if attestation.evidence_identity != supported_evidence.identity: + return _envelope("unsupported_evidence_identity") + if not all( + ( + supported_evidence.identity, + supported_evidence.host_channel, + supported_evidence.cli_identity, + supported_evidence.selected_limit_derivation, + ) + ): + return _envelope("supported_evidence_incomplete") + if supported_evidence.contract_digest != budget.contract_digest: + return _envelope("evidence_contract_digest_mismatch") + if supported_evidence.parser_version != budget.parser_version: + return _envelope("evidence_parser_version_mismatch") + if supported_evidence.evidence_schema_version != budget.evidence_version: + return _envelope("evidence_schema_version_mismatch") + if supported_evidence.selected_result_token_limit != ( + budget.authoritative_attested_recipe_result_token_limit + ): + return _envelope("evidence_selected_limit_mismatch") -def resolve_effective_delivery_bound(caps: BackendCapabilities) -> int: - """Return the worst-case operative token bound for ``caps``'s backend transport.""" - return caps.effective_delivery_token_limit + return _decision( + RecipeDeliveryMode.ATTESTED_INLINE, + selected_limit=supported_evidence.selected_result_token_limit, + reason="supported_host_evidence", + receipt_status="reservation_required", + ) diff --git a/src/autoskillit/core/types/AGENTS.md b/src/autoskillit/core/types/AGENTS.md index ebf90f5577..b261fec55c 100644 --- a/src/autoskillit/core/types/AGENTS.md +++ b/src/autoskillit/core/types/AGENTS.md @@ -29,6 +29,7 @@ Type re-export hub and all typed building blocks for the autoskillit package (IL | `_type_protocols_backend.py` | Protocols: `StreamParser`, `ResultParser`, `EnvPolicy`, `SessionLocator`, `CodingAgentBackend` | | `_type_checkpoint.py` | `SessionCheckpoint` frozen dataclass and `compute_remaining()` helper for session resume | | `_type_backend.py` | `BackendCapabilities` frozen dataclass, `CLAUDE_CODE_CAPABILITIES` constant, `CmdSpec`, `SkillSessionConfig`, `ClaudeEventData`, `CodexEventData`, `SessionEvent`, `AgentSessionResult` | +| `_type_recipe_delivery.py` | Typed Codex recipe budgets, protected-host evidence definitions, requests, attestations, and delivery decisions | | `_type_capture.py` | `CaptureEntrySpec` and `CaptureValueTypeError` for typed capture contract enforcement | | `_type_dispatch_identity.py` | `DispatchIdentity` frozen value object, `PromptContractError`, and `assert_prompt_sentinel` for sentinel contract enforcement | | `_type_helpers.py` | Text processing and skill-name extraction utilities | diff --git a/src/autoskillit/core/types/__init__.py b/src/autoskillit/core/types/__init__.py index 6c0410bcc1..0c39ab4bd7 100644 --- a/src/autoskillit/core/types/__init__.py +++ b/src/autoskillit/core/types/__init__.py @@ -54,6 +54,8 @@ from ._type_protocols_recipe import __all__ as _protocols_recipe_all from ._type_protocols_workspace import * # noqa: F401, F403 from ._type_protocols_workspace import __all__ as _protocols_workspace_all +from ._type_recipe_delivery import * # noqa: F401, F403 +from ._type_recipe_delivery import __all__ as _recipe_delivery_all from ._type_results import * # noqa: F401, F403 from ._type_results import __all__ as _results_all from ._type_results_execution import * # noqa: F401, F403 @@ -96,6 +98,7 @@ + _protocols_backend_all + _results_all + _results_execution_all + + _recipe_delivery_all + _resume_all + _session_env_all + _subprocess_all diff --git a/src/autoskillit/core/types/_type_backend.py b/src/autoskillit/core/types/_type_backend.py index 2923c0d159..3e8ce9c8b9 100644 --- a/src/autoskillit/core/types/_type_backend.py +++ b/src/autoskillit/core/types/_type_backend.py @@ -12,6 +12,7 @@ from ._type_checkpoint import SessionCheckpoint from ._type_enums import BackendEventKind, OutputFormat from ._type_plugin_source import PluginSource +from ._type_recipe_delivery import RecipeDeliveryBudgetDef from ._type_results import ValidatedAddDir __all__ = [ @@ -162,11 +163,10 @@ class BackendCapabilities: # directory rather than written with gating frontmatter that the backend # would ignore. supports_model_invocation_gating: bool = True - # Worst-case delivery bound in tokens: the lowest operative bound that - # could apply to this backend's tool-output transport. Distinct from any - # config-file ceiling (e.g. `tool_output_token_limit`), which code-mode - # models may bypass. Used by `enforce_response_budget` to spill payloads - # the downstream transport cannot deliver at full size. + # Unnegotiated tool-result bound in tokens: the lowest operative bound + # when a caller has not supplied protected host evidence for a larger + # result. Distinct from history-retention configuration and negotiated + # recipe delivery decisions. # # Default of 10,000 matches the smallest registered backend bound # (Codex code-mode). Any new backend that omits this field inherits @@ -174,7 +174,15 @@ class BackendCapabilities: # bounding" behavior — preventing a silent opt-out where a future # backend without an explicit capability setting would be delivered # without any delivery-bound enforcement. - effective_delivery_token_limit: int = 10_000 + unnegotiated_tool_result_token_limit: int = 10_000 + # True only when a protected host channel can attest the selected outer + # result limit before nested MCP execution. Current backends remain False + # until a version-pinned conformance report enables an evidence identity. + protected_recipe_delivery_capable: bool = False + # Backend-owned authority for ordinary and protected recipe delivery. + # None means the backend has no version-pinned recipe-delivery contract; + # protected delivery must then fail closed even if capability data drifts. + recipe_delivery_budget: RecipeDeliveryBudgetDef | None = None ALL_PROJECT_LOCAL_SKILL_SEARCH_DIRS: tuple[str, ...] = ( @@ -304,7 +312,9 @@ def model_class(model: str) -> str: skill_sigil="/", session_dir_persistent=False, supports_model_invocation_gating=True, - effective_delivery_token_limit=46_500, + unnegotiated_tool_result_token_limit=46_500, + protected_recipe_delivery_capable=False, + recipe_delivery_budget=None, ) diff --git a/src/autoskillit/core/types/_type_constants_registries.py b/src/autoskillit/core/types/_type_constants_registries.py index 3131887d7e..ca76b35485 100644 --- a/src/autoskillit/core/types/_type_constants_registries.py +++ b/src/autoskillit/core/types/_type_constants_registries.py @@ -9,7 +9,9 @@ import hashlib import json +from collections.abc import Mapping from dataclasses import dataclass, fields +from types import MappingProxyType from typing import Literal, NamedTuple # _type_backend.py itself imports several other core/types siblings, but all @@ -44,7 +46,9 @@ "CORE_PACKS", "TOOL_SUBSET_TAGS", "ALL_VISIBILITY_TAGS", - "SERVE_SURFACES", + "RecipeDeliverySurfaceDef", + "RECIPE_DELIVERY_SURFACE_REGISTRY", + "RECIPE_DELIVERY_SURFACE_REGISTRY_DIGEST", "SkillCapabilityDef", "HardCapabilityMismatch", "SKILL_CAPABILITY_REGISTRY", @@ -171,15 +175,6 @@ UNGATED_TOOLS: frozenset[str] = FREE_RANGE_TOOLS -SERVE_SURFACES: frozenset[str] = frozenset( - { - "open_kitchen", # S1 — initial serve, sets session snapshot - "open_kitchen_deferred_recall", # S2 — deferred-recall re-serve - "load_recipe", # S3 — re-serve tool - "get_recipe", # S4 — MCP resource handler - } -) - class ResponseBackstopExemptionDef(NamedTuple): """Measured ceiling for a tool that bypasses universal response shaping.""" @@ -189,18 +184,98 @@ class ResponseBackstopExemptionDef(NamedTuple): measurement_id: str -RESPONSE_BACKSTOP_EXEMPTION_REGISTRY: dict[str, ResponseBackstopExemptionDef] = { - "load_recipe": ResponseBackstopExemptionDef( - max_chars=195_000, - max_utf8_bytes=195_000, - measurement_id="bundled-recipes-all-modes-2026-07-22/load-recipe", - ), - "open_kitchen": ResponseBackstopExemptionDef( - max_chars=195_000, - max_utf8_bytes=195_000, - measurement_id="bundled-recipes-all-modes-2026-07-22/open-kitchen", - ), -} +class RecipeDeliverySurfaceDef(NamedTuple): + """Static definition of one recipe-bearing public delivery route.""" + + producer_tool: str + route_kind: Literal["tool", "deferred_tool", "resource"] + negotiation_eligible: bool + pull_eligible: bool + recreation_eligible: bool + response_exemption_tool: str | None + response_exemption: ResponseBackstopExemptionDef | None + + +_RECIPE_RESPONSE_MAX_UTF8_BYTES = 195_000 + +RECIPE_DELIVERY_SURFACE_REGISTRY: Mapping[str, RecipeDeliverySurfaceDef] = MappingProxyType( + { + "open_kitchen": RecipeDeliverySurfaceDef( + producer_tool="open_kitchen", + route_kind="tool", + negotiation_eligible=True, + pull_eligible=True, + recreation_eligible=True, + response_exemption_tool="open_kitchen", + response_exemption=ResponseBackstopExemptionDef( + max_chars=_RECIPE_RESPONSE_MAX_UTF8_BYTES, + max_utf8_bytes=_RECIPE_RESPONSE_MAX_UTF8_BYTES, + measurement_id="bundled-recipes-all-modes-2026-07-22/open-kitchen", + ), + ), + "open_kitchen_deferred_recall": RecipeDeliverySurfaceDef( + producer_tool="open_kitchen", + route_kind="deferred_tool", + negotiation_eligible=True, + pull_eligible=True, + recreation_eligible=True, + response_exemption_tool="open_kitchen", + response_exemption=None, + ), + "load_recipe": RecipeDeliverySurfaceDef( + producer_tool="load_recipe", + route_kind="tool", + negotiation_eligible=True, + pull_eligible=True, + recreation_eligible=False, + response_exemption_tool="load_recipe", + response_exemption=ResponseBackstopExemptionDef( + max_chars=_RECIPE_RESPONSE_MAX_UTF8_BYTES, + max_utf8_bytes=_RECIPE_RESPONSE_MAX_UTF8_BYTES, + measurement_id="bundled-recipes-all-modes-2026-07-22/load-recipe", + ), + ), + "get_recipe": RecipeDeliverySurfaceDef( + producer_tool="get_recipe", + route_kind="resource", + negotiation_eligible=False, + pull_eligible=True, + recreation_eligible=True, + response_exemption_tool=None, + response_exemption=None, + ), + } +) + + +def _recipe_delivery_surface_registry_digest() -> str: + canonical = { + surface: { + **definition._asdict(), + "response_exemption": ( + definition.response_exemption._asdict() + if definition.response_exemption is not None + else None + ), + } + for surface, definition in sorted(RECIPE_DELIVERY_SURFACE_REGISTRY.items()) + } + payload = json.dumps(canonical, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + return hashlib.sha256(payload.encode("ascii")).hexdigest() + + +RECIPE_DELIVERY_SURFACE_REGISTRY_DIGEST: str = _recipe_delivery_surface_registry_digest() + +RESPONSE_BACKSTOP_EXEMPTION_REGISTRY: Mapping[str, ResponseBackstopExemptionDef] = ( + MappingProxyType( + { + definition.response_exemption_tool: definition.response_exemption + for definition in RECIPE_DELIVERY_SURFACE_REGISTRY.values() + if definition.response_exemption_tool is not None + and definition.response_exemption is not None + } + ) +) def _response_backstop_exemption_registry_digest() -> str: diff --git a/src/autoskillit/core/types/_type_recipe_delivery.py b/src/autoskillit/core/types/_type_recipe_delivery.py new file mode 100644 index 0000000000..844aeef885 --- /dev/null +++ b/src/autoskillit/core/types/_type_recipe_delivery.py @@ -0,0 +1,107 @@ +"""Typed recipe-delivery budget, provenance, and decision contracts.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum +from typing import NamedTuple + +__all__ = [ + "RECIPE_DELIVERY_ATTESTATION_AUDIENCE", + "RecipeDeliveryAttestation", + "RecipeDeliveryBudgetDef", + "RecipeDeliveryDecision", + "RecipeDeliveryEvidenceDef", + "RecipeDeliveryMode", + "RecipeDeliveryRequest", +] + +RECIPE_DELIVERY_ATTESTATION_AUDIENCE = "autoskillit.recipe-delivery" + + +class RecipeDeliveryMode(StrEnum): + """Wire-shaping decision for one finalized recipe response.""" + + ORDINARY_INLINE = "ordinary_inline" + ATTESTED_INLINE = "attested_inline" + ENVELOPE = "envelope" + + +class RecipeDeliveryBudgetDef(NamedTuple): + """Backend-selected limits whose byte and token domains remain distinct.""" + + ordinary_omitted_result_token_limit: int + authoritative_attested_recipe_result_token_limit: int + history_retention_token_limit: int + measured_recipe_exemption_max_utf8_bytes: int + headroom_tokens: int + contract_version: int + parser_version: int + evidence_version: int + contract_digest: str + + +class RecipeDeliveryEvidenceDef(NamedTuple): + """Version-pinned identity of a protected host evidence channel.""" + + identity: str + host_channel: str + evidence_schema_version: int + parser_version: int + cli_identity: str + selected_limit_derivation: str + selected_result_token_limit: int + contract_digest: str + + +@dataclass(frozen=True, slots=True) +class RecipeDeliveryRequest: + """Immutable nested request; caller fields are claims, not attestation.""" + + audience: str + delivery_call_id: str + contract_version: int + contract_digest: str + caller_requested_outer_tokens: int + code_digest: str + + +@dataclass(frozen=True, slots=True) +class RecipeDeliveryAttestation: + """Host observation bound to one protected outer and nested call.""" + + audience: str + thread_id: str + turn_id: str + outer_call_id: str + code_mode_cell_id: str + delivery_call_id: str + host_observed_requested_outer_tokens: int + selected_result_token_limit: int + code_digest: str + request_digest: str + nonce: str + expires_at_unix: int + contract_version: int + contract_digest: str + parser_version: int + evidence_version: int + evidence_identity: str + + +@dataclass(frozen=True, slots=True) +class RecipeDeliveryDecision: + """Resolved recipe delivery outcome with explicit provenance domains.""" + + mode: RecipeDeliveryMode + caller_requested_outer_tokens: int | None + host_observed_requested_outer_tokens: int | None + required_outer_tokens: int + unnegotiated_tool_result_token_limit: int + selected_result_token_limit: int + contract_digest: str + evidence_identity: str | None + reason: str + producer: str + payload_sha256: str + receipt_status: str diff --git a/src/autoskillit/execution/__init__.py b/src/autoskillit/execution/__init__.py index 34ce6816e3..a7d3f585c7 100644 --- a/src/autoskillit/execution/__init__.py +++ b/src/autoskillit/execution/__init__.py @@ -20,21 +20,38 @@ from autoskillit.execution.backends import ( BACKEND_REGISTRY, CODEX_AUTO_COMPACT_LIMIT, + CODEX_HISTORY_RETENTION_TOKEN_LIMIT, CODEX_LIMITS_LAST_VERIFIED_VERSION, CODEX_MCP_REQUIRED_KEYS, CODEX_MCP_STARTUP_TIMEOUT_SEC, CODEX_MCP_TOOL_TIMEOUT_FLOOR, - CODEX_TOOL_OUTPUT_TOKEN_LIMIT, + CODEX_RECIPE_DELIVERY_BUDGET, + CODEX_RECIPE_DELIVERY_CALLING_CONTRACT, + CODEX_RECIPE_DELIVERY_CALLING_CONTRACT_DIGEST, + SUPPORTED_CODEX_RECIPE_EVIDENCE_REGISTRY, ClaudeCodeBackend, + CodexAttestationResult, CodexBackend, + CodexHostCorrelation, + CodexOuterBudgetAttestor, + NullProtectedHostAttestationProvider, + ProtectedHostAttestationProvider, + ProtectedStoreAuthority, + RecipeDeliveryReceiptLedger, + RecipeReceiptHandle, + RecipeReservationResult, _is_autoskillit_hook_entry, _is_autoskillit_registered, _read_codex_config, _serialize_toml, _write_codex_config, + codex_recipe_delivery_calling_contract, ensure_codex_mcp_registered, + enumerate_fresh_codex_marker_ids, generate_codex_hooks_config, get_backend, + read_rollout_thread_id, + resolve_unique_codex_host_correlation, sync_hooks_to_codex_config, ) from autoskillit.execution.ci import DefaultCIWatcher @@ -232,19 +249,36 @@ "CODEX_MCP_REQUIRED_KEYS", "CODEX_MCP_STARTUP_TIMEOUT_SEC", "CODEX_MCP_TOOL_TIMEOUT_FLOOR", - "CODEX_TOOL_OUTPUT_TOKEN_LIMIT", + "CODEX_HISTORY_RETENTION_TOKEN_LIMIT", + "CODEX_RECIPE_DELIVERY_BUDGET", + "CODEX_RECIPE_DELIVERY_CALLING_CONTRACT", + "CODEX_RECIPE_DELIVERY_CALLING_CONTRACT_DIGEST", + "SUPPORTED_CODEX_RECIPE_EVIDENCE_REGISTRY", "CODEX_LIMITS_LAST_VERIFIED_VERSION", "CODEX_AUTO_COMPACT_LIMIT", "ClaudeCodeBackend", + "CodexAttestationResult", "CodexBackend", + "CodexHostCorrelation", + "CodexOuterBudgetAttestor", + "NullProtectedHostAttestationProvider", + "ProtectedHostAttestationProvider", + "ProtectedStoreAuthority", + "RecipeDeliveryReceiptLedger", + "RecipeReceiptHandle", + "RecipeReservationResult", "_is_autoskillit_hook_entry", "_is_autoskillit_registered", "_read_codex_config", "_serialize_toml", "_write_codex_config", + "codex_recipe_delivery_calling_contract", "ensure_codex_mcp_registered", + "enumerate_fresh_codex_marker_ids", "generate_codex_hooks_config", "sync_hooks_to_codex_config", + "read_rollout_thread_id", + "resolve_unique_codex_host_correlation", # anomaly_detection "detect_anomalies", "AnomalyKind", diff --git a/src/autoskillit/execution/backends/AGENTS.md b/src/autoskillit/execution/backends/AGENTS.md index ba41e09b78..8ebb827293 100644 --- a/src/autoskillit/execution/backends/AGENTS.md +++ b/src/autoskillit/execution/backends/AGENTS.md @@ -16,6 +16,7 @@ IL-1 backend abstraction layer — concrete `CodingAgentBackend` implementations | `_codex_config.py` | TOML serialization with `[[key]]` array-of-tables support, MCP registration (`ensure_codex_mcp_registered`, `_serialize_toml`) | | `_codex_hooks.py` | Codex config.toml hook generation, sync, and upsert (`generate_codex_hooks_config`, `sync_hooks_to_codex_config`) | | `_codex_parse.py` | `CodexStreamParser`, `CodexResultParser`, `_scan_codex_ndjson`, `_CodexParseAccumulator` | +| `_codex_recipe_delivery.py` | Protected outer-budget provider seam, bounded diagnostic correlation, and durable consumed-call/receipt ledger | | `codex_scenario_player.py` | `CodexScenarioPlayer`, `make_codex_scenario_player`, `CodexStepRecord`, `CodexScenario`, `_load_manifest`, `_FakeCodexCLI`, `_write_shim_script` — scenario replay data layer for Codex backend | | `_cmd_builder.py` | `CmdBuilder`, `CmdOrderingError` — typed builder that enforces positional-before-variadic ordering by construction | diff --git a/src/autoskillit/execution/backends/__init__.py b/src/autoskillit/execution/backends/__init__.py index d7ba36891a..b7e402b80a 100644 --- a/src/autoskillit/execution/backends/__init__.py +++ b/src/autoskillit/execution/backends/__init__.py @@ -4,15 +4,20 @@ from ._codex_config import ( CODEX_AUTO_COMPACT_LIMIT, + CODEX_HISTORY_RETENTION_TOKEN_LIMIT, CODEX_LIMITS_LAST_VERIFIED_VERSION, CODEX_MCP_REQUIRED_KEYS, CODEX_MCP_STARTUP_TIMEOUT_SEC, CODEX_MCP_TOOL_TIMEOUT_FLOOR, - CODEX_TOOL_OUTPUT_TOKEN_LIMIT, + CODEX_RECIPE_DELIVERY_BUDGET, + CODEX_RECIPE_DELIVERY_CALLING_CONTRACT, + CODEX_RECIPE_DELIVERY_CALLING_CONTRACT_DIGEST, + SUPPORTED_CODEX_RECIPE_EVIDENCE_REGISTRY, _is_autoskillit_registered, _read_codex_config, _serialize_toml, _write_codex_config, + codex_recipe_delivery_calling_contract, ensure_codex_mcp_registered, ) from ._codex_hooks import ( @@ -21,6 +26,20 @@ sync_hooks_to_codex_config, ) from ._codex_parse import CodexResultParser, CodexStreamParser +from ._codex_recipe_delivery import ( + CodexAttestationResult, + CodexHostCorrelation, + CodexOuterBudgetAttestor, + NullProtectedHostAttestationProvider, + ProtectedHostAttestationProvider, + ProtectedStoreAuthority, + RecipeDeliveryReceiptLedger, + RecipeReceiptHandle, + RecipeReservationResult, + enumerate_fresh_codex_marker_ids, + read_rollout_thread_id, + resolve_unique_codex_host_correlation, +) from ._composite_locator import CompositeSessionLocator from .claude import ( ClaudeCodeBackend, @@ -77,22 +96,39 @@ def get_backend(name: str) -> CodingAgentBackend: "CodexEnvPolicy", "CodexFlags", "CodexResultParser", + "CodexAttestationResult", + "CodexHostCorrelation", + "CodexOuterBudgetAttestor", "CodexScenarioPlayer", "CodexSessionLocator", "CodexStreamParser", "CODEX_MCP_STARTUP_TIMEOUT_SEC", "CODEX_MCP_TOOL_TIMEOUT_FLOOR", "CODEX_MCP_REQUIRED_KEYS", - "CODEX_TOOL_OUTPUT_TOKEN_LIMIT", + "CODEX_HISTORY_RETENTION_TOKEN_LIMIT", + "CODEX_RECIPE_DELIVERY_BUDGET", + "CODEX_RECIPE_DELIVERY_CALLING_CONTRACT", + "CODEX_RECIPE_DELIVERY_CALLING_CONTRACT_DIGEST", + "SUPPORTED_CODEX_RECIPE_EVIDENCE_REGISTRY", "CODEX_LIMITS_LAST_VERIFIED_VERSION", "CODEX_AUTO_COMPACT_LIMIT", "NON_VARIADIC_CODEX_FLAGS", + "NullProtectedHostAttestationProvider", + "ProtectedHostAttestationProvider", + "ProtectedStoreAuthority", + "RecipeDeliveryReceiptLedger", + "RecipeReceiptHandle", + "RecipeReservationResult", "VARIADIC_CODEX_FLAGS", "_is_autoskillit_registered", "_read_codex_config", "_serialize_toml", "_write_codex_config", + "codex_recipe_delivery_calling_contract", "ensure_codex_mcp_registered", + "enumerate_fresh_codex_marker_ids", "get_backend", "make_codex_scenario_player", + "read_rollout_thread_id", + "resolve_unique_codex_host_correlation", ] diff --git a/src/autoskillit/execution/backends/_claude_prompt.py b/src/autoskillit/execution/backends/_claude_prompt.py index 8a045bd030..dd2e978b0e 100644 --- a/src/autoskillit/execution/backends/_claude_prompt.py +++ b/src/autoskillit/execution/backends/_claude_prompt.py @@ -16,6 +16,9 @@ extract_skill_name, temp_dir_display_str, ) +from autoskillit.execution.backends._codex_config import ( + CODEX_RECIPE_DELIVERY_CALLING_CONTRACT, +) def _extract_write_artifacts(tool_uses: list[dict[str, Any]]) -> list[str]: @@ -186,7 +189,10 @@ def _inject_narration_suppression(skill_command: str, *, has_skill_prefix: bool def codex_discipline_suffix() -> str: """Canonical combined discipline suffix: output-discipline + intake-discipline.""" - return f"{OUTPUT_DISCIPLINE_DIGEST}\n\n{CODEX_INTAKE_DISCIPLINE_DIGEST}" + return ( + f"{OUTPUT_DISCIPLINE_DIGEST}\n\n{CODEX_INTAKE_DISCIPLINE_DIGEST}\n\n" + f"{CODEX_RECIPE_DELIVERY_CALLING_CONTRACT}" + ) def _inject_output_discipline(prompt: str, *, include: bool = False) -> str: diff --git a/src/autoskillit/execution/backends/_codex_config.py b/src/autoskillit/execution/backends/_codex_config.py index e9eeadc1a2..4371fb0ec2 100644 --- a/src/autoskillit/execution/backends/_codex_config.py +++ b/src/autoskillit/execution/backends/_codex_config.py @@ -2,7 +2,11 @@ from __future__ import annotations +import hashlib +import json +from collections.abc import Mapping from pathlib import Path +from types import MappingProxyType from typing import Any import regex as _re @@ -10,8 +14,14 @@ from autoskillit.core import ( CODEX_MCP_ENV_FORWARD_VARS, HEADLESS_AUTO_GATE_ENV_VAR, + RECIPE_DELIVERY_ATTESTATION_AUDIENCE, + RECIPE_DELIVERY_SURFACE_REGISTRY, + RECIPE_DELIVERY_SURFACE_REGISTRY_DIGEST, RESPONSE_BACKSTOP_EXEMPTION_REGISTRY, + RESPONSE_BACKSTOP_EXEMPTION_REGISTRY_DIGEST, ReadResult, + RecipeDeliveryBudgetDef, + RecipeDeliveryEvidenceDef, atomic_write, get_logger, safe_upsert_section, @@ -24,14 +34,115 @@ CODEX_MCP_STARTUP_TIMEOUT_SEC: float = 30.0 -# Damage bound for all tool/function output stored in Codex context, including -# native shell and MCP output. Codex currently applies a coarse four-UTF-8-byte -# truncation heuristic; the extra 8,000 tokens provide serialized-payload -# headroom. Guard and spill layers remain the load-bearing mechanisms. +# The configured history-retention requirement is derived from the largest +# measured recipe exemption plus explicit serialized-response headroom. _MAX_RESPONSE_BACKSTOP_EXEMPTION_BYTES: int = max( definition.max_utf8_bytes for definition in RESPONSE_BACKSTOP_EXEMPTION_REGISTRY.values() ) -CODEX_TOOL_OUTPUT_TOKEN_LIMIT: int = ((_MAX_RESPONSE_BACKSTOP_EXEMPTION_BYTES + 3) // 4) + 8_000 +_CODEX_RECIPE_DELIVERY_HEADROOM_TOKENS: int = 8_000 +_CODEX_ORDINARY_RESULT_TOKEN_LIMIT: int = 10_000 +_CODEX_ATTESTED_RECIPE_RESULT_TOKEN_LIMIT: int = ( + (_MAX_RESPONSE_BACKSTOP_EXEMPTION_BYTES + 3) // 4 +) + _CODEX_RECIPE_DELIVERY_HEADROOM_TOKENS + +_CODEX_RECIPE_DELIVERY_CONTRACT_DIGEST = ( + "sha256:" + + hashlib.sha256( + json.dumps( + { + "attested_result_tokens": _CODEX_ATTESTED_RECIPE_RESULT_TOKEN_LIMIT, + "contract_version": 1, + "evidence_version": 1, + "headroom_tokens": _CODEX_RECIPE_DELIVERY_HEADROOM_TOKENS, + "history_retention_tokens": _CODEX_ATTESTED_RECIPE_RESULT_TOKEN_LIMIT, + "measured_recipe_bytes": _MAX_RESPONSE_BACKSTOP_EXEMPTION_BYTES, + "ordinary_result_tokens": _CODEX_ORDINARY_RESULT_TOKEN_LIMIT, + "parser_version": 1, + "response_exemption_registry": RESPONSE_BACKSTOP_EXEMPTION_REGISTRY_DIGEST, + "surface_registry": RECIPE_DELIVERY_SURFACE_REGISTRY_DIGEST, + }, + ensure_ascii=True, + separators=(",", ":"), + sort_keys=True, + ).encode("ascii") + ).hexdigest() +) + +CODEX_RECIPE_DELIVERY_BUDGET: RecipeDeliveryBudgetDef = RecipeDeliveryBudgetDef( + ordinary_omitted_result_token_limit=_CODEX_ORDINARY_RESULT_TOKEN_LIMIT, + authoritative_attested_recipe_result_token_limit=(_CODEX_ATTESTED_RECIPE_RESULT_TOKEN_LIMIT), + history_retention_token_limit=_CODEX_ATTESTED_RECIPE_RESULT_TOKEN_LIMIT, + measured_recipe_exemption_max_utf8_bytes=_MAX_RESPONSE_BACKSTOP_EXEMPTION_BYTES, + headroom_tokens=_CODEX_RECIPE_DELIVERY_HEADROOM_TOKENS, + contract_version=1, + parser_version=1, + evidence_version=1, + contract_digest=_CODEX_RECIPE_DELIVERY_CONTRACT_DIGEST, +) +CODEX_HISTORY_RETENTION_TOKEN_LIMIT: int = ( + CODEX_RECIPE_DELIVERY_BUDGET.history_retention_token_limit +) + +# A protected evidence identity is enabled only with its passing conformance +# report. Writable rollout and trace formats are intentionally absent. +SUPPORTED_CODEX_RECIPE_EVIDENCE_REGISTRY: Mapping[str, RecipeDeliveryEvidenceDef] = ( + MappingProxyType({}) +) + + +def codex_recipe_delivery_calling_contract(*, mcp_prefix: str = "") -> str: + """Generate the only permitted Codex high-budget recipe calling contract.""" + budget = CODEX_RECIPE_DELIVERY_BUDGET + full_recipe_tools = ", ".join( + f"{mcp_prefix}{name}" + for name in sorted( + { + definition.producer_tool + for definition in RECIPE_DELIVERY_SURFACE_REGISTRY.values() + if definition.negotiation_eligible + } + ) + ) + return "\n".join( + ( + f"Codex recipe delivery calling contract v{budget.contract_version}:", + ( + "- Ordinary functions.exec results must not request more than " + f"{budget.ordinary_omitted_result_token_limit} tokens." + ), + ( + "- The sole exception is a full recipe call to " + f"{full_recipe_tools} with a protected host-provided delivery request." + ), + ( + "- The functions.exec cell must start with exactly: // @exec: " + f'{{"max_output_tokens": ' + f"{budget.authoritative_attested_recipe_result_token_limit}" + "}" + ), + "- Pass delivery_request unchanged with exactly these fields:", + f" audience={RECIPE_DELIVERY_ATTESTATION_AUDIENCE}", + " delivery_call_id=", + f" contract_version={budget.contract_version}", + f" contract_digest={budget.contract_digest}", + ( + " caller_requested_outer_tokens=" + f"{budget.authoritative_attested_recipe_result_token_limit}" + ), + " code_digest=", + ( + "- Never synthesize, infer, alter, or replay delivery_request fields. " + "If protected host values are unavailable, omit delivery_request and use " + "the bounded recipe_pull path." + ), + ("- ingredients_only calls and recipe resources are not eligible for the exception."), + ) + ) + + +CODEX_RECIPE_DELIVERY_CALLING_CONTRACT: str = codex_recipe_delivery_calling_contract() +CODEX_RECIPE_DELIVERY_CALLING_CONTRACT_DIGEST: str = hashlib.sha256( + CODEX_RECIPE_DELIVERY_CALLING_CONTRACT.encode("utf-8") +).hexdigest() # Disable Codex auto-compaction by setting the limit to an unreachable value. # Auto-compaction at 90% of 258K context window can destroy recipe content @@ -42,7 +153,7 @@ CODEX_AUTO_COMPACT_LIMIT: int = 999_999_999 # Codex CLI version against which the numeric limits above were last verified: -# the 4-bytes-per-token truncation heuristic behind CODEX_TOOL_OUTPUT_TOKEN_LIMIT +# the history-retention requirement behind CODEX_HISTORY_RETENTION_TOKEN_LIMIT # and the context-window assumption behind CODEX_AUTO_COMPACT_LIMIT. The doctor # check `codex_limits_verified` warns when the installed CLI is newer. CODEX_LIMITS_LAST_VERIFIED_VERSION: tuple[int, int, int] = (0, 144, 1) @@ -251,7 +362,7 @@ def _is_autoskillit_registered( return False if entry.get("startup_timeout_sec") != CODEX_MCP_STARTUP_TIMEOUT_SEC: return False - if config.get("tool_output_token_limit") != CODEX_TOOL_OUTPUT_TOKEN_LIMIT: + if config.get("tool_output_token_limit") != CODEX_HISTORY_RETENTION_TOKEN_LIMIT: return False if config.get("model_auto_compact_token_limit", 0) < CODEX_AUTO_COMPACT_LIMIT: return False @@ -343,7 +454,7 @@ def ensure_codex_mcp_registered( _upsert_top_level_key_exact( config_path, key="tool_output_token_limit", - value=CODEX_TOOL_OUTPUT_TOKEN_LIMIT, + value=CODEX_HISTORY_RETENTION_TOKEN_LIMIT, ) _ensure_top_level_key( config_path, @@ -360,7 +471,7 @@ def ensure_codex_mcp_registered( ): return False config.setdefault("mcp_servers", {})["autoskillit"] = entry - config["tool_output_token_limit"] = CODEX_TOOL_OUTPUT_TOKEN_LIMIT + config["tool_output_token_limit"] = CODEX_HISTORY_RETENTION_TOKEN_LIMIT existing_compact_limit = config.get("model_auto_compact_token_limit", 0) if not isinstance(existing_compact_limit, int): existing_compact_limit = 0 diff --git a/src/autoskillit/execution/backends/_codex_recipe_delivery.py b/src/autoskillit/execution/backends/_codex_recipe_delivery.py new file mode 100644 index 0000000000..d0e7938a88 --- /dev/null +++ b/src/autoskillit/execution/backends/_codex_recipe_delivery.py @@ -0,0 +1,691 @@ +"""Protected Codex recipe attestation and durable insertion receipts. + +Rollout files and kitchen markers are correlation inputs only. They are +same-user writable and never authorize an elevated result limit. +""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import stat +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Protocol, runtime_checkable +from uuid import uuid4 + +from autoskillit.core import ( + BackendCapabilities, + RecipeDeliveryAttestation, + RecipeDeliveryBudgetDef, + RecipeDeliveryEvidenceDef, + RecipeDeliveryMode, + RecipeDeliveryRequest, + get_logger, + resolve_recipe_delivery_decision, +) + +_MARKER_MAX_BYTES = 64 * 1024 +_MARKER_MAX_CANDIDATES = 256 +_ROLLOUT_RECORD_MAX_BYTES = 256 * 1024 +_LEDGER_SCHEMA_VERSION = 1 +_DATABASE_NAME = "codex-recipe-delivery.sqlite3" + + +def _is_private_regular_file(path: Path) -> bool: + try: + descriptor = path.lstat() + except OSError: + return False + return stat.S_ISREG(descriptor.st_mode) and stat.S_IMODE(descriptor.st_mode) == 0o600 + + +@dataclass(frozen=True, slots=True) +class CodexHostCorrelation: + """Diagnostic marker-to-rollout correlation for one canonical thread.""" + + thread_id: str + marker_path: Path + rollout_path: Path + + +@dataclass(frozen=True, slots=True) +class ProtectedStoreAuthority: + """Host assertion that a local store is outside model/direct-MCP authority.""" + + root: Path + security_identity: str + local_filesystem: bool + caller_writable: bool + initialized_by_host: bool + + +@dataclass(frozen=True, slots=True) +class CodexAttestationResult: + """Fail-closed protected-host attestation result.""" + + attestation: RecipeDeliveryAttestation | None + evidence: RecipeDeliveryEvidenceDef | None + correlation: CodexHostCorrelation | None + reason: str + + +@dataclass(frozen=True, slots=True) +class RecipeReceiptHandle: + """Opaque owner-bound handle kept internal to finalization/enforcement.""" + + thread_id: str + turn_id: str + outer_call_id: str + code_digest: str + reservation_id: str + owner_token: str + + +@dataclass(frozen=True, slots=True) +class RecipeReservationResult: + """Outcome of atomically consuming evidence and reserving a receipt.""" + + handle: RecipeReceiptHandle | None + reason: str + + +@runtime_checkable +class ProtectedHostAttestationProvider(Protocol): + """Unforgeable pre-call host channel; implementations live outside MCP input.""" + + def attest( + self, + *, + request: RecipeDeliveryRequest, + correlation: CodexHostCorrelation, + now_unix: int, + ) -> RecipeDeliveryAttestation | None: ... + + def store_authority(self, *, thread_id: str) -> ProtectedStoreAuthority | None: ... + + +@dataclass(frozen=True, slots=True) +class NullProtectedHostAttestationProvider: + """Default provider: current Codex exposes no protected selected-limit channel.""" + + def attest( + self, + *, + request: RecipeDeliveryRequest, + correlation: CodexHostCorrelation, + now_unix: int, + ) -> RecipeDeliveryAttestation | None: + del request, correlation, now_unix + return None + + def store_authority(self, *, thread_id: str) -> ProtectedStoreAuthority | None: + del thread_id + return None + + +def _marker_state_dir(project_dir: Path) -> Path: + override = os.environ.get("AUTOSKILLIT_STATE_DIR") + if override: + return Path(override) / "kitchen_state" + base = project_dir / ".autoskillit" / "temp" / "kitchen_state" + campaign_id = os.environ.get("AUTOSKILLIT_CAMPAIGN_ID", "") + if campaign_id: + if campaign_id in {".", ".."} or "/" in campaign_id or os.sep in campaign_id: + return base / "__invalid_campaign__" + return base / campaign_id + return base + + +def _fresh_marker_session_id(path: Path, *, now_unix: int, ttl_seconds: int) -> str | None: + try: + with path.open("rb") as stream: + descriptor = os.fstat(stream.fileno()) + if ( + not stat.S_ISREG(descriptor.st_mode) + or descriptor.st_size <= 0 + or descriptor.st_size > _MARKER_MAX_BYTES + ): + return None + raw = stream.read(_MARKER_MAX_BYTES + 1) + if len(raw) != descriptor.st_size: + return None + record = json.loads(raw.decode("utf-8")) + session_id = record["session_id"] + opened_at = datetime.fromisoformat(record["opened_at"]) + except (OSError, UnicodeDecodeError, json.JSONDecodeError, KeyError, TypeError, ValueError): + return None + if not isinstance(session_id, str) or not session_id or path.stem != session_id: + return None + if opened_at.tzinfo is None: + opened_at = opened_at.replace(tzinfo=UTC) + age = now_unix - int(opened_at.timestamp()) + if age < 0 or age >= ttl_seconds: + return None + return session_id + + +def enumerate_fresh_codex_marker_ids( + project_dir: Path, + *, + now_unix: int, + ttl_seconds: int = 24 * 60 * 60, +) -> tuple[tuple[str, Path], ...]: + """Return fresh markers within a strict diagnostic-work ceiling.""" + state_dir = _marker_state_dir(project_dir) + try: + fresh: list[tuple[str, Path]] = [] + for candidate_count, path in enumerate(state_dir.glob("*.json"), start=1): + if candidate_count > _MARKER_MAX_CANDIDATES: + return () + session_id = _fresh_marker_session_id( + path, + now_unix=now_unix, + ttl_seconds=ttl_seconds, + ) + if session_id is not None: + fresh.append((session_id, path)) + except OSError: + return () + return tuple(sorted(fresh, key=lambda item: (item[0], str(item[1])))) + + +def read_rollout_thread_id( + path: Path, *, max_record_bytes: int = _ROLLOUT_RECORD_MAX_BYTES +) -> str: + """Read only the bounded first rollout record and return its canonical thread ID.""" + try: + with path.open("rb") as stream: + raw = stream.readline(max_record_bytes + 1) + except OSError: + return "" + if not raw or len(raw) > max_record_bytes or not raw.endswith(b"\n"): + return "" + try: + record = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return "" + if not isinstance(record, dict): + return "" + record_type = record.get("type") + if record_type == "thread.started": + thread_id = record.get("thread_id") + elif record_type == "session_meta": + payload = record.get("payload") + thread_id = payload.get("id") if isinstance(payload, dict) else None + else: + return "" + return thread_id if isinstance(thread_id, str) else "" + + +def resolve_unique_codex_host_correlation( + project_dir: Path, + *, + locate_rollout: Callable[[str], Path | None], + now_unix: int, +) -> CodexHostCorrelation | None: + """Require exactly one fresh marker whose rollout first event has the same thread.""" + matches: list[CodexHostCorrelation] = [] + for session_id, marker_path in enumerate_fresh_codex_marker_ids( + project_dir, + now_unix=now_unix, + ): + try: + rollout_path = locate_rollout(session_id) + except (OSError, RuntimeError, ValueError): + continue + if rollout_path is None: + continue + thread_id = read_rollout_thread_id(rollout_path) + if thread_id != session_id: + continue + matches.append( + CodexHostCorrelation( + thread_id=thread_id, + marker_path=marker_path, + rollout_path=rollout_path, + ) + ) + return matches[0] if len(matches) == 1 else None + + +@dataclass(frozen=True, slots=True) +class CodexOuterBudgetAttestor: + """Combine diagnostic correlation with a protected provider, failing closed.""" + + provider: ProtectedHostAttestationProvider + locate_rollout: Callable[[str], Path | None] + supported_evidence: Mapping[str, RecipeDeliveryEvidenceDef] + + def attest( + self, + *, + request: RecipeDeliveryRequest, + project_dir: Path, + now_unix: int, + ) -> CodexAttestationResult: + correlation = resolve_unique_codex_host_correlation( + project_dir, + locate_rollout=self.locate_rollout, + now_unix=now_unix, + ) + if correlation is None: + return CodexAttestationResult(None, None, None, "host_correlation_ambiguous") + try: + attestation = self.provider.attest( + request=request, + correlation=correlation, + now_unix=now_unix, + ) + except (OSError, RuntimeError, ValueError, TypeError) as exc: + get_logger(__name__).warning( + "codex_protected_provider_failed", + exception_type=type(exc).__name__, + exc_info=True, + ) + return CodexAttestationResult(None, None, correlation, "protected_provider_failed") + if attestation is None: + return CodexAttestationResult( + None, None, correlation, "protected_evidence_unavailable" + ) + if attestation.thread_id != correlation.thread_id: + return CodexAttestationResult(None, None, correlation, "protected_thread_mismatch") + evidence = self.supported_evidence.get(attestation.evidence_identity) + if evidence is None: + return CodexAttestationResult(None, None, correlation, "unsupported_evidence_identity") + return CodexAttestationResult(attestation, evidence, correlation, "attested") + + +class RecipeDeliveryReceiptLedger: + """Atomic SQLite consumed-call journal and per-thread insertion receipt.""" + + def __init__(self, authority: ProtectedStoreAuthority) -> None: + if not self._authority_valid(authority): + raise ValueError("protected local store authority required") + self._authority = authority + self._path = authority.root / _DATABASE_NAME + + @staticmethod + def _authority_valid(authority: ProtectedStoreAuthority) -> bool: + return bool( + authority.root.is_absolute() + and authority.security_identity + and authority.local_filesystem + and not authority.caller_writable + and authority.initialized_by_host + ) + + @property + def path(self) -> Path: + return self._path + + @classmethod + def initialize_protected( + cls, authority: ProtectedStoreAuthority + ) -> RecipeDeliveryReceiptLedger: + """Initialize a store only after the protected host attests its boundary.""" + ledger = cls(authority) + authority.root.mkdir(mode=0o700, parents=True, exist_ok=True) + if ledger._path.exists(): + existing = cls.open_existing(authority) + if existing is None: + raise RuntimeError("existing protected receipt store is invalid") + return existing + temporary_path = authority.root / f".{_DATABASE_NAME}.{uuid4().hex}.tmp" + try: + connection = sqlite3.connect(temporary_path, timeout=0, isolation_level=None) + try: + connection.execute("PRAGMA journal_mode=DELETE") + connection.execute("PRAGMA synchronous=FULL") + connection.execute("PRAGMA busy_timeout=0") + connection.executescript( + """ + BEGIN IMMEDIATE; + CREATE TABLE IF NOT EXISTS metadata ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + CREATE TABLE IF NOT EXISTS consumed_calls ( + thread_id TEXT NOT NULL, + turn_id TEXT NOT NULL, + outer_call_id TEXT NOT NULL, + code_digest TEXT NOT NULL, + delivery_call_id TEXT NOT NULL, + consumed_at_unix INTEGER NOT NULL, + PRIMARY KEY (thread_id, turn_id, outer_call_id, code_digest) + ); + CREATE TABLE IF NOT EXISTS receipts ( + thread_id TEXT PRIMARY KEY, + status TEXT NOT NULL CHECK (status IN ('pending', 'committed')), + reservation_id TEXT NOT NULL UNIQUE, + owner_token TEXT NOT NULL, + producer TEXT NOT NULL, + payload_sha256 TEXT NOT NULL, + evidence_identity TEXT NOT NULL, + caller_requested_outer_tokens INTEGER NOT NULL, + host_observed_requested_outer_tokens INTEGER NOT NULL, + selected_result_token_limit INTEGER NOT NULL, + updated_at_unix INTEGER NOT NULL + ); + COMMIT; + """ + ) + connection.execute( + "INSERT OR REPLACE INTO metadata(key, value) VALUES('schema_version', ?)", + (str(_LEDGER_SCHEMA_VERSION),), + ) + connection.execute( + "INSERT OR REPLACE INTO metadata(key, value) VALUES('security_identity', ?)", + (authority.security_identity,), + ) + finally: + connection.close() + try: + os.chmod(temporary_path, 0o600) + except OSError as exc: + raise RuntimeError("protected receipt store permissions unavailable") from exc + if not _is_private_regular_file(temporary_path): + raise RuntimeError("protected receipt store permissions unavailable") + try: + os.link(temporary_path, ledger._path) + except FileExistsError: + existing = cls.open_existing(authority) + if existing is None: + raise RuntimeError("existing protected receipt store is invalid") + return existing + return ledger + finally: + try: + temporary_path.unlink(missing_ok=True) + except OSError: + pass + + @classmethod + def open_existing( + cls, authority: ProtectedStoreAuthority + ) -> RecipeDeliveryReceiptLedger | None: + try: + ledger = cls(authority) + except ValueError: + return None + if not _is_private_regular_file(ledger._path): + return None + connection = ledger._connect() + if connection is None: + return None + try: + rows = dict(connection.execute("SELECT key, value FROM metadata")) + except sqlite3.Error: + return None + finally: + connection.close() + if rows.get("schema_version") != str(_LEDGER_SCHEMA_VERSION): + return None + if rows.get("security_identity") != authority.security_identity: + return None + return ledger + + def _connect(self) -> sqlite3.Connection | None: + try: + connection = sqlite3.connect( + f"file:{self._path}?mode=rw", + uri=True, + timeout=0, + isolation_level=None, + ) + connection.execute("PRAGMA busy_timeout=0") + if connection.execute("PRAGMA journal_mode").fetchone() != ("delete",): + connection.close() + return None + if connection.execute("PRAGMA synchronous").fetchone() != (2,): + connection.close() + return None + return connection + except sqlite3.Error: + return None + + def reserve( + self, + *, + capabilities: BackendCapabilities, + required_serialized_tokens: int, + budget: RecipeDeliveryBudgetDef, + request: RecipeDeliveryRequest, + attestation: RecipeDeliveryAttestation, + supported_evidence: RecipeDeliveryEvidenceDef, + producer: str, + payload_sha256: str, + now_unix: int, + ) -> RecipeReservationResult: + """Consume one host call and create its pending receipt in one transaction.""" + validation = resolve_recipe_delivery_decision( + capabilities=capabilities, + required_serialized_tokens=required_serialized_tokens, + budget=budget, + producer=producer, + payload_sha256=payload_sha256, + request=request, + attestation=attestation, + supported_evidence=supported_evidence, + now_unix=now_unix, + ) + if validation.mode is not RecipeDeliveryMode.ATTESTED_INLINE: + return RecipeReservationResult(None, validation.reason) + connection = self._connect() + if connection is None: + return RecipeReservationResult(None, "store_unavailable") + reservation_id = uuid4().hex + owner_token = uuid4().hex + handle = RecipeReceiptHandle( + thread_id=attestation.thread_id, + turn_id=attestation.turn_id, + outer_call_id=attestation.outer_call_id, + code_digest=attestation.code_digest, + reservation_id=reservation_id, + owner_token=owner_token, + ) + try: + connection.execute("BEGIN IMMEDIATE") + existing = connection.execute( + "SELECT status FROM receipts WHERE thread_id = ?", + (attestation.thread_id,), + ).fetchone() + if existing is not None: + connection.execute("ROLLBACK") + return RecipeReservationResult(None, f"receipt_{existing[0]}") + connection.execute( + """ + INSERT INTO consumed_calls( + thread_id, turn_id, outer_call_id, code_digest, + delivery_call_id, consumed_at_unix + ) VALUES (?, ?, ?, ?, ?, ?) + """, + ( + attestation.thread_id, + attestation.turn_id, + attestation.outer_call_id, + attestation.code_digest, + attestation.delivery_call_id, + now_unix, + ), + ) + connection.execute( + """ + INSERT INTO receipts( + thread_id, status, reservation_id, owner_token, producer, + payload_sha256, evidence_identity, + caller_requested_outer_tokens, + host_observed_requested_outer_tokens, + selected_result_token_limit, updated_at_unix + ) VALUES (?, 'pending', ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + attestation.thread_id, + reservation_id, + owner_token, + producer, + payload_sha256, + attestation.evidence_identity, + request.caller_requested_outer_tokens, + attestation.host_observed_requested_outer_tokens, + attestation.selected_result_token_limit, + now_unix, + ), + ) + connection.execute("COMMIT") + return RecipeReservationResult(handle, "reserved") + except sqlite3.IntegrityError: + self._rollback(connection) + return RecipeReservationResult(None, "host_call_replayed") + except sqlite3.OperationalError as exc: + self._rollback(connection) + reason = "store_busy" if "locked" in str(exc).lower() else "store_operational_error" + return RecipeReservationResult(None, reason) + except sqlite3.Error: + self._rollback(connection) + return RecipeReservationResult(None, "store_error") + finally: + connection.close() + + @staticmethod + def _rollback(connection: sqlite3.Connection) -> None: + try: + connection.execute("ROLLBACK") + except sqlite3.Error: + pass + + def commit(self, handle: RecipeReceiptHandle, *, now_unix: int) -> bool: + connection = self._connect() + if connection is None: + return False + try: + connection.execute("BEGIN IMMEDIATE") + cursor = connection.execute( + """ + UPDATE receipts SET status = 'committed', updated_at_unix = ? + WHERE thread_id = ? AND reservation_id = ? AND owner_token = ? + AND status = 'pending' + """, + ( + now_unix, + handle.thread_id, + handle.reservation_id, + handle.owner_token, + ), + ) + if cursor.rowcount != 1: + connection.execute("ROLLBACK") + return False + connection.execute("COMMIT") + return True + except sqlite3.Error: + self._rollback(connection) + return self._status_matches(handle, "committed") + finally: + connection.close() + + def abort(self, handle: RecipeReceiptHandle) -> bool: + """Delete only the owner's pending receipt; consumed evidence remains durable.""" + connection = self._connect() + if connection is None: + return False + try: + connection.execute("BEGIN IMMEDIATE") + cursor = connection.execute( + """ + DELETE FROM receipts + WHERE thread_id = ? AND reservation_id = ? AND owner_token = ? + AND status = 'pending' + """, + (handle.thread_id, handle.reservation_id, handle.owner_token), + ) + if cursor.rowcount != 1: + connection.execute("ROLLBACK") + return False + connection.execute("COMMIT") + return True + except sqlite3.Error: + self._rollback(connection) + return self._receipt_state_exact(handle) == "absent" + finally: + connection.close() + + def _status_matches(self, handle: RecipeReceiptHandle, expected: str) -> bool: + connection = self._connect() + if connection is None: + return False + try: + row = connection.execute( + """ + SELECT status FROM receipts + WHERE thread_id = ? AND reservation_id = ? AND owner_token = ? + """, + (handle.thread_id, handle.reservation_id, handle.owner_token), + ).fetchone() + return row == (expected,) + except sqlite3.Error: + return False + finally: + connection.close() + + def _receipt_state_exact(self, handle: RecipeReceiptHandle) -> str: + connection = self._connect() + if connection is None: + return "unknown" + try: + row = connection.execute( + """ + SELECT status, reservation_id, owner_token FROM receipts + WHERE thread_id = ? + """, + (handle.thread_id,), + ).fetchone() + if row is None: + return "absent" + if row[1:] != (handle.reservation_id, handle.owner_token): + return "different_owner" + return str(row[0]) + except sqlite3.Error: + return "unknown" + finally: + connection.close() + + def receipt_status(self, thread_id: str) -> str | None: + connection = self._connect() + if connection is None: + return None + try: + row = connection.execute( + "SELECT status FROM receipts WHERE thread_id = ?", + (thread_id,), + ).fetchone() + return row[0] if row is not None else None + except sqlite3.Error: + return None + finally: + connection.close() + + def recover_stale_pending(self, *, thread_id: str, before_unix: int) -> bool: + """Release a stale pending receipt while retaining consumed-call evidence.""" + connection = self._connect() + if connection is None: + return False + try: + connection.execute("BEGIN IMMEDIATE") + cursor = connection.execute( + """ + DELETE FROM receipts + WHERE thread_id = ? AND status = 'pending' AND updated_at_unix < ? + """, + (thread_id, before_unix), + ) + connection.execute("COMMIT") + return cursor.rowcount == 1 + except sqlite3.Error: + self._rollback(connection) + return False + finally: + connection.close() diff --git a/src/autoskillit/execution/backends/_probe_cache.py b/src/autoskillit/execution/backends/_probe_cache.py index ade9bc3839..893e562c68 100644 --- a/src/autoskillit/execution/backends/_probe_cache.py +++ b/src/autoskillit/execution/backends/_probe_cache.py @@ -8,6 +8,7 @@ from __future__ import annotations import hashlib +import json from dataclasses import dataclass from datetime import UTC, datetime, timedelta from pathlib import Path @@ -15,11 +16,18 @@ from autoskillit.core import ( OUTPUT_DISCIPLINE_COMBINED_SHA256, OUTPUT_DISCIPLINE_POLICY_VERSION, + RECIPE_DELIVERY_SURFACE_REGISTRY_DIGEST, RESPONSE_BACKSTOP_EXEMPTION_REGISTRY_DIGEST, get_logger, read_versioned_json, write_versioned_json, ) +from autoskillit.execution.backends._codex_config import ( + CODEX_LIMITS_LAST_VERIFIED_VERSION, + CODEX_RECIPE_DELIVERY_BUDGET, + CODEX_RECIPE_DELIVERY_CALLING_CONTRACT_DIGEST, + SUPPORTED_CODEX_RECIPE_EVIDENCE_REGISTRY, +) logger = get_logger(__name__) @@ -28,15 +36,45 @@ "generated-codex-child-v1", "deep-investigate-codex-v2", "deep-investigate-claude-200k-v2", + "codex-recipe-delivery-v1", ) PROBE_SUITE_CONTRACT_DIGEST: str = hashlib.sha256( "\n".join(PROBE_SUITE_CONTRACT).encode("utf-8") ).hexdigest() +CODEX_RECIPE_PROBE_MODEL_IDENTITY = "gpt-5.6-sol" +_SUPPORTED_RECIPE_EVIDENCE_DIGEST = hashlib.sha256( + json.dumps( + { + identity: definition._asdict() + for identity, definition in sorted(SUPPORTED_CODEX_RECIPE_EVIDENCE_REGISTRY.items()) + }, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ).encode("ascii") +).hexdigest() +CODEX_RECIPE_PROBE_POLICY_COMPONENTS: tuple[str, ...] = ( + f"budget:{CODEX_RECIPE_DELIVERY_BUDGET.contract_digest}", + f"parser:{CODEX_RECIPE_DELIVERY_BUDGET.parser_version}", + f"evidence-schema:{CODEX_RECIPE_DELIVERY_BUDGET.evidence_version}", + f"attestation-registry:{_SUPPORTED_RECIPE_EVIDENCE_DIGEST}", + f"surface-registry:{RECIPE_DELIVERY_SURFACE_REGISTRY_DIGEST}", + f"prompt:{CODEX_RECIPE_DELIVERY_CALLING_CONTRACT_DIGEST}", + f"response-exemptions:{RESPONSE_BACKSTOP_EXEMPTION_REGISTRY_DIGEST}", + "cli-pin:" + ".".join(str(value) for value in CODEX_LIMITS_LAST_VERIFIED_VERSION), + f"model:{CODEX_RECIPE_PROBE_MODEL_IDENTITY}", + "fixtures:protected-v1+diagnostic-v1", + "attestation-provider:null-protected-host-v1", +) +CODEX_RECIPE_PROBE_POLICY_DIGEST: str = hashlib.sha256( + "\n".join(CODEX_RECIPE_PROBE_POLICY_COMPONENTS).encode("utf-8") +).hexdigest() PROBE_POLICY_IDENTITY: str = ( f"v{OUTPUT_DISCIPLINE_POLICY_VERSION}-{OUTPUT_DISCIPLINE_COMBINED_SHA256}-" - f"{RESPONSE_BACKSTOP_EXEMPTION_REGISTRY_DIGEST}-{PROBE_SUITE_CONTRACT_DIGEST}" + f"{RESPONSE_BACKSTOP_EXEMPTION_REGISTRY_DIGEST}-{PROBE_SUITE_CONTRACT_DIGEST}-" + f"{CODEX_RECIPE_PROBE_POLICY_DIGEST}" ) -_SCHEMA_VERSION: int = 2 +_SCHEMA_VERSION: int = 3 @dataclass(frozen=True, slots=True) diff --git a/src/autoskillit/execution/backends/codex.py b/src/autoskillit/execution/backends/codex.py index 54106f4c0d..ac0af7d491 100644 --- a/src/autoskillit/execution/backends/codex.py +++ b/src/autoskillit/execution/backends/codex.py @@ -74,12 +74,11 @@ ) from autoskillit.execution.backends._cmd_builder import CmdBuilder from autoskillit.execution.backends._codex_config import ( + CODEX_RECIPE_DELIVERY_BUDGET, _format_toml_value, ensure_codex_mcp_registered, ) -from autoskillit.execution.backends._codex_hooks import ( - sync_hooks_to_codex_config, -) +from autoskillit.execution.backends._codex_hooks import sync_hooks_to_codex_config from autoskillit.execution.backends._codex_parse import CodexResultParser, CodexStreamParser __all__ = [ @@ -639,7 +638,11 @@ def capabilities(self) -> BackendCapabilities: skill_sigil="$", session_dir_persistent=True, supports_model_invocation_gating=False, - effective_delivery_token_limit=10_000, + unnegotiated_tool_result_token_limit=( + CODEX_RECIPE_DELIVERY_BUDGET.ordinary_omitted_result_token_limit + ), + protected_recipe_delivery_capable=False, + recipe_delivery_budget=CODEX_RECIPE_DELIVERY_BUDGET, ) @property diff --git a/src/autoskillit/hooks/formatters/_fmt_recipe.py b/src/autoskillit/hooks/formatters/_fmt_recipe.py index 07576992a0..35551f89f9 100644 --- a/src/autoskillit/hooks/formatters/_fmt_recipe.py +++ b/src/autoskillit/hooks/formatters/_fmt_recipe.py @@ -23,7 +23,10 @@ ) if TYPE_CHECKING: + from autoskillit._recipe_delivery_framing import is_attested_recipe_delivery from autoskillit.recipe import ListRecipesResult, LoadRecipeResult, OpenKitchenResult +else: + from _recipe_delivery_framing import is_attested_recipe_delivery # Field coverage contract for _fmt_load_recipe ↔ LoadRecipeResult @@ -253,6 +256,8 @@ def _fmt_open_kitchen(data: OpenKitchenResult, pipeline: bool) -> str: def _fmt_open_kitchen_plain_text(text: str, _pipeline: bool) -> str: """Format open_kitchen plain-text response (no recipe attached).""" + if is_attested_recipe_delivery(text): + return text return f"## open_kitchen\n\n{text}" diff --git a/src/autoskillit/hooks/formatters/pretty_output_hook.py b/src/autoskillit/hooks/formatters/pretty_output_hook.py index be3fa12e79..1bf425c200 100644 --- a/src/autoskillit/hooks/formatters/pretty_output_hook.py +++ b/src/autoskillit/hooks/formatters/pretty_output_hook.py @@ -37,8 +37,10 @@ # resolve the helpers to the same top-level modules so identity stays # consistent across both invocation modes. _HOOKS_DIR = str(Path(__file__).resolve().parent) -if _HOOKS_DIR not in sys.path: - sys.path.insert(0, _HOOKS_DIR) +_PACKAGE_ROOT = str(Path(__file__).resolve().parents[2]) +for _import_dir in (_HOOKS_DIR, _PACKAGE_ROOT): + if _import_dir not in sys.path: + sys.path.insert(0, _import_dir) from _fmt_dispatch import ( # type: ignore[import-not-found] # noqa: E402, F401 _FMT_DISPATCH_FOOD_TRUCK_RENDERED, diff --git a/src/autoskillit/recipe/rules/rules_tools.py b/src/autoskillit/recipe/rules/rules_tools.py index 0dc94c2de1..914b6c1461 100644 --- a/src/autoskillit/recipe/rules/rules_tools.py +++ b/src/autoskillit/recipe/rules/rules_tools.py @@ -66,7 +66,7 @@ # --- Recipe tools --- "validate_recipe": frozenset({"script_path"}), "migrate_recipe": frozenset({"name"}), - "load_recipe": frozenset({"name", "overrides", "ingredients_only"}), + "load_recipe": frozenset({"name", "overrides", "ingredients_only", "delivery_request"}), "list_recipes": frozenset(), # --- Clone tools --- "clone_repo": frozenset( diff --git a/src/autoskillit/server/AGENTS.md b/src/autoskillit/server/AGENTS.md index 05e89d6d7e..aba586e1a0 100644 --- a/src/autoskillit/server/AGENTS.md +++ b/src/autoskillit/server/AGENTS.md @@ -16,6 +16,7 @@ Sub-package: tools/ (see tools/AGENTS.md). | `_notify.py` | MCP notification dispatch and response-size tracking | | `_response_budget.py` | Lossless response spill, exact canonical projection finalization, measured exemptions, and privacy-safe budget telemetry | | `_response_conformance.py` | Post-FastMCP-conversion conformance gate for registered string tool responses | +| `_recipe_delivery.py` | Unified recipe finalization, content-addressed generations, pull integrity, and receipt completion | | `_session_type.py` | Session-type tag visibility dispatcher — controls which tools are visible per session type | | `_state.py` | Mutable singleton state and context accessor functions (`_ctx` sentinel, `get_ctx`, `set_ctx`) | | `_subprocess.py` | Subprocess execution helpers for MCP tools | diff --git a/src/autoskillit/server/_notify.py b/src/autoskillit/server/_notify.py index d68b902bd8..de7137f372 100644 --- a/src/autoskillit/server/_notify.py +++ b/src/autoskillit/server/_notify.py @@ -16,7 +16,11 @@ RESERVED_LOG_RECORD_KEYS, BackendCapabilities, get_logger, - resolve_effective_delivery_bound, + resolve_general_output_token_limit, +) +from autoskillit.server._recipe_delivery import ( + FinalizedRecipeResponse, + complete_finalized_recipe_response, ) from autoskillit.server._response_budget import ( bounded_response_budget_failure, @@ -108,9 +112,15 @@ async def wrapper(*args: Any, **kwargs: Any) -> Any: } ) logger.exception("track_response_size_handler_failed", tool_name=tool_name) + finalized = result if isinstance(result, FinalizedRecipeResponse) else None + response_value = finalized.rendered if finalized is not None else result ctx = _get_ctx_or_none() try: - response_str = result if isinstance(result, str) else json.dumps(result) + response_str = ( + response_value + if isinstance(response_value, str) + else json.dumps(response_value) + ) except (TypeError, ValueError): logger.warning( "track_response_size_nonserializable_result", @@ -142,19 +152,21 @@ async def wrapper(*args: Any, **kwargs: Any) -> Any: artifact_dir = ( temp_dir / "responses" / tool_name if isinstance(temp_dir, Path) else None ) - effective_delivery_token_limit: int | None = None - if ctx is not None: + selected_result_token_limit: int | None = None + if finalized is not None: + selected_result_token_limit = finalized.decision.selected_result_token_limit + elif ctx is not None: backend = getattr(ctx, "backend", None) caps = getattr(backend, "capabilities", None) if backend is not None else None if isinstance(caps, BackendCapabilities): - effective_delivery_token_limit = resolve_effective_delivery_bound(caps) + selected_result_token_limit = resolve_general_output_token_limit(caps) try: result = enforce_response_budget( - result, + response_value, tool_name=tool_name, artifact_dir=artifact_dir, config=budget, - effective_delivery_token_limit=effective_delivery_token_limit, + selected_result_token_limit=selected_result_token_limit, ) except Exception: logger.error("track_response_size_enforcement_failed", tool_name=tool_name) @@ -192,6 +204,8 @@ async def wrapper(*args: Any, **kwargs: Any) -> Any: "track_response_size_notification_failed", tool_name=tool_name, ) + if finalized is not None: + result = complete_finalized_recipe_response(finalized, result) return result return wrapper diff --git a/src/autoskillit/server/_recipe_delivery.py b/src/autoskillit/server/_recipe_delivery.py new file mode 100644 index 0000000000..ba2b7d0d9f --- /dev/null +++ b/src/autoskillit/server/_recipe_delivery.py @@ -0,0 +1,970 @@ +"""Unified recipe finalization and immutable pull-generation storage.""" + +from __future__ import annotations + +import fcntl +import hashlib +import json +import shutil +import time +from collections.abc import Iterator +from contextlib import contextmanager +from dataclasses import dataclass, replace +from pathlib import Path +from typing import TYPE_CHECKING, Any + +from autoskillit._recipe_delivery_framing import ( + RECIPE_BODY_END, + RECIPE_BODY_START, + RECIPE_COMPLETION_SENTINEL, +) +from autoskillit.config import OutputBudgetConfig +from autoskillit.core import ( + CLAUDE_CODE_CAPABILITIES, + RECIPE_DELIVERY_SURFACE_REGISTRY, + BackendCapabilities, + RecipeDeliveryAttestation, + RecipeDeliveryDecision, + RecipeDeliveryMode, + RecipeDeliveryRequest, + atomic_write, + get_logger, + resolve_general_output_token_limit, + resolve_recipe_delivery_decision, +) +from autoskillit.execution import ( + RecipeDeliveryReceiptLedger, + RecipeReceiptHandle, + codex_recipe_delivery_calling_contract, +) +from autoskillit.recipe import _extract_routing_edges, step_byte_ranges_from_yaml +from autoskillit.server._response_budget import enforce_response_budget + +if TYPE_CHECKING: + from autoskillit.core import RecipeDeliveryBudgetDef, RecipeDeliveryEvidenceDef + from autoskillit.pipeline import ToolContext + +RECIPE_ARTIFACT_DESCRIPTOR_VERSION = 1 +RECIPE_ARTIFACT_SCHEMA_VERSION = 1 +RECIPE_ARTIFACT_MAX_BLOB_BYTES = 1_000_000 +RECIPE_ARTIFACT_MAX_DESCRIPTOR_BYTES = 16_384 + + +def document_recipe_delivery_contract(function: Any) -> Any: + """Append the generated Codex contract before FastMCP reads a tool docstring.""" + description = function.__doc__ or "" + function.__doc__ = f"{description.rstrip()}\n\n{codex_recipe_delivery_calling_contract()}\n" + return function + + +class RecipeArtifactError(RuntimeError): + """A requested immutable recipe generation is absent or corrupt.""" + + +@dataclass(frozen=True, slots=True) +class RecipeArtifactGeneration: + """Exact identities for one immutable canonical recipe payload.""" + + producer_tool: str + recipe_name: str + descriptor_version: int + schema_version: int + payload_sha256: str + artifact_blob_sha256: str + artifact_blob_size_bytes: int + body_sha256: str + body_size_bytes: int + + def has_valid_read_bounds(self) -> bool: + """Return whether caller-provided sizes stay within server ceilings.""" + return ( + self.descriptor_version > 0 + and self.schema_version > 0 + and 0 < self.artifact_blob_size_bytes <= RECIPE_ARTIFACT_MAX_BLOB_BYTES + and 0 <= self.body_size_bytes <= self.artifact_blob_size_bytes + ) + + def pull_identity(self) -> dict[str, str | int]: + return { + "producer_tool": self.producer_tool, + "recipe_name": self.recipe_name, + "descriptor_version": self.descriptor_version, + "schema_version": self.schema_version, + "payload_sha256": self.payload_sha256, + "artifact_blob_sha256": self.artifact_blob_sha256, + "artifact_blob_size_bytes": self.artifact_blob_size_bytes, + "body_sha256": self.body_sha256, + "body_size_bytes": self.body_size_bytes, + "pull_tool": "get_recipe_section", + } + + +@dataclass(frozen=True, slots=True) +class FinalizedRecipeResponse: + """Internal carrier consumed before FastMCP result conversion.""" + + rendered: str + decision: RecipeDeliveryDecision + receipt_handle: RecipeReceiptHandle | None = None + receipt_ledger: RecipeDeliveryReceiptLedger | None = None + + +def _qualified_sha256(data: bytes) -> str: + return f"sha256:{hashlib.sha256(data).hexdigest()}" + + +def _domain_sha256(domain: str, data: bytes) -> str: + return _qualified_sha256(domain.encode("ascii") + b"\0" + data) + + +def _canonical_payload(payload: dict[str, Any]) -> bytes: + return json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + +def _read_bounded_bytes(path: Path, *, max_bytes: int, error: str) -> bytes: + """Read through one descriptor with an explicit allocation ceiling.""" + try: + with path.open("rb") as source: + data = source.read(max_bytes + 1) + except OSError as exc: + raise RecipeArtifactError(error) from exc + if len(data) > max_bytes: + raise RecipeArtifactError(error) + return data + + +def _safe_component(value: str) -> str: + if not value: + return "~" + if value in {".", ".."}: + raise RecipeArtifactError("unsafe recipe artifact path component") + encoded: list[str] = [] + for byte in value.encode("utf-8"): + character = chr(byte) + if character.isascii() and (character.isalnum() or character in "._-"): + encoded.append(character) + else: + encoded.append(f"~{byte:02x}") + return "".join(encoded) + + +def _artifact_root(temp_dir: Path) -> Path: + if not isinstance(temp_dir, Path): + raise RecipeArtifactError("recipe artifact temp directory is unavailable") + return temp_dir / "recipe-delivery" + + +def _generation_dir( + temp_dir: Path, + *, + kitchen_id: str, + producer_tool: str, + recipe_name: str, + descriptor_version: int, + schema_version: int, + payload_sha256: str, +) -> Path: + return ( + _artifact_root(temp_dir) + / _safe_component(kitchen_id) + / _safe_component(producer_tool) + / _safe_component(recipe_name) + / f"descriptor-{descriptor_version}-schema-{schema_version}" + / _safe_component(payload_sha256) + ) + + +def _retired_namespace_marker(temp_dir: Path, *, kitchen_id: str) -> Path: + return _artifact_root(temp_dir) / ".retired" / f"{_safe_component(kitchen_id)}.retired" + + +@contextmanager +def _generation_lock(temp_dir: Path, *, exclusive: bool) -> Iterator[None]: + root = _artifact_root(temp_dir) + root.mkdir(parents=True, exist_ok=True) + with (root / ".generation.lock").open("a+b") as lock_file: + fcntl.flock(lock_file, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH) + try: + yield + finally: + fcntl.flock(lock_file, fcntl.LOCK_UN) + + +def _generation_from_payload( + *, + producer_tool: str, + recipe_name: str, + blob: bytes, + payload: dict[str, Any], + descriptor_version: int | None = None, + schema_version: int | None = None, +) -> RecipeArtifactGeneration: + body = payload.get("content") + body_bytes = body.encode("utf-8") if isinstance(body, str) else b"" + return RecipeArtifactGeneration( + producer_tool=producer_tool, + recipe_name=recipe_name, + descriptor_version=( + RECIPE_ARTIFACT_DESCRIPTOR_VERSION + if descriptor_version is None + else descriptor_version + ), + schema_version=( + RECIPE_ARTIFACT_SCHEMA_VERSION if schema_version is None else schema_version + ), + payload_sha256=_domain_sha256("autoskillit.recipe-payload.v1", blob), + artifact_blob_sha256=_qualified_sha256(blob), + artifact_blob_size_bytes=len(blob), + body_sha256=_qualified_sha256(body_bytes), + body_size_bytes=len(body_bytes), + ) + + +def persist_recipe_artifact( + temp_dir: Path, + *, + kitchen_id: str, + producer_tool: str, + recipe_name: str, + payload: dict[str, Any], +) -> RecipeArtifactGeneration: + """Publish an immutable canonical payload and its generation descriptor.""" + blob = _canonical_payload(payload) + if len(blob) > RECIPE_ARTIFACT_MAX_BLOB_BYTES: + raise RecipeArtifactError("recipe artifact blob exceeds persistence limit") + generation = _generation_from_payload( + producer_tool=producer_tool, + recipe_name=recipe_name, + blob=blob, + payload=payload, + ) + directory = _generation_dir( + temp_dir, + kitchen_id=kitchen_id, + producer_tool=producer_tool, + recipe_name=recipe_name, + descriptor_version=generation.descriptor_version, + schema_version=generation.schema_version, + payload_sha256=generation.payload_sha256, + ) + descriptor = _canonical_payload(generation.pull_identity()) + with _generation_lock(temp_dir, exclusive=True): + if _retired_namespace_marker(temp_dir, kitchen_id=kitchen_id).exists(): + raise RecipeArtifactError("recipe artifact namespace is retired") + directory.mkdir(parents=True, exist_ok=True) + blob_path = directory / "payload.json" + descriptor_path = directory / "descriptor.json" + if blob_path.exists(): + existing_blob = _read_bounded_bytes( + blob_path, + max_bytes=len(blob), + error="content-addressed payload collision", + ) + if existing_blob != blob: + raise RecipeArtifactError("content-addressed payload collision") + if descriptor_path.exists(): + existing_descriptor = _read_bounded_bytes( + descriptor_path, + max_bytes=len(descriptor), + error="content-addressed descriptor collision", + ) + if existing_descriptor != descriptor: + raise RecipeArtifactError("content-addressed descriptor collision") + if not blob_path.exists(): + atomic_write(blob_path, blob.decode("utf-8")) + if not descriptor_path.exists(): + atomic_write(descriptor_path, descriptor.decode("utf-8")) + return generation + + +def load_recipe_artifact( + temp_dir: Path, + *, + kitchen_id: str, + identity: RecipeArtifactGeneration, +) -> dict[str, Any]: + """Read and independently verify an exact immutable payload generation.""" + if not identity.has_valid_read_bounds(): + raise RecipeArtifactError("invalid recipe artifact identity bounds") + directory = _generation_dir( + temp_dir, + kitchen_id=kitchen_id, + producer_tool=identity.producer_tool, + recipe_name=identity.recipe_name, + descriptor_version=identity.descriptor_version, + schema_version=identity.schema_version, + payload_sha256=identity.payload_sha256, + ) + with _generation_lock(temp_dir, exclusive=False): + blob_path = directory / "payload.json" + descriptor_path = directory / "descriptor.json" + blob = _read_bounded_bytes( + blob_path, + max_bytes=identity.artifact_blob_size_bytes, + error="artifact blob size mismatch", + ) + descriptor_bytes = _read_bounded_bytes( + descriptor_path, + max_bytes=RECIPE_ARTIFACT_MAX_DESCRIPTOR_BYTES, + error="recipe generation descriptor exceeds read limit", + ) + if len(blob) != identity.artifact_blob_size_bytes: + raise RecipeArtifactError("artifact blob size mismatch") + if _qualified_sha256(blob) != identity.artifact_blob_sha256: + raise RecipeArtifactError("artifact blob digest mismatch") + if _domain_sha256("autoskillit.recipe-payload.v1", blob) != identity.payload_sha256: + raise RecipeArtifactError("semantic payload digest mismatch") + try: + descriptor_raw = descriptor_bytes.decode("utf-8") + descriptor = json.loads(descriptor_raw) + payload = json.loads(blob) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RecipeArtifactError("recipe generation is not valid JSON") from exc + if descriptor != identity.pull_identity(): + raise RecipeArtifactError("generation descriptor mismatch") + if not isinstance(payload, dict): + raise RecipeArtifactError("persisted recipe payload is not a mapping") + expected = _generation_from_payload( + producer_tool=identity.producer_tool, + recipe_name=identity.recipe_name, + blob=blob, + payload=payload, + descriptor_version=identity.descriptor_version, + schema_version=identity.schema_version, + ) + if expected != identity: + raise RecipeArtifactError("recipe body identity mismatch") + return payload + + +def retire_recipe_artifacts(temp_dir: Path, *, kitchen_id: str) -> bool: + """Retire one kitchen namespace after all shared-lock readers finish.""" + try: + namespace = _artifact_root(temp_dir) / _safe_component(kitchen_id) + retired_marker = _retired_namespace_marker(temp_dir, kitchen_id=kitchen_id) + with _generation_lock(temp_dir, exclusive=True): + retired_marker.parent.mkdir(parents=True, exist_ok=True) + atomic_write(retired_marker, "retired\n") + if namespace.exists(): + shutil.rmtree(namespace) + except (OSError, RecipeArtifactError, TypeError): + return False + return True + + +def _validate_producer_authorization_policies() -> None: + policies: dict[str, tuple[bool, bool]] = {} + for definition in RECIPE_DELIVERY_SURFACE_REGISTRY.values(): + policy = (definition.pull_eligible, definition.recreation_eligible) + existing = policies.setdefault(definition.producer_tool, policy) + if existing != policy: + raise RecipeArtifactError( + "recipe delivery surfaces sharing a producer must share pull policies" + ) + + +def recipe_pull_producers() -> frozenset[str]: + """Return public producers authorized to resolve immutable pull generations.""" + _validate_producer_authorization_policies() + return frozenset( + definition.producer_tool + for definition in RECIPE_DELIVERY_SURFACE_REGISTRY.values() + if definition.pull_eligible + ) + + +def recipe_recreation_producers() -> frozenset[str]: + """Return producers whose missing generations may be rebuilt in-session.""" + _validate_producer_authorization_policies() + return frozenset( + definition.producer_tool + for definition in RECIPE_DELIVERY_SURFACE_REGISTRY.values() + if definition.recreation_eligible + ) + + +def _step_one_line_summary(step: Any) -> str: + """Return a compact single-line summary for one recipe step.""" + description = getattr(step, "description", "") or "" + if description.strip(): + return description.strip().splitlines()[0][:160] + message = getattr(step, "message", None) + if isinstance(message, str) and message.strip(): + return message.strip().splitlines()[0][:160] + tool = getattr(step, "tool", None) + action = getattr(step, "action", None) + if tool: + return f"tool={tool}" + if action: + return f"action={action}" + return "" + + +def _compute_step_byte_ranges(content: str) -> dict[str, tuple[int, int]]: + """Return UTF-8 step-body offsets from the canonical YAML helper.""" + return step_byte_ranges_from_yaml(content) + + +def extract_step_skeleton( + post_prune_step_names: list[str], + routing_edges_by_step: dict[str, list[tuple[str, str]]], + step_summaries: dict[str, str] | None = None, + byte_ranges: dict[str, tuple[int, int]] | None = None, +) -> dict[str, Any]: + """Build the compact routing and byte-range index carried by an envelope.""" + skeleton: list[dict[str, Any]] = [] + for name in post_prune_step_names: + entry: dict[str, Any] = { + "name": name, + "edges": [ + {"type": edge_type, "target": target} + for edge_type, target in routing_edges_by_step.get(name) or [] + if target + ], + } + summary = (step_summaries or {}).get(name) or "" + if summary: + entry["summary"] = summary + span = (byte_ranges or {}).get(name) + if span is not None: + entry["byte_range"] = list(span) + skeleton.append(entry) + return {"step_count": len(skeleton), "steps": skeleton} + + +def _safe_utf8_truncate(data: bytes) -> str: + """Decode a byte prefix without retaining a partial UTF-8 codepoint.""" + while data: + try: + return data.decode("utf-8") + except UnicodeDecodeError as exc: + data = data[: exc.start] + return "" + + +def build_step_summaries(active_recipe_steps: Any) -> dict[str, str]: + """Build step-name to one-line-summary metadata from parsed recipe steps.""" + if not isinstance(active_recipe_steps, dict) or not active_recipe_steps: + return {} + return { + name: _step_one_line_summary(step) + for name, step in active_recipe_steps.items() + if isinstance(name, str) and name + } + + +def build_routing_edges_by_step( + active_recipe_steps: Any, + *, + edge_extractor: Any = _extract_routing_edges, +) -> dict[str, list[tuple[str, str]]]: + """Build outgoing routing-edge metadata from parsed recipe steps.""" + if not isinstance(active_recipe_steps, dict) or not active_recipe_steps: + return {} + edges_by_step: dict[str, list[tuple[str, str]]] = {} + for name, step in active_recipe_steps.items(): + if not isinstance(name, str) or not name: + continue + extracted = edge_extractor(step) if edge_extractor is not None else [] + edges_by_step[name] = [ + (edge.edge_type, edge.target) + for edge in (extracted or []) + if getattr(edge, "target", None) + ] + return edges_by_step + + +def build_recipe_envelope( + payload: dict[str, Any], + *, + recipe_name: str, + generation: RecipeArtifactGeneration, + skeleton_source: ToolContext, + bound_bytes: int, +) -> dict[str, Any]: + """Build the bounded pull envelope used by every recipe delivery surface.""" + post_prune_raw = payload.get("post_prune_step_names") or [] + if not isinstance(post_prune_raw, list): + post_prune_raw = [] + post_prune_names = [name for name in post_prune_raw if isinstance(name, str)] + active_recipe_steps: dict[str, Any] | None = None + if getattr(skeleton_source, "recipe_name", "") == recipe_name: + active_recipe_steps = skeleton_source.active_recipe_steps + summaries = build_step_summaries(active_recipe_steps) + edges = build_routing_edges_by_step(active_recipe_steps) + byte_ranges = _compute_step_byte_ranges(payload.get("content") or "") + skeleton = extract_step_skeleton( + post_prune_names, + edges, + summaries, + byte_ranges=byte_ranges, + ) + pull_identity = generation.pull_identity() + + def _pullable_skeleton_size(candidate: dict[str, Any]) -> int: + pullable = { + "success": payload.get("success", True), + "step_flow_skeleton": candidate, + "recipe_pull": pull_identity, + "delivery_bound_spill": True, + } + return len(json.dumps(pullable, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) + + if _pullable_skeleton_size(skeleton) > bound_bytes: + for summary_limit in (120, 80, 64, 48, 32, 24, 16, 8, 0): + bounded_summaries = ( + {name: summary[:summary_limit] for name, summary in summaries.items()} + if summary_limit + else {} + ) + skeleton = extract_step_skeleton( + post_prune_names, + edges, + bounded_summaries, + byte_ranges=byte_ranges, + ) + if _pullable_skeleton_size(skeleton) <= bound_bytes: + break + + envelope: dict[str, Any] = {"success": payload.get("success", True)} + envelope_bytes = len(json.dumps(envelope["success"]).encode("utf-8")) + for key in ( + "kitchen", + "version", + "valid", + "dispatch_feasible", + "errors", + "warnings", + "hooks", + "post_prune_step_names", + "post_prune_routing_edges", + "requires_packs", + "requires_features", + ): + if key in payload and payload[key] is not None: + envelope[key] = payload[key] + envelope_bytes += len( + json.dumps({key: payload[key]}, ensure_ascii=False).encode("utf-8") + ) + + skeleton_overhead = len( + json.dumps({"step_flow_skeleton": skeleton}, ensure_ascii=False).encode("utf-8") + ) + pull_overhead = len( + json.dumps( + {"recipe_pull": pull_identity, "delivery_bound_spill": True}, + ensure_ascii=False, + ).encode("utf-8") + ) + remaining = max( + 0, + bound_bytes - skeleton_overhead - pull_overhead - envelope_bytes - 64, + ) + + def _project_priority_strings(keys: tuple[str, ...]) -> None: + nonlocal remaining + candidates = [ + key for key in keys if isinstance(payload.get(key), str) and payload.get(key) + ] + overhead = { + key: len(json.dumps({key: ""}, ensure_ascii=False).encode("utf-8")) + for key in candidates + } + present: list[str] = [] + budget = remaining + for key in candidates: + if overhead[key] < budget: + present.append(key) + budget -= overhead[key] + if not present: + return + + lengths = {key: len(payload[key].encode("utf-8")) for key in present} + allocation: dict[str, int] = dict.fromkeys(present, 0) + active = list(present) + pool = remaining - sum(overhead[key] for key in present) + while active and pool > 0: + share, extra = divmod(pool, len(active)) + still_active: list[str] = [] + for index, key in enumerate(active): + give = share + (1 if index < extra else 0) + take = min(give, lengths[key] - allocation[key], pool) + allocation[key] += take + pool -= take + if allocation[key] < lengths[key]: + still_active.append(key) + active = still_active + + for key in present: + take = allocation[key] + if take <= 0: + continue + value_bytes = payload[key].encode("utf-8") + if len(value_bytes) <= take: + envelope[key] = payload[key] + remaining -= len(value_bytes) + overhead[key] + else: + envelope[key] = _safe_utf8_truncate(value_bytes[:take]) + remaining -= take + overhead[key] + get_logger(__name__).warning( + "recipe_envelope_priority_field_truncated", + recipe_name=recipe_name, + field=key, + alloc_bytes=take, + ) + + _project_priority_strings(("orchestration_rules", "stop_step_semantics")) + for key in ("ingredients_table", "suggestions"): + value = payload.get(key) + if value is None: + continue + serialized_value = json.dumps(value, ensure_ascii=False).encode("utf-8") + if len(serialized_value) + 32 <= remaining: + envelope[key] = value + remaining -= len(serialized_value) + 32 + + envelope["step_flow_skeleton"] = skeleton + envelope["recipe_pull"] = pull_identity + envelope["delivery_bound_spill"] = True + if ( + len(json.dumps(envelope, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) + <= bound_bytes + ): + return envelope + + fallback_candidates: tuple[dict[str, Any], ...] = ( + { + "success": payload.get("success", True), + "step_flow_skeleton": skeleton, + "recipe_pull": pull_identity, + "delivery_bound_spill": True, + }, + { + "success": False, + "error": "recipe_envelope_exceeds_delivery_bound", + "recipe_pull": pull_identity, + }, + {"success": False, "error": "recipe_envelope_exceeds_delivery_bound"}, + {}, + ) + for fallback in fallback_candidates: + if ( + len(json.dumps(fallback, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) + <= bound_bytes + ): + return fallback + raise ValueError("delivery bound is too small for a JSON object") + + +def _conservative_token_upper_bound(rendered: str) -> int: + """Bound tokenizer output without assuming four UTF-8 bytes per token. + + Codex tokenization can merge bytes into one token, but it cannot require + more tokens than the number of input bytes. Using the exact byte count is + intentionally conservative for delivery-mode admission. + """ + return len(rendered.encode("utf-8")) + + +def _attested_render( + payload: dict[str, Any], + generation: RecipeArtifactGeneration, + *, + budget: RecipeDeliveryBudgetDef, + evidence_identity: str, +) -> str: + body = payload.get("content") if isinstance(payload.get("content"), str) else "" + metadata = {key: value for key, value in payload.items() if key != "content"} + control = { + "recipe_delivery": { + "mode": RecipeDeliveryMode.ATTESTED_INLINE.value, + "contract_digest": budget.contract_digest, + "evidence_identity": evidence_identity, + "selected_result_token_limit": ( + budget.authoritative_attested_recipe_result_token_limit + ), + "recipe_pull": generation.pull_identity(), + "payload_metadata": metadata, + } + } + prefix = json.dumps(control, ensure_ascii=False, separators=(",", ":")) + return ( + f"{prefix}\n{RECIPE_BODY_START}\n{body}\n{RECIPE_BODY_END}\n" + f"{RECIPE_COMPLETION_SENTINEL} {generation.body_sha256}" + ) + + +def _failure_decision( + *, producer: str, reason: str, selected_limit: int, contract_digest: str +) -> RecipeDeliveryDecision: + return RecipeDeliveryDecision( + mode=RecipeDeliveryMode.ENVELOPE, + caller_requested_outer_tokens=None, + host_observed_requested_outer_tokens=None, + required_outer_tokens=0, + unnegotiated_tool_result_token_limit=selected_limit, + selected_result_token_limit=selected_limit, + contract_digest=contract_digest, + evidence_identity=None, + reason=reason, + producer=producer, + payload_sha256="sha256:" + ("0" * 64), + receipt_status="not_reserved", + ) + + +def finalize_recipe_delivery( + payload: dict[str, Any], + *, + surface: str, + recipe_name: str, + tool_ctx: ToolContext, + delivery_request: RecipeDeliveryRequest | None = None, + attestation: RecipeDeliveryAttestation | None = None, + supported_evidence: RecipeDeliveryEvidenceDef | None = None, + receipt_ledger: RecipeDeliveryReceiptLedger | None = None, + now_unix: int | None = None, +) -> FinalizedRecipeResponse: + """Persist, decide, shape, and transactionally reserve one recipe response.""" + surface_definition = RECIPE_DELIVERY_SURFACE_REGISTRY[surface] + candidate_capabilities = ( + getattr(tool_ctx.backend, "capabilities", None) if tool_ctx.backend is not None else None + ) + capabilities = ( + candidate_capabilities + if isinstance(candidate_capabilities, BackendCapabilities) + else replace( + CLAUDE_CODE_CAPABILITIES, + unnegotiated_tool_result_token_limit=( + CLAUDE_CODE_CAPABILITIES.unnegotiated_tool_result_token_limit + ), + protected_recipe_delivery_capable=False, + recipe_delivery_budget=None, + ) + ) + delivery_budget = capabilities.recipe_delivery_budget + ordinary_limit = resolve_general_output_token_limit(capabilities) + try: + generation = persist_recipe_artifact( + tool_ctx.temp_dir, + kitchen_id=tool_ctx.kitchen_id, + producer_tool=surface_definition.producer_tool, + recipe_name=recipe_name, + payload=payload, + ) + except (OSError, RecipeArtifactError, TypeError, ValueError): + decision = _failure_decision( + producer=surface_definition.producer_tool, + reason="recipe_artifact_persistence_failed", + selected_limit=ordinary_limit, + contract_digest=(delivery_budget.contract_digest if delivery_budget else ""), + ) + return FinalizedRecipeResponse( + rendered=json.dumps( + {"success": False, "error": "recipe_artifact_unavailable"}, + separators=(",", ":"), + ), + decision=decision, + ) + + ordinary_rendered = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + candidate_evidence = supported_evidence if surface_definition.negotiation_eligible else None + candidate_attestation = attestation if surface_definition.negotiation_eligible else None + candidate_request = delivery_request if surface_definition.negotiation_eligible else None + high_rendered = ( + _attested_render( + payload, + generation, + budget=delivery_budget, + evidence_identity=( + candidate_evidence.identity if candidate_evidence is not None else "unsupported" + ), + ) + if delivery_budget is not None + else ordinary_rendered + ) + ordinary_required_tokens = _conservative_token_upper_bound(ordinary_rendered) + required_tokens = ( + ordinary_required_tokens + if ordinary_required_tokens <= ordinary_limit + else _conservative_token_upper_bound(high_rendered) + ) + decision = resolve_recipe_delivery_decision( + capabilities=capabilities, + required_serialized_tokens=required_tokens, + budget=delivery_budget, + producer=surface_definition.producer_tool, + payload_sha256=generation.payload_sha256, + request=candidate_request, + attestation=candidate_attestation, + supported_evidence=candidate_evidence, + now_unix=now_unix, + ) + response_budget = getattr(getattr(tool_ctx, "config", None), "output_budget", None) + response_max_bytes = getattr(response_budget, "response_max_bytes", None) + response_ceiling_bytes = ( + response_max_bytes + if isinstance(response_max_bytes, int) and response_max_bytes > 0 + else None + ) + if ( + decision.mode is RecipeDeliveryMode.ORDINARY_INLINE + and surface_definition.response_exemption_tool is None + and response_ceiling_bytes is not None + and len(ordinary_rendered.encode("utf-8")) > response_ceiling_bytes + ): + decision = replace( + decision, + mode=RecipeDeliveryMode.ENVELOPE, + reason="server_response_budget_requires_envelope", + receipt_status="not_reserved", + ) + receipt_handle: RecipeReceiptHandle | None = None + if decision.mode is RecipeDeliveryMode.ATTESTED_INLINE: + if ( + delivery_budget is None + or receipt_ledger is None + or candidate_request is None + or candidate_attestation is None + or candidate_evidence is None + ): + decision = replace( + decision, + mode=RecipeDeliveryMode.ENVELOPE, + selected_result_token_limit=ordinary_limit, + reason="protected_receipt_store_unavailable", + receipt_status="not_reserved", + ) + else: + reservation = receipt_ledger.reserve( + capabilities=capabilities, + required_serialized_tokens=required_tokens, + budget=delivery_budget, + request=candidate_request, + attestation=candidate_attestation, + supported_evidence=candidate_evidence, + producer=surface_definition.producer_tool, + payload_sha256=generation.payload_sha256, + now_unix=int(time.time()) if now_unix is None else now_unix, + ) + if reservation.handle is None: + decision = replace( + decision, + mode=RecipeDeliveryMode.ENVELOPE, + selected_result_token_limit=ordinary_limit, + reason=reservation.reason, + receipt_status="not_reserved", + ) + else: + receipt_handle = reservation.handle + decision = replace(decision, receipt_status="pending") + + if decision.mode is RecipeDeliveryMode.ORDINARY_INLINE: + rendered = ordinary_rendered + elif decision.mode is RecipeDeliveryMode.ATTESTED_INLINE: + rendered = high_rendered + else: + envelope_bound_bytes = ordinary_limit + if ( + surface_definition.response_exemption_tool is None + and response_ceiling_bytes is not None + ): + envelope_bound_bytes = min(envelope_bound_bytes, response_ceiling_bytes) + rendered = json.dumps( + build_recipe_envelope( + payload, + recipe_name=recipe_name, + generation=generation, + skeleton_source=tool_ctx, + bound_bytes=envelope_bound_bytes, + ), + ensure_ascii=False, + separators=(",", ":"), + ) + return FinalizedRecipeResponse( + rendered=rendered, + decision=decision, + receipt_handle=receipt_handle, + receipt_ledger=receipt_ledger if receipt_handle is not None else None, + ) + + +def complete_finalized_recipe_response( + finalized: FinalizedRecipeResponse, + enforced: Any, + *, + now_unix: int | None = None, +) -> Any: + """Commit only an exact enforced response; otherwise abort its pending receipt.""" + handle = finalized.receipt_handle + ledger = finalized.receipt_ledger + if enforced == finalized.rendered: + if handle is None: + return enforced + if ledger is not None and ledger.commit( + handle, + now_unix=int(time.time()) if now_unix is None else now_unix, + ): + return enforced + enforced = json.dumps( + {"success": False, "error": "recipe_delivery_receipt_commit_failed"}, + separators=(",", ":"), + ) + if handle is not None and (ledger is None or not ledger.abort(handle)): + return json.dumps( + {"success": False, "error": "recipe_delivery_receipt_abort_failed"}, + separators=(",", ":"), + ) + return enforced + + +def enforce_recipe_resource_response( + finalized: FinalizedRecipeResponse, + *, + tool_ctx: ToolContext, +) -> str: + """Apply the ordinary response backstop and complete the receipt transaction.""" + configured_budget = getattr(tool_ctx.config, "output_budget", None) + output_budget = ( + configured_budget + if isinstance(configured_budget, OutputBudgetConfig) + else OutputBudgetConfig() + ) + temp_dir = getattr(tool_ctx, "temp_dir", None) + enforced = enforce_response_budget( + finalized.rendered, + tool_name="get_recipe", + artifact_dir=( + temp_dir / "responses" / "get_recipe" if isinstance(temp_dir, Path) else None + ), + config=output_budget, + selected_result_token_limit=finalized.decision.selected_result_token_limit, + ) + completed = complete_finalized_recipe_response(finalized, enforced) + if isinstance(completed, str): + return completed + return json.dumps(completed, ensure_ascii=False, separators=(",", ":")) + + +__all__ = [ + "FinalizedRecipeResponse", + "RECIPE_ARTIFACT_DESCRIPTOR_VERSION", + "RECIPE_ARTIFACT_SCHEMA_VERSION", + "RECIPE_BODY_END", + "RECIPE_BODY_START", + "RECIPE_COMPLETION_SENTINEL", + "RecipeArtifactError", + "RecipeArtifactGeneration", + "complete_finalized_recipe_response", + "enforce_recipe_resource_response", + "finalize_recipe_delivery", + "load_recipe_artifact", + "persist_recipe_artifact", + "recipe_pull_producers", + "retire_recipe_artifacts", +] diff --git a/src/autoskillit/server/_response_budget.py b/src/autoskillit/server/_response_budget.py index 75f3f7d7de..b3f04aaa5b 100644 --- a/src/autoskillit/server/_response_budget.py +++ b/src/autoskillit/server/_response_budget.py @@ -75,9 +75,9 @@ def _canonical_json(value: Any) -> str: def _estimated_tokens(original_size: int) -> int: """Estimate tokens via the four-UTF-8-byte ceiling-division heuristic. - Matches the ``CODEX_TOOL_OUTPUT_TOKEN_LIMIT`` derivation: a coarse + Uses the general output token limit as a coarse transport-layer estimate, not a tokenizer count. Used to compare - payload size against ``effective_delivery_token_limit``. + payload size against ``selected_result_token_limit``. """ return (original_size + 3) // 4 @@ -225,23 +225,6 @@ def _artifact_path(artifact_dir: Path, tool_name: str) -> Path: return artifact_dir / f"{safe_name or 'response'}_{uuid.uuid4().hex[:8]}.log" -def _recipe_artifact_path(artifact_dir: Path, tool_name: str, recipe_name: str) -> Path: - """Deterministic artifact path for recipe-bearing tools. - - Unlike ``_artifact_path`` (which uses UUID for uniqueness), the recipe - artifact path is deterministic so that the ``get_recipe_section`` pull - tool can reconstruct the same path without needing it passed back in - the envelope. Re-opening the same recipe overwrites the artifact - (idempotent) rather than scattering UUID-suffixed copies. - - The path is namespaced by both tool and recipe so two recipes loaded - from different surfaces (open_kitchen vs load_recipe) do not collide. - """ - safe_tool = "".join(c if c.isalnum() or c in "._-" else "_" for c in tool_name) - safe_recipe = "".join(c if c.isalnum() or c in "._-" else "_" for c in recipe_name) - return artifact_dir / f"{safe_tool or 'response'}_{safe_recipe or 'recipe'}.log" - - def _finalize_envelope(envelope: dict[str, Any], *, max_bytes: int) -> str: metadata = envelope.get(RESPONSE_SPILL_METADATA_KEY) if not isinstance(metadata, dict) or "projected_utf8_bytes" not in metadata: @@ -826,7 +809,7 @@ def _spill_for_delivery_bound( artifact_dir: Path | None, original: str, original_size: int, - effective_delivery_token_limit: int, + selected_result_token_limit: int, ) -> Any: """Persist ``original`` and return a bounded projection honoring the delivery bound. @@ -870,7 +853,7 @@ def _spill_for_delivery_bound( parsed = None else: parsed = result - bound = effective_delivery_token_limit * 4 + bound = selected_result_token_limit * 4 floor_bytes = min(bound, config.response_max_bytes) rendered: str | None try: @@ -937,7 +920,7 @@ def enforce_response_budget( artifact_dir: Path | None, config: OutputBudgetConfig, force_spill: bool = False, - effective_delivery_token_limit: int | None = None, + selected_result_token_limit: int | None = None, ) -> Any: """Return a bounded response of the same handler type. @@ -945,11 +928,11 @@ def enforce_response_budget( Artifact failure and missing-context cases fail closed without echoing the original payload. - ``effective_delivery_token_limit`` is the worst-case operative bound on the - downstream transport (e.g. Codex code-mode default ~10K). When set, payloads - whose estimated token count exceeds it are spilled even if they pass the - server-side exemption or response-byte ceilings, because the transport - cannot deliver them at full size. + ``selected_result_token_limit`` is the authoritative bound selected for the + current downstream transport. For ordinary calls this is the backend's + unnegotiated limit; protected recipe calls may supply an attested limit. + Payloads whose estimated token count exceeds it are spilled even if they + pass the server-side exemption or response-byte ceilings. """ try: original = _serialized(result) @@ -964,9 +947,9 @@ def enforce_response_budget( original_bytes = original.encode("utf-8") original_size = len(original_bytes) over_delivery_bound = ( - effective_delivery_token_limit is not None - and effective_delivery_token_limit > 0 - and _estimated_tokens(original_size) > effective_delivery_token_limit + selected_result_token_limit is not None + and selected_result_token_limit > 0 + and _estimated_tokens(original_size) > selected_result_token_limit ) exemption = RESPONSE_BACKSTOP_EXEMPTION_REGISTRY.get(tool_name) if exemption is not None: @@ -982,7 +965,7 @@ def enforce_response_budget( original_utf8_bytes=original_size, ) if over_delivery_bound: - assert effective_delivery_token_limit is not None # narrowed by over_delivery_bound + assert selected_result_token_limit is not None return _spill_for_delivery_bound( result, tool_name=tool_name, @@ -990,7 +973,7 @@ def enforce_response_budget( artifact_dir=artifact_dir, original=original, original_size=original_size, - effective_delivery_token_limit=effective_delivery_token_limit, + selected_result_token_limit=selected_result_token_limit, ) _emit_response_budget_event( "response_budget_exemption", @@ -1037,8 +1020,8 @@ def enforce_response_budget( } delivery_bound_bytes = ( - effective_delivery_token_limit * 4 - if effective_delivery_token_limit is not None and effective_delivery_token_limit > 0 + selected_result_token_limit * 4 + if selected_result_token_limit is not None and selected_result_token_limit > 0 else None ) projection_max_bytes = ( @@ -1130,7 +1113,7 @@ def shape_json_response( tool_name: str, artifact_dir: Path, config: OutputBudgetConfig, - effective_delivery_token_limit: int | None = None, + selected_result_token_limit: int | None = None, ) -> str: """Serialize a tool dict, spilling once it crosses the source threshold.""" rendered = json.dumps(payload) @@ -1142,7 +1125,7 @@ def shape_json_response( artifact_dir=artifact_dir, config=config, force_spill=True, - effective_delivery_token_limit=effective_delivery_token_limit, + selected_result_token_limit=selected_result_token_limit, ) return shaped if isinstance(shaped, str) else json.dumps(shaped) diff --git a/src/autoskillit/server/tools/_execution_helpers.py b/src/autoskillit/server/tools/_execution_helpers.py index 405587b179..2e7f917c7b 100644 --- a/src/autoskillit/server/tools/_execution_helpers.py +++ b/src/autoskillit/server/tools/_execution_helpers.py @@ -19,7 +19,7 @@ SpillSpec, SubprocessResult, get_logger, - resolve_effective_delivery_bound, + resolve_general_output_token_limit, resolve_temp_dir, spill_output, ) @@ -203,18 +203,18 @@ def shape_execution_response( if work_dir and Path(work_dir).is_absolute() else tool_ctx.temp_dir / tool_name ) - effective_delivery_token_limit: int | None = None + selected_result_token_limit: int | None = None backend = getattr(tool_ctx, "backend", None) caps = getattr(backend, "capabilities", None) if backend is not None else None if isinstance(caps, BackendCapabilities): - effective_delivery_token_limit = resolve_effective_delivery_bound(caps) + selected_result_token_limit = resolve_general_output_token_limit(caps) return shape_json_response( payload, tool_name=tool_name, artifact_dir=artifact_root, config=tool_ctx.config.output_budget, - effective_delivery_token_limit=effective_delivery_token_limit, + selected_result_token_limit=selected_result_token_limit, ) diff --git a/src/autoskillit/server/tools/_serve_helpers.py b/src/autoskillit/server/tools/_serve_helpers.py index a2ae609959..d7dab9cbeb 100644 --- a/src/autoskillit/server/tools/_serve_helpers.py +++ b/src/autoskillit/server/tools/_serve_helpers.py @@ -9,7 +9,6 @@ from __future__ import annotations -import hashlib import json from pathlib import Path from typing import TYPE_CHECKING, Any @@ -17,18 +16,11 @@ from autoskillit.core import ( RESPONSE_BACKSTOP_EXEMPTION_REGISTRY, RESPONSE_BACKSTOP_EXEMPTION_REGISTRY_DIGEST, - atomic_write, - get_logger, -) -from autoskillit.recipe import ( # noqa: F401 — canonical extractor reused by build_routing_edges_by_step default - _extract_routing_edges, - step_byte_ranges_from_yaml, ) if TYPE_CHECKING: from autoskillit.core import BackendCapabilities, CodingAgentBackend from autoskillit.pipeline.context import ToolContext - from autoskillit.recipe.schema import RecipeStep def build_backend_capabilities_map( @@ -193,530 +185,3 @@ def serve_recipe( if ctx.recipes is None: raise RuntimeError("serve_recipe() called with ctx.recipes=None") return ctx.recipes.load_and_validate(name, ctx.project_dir, **kwargs) - - -# --------------------------------------------------------------------------- -# Bounded envelope (Part B of #4304) -# --------------------------------------------------------------------------- - - -def _step_one_line_summary(step: RecipeStep) -> str: - """Return a compact one-line summary of *step* for the envelope skeleton. - - Used by the open_kitchen / load_recipe integration paths to populate - ``step_summaries`` before calling ``extract_step_skeleton``. Prefers - an explicit ``description`` field; falls back to a 160-char head of - the step's ``message`` or a tool/action signature. Always single-line. - """ - desc = getattr(step, "description", "") or "" - if desc.strip(): - return desc.strip().splitlines()[0][:160] - msg = getattr(step, "message", None) - if isinstance(msg, str) and msg.strip(): - return msg.strip().splitlines()[0][:160] - tool = getattr(step, "tool", None) - action = getattr(step, "action", None) - if tool: - return f"tool={tool}" - if action: - return f"action={action}" - return "" - - -def _compute_step_byte_ranges(content: str) -> dict[str, tuple[int, int]]: - """Return ``{step_name: (start, end)}`` UTF-8 byte offsets within *content*. - - Thin wrapper over - :func:`autoskillit.recipe.step_byte_ranges_from_yaml` that keeps the - ``_serve_helpers``-local symbol stable for downstream callers and - tests. The yaml-handling and ``isinstance(..., MappingNode)`` - fail-open logic lives in ``core.io`` so that yaml imports are - confined to the single canonical module (the package-wide invariant - enforced by ``tests/arch/test_subpackage_isolation.py`` and - ``tests/core/test_io.py::TestYamlConsolidationArchitecture``). - """ - return step_byte_ranges_from_yaml(content) - - -def extract_step_skeleton( - post_prune_step_names: list[str], - routing_edges_by_step: dict[str, list[tuple[str, str]]], - step_summaries: dict[str, str] | None = None, - byte_ranges: dict[str, tuple[int, int]] | None = None, -) -> dict[str, Any]: - """Build a compact step-flow skeleton from parsed post-prune step data. - - The skeleton is a list of per-step dicts (name + one-line summary + - outgoing routing edges + optional byte range). Combined with the - byte-range index in the persisted artifact, the orchestrator can - route without pulling a full step body, and pull-on-demand only the - step it is about to execute. - - *post_prune_step_names* — order-preserving list of step names (from - ``load_and_validate``'s ``post_prune_step_names`` result field). - *routing_edges_by_step* — name → list of (edge_type, target) tuples - derived from ``_extract_routing_edges`` for each step. - *step_summaries* — optional name → one-line summary override. A step - with no summary (or an empty one) omits the ``summary`` key - entirely rather than emitting it as ``""`` — saves bytes at scale - and matches the byte_range fail-open/omit convention below. - *byte_ranges* — optional name → (start, end) UTF-8 byte offsets - within the persisted ``content`` field. When provided for a - given step, the skeleton's per-step entry gains a - ``byte_range`` field carrying ``[start, end]``; steps absent - from the map get no such key (fail-open, matching the - edges/summary fallback pattern above). - """ - skeleton: list[dict[str, Any]] = [] - for name in post_prune_step_names: - edges = routing_edges_by_step.get(name) or [] - summary = (step_summaries or {}).get(name) or "" - entry: dict[str, Any] = { - "name": name, - "edges": [ - {"type": edge_type, "target": target} for edge_type, target in edges if target - ], - } - if summary: - entry["summary"] = summary - span = (byte_ranges or {}).get(name) - if span is not None: - entry["byte_range"] = list(span) - skeleton.append(entry) - return { - "step_count": len(skeleton), - "steps": skeleton, - } - - -def build_recipe_envelope( - payload: dict[str, Any], - *, - recipe_name: str, - artifact_path: str, - artifact_sha256: str, - skeleton: dict[str, Any], - bound_bytes: int, - producer_tool: str = "open_kitchen", -) -> dict[str, Any]: - """Build a bounded envelope that fits the smallest backend delivery bound. - - The envelope is the orchestrator-visible replacement for the full - recipe payload when the payload exceeds a backend's effective delivery - token limit. It carries: - - - routing metadata (``success``, ``kitchen``, ``version``, ``valid``, - ``dispatch_feasible``); - - verbatim priority content (``orchestration_rules``, - ``stop_step_semantics``, ``errors``, ``warnings``, ``hooks``); - - ingredients schema (``ingredients_table``, ``suggestions``); - - the step-flow skeleton (post-prune step names, summaries, routing - edges) — enough for the orchestrator to route between steps without - pulling a full step body; - - the post-prune step list and routing edge list (so callers that - only need the routing graph don't need to parse the skeleton); - - a pull reference pointing to the deterministic artifact path - (overwritten by every open_kitchen / load_recipe call) and the - ``get_recipe_section`` MCP tool name. - - The envelope omits the full ``content`` field. The orchestrator pulls - each step's body via ``get_recipe_section(section=)`` at - execution time, bounded to the delivery limit and chunked with a - continuation token for steps larger than one chunk. - - ``bound_bytes`` is the effective delivery byte ceiling (smallest - backend bound × 4). Large string fields like ``orchestration_rules`` - are projected to fit alongside the skeleton + pull reference via a - two-phase allocator: phase 1 drops (omits, never emits as ``""``) - any key that can't even afford its JSON-key overhead under the - current ``remaining`` budget; phase 2 water-fills the leftover - content budget across survivors so neither priority field can - monopolize the pool. Truncation goes through ``_safe_utf8_truncate`` - so a multi-byte UTF-8 codepoint is never split into an - undecodable byte sequence. ``recipe_name`` is required (keyword-only) - to match the sibling-function convention in this module. - """ - envelope: dict[str, Any] = {} - envelope_bytes = 0 - serialized = json.dumps(payload.get("success", True)).encode("utf-8") - envelope_bytes += len(serialized) - envelope["success"] = payload.get("success", True) - for key in ( - "kitchen", - "version", - "valid", - "dispatch_feasible", - "errors", - "warnings", - "hooks", - "post_prune_step_names", - "post_prune_routing_edges", - "requires_packs", - "requires_features", - ): - if key in payload and payload[key] is not None: - envelope[key] = payload[key] - envelope_bytes += len( - json.dumps({key: payload[key]}, ensure_ascii=False).encode("utf-8") - ) - - # Fixed-size overhead: skeleton JSON + pull reference + delivery_bound_spill. - skeleton_overhead = len( - json.dumps({"step_flow_skeleton": skeleton}, ensure_ascii=False).encode("utf-8") - ) - pull_overhead = len( - json.dumps( - { - "recipe_pull": { - "recipe_name": recipe_name, - "producer_tool": producer_tool, - "artifact_path": artifact_path, - "sha256": artifact_sha256, - "pull_tool": "get_recipe_section", - }, - "delivery_bound_spill": True, - }, - ensure_ascii=False, - ).encode("utf-8") - ) - - remaining = max( - 0, - bound_bytes - skeleton_overhead - pull_overhead - envelope_bytes - 64, - ) - - def _project_priority_strings(keys: tuple[str, ...]) -> None: - nonlocal remaining - candidates = [k for k in keys if isinstance(payload.get(k), str) and payload.get(k)] - if not candidates: - return - - overhead = { - k: len(json.dumps({k: ""}, ensure_ascii=False).encode("utf-8")) for k in candidates - } - # Phase 1: a key survives only if its JSON-key overhead plus at least - # one content byte fits what's left, checked in priority (declaration) - # order. Keys that don't clear this bar are omitted entirely — never - # emitted as "". - present: list[str] = [] - budget = remaining - for k in candidates: - if overhead[k] < budget: - present.append(k) - budget -= overhead[k] - if not present: - return - - # Phase 2: water-fill the leftover content budget evenly across - # survivors, round by round. A key whose value is shorter than its - # share drops out and its unused share carries over to the *other* - # survivors in the next round — no single key can monopolize the - # pool the way a sequential first-come-first-served allocator can. - content_budget = remaining - sum(overhead[k] for k in present) - lengths = {k: len(payload[k].encode("utf-8")) for k in present} - alloc: dict[str, int] = dict.fromkeys(present, 0) - active = list(present) - pool = content_budget - while active and pool > 0: - share, extra = divmod(pool, len(active)) - still_active: list[str] = [] - for index, key in enumerate(active): - give = share + (1 if index < extra else 0) - need = lengths[key] - alloc[key] - take = min(give, need, pool) - alloc[key] += take - pool -= take - if alloc[key] < lengths[key]: - still_active.append(key) - active = still_active - - for key in present: - take = alloc[key] - if take <= 0: - continue # no content budget survives for this field; omit it - value_bytes = payload[key].encode("utf-8") - if len(value_bytes) <= take: - envelope[key] = payload[key] - remaining -= len(value_bytes) + overhead[key] - else: - envelope[key] = _safe_utf8_truncate(value_bytes[:take]) - remaining -= take + overhead[key] - get_logger(__name__).warning( - "recipe_envelope_priority_field_truncated", - recipe_name=recipe_name, - field=key, - alloc_bytes=take, - ) - - _project_priority_strings(("orchestration_rules", "stop_step_semantics")) - - # ingredients_table and suggestions are deprioritized — serialize them - # only if budget allows; otherwise omit. The orchestrator can pull the - # full ingredients_table via ``get_recipe_section(section="ingredients_table")``. - for key in ("ingredients_table", "suggestions"): - value = payload.get(key) - if value is None: - continue - serialized_value = json.dumps(value, ensure_ascii=False).encode("utf-8") - if len(serialized_value) + 32 <= remaining: - envelope[key] = value - remaining -= len(serialized_value) + 32 - - envelope["step_flow_skeleton"] = skeleton - envelope["recipe_pull"] = { - "recipe_name": recipe_name, - "producer_tool": producer_tool, - "artifact_path": artifact_path, - "sha256": artifact_sha256, - "pull_tool": "get_recipe_section", - } - envelope["delivery_bound_spill"] = True - if ( - len(json.dumps(envelope, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) - <= bound_bytes - ): - return envelope - - fallback_candidates: tuple[dict[str, Any], ...] = ( - { - "success": payload.get("success", True), - "step_flow_skeleton": skeleton, - "recipe_pull": { - "recipe_name": recipe_name, - "producer_tool": producer_tool, - "artifact_path": artifact_path, - "sha256": artifact_sha256, - "pull_tool": "get_recipe_section", - }, - "delivery_bound_spill": True, - }, - { - "success": False, - "error": "recipe_envelope_exceeds_delivery_bound", - "recipe_pull": { - "recipe_name": recipe_name, - "producer_tool": producer_tool, - "artifact_path": artifact_path, - "sha256": artifact_sha256, - "pull_tool": "get_recipe_section", - }, - }, - {"success": False, "error": "recipe_envelope_exceeds_delivery_bound"}, - {}, - ) - for fallback in fallback_candidates: - if ( - len(json.dumps(fallback, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) - <= bound_bytes - ): - return fallback - raise ValueError("delivery bound is too small for a JSON object") - - -def _safe_utf8_truncate(data: bytes) -> str: - """Decode *data* as UTF-8, backing off byte-by-byte from the end on failure. - - Handles both a dangling continuation byte and a dangling multi-byte lead - byte — the two ways a naive byte-index slice can land mid-codepoint. - Used by the envelope priority-field allocator to guarantee that a - multi-byte codepoint is never split into an undecodable byte sequence. - """ - while data: - try: - return data.decode("utf-8") - except UnicodeDecodeError as exc: - data = data[: exc.start] - return "" - - -def persist_recipe_artifact( - artifact_dir: Path, - *, - tool_name: str, - recipe_name: str, - payload: dict[str, Any], -) -> tuple[str, str]: - """Atomically persist the full recipe payload to the deterministic path. - - Returns (artifact_path, sha256) for inclusion in the envelope's - ``recipe_pull`` block. Uses ``atomic_write`` so concurrent open_kitchen - / load_recipe calls do not see a half-written file. - - The path is deterministic per (tool, recipe_name) so the pull tool - can reconstruct it from ``tool_ctx.recipe_name`` without the caller - having to thread it through every surface. Re-opening the same recipe - overwrites the artifact (idempotent). - """ - from autoskillit.server._response_budget import _recipe_artifact_path # circular-break - - path = _recipe_artifact_path(artifact_dir, tool_name, recipe_name) - serialized = json.dumps(payload, ensure_ascii=False) - atomic_write(path, serialized) - sha256 = hashlib.sha256(serialized.encode("utf-8")).hexdigest() - return str(path.resolve()), sha256 - - -def build_step_summaries(active_recipe_steps: Any) -> dict[str, str]: - """Build a {step_name: one_line_summary} dict from the parsed Recipe.steps. - - Falls back to an empty string per step if the parsed structure is - unavailable (e.g. the recipe was loaded but post-prune filtering - stripped the steps out, or the serve path passed no Recipe object). - """ - if not isinstance(active_recipe_steps, dict) or not active_recipe_steps: - return {} - summaries: dict[str, str] = {} - for name, step in active_recipe_steps.items(): - if not isinstance(name, str) or not name: - continue - summaries[name] = _step_one_line_summary(step) - return summaries - - -def build_routing_edges_by_step( - active_recipe_steps: Any, - *, - edge_extractor: Any = _extract_routing_edges, -) -> dict[str, list[tuple[str, str]]]: - """Build a {step_name: [(edge_type, target), ...]} dict via edge_extractor. - - ``edge_extractor`` is the existing ``_extract_routing_edges`` callable - from ``recipe/_analysis_graph.py``. We avoid importing it here to keep - the helper importable from lightweight contexts; the caller passes the - callable in. Steps whose extractor returns nothing map to an empty - list (not omitted) so callers don't have to ``.get(name) or []``. - """ - if not isinstance(active_recipe_steps, dict) or not active_recipe_steps: - return {} - edges_by_step: dict[str, list[tuple[str, str]]] = {} - for name, step in active_recipe_steps.items(): - if not isinstance(name, str) or not name: - continue - extracted = edge_extractor(step) if edge_extractor is not None else [] - edges_by_step[name] = [ - (edge.edge_type, edge.target) - for edge in (extracted or []) - if getattr(edge, "target", None) - ] - return edges_by_step - - -def maybe_envelope_recipe_response( - payload: dict[str, Any], - *, - tool_name: str, - recipe_name: str, - tool_ctx: ToolContext, - effective_delivery_token_limit: int | None, -) -> dict[str, Any]: - """Conditionally replace a recipe payload with a bounded envelope. - - Persists the full payload to the deterministic artifact path - UNCONDITIONALLY (gated only by ``temp_dir`` being a ``Path``), so the - ``get_recipe_section`` pull tool always has a backing store. If the - payload's estimated token count exceeds - ``effective_delivery_token_limit``, returns ``build_recipe_envelope(...)`` - so the orchestrator can pull each step's body on demand. - - Otherwise returns the payload unchanged (Claude backend path: the - full payload fits inline; backward compatible) — with the artifact - already on disk for the pull tool. - - On persistence failure: returns the original payload unchanged — - the caller is then subject to ``track_response_size``'s spill path - (which is more permissive but loses the pull guarantee). This is - a fail-open at the persistence layer; the spill path itself remains - fail-closed for shape violations. - - Persistence happens exactly once per call: the single early call's - return value is reused by ``build_recipe_envelope`` via its - ``artifact_path`` / ``artifact_sha256`` keyword args. - """ - artifact_dir = tool_ctx.temp_dir - if not isinstance(artifact_dir, Path): - return payload - - try: - artifact_path, artifact_sha256 = persist_recipe_artifact( - artifact_dir, - tool_name=tool_name, - recipe_name=recipe_name, - payload=payload, - ) - except OSError: - return payload - - if effective_delivery_token_limit is None or effective_delivery_token_limit <= 0: - return payload - - serialized = json.dumps(payload, ensure_ascii=False) - estimated_tokens = (len(serialized.encode("utf-8")) + 3) // 4 - if estimated_tokens <= effective_delivery_token_limit: - return payload - - # existing skeleton/edges construction (post_prune_names, summaries, - # edges via build_routing_edges_by_step) feeds extract_step_skeleton - # plus the new byte-range field computed from the persisted - # ``content`` payload field. - post_prune_raw = payload.get("post_prune_step_names") or [] - if not isinstance(post_prune_raw, list): - post_prune_raw = [] - post_prune_names = [n for n in post_prune_raw if isinstance(n, str)] - # ``active_recipe_steps`` is the kitchen's currently-open recipe — NOT - # necessarily the recipe being delivered here. When the caller - # (e.g. ``load_recipe``) targets a different recipe than the active - # one, mixing the other recipe's parsed step summary/edges into the - # skeleton bloats the envelope past the delivery bound even for a - # small payload. Use the active recipe's parsed steps only when it - # matches the payload's recipe_name; otherwise emit a name-only - # skeleton (orchestrator still has the artifact + pull reference). - active_recipe_steps: dict[str, Any] | None = None - active_recipe_name = getattr(tool_ctx, "recipe_name", "") or "" - if active_recipe_name and active_recipe_name == recipe_name: - active_recipe_steps = tool_ctx.active_recipe_steps - summaries = build_step_summaries(active_recipe_steps) - edges = build_routing_edges_by_step(active_recipe_steps) - byte_ranges = _compute_step_byte_ranges(payload.get("content") or "") - bound_bytes = effective_delivery_token_limit * 4 - skeleton = extract_step_skeleton(post_prune_names, edges, summaries, byte_ranges=byte_ranges) - - def _pullable_skeleton_size(candidate: dict[str, Any]) -> int: - pullable = { - "success": payload.get("success", True), - "step_flow_skeleton": candidate, - "recipe_pull": { - "recipe_name": recipe_name, - "producer_tool": tool_name, - "artifact_path": artifact_path, - "sha256": artifact_sha256, - "pull_tool": "get_recipe_section", - }, - "delivery_bound_spill": True, - } - return len(json.dumps(pullable, ensure_ascii=False, separators=(",", ":")).encode("utf-8")) - - if _pullable_skeleton_size(skeleton) > bound_bytes: - for summary_limit in (120, 80, 64, 48, 32, 24, 16, 8, 0): - bounded_summaries = ( - {name: summary[:summary_limit] for name, summary in summaries.items()} - if summary_limit - else {} - ) - skeleton = extract_step_skeleton( - post_prune_names, - edges, - bounded_summaries, - byte_ranges=byte_ranges, - ) - if _pullable_skeleton_size(skeleton) <= bound_bytes: - break - - return build_recipe_envelope( - payload, - artifact_path=artifact_path, - artifact_sha256=artifact_sha256, - skeleton=skeleton, - bound_bytes=bound_bytes, - recipe_name=recipe_name, - producer_tool=tool_name, - ) diff --git a/src/autoskillit/server/tools/tools_kitchen.py b/src/autoskillit/server/tools/tools_kitchen.py index d6d67d2d0b..6644fa48fb 100644 --- a/src/autoskillit/server/tools/tools_kitchen.py +++ b/src/autoskillit/server/tools/tools_kitchen.py @@ -7,7 +7,7 @@ import os from datetime import UTC, datetime from pathlib import Path -from typing import TYPE_CHECKING, Any, TypedDict +from typing import TYPE_CHECKING, Any, TypedDict, cast if TYPE_CHECKING: from autoskillit.config.settings import OutputBudgetConfig, QuotaGuardConfig @@ -27,9 +27,9 @@ from autoskillit.core import ( DISPATCH_ID_ENV_VAR, PIPELINE_FORBIDDEN_TOOLS, - BackendCapabilities, CapabilityResolutionDetail, ProcessStaleError, + RecipeDeliveryRequest, _collect_disabled_feature_tags, atomic_write, clear_kitchens_for_pid, @@ -43,7 +43,6 @@ read_active_kitchens_registry, read_marker, register_active_kitchen, - resolve_effective_delivery_bound, resolve_kitchen_id, sweep_stale_markers, unregister_active_kitchen, @@ -69,6 +68,12 @@ strip_ingredients_only_keys, ) from autoskillit.server._notify import track_response_size +from autoskillit.server._recipe_delivery import ( + document_recipe_delivery_contract, + enforce_recipe_resource_response, + finalize_recipe_delivery, + retire_recipe_artifacts, +) from autoskillit.server.tools._authority_feedback import ( build_authority_clobber_warnings, build_authority_rejection_envelope, @@ -87,7 +92,6 @@ from autoskillit.server.tools._serve_helpers import ( build_backend_capabilities_map, build_open_kitchen_recipe_payload, - maybe_envelope_recipe_response, render_served_response, response_backstop_tool_meta, serve_recipe, @@ -575,6 +579,13 @@ def _close_kitchen_handler() -> None: unregister_active_kitchen(ctx.kitchen_id) except Exception: logger.warning("close_kitchen_registry_failed", exc_info=True) + if ( + isinstance(ctx.temp_dir, Path) + and isinstance(ctx.kitchen_id, str) + and ctx.kitchen_id + and not retire_recipe_artifacts(ctx.temp_dir, kitchen_id=ctx.kitchen_id) + ): + logger.warning("close_kitchen_recipe_artifact_retirement_failed") ctx.active_recipe_packs = None ctx.active_recipe_features = None ctx.active_recipe_steps = None @@ -730,7 +741,13 @@ def get_recipe(name: str) -> str: "infeasible_steps": result.get("infeasible_steps", []), } ) - return result.get("content", json.dumps({"error": "Recipe composition failed."})) + finalized = finalize_recipe_delivery( + result, + surface="get_recipe", + recipe_name=name, + tool_ctx=ctx, + ) + return enforce_recipe_resource_response(finalized, tool_ctx=ctx) def _build_tool_category_listing( @@ -770,12 +787,14 @@ def _check_override_keys( annotations={"readOnlyHint": True}, meta=response_backstop_tool_meta("open_kitchen", always_load=True), ) +@document_recipe_delivery_contract @_cancellation_shield() @track_response_size("open_kitchen") async def open_kitchen( name: str | None = None, overrides: dict[str, str] | None = None, ingredients_only: bool = False, + delivery_request: RecipeDeliveryRequest | None = None, ctx: Context = CurrentContext(), ) -> str: """Open the AutoSkillit kitchen for service. @@ -1063,26 +1082,15 @@ async def open_kitchen( tool_ctx.session_serve_overrides = dict(overrides) tool_ctx.session_serve_defer_unresolved = not bool(overrides) if not ingredients_only: - _backend_caps = ( - tool_ctx.backend.capabilities - if tool_ctx.backend is not None - and isinstance( - getattr(tool_ctx.backend, "capabilities", None), - BackendCapabilities, - ) - else None - ) - _edtl = ( - resolve_effective_delivery_bound(_backend_caps) - if _backend_caps is not None - else None - ) - result = maybe_envelope_recipe_response( - result, - tool_name="open_kitchen", - recipe_name=name, - tool_ctx=tool_ctx, - effective_delivery_token_limit=_edtl, + return cast( + str, + finalize_recipe_delivery( + result, + surface="open_kitchen_deferred_recall", + recipe_name=name, + tool_ctx=tool_ctx, + delivery_request=delivery_request, + ), ) return render_served_response(result) try: @@ -1237,26 +1245,15 @@ async def open_kitchen( return _validation_err if not ingredients_only: - _backend_caps_normal = ( - tool_ctx.backend.capabilities - if tool_ctx.backend is not None - and isinstance( - getattr(tool_ctx.backend, "capabilities", None), - BackendCapabilities, - ) - else None - ) - _edtl_normal = ( - resolve_effective_delivery_bound(_backend_caps_normal) - if _backend_caps_normal is not None - else None - ) - result = maybe_envelope_recipe_response( - result, - tool_name="open_kitchen", - recipe_name=name, - tool_ctx=tool_ctx, - effective_delivery_token_limit=_edtl_normal, + return cast( + str, + finalize_recipe_delivery( + result, + surface="open_kitchen", + recipe_name=name, + tool_ctx=tool_ctx, + delivery_request=delivery_request, + ), ) return render_served_response(result) diff --git a/src/autoskillit/server/tools/tools_recipe.py b/src/autoskillit/server/tools/tools_recipe.py index a862f2c0ba..46a495c6b9 100644 --- a/src/autoskillit/server/tools/tools_recipe.py +++ b/src/autoskillit/server/tools/tools_recipe.py @@ -2,10 +2,9 @@ from __future__ import annotations -import hashlib import json from pathlib import Path -from typing import Any +from typing import Any, cast import structlog from fastmcp import Context @@ -19,10 +18,11 @@ ) from autoskillit.core import ( BackendCapabilities, + RecipeDeliveryRequest, fast_dumps, get_logger, load_yaml, - resolve_effective_delivery_bound, + resolve_general_output_token_limit, temp_dir_display_str, ) # noqa: F401 from autoskillit.pipeline import GATED_TOOLS, UNGATED_TOOLS # noqa: F401 @@ -34,6 +34,16 @@ strip_ingredients_only_keys, ) from autoskillit.server._notify import _notify, track_response_size +from autoskillit.server._recipe_delivery import ( + RecipeArtifactError, + RecipeArtifactGeneration, + document_recipe_delivery_contract, + finalize_recipe_delivery, + load_recipe_artifact, + persist_recipe_artifact, + recipe_pull_producers, + recipe_recreation_producers, +) from autoskillit.server._state import _get_ctx_or_none from autoskillit.server.tools._authority_feedback import build_authority_clobber_warnings from autoskillit.server.tools._auto_overrides import ( @@ -45,8 +55,6 @@ from autoskillit.server.tools._serve_helpers import ( build_backend_capabilities_map, build_open_kitchen_recipe_payload, - maybe_envelope_recipe_response, - persist_recipe_artifact, render_served_response, response_backstop_tool_meta, serve_recipe, @@ -65,7 +73,12 @@ def __init__(self, code: str, detail: str) -> None: def _bounded_recipe_section_response( - section: str, content: str, *, part: int, bound_bytes: int + section: str, + content: str, + *, + part: int, + bound_bytes: int, + generation: RecipeArtifactGeneration, ) -> str: """Render one continuation chunk whose serialized UTF-8 size fits the bound.""" @@ -76,10 +89,14 @@ def _render(start: int, end: int) -> str: "section": section, "content": content[start:end], "has_more": has_more, + "byte_start": len(content[:start].encode("utf-8")), + "byte_end": len(content[:end].encode("utf-8")), + "byte_total": len(content.encode("utf-8")), + "payload_sha256": generation.payload_sha256, + "body_sha256": generation.body_sha256, } if has_more: response["next_part"] = chunk_index + 1 - response["total_size"] = len(content) return json.dumps(response, ensure_ascii=False) start = 0 @@ -108,14 +125,7 @@ def _render(start: int, end: int) -> str: return rendered start = end if start >= len(content): - return json.dumps( - { - "success": True, - "section": section, - "content": "", - "has_more": False, - } - ) + return _render(start, start) raise AssertionError("continuation loop must return") @@ -164,10 +174,14 @@ async def list_recipes() -> str: annotations={"readOnlyHint": True}, meta=response_backstop_tool_meta("load_recipe"), ) +@document_recipe_delivery_contract @_cancellation_shield() @track_response_size("load_recipe") async def load_recipe( - name: str, overrides: dict[str, str] | None = None, ingredients_only: bool = False + name: str, + overrides: dict[str, str] | None = None, + ingredients_only: bool = False, + delivery_request: RecipeDeliveryRequest | None = None, ) -> str: """Load a recipe by name and return its raw YAML content. @@ -400,26 +414,15 @@ async def load_recipe( ) return _validation_err if not ingredients_only: - _lr_backend_caps = ( - tool_ctx.backend.capabilities - if tool_ctx.backend is not None - and isinstance( - getattr(tool_ctx.backend, "capabilities", None), - BackendCapabilities, - ) - else None - ) - _lr_edtl = ( - resolve_effective_delivery_bound(_lr_backend_caps) - if _lr_backend_caps is not None - else None - ) - result = maybe_envelope_recipe_response( - result, - tool_name="load_recipe", - recipe_name=name, - tool_ctx=tool_ctx, - effective_delivery_token_limit=_lr_edtl, + return cast( + str, + finalize_recipe_delivery( + result, + surface="load_recipe", + recipe_name=name, + tool_ctx=tool_ctx, + delivery_request=delivery_request, + ), ) return render_served_response(result) except Exception as exc: @@ -432,11 +435,16 @@ async def load_recipe( @track_response_size("get_recipe_section") async def get_recipe_section( section: str, + recipe_name: str, + producer_tool: str, + descriptor_version: int, + schema_version: int, + payload_sha256: str, + artifact_blob_sha256: str, + artifact_blob_size_bytes: int, + body_sha256: str, + body_size_bytes: int, part: int = 0, - recipe_name: str | None = None, - producer_tool: str | None = None, - artifact_path: str | None = None, - artifact_sha256: str | None = None, ) -> str: """Retrieve a recipe step or section from the persisted recipe artifact. @@ -461,9 +469,11 @@ async def get_recipe_section( ``recipe_pull.recipe_name`` field. producer_tool: Producer identity copied from the envelope's ``recipe_pull.producer_tool`` field. - artifact_sha256: Artifact digest copied from the envelope's - ``recipe_pull.sha256`` field. Required to bind the pull to - the exact composition that produced the envelope. + payload_sha256: Domain-labelled semantic payload identity. + artifact_blob_sha256: Digest of the exact persisted blob bytes. + artifact_blob_size_bytes: Exact persisted blob byte size. + body_sha256: Digest of the recipe body bytes. + body_size_bytes: Exact recipe body byte size. Returns: JSON with ``success``, ``section``, ``content``, ``has_more``, @@ -481,30 +491,39 @@ async def get_recipe_section( if tool_ctx is None or tool_ctx.recipes is None: return json.dumps({"success": False, "error": "kitchen not open"}) - if not recipe_name or producer_tool is None or not artifact_sha256: + if not recipe_name or not producer_tool: return json.dumps({"success": False, "error": "recipe_artifact_identity_required"}) requested_recipe_name = recipe_name - if producer_tool not in {"open_kitchen", "load_recipe"}: + if producer_tool not in recipe_pull_producers(): return json.dumps({"success": False, "error": "invalid_recipe_artifact_identity"}) artifact_dir = getattr(tool_ctx, "temp_dir", None) if not isinstance(artifact_dir, Path): return json.dumps({"success": False, "error": "kitchen temp_dir not available"}) - from autoskillit.server._response_budget import _recipe_artifact_path # circular-break - - resolved_artifact_path = _recipe_artifact_path( - artifact_dir, producer_tool, requested_recipe_name + identity = RecipeArtifactGeneration( + producer_tool=producer_tool, + recipe_name=requested_recipe_name, + descriptor_version=descriptor_version, + schema_version=schema_version, + payload_sha256=payload_sha256, + artifact_blob_sha256=artifact_blob_sha256, + artifact_blob_size_bytes=artifact_blob_size_bytes, + body_sha256=body_sha256, + body_size_bytes=body_size_bytes, ) - if ( - artifact_path is not None - and Path(artifact_path).resolve() != resolved_artifact_path.resolve() - ): + if not identity.has_valid_read_bounds(): return json.dumps({"success": False, "error": "invalid_recipe_artifact_identity"}) - if not resolved_artifact_path.exists(): - if producer_tool != "open_kitchen": + try: + persisted = load_recipe_artifact( + artifact_dir, + kitchen_id=tool_ctx.kitchen_id, + identity=identity, + ) + except RecipeArtifactError: + if producer_tool not in recipe_recreation_producers(): return json.dumps({"success": False, "error": "recipe_artifact_unavailable"}) # Recreation path: re-invoke the same serve pipeline that # built the artifact originally. This handles the case @@ -547,16 +566,20 @@ async def get_recipe_section( "detail": "recreation returned invalid recipe", } ) - _recreate = build_open_kitchen_recipe_payload(_recreate, version=__version__) + if producer_tool == "open_kitchen": + _recreate = build_open_kitchen_recipe_payload( + _recreate, version=__version__ + ) try: - persist_recipe_artifact( + recreated_generation = persist_recipe_artifact( artifact_dir, - tool_name=producer_tool, + kitchen_id=tool_ctx.kitchen_id, + producer_tool=producer_tool, recipe_name=requested_recipe_name, payload=_recreate, ) - except OSError: + except (OSError, RecipeArtifactError): return json.dumps( { "success": False, @@ -564,6 +587,13 @@ async def get_recipe_section( "detail": "recreation write failed", } ) + if recreated_generation != identity: + return json.dumps( + { + "success": False, + "error": "invalid_recipe_artifact_identity", + } + ) except Exception as exc: logger.warning( "get_recipe_section_recreate_failed", @@ -580,31 +610,20 @@ async def get_recipe_section( } ) - # Read the persisted full payload. The pull tool returns a - # bounded chunk from the requested section of the recipe YAML. - try: - persisted_raw = resolved_artifact_path.read_text(encoding="utf-8") - if hashlib.sha256(persisted_raw.encode("utf-8")).hexdigest() != artifact_sha256: + try: + persisted = load_recipe_artifact( + artifact_dir, + kitchen_id=tool_ctx.kitchen_id, + identity=identity, + ) + except RecipeArtifactError as exc: return json.dumps( - {"success": False, "error": "invalid_recipe_artifact_identity"} + { + "success": False, + "error": "recipe_artifact_unavailable", + "detail": str(exc), + } ) - persisted = json.loads(persisted_raw) - except (OSError, json.JSONDecodeError) as exc: - return json.dumps( - { - "success": False, - "error": "recipe_artifact_unavailable", - "detail": f"{type(exc).__name__}: {exc}", - } - ) - if not isinstance(persisted, dict): - return json.dumps( - { - "success": False, - "error": "recipe_artifact_unavailable", - "detail": "persisted recipe artifact is not a mapping", - } - ) content = "" if section == "content": @@ -669,16 +688,27 @@ async def get_recipe_section( else None ) bound_tokens = ( - resolve_effective_delivery_bound(_backend_caps) + resolve_general_output_token_limit(_backend_caps) if _backend_caps is not None else 10_000 ) - bound_bytes = bound_tokens * 4 + bound_bytes = bound_tokens + configured_response_max = getattr( + getattr(tool_ctx.config, "output_budget", None), + "response_max_bytes", + None, + ) + if isinstance(configured_response_max, int) and configured_response_max > 0: + bound_bytes = min(bound_bytes, configured_response_max) if part < 0: part = 0 return _bounded_recipe_section_response( - section, content, part=part, bound_bytes=bound_bytes + section, + content, + part=part, + bound_bytes=bound_bytes, + generation=identity, ) except Exception as exc: logger.error("get_recipe_section unhandled exception", exc_info=True) diff --git a/src/autoskillit/skills_extended/resolve-review/SKILL.md b/src/autoskillit/skills_extended/resolve-review/SKILL.md index 8eb6443610..ccdd7c9e07 100644 --- a/src/autoskillit/skills_extended/resolve-review/SKILL.md +++ b/src/autoskillit/skills_extended/resolve-review/SKILL.md @@ -488,6 +488,7 @@ classified `REJECT` with `category: "arch_violation"`. | Interactive ordering gate | `test_interactive_ordering_gate.py` | Interactive launch sites that skip `assert_interactive_ordering()` before `_session_launch` | | Kitchen guard scoping | `test_kitchen_guard_scoping.py` | `any_kitchen_open()` call sites not passing `project_path` — must use scoped check | | Model identity contract | `test_model_identity_contract.py` | `detect_model_drift()` using raw string comparison instead of `normalize_model_id()` and `_models_match()`, missing `profile_name` suppression guard with `normalize_model_id` normalization, or `profile_name` guard calling `_is_non_anthropic` more than once or on `configured_model` instead of `observed_model` (over-restriction that kills the guard for the standard Anthropic-configured + non-Anthropic-observed production path) | +| Recipe delivery provenance | `test_recipe_delivery_provenance.py` | Caller claims, host observations, selected outer limits, history retention, and measured byte ceilings must remain separately sourced | | No hardcoded model IDs in translation tests | `test_no_hardcoded_model_ids_in_translation_tests.py` | String literal alias-resolved model IDs in `assert` comparisons in `test_model_translation.py` — assertions must reference `CODEX_MODEL_ALIASES[key]` to prevent co-authoring of wrong values | | No error-dict returns | `test_no_error_dict_return.py` | `load_and_validate()` returning `{"error": ...}` dict — errors must propagate via exceptions | | No hook tracker writes | `test_tracker_write_provenance.py` | Hook scripts referencing `pipeline_tracker` and performing file writes — step completion is server-authoritative, only `server/tools/` may mutate tracker state | diff --git a/tests/_test_filter.py b/tests/_test_filter.py index 243a350182..4782fe21f5 100644 --- a/tests/_test_filter.py +++ b/tests/_test_filter.py @@ -235,6 +235,7 @@ class ImportContext(enum.StrEnum): ), "_type_results_execution": frozenset({"core", "execution", "server", "pipeline"}), "_type_backend": frozenset({"core", "execution", "cli", "recipe", "server", "workspace"}), + "_type_recipe_delivery": frozenset({"core", "execution", "server"}), "_type_dispatch_identity": frozenset({"core", "fleet", "execution"}), "_type_figure_spec": frozenset({"core", "report"}), "_type_session_env": frozenset({"core", "cli"}), @@ -787,7 +788,6 @@ class ImportContext(enum.StrEnum): "server/test_admission_dispatch_agreement.py", "server/test_pipeline_deps_derivation.py", "server/test_pipeline_tracker.py", - "server/test_tools_recipe_pull.py", # CLI file-level entries (6 of 38 import autoskillit.recipe): "cli/test_cli_prompts.py", "cli/test_l3_orchestrator_prompt.py", @@ -943,6 +943,7 @@ class ImportContext(enum.StrEnum): } ), "_llm_triage": frozenset({"test_llm_triage.py", "server"}), + "_recipe_delivery_framing": frozenset({"server", "hooks", "infra"}), "_test_filter": frozenset({"arch", "infra", "contracts"}), "smoke_utils": frozenset({"test_smoke_utils.py", "recipe", "smoke_utils"}), "version": frozenset({"test_version.py", "server", "cli"}), @@ -1046,6 +1047,7 @@ class ImportContext(enum.StrEnum): "hook_registry": frozenset({"hooks"}), "planner": frozenset({"planner"}), "_llm_triage": frozenset({"test_llm_triage.py"}), + "_recipe_delivery_framing": frozenset({"server", "hooks", "infra"}), "smoke_utils": frozenset({"test_smoke_utils.py"}), "version": frozenset({"test_version.py"}), "_test_filter": frozenset({"arch", "contracts"}), diff --git a/tests/arch/AGENTS.md b/tests/arch/AGENTS.md index 3dee0ee30d..aa21ceb774 100644 --- a/tests/arch/AGENTS.md +++ b/tests/arch/AGENTS.md @@ -107,10 +107,11 @@ AST enforcement, sub-package layer contracts, and architectural invariant tests. | `test_cross_skill_handoff_paths.py` | Architectural test: cross-skill artifact read paths (e.g., resolve-review reading review-pr outputs) must be compatible with the producer skill's recipe output_dir | | `test_skill_family_parity.py` | Parametrized parity tests: every GITHUB_API_SKILL_FAMILIES member must satisfy its family's required API pattern contracts | | `test_model_identity_contract.py` | AST guard: detect_model_drift must use normalize_model_id and _models_match, and must have a profile_name suppression guard with normalize_model_id — prevents raw-string false positives and profile-routed false MODEL_DRIFT | +| `test_recipe_delivery_provenance.py` | AST guard: caller-requested, host-observed, selected-result, history-retention, and measured-byte authorities remain independent | | `test_helpers_exports.py` | Asserts shared test helpers export required symbols (strip_markdown_code_regions) | | `test_hook_efficacy_contract.py` | Arch contracts: HookDef mechanism presence, enforcement_strength backend coverage, codex hard-claim consistency, codex_status/strength coherence | | `test_hook_env_var_authority.py` | AST guard: every hook script that reads `AUTOSKILLIT_PROVIDER_PROFILE` must also read `AUTOSKILLIT_AGENT_BACKEND` so backend identity is checked independently of provider routing | -| `test_serve_surface_registry.py` | Structural tests for `SERVE_SURFACES` frozenset: membership completeness, AST guard that `load_and_validate` is only called from `_serve_helpers.py` | +| `test_serve_surface_registry.py` | Structural tests for the typed recipe-delivery surface registry and the `load_and_validate` call boundary | | `test_session_type_exhaustive.py` | AST guard: _apply_session_type_visibility must use exhaustive match/assert_never dispatch | | `test_input_spec_type_exhaustive.py` | AST guard: InputSpec type dispatch (match/assert_never) and YAML coverage of VALID_INPUT_SPEC_TYPES | | `test_notification_fallback_coverage.py` | AST guard: every ctx.enable_components call in server/tools/ must have a non-notification fallback or be in an exempt session-scoped unlock function | diff --git a/tests/arch/test_recipe_delivery_provenance.py b/tests/arch/test_recipe_delivery_provenance.py new file mode 100644 index 0000000000..50573127ad --- /dev/null +++ b/tests/arch/test_recipe_delivery_provenance.py @@ -0,0 +1,74 @@ +"""AST guards for independent recipe-delivery provenance domains.""" + +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +pytestmark = [pytest.mark.layer("arch"), pytest.mark.small] + +_DELIVERY_BOUNDS = ( + Path(__file__).resolve().parents[2] / "src" / "autoskillit" / "core" / "_delivery_bounds.py" +) + + +def _resolver() -> ast.FunctionDef: + tree = ast.parse(_DELIVERY_BOUNDS.read_text(encoding="utf-8")) + for node in tree.body: + if isinstance(node, ast.FunctionDef) and node.name == "resolve_recipe_delivery_decision": + return node + pytest.fail("resolve_recipe_delivery_decision not found") + + +def _assigned_expression(function: ast.FunctionDef, target_name: str) -> ast.expr: + for node in ast.walk(function): + if ( + isinstance(node, ast.Assign) + and len(node.targets) == 1 + and isinstance(node.targets[0], ast.Name) + and node.targets[0].id == target_name + ): + return node.value + pytest.fail(f"assignment to {target_name!r} not found") + + +def test_requested_and_observed_limits_have_independent_sources() -> None: + function = _resolver() + requested = ast.dump(_assigned_expression(function, "requested")) + observed = ast.dump(_assigned_expression(function, "observed")) + + assert "caller_requested_outer_tokens" in requested + assert "request" in requested + assert "host_observed_requested_outer_tokens" not in requested + + assert "host_observed_requested_outer_tokens" in observed + assert "attestation" in observed + assert "caller_requested_outer_tokens" not in observed + + +def test_selected_limit_never_comes_from_history_or_measured_size() -> None: + function = _resolver() + body = ast.dump(function) + assert "history_retention_token_limit" not in body + assert "measured_recipe_exemption_max_utf8_bytes" not in body + + selected_values = [ + keyword.value + for call in ast.walk(function) + if isinstance(call, ast.Call) + for keyword in call.keywords + if keyword.arg == "selected_limit" + ] + assert selected_values, "resolver must pass an explicit selected limit on every decision path" + for value in selected_values: + value_dump = ast.dump(value) + assert "required_serialized_tokens" not in value_dump + assert "history_retention_token_limit" not in value_dump + + +def test_resolver_does_not_treat_wire_metadata_or_rollouts_as_authority() -> None: + body = ast.dump(_resolver()).lower() + for untrusted_source in ("_meta", "rollout", "trace", "tool_output_token_limit"): + assert untrusted_source not in body diff --git a/tests/arch/test_serve_surface_registry.py b/tests/arch/test_serve_surface_registry.py index a1da39f86e..33700be789 100644 --- a/tests/arch/test_serve_surface_registry.py +++ b/tests/arch/test_serve_surface_registry.py @@ -1,12 +1,14 @@ -"""Structural tests for SERVE_SURFACES frozenset and load_and_validate call-site enforcement. +"""Structural tests for the delivery registry and load-and-validate call sites. AST guard: load_and_validate must only be called from _serve_helpers.py within server/tools/. -SERVE_SURFACES membership: all four surfaces registered, no phantom entries. +Registry membership: all four surfaces registered, no phantom entries. """ from __future__ import annotations import ast +import operator +from collections.abc import Mapping from pathlib import Path import pytest @@ -54,18 +56,29 @@ def test_serve_surfaces_registry_is_exhaustive() -> None: ) -def test_serve_surfaces_frozenset_defined() -> None: - """SERVE_SURFACES must be importable from autoskillit.core and be a frozenset.""" - from autoskillit.core import SERVE_SURFACES +def test_recipe_delivery_surface_registry_is_typed() -> None: + """The sole route authority is a typed core registry.""" + from autoskillit.core import ( + RECIPE_DELIVERY_SURFACE_REGISTRY, + RecipeDeliverySurfaceDef, + ) - assert isinstance(SERVE_SURFACES, frozenset), ( - f"SERVE_SURFACES must be a frozenset, got {type(SERVE_SURFACES)}" + assert isinstance(RECIPE_DELIVERY_SURFACE_REGISTRY, Mapping) + assert all( + isinstance(definition, RecipeDeliverySurfaceDef) + for definition in RECIPE_DELIVERY_SURFACE_REGISTRY.values() ) + with pytest.raises(TypeError): + operator.setitem( + RECIPE_DELIVERY_SURFACE_REGISTRY, + "mutated", + next(iter(RECIPE_DELIVERY_SURFACE_REGISTRY.values())), + ) def test_serve_surfaces_contains_expected_members() -> None: - """SERVE_SURFACES must list all four known serve surfaces.""" - from autoskillit.core import SERVE_SURFACES + """The registry lists exactly the four recipe-bearing serve routes.""" + from autoskillit.core import RECIPE_DELIVERY_SURFACE_REGISTRY expected = { "open_kitchen", @@ -73,8 +86,21 @@ def test_serve_surfaces_contains_expected_members() -> None: "load_recipe", "get_recipe", } - assert expected == SERVE_SURFACES, ( - f"SERVE_SURFACES mismatch. " - f"Missing: {expected - SERVE_SURFACES}. " - f"Extra: {SERVE_SURFACES - expected}." + actual = set(RECIPE_DELIVERY_SURFACE_REGISTRY) + assert expected == actual, ( + f"RECIPE_DELIVERY_SURFACE_REGISTRY mismatch. " + f"Missing: {expected - actual}. " + f"Extra: {actual - expected}." ) + + +def test_recipe_recreation_policy_is_registry_owned() -> None: + from autoskillit.core import RECIPE_DELIVERY_SURFACE_REGISTRY + + eligible = { + surface + for surface, definition in RECIPE_DELIVERY_SURFACE_REGISTRY.items() + if definition.recreation_eligible + } + + assert eligible == {"open_kitchen", "open_kitchen_deferred_recall", "get_recipe"} diff --git a/tests/arch/test_subpackage_isolation.py b/tests/arch/test_subpackage_isolation.py index 2203f7bbda..d8f4782ee7 100644 --- a/tests/arch/test_subpackage_isolation.py +++ b/tests/arch/test_subpackage_isolation.py @@ -884,11 +884,11 @@ def test_no_subpackage_exceeds_10_files() -> None: logic on DispatchRecord.from_dict. Exempt at 15 files. """ EXEMPTIONS: dict[str, int] = { - "server": 14, + "server": 15, # +_recipe_delivery unified finalizer and immutable generation store "recipe": 42, # was 33; +9 from CI/graph/dataflow splits "execution": 18, - "core": 25, # +_delivery_bounds (resolve_effective_delivery_bound) - "core/types": 32, # +invariant_registry (INVARIANT_REGISTRY) +closure_report + "core": 25, # +_delivery_bounds (resolve_general_output_token_limit) + "core/types": 33, # +recipe_delivery typed budget/provenance contracts "cli": 21, "hooks": 18, # +recipe_confirmed_post_hook, +quota_guard_state_post_hook, +_policy_event, +shell_capture_hook (#4286), +_capture_cleanup.py # noqa: E501 "pipeline": 12, @@ -899,7 +899,7 @@ def test_no_subpackage_exceeds_10_files() -> None: # +_backend_compat.py (shared target-resolution + fail-closed compatibility gate # for direct headless executor callers — report_bug, prepare_issue, enrich_issues) "hooks/guards": 32, # -output_budget_guard (#4286) - "execution/backends": 12, # +_composite_locator.py, +_probe_cache.py + "execution/backends": 13, # +_codex_recipe_delivery protected attestation broker } violations: list[str] = [] dirs_to_check: list[Path] = [] @@ -1035,7 +1035,7 @@ def test_data_directories_are_not_python_packages() -> None: "fresh-open and deferred-recall open_kitchen paths, plus overlay lock sidecar " "cleanup at close_kitchen (#4293 pipeline tracker split-brain, +42 net lines)" "; envelope integration on both deferred-recall and normal open_kitchen paths: " - "resolve_effective_delivery_bound + BackendCapabilities isinstance guard + " + "resolve_general_output_token_limit + BackendCapabilities isinstance guard + " "maybe_envelope_recipe_response call (#4304 Part B, +24 net lines)", ), "tools_execution.py": ( diff --git a/tests/arch/test_subpackage_structure.py b/tests/arch/test_subpackage_structure.py index fa692b8b55..33e1e33422 100644 --- a/tests/arch/test_subpackage_structure.py +++ b/tests/arch/test_subpackage_structure.py @@ -41,6 +41,7 @@ def test_core_types_has_all_type_modules(self): "_type_protocols_workspace", "_type_results", "_type_results_execution", + "_type_recipe_delivery", "_type_resume", "_type_session_env", "_type_subprocess", @@ -61,8 +62,8 @@ def test_type_constants_split_completeness(self): assert len(combined) == len(remaining) + len(env) + len(features) + len(registries), ( "Duplicate symbols across split modules" ) - assert len(combined) == 112, ( - f"Expected 112 symbols total, got {len(combined)} " + assert len(combined) == 114, ( + f"Expected 114 symbols total, got {len(combined)} " f"(remaining={len(remaining)}, env={len(env)}, " f"features={len(features)}, registries={len(registries)})" ) diff --git a/tests/cli/test_doctor_backend_guards.py b/tests/cli/test_doctor_backend_guards.py index e4524e3a4a..62871e85c9 100644 --- a/tests/cli/test_doctor_backend_guards.py +++ b/tests/cli/test_doctor_backend_guards.py @@ -192,9 +192,9 @@ def test_codex_backend_ok_when_registered( ) from autoskillit.execution import ( CODEX_AUTO_COMPACT_LIMIT, + CODEX_HISTORY_RETENTION_TOKEN_LIMIT, CODEX_MCP_STARTUP_TIMEOUT_SEC, CODEX_MCP_TOOL_TIMEOUT_FLOOR, - CODEX_TOOL_OUTPUT_TOKEN_LIMIT, ) from autoskillit.execution.backends.codex import CodexBackend @@ -204,7 +204,7 @@ def test_codex_backend_ok_when_registered( f'"{v}"' for v in sorted(CODEX_MCP_ENV_FORWARD_VARS - {HEADLESS_AUTO_GATE_ENV_VAR}) ) (codex_dir / "config.toml").write_text( - f"tool_output_token_limit = {CODEX_TOOL_OUTPUT_TOKEN_LIMIT}\n" + f"tool_output_token_limit = {CODEX_HISTORY_RETENTION_TOKEN_LIMIT}\n" f"model_auto_compact_token_limit = {CODEX_AUTO_COMPACT_LIMIT}\n" f'[mcp_servers.autoskillit]\ncommand = "autoskillit"\n' f"env_vars = [{env_vars_str}]\n" diff --git a/tests/cli/test_doctor_codex_mcp.py b/tests/cli/test_doctor_codex_mcp.py index f6f4b2c4be..61972f3b02 100644 --- a/tests/cli/test_doctor_codex_mcp.py +++ b/tests/cli/test_doctor_codex_mcp.py @@ -12,9 +12,9 @@ from autoskillit.core import ReadResult, Severity from autoskillit.execution.backends._codex_config import ( CODEX_AUTO_COMPACT_LIMIT, + CODEX_HISTORY_RETENTION_TOKEN_LIMIT, CODEX_MCP_STARTUP_TIMEOUT_SEC, CODEX_MCP_TOOL_TIMEOUT_FLOOR, - CODEX_TOOL_OUTPUT_TOKEN_LIMIT, ) pytestmark = [pytest.mark.layer("cli"), pytest.mark.small] @@ -41,7 +41,7 @@ def test_ok_when_valid_config(self, monkeypatch: pytest.MonkeyPatch) -> None: "tool_timeout_sec": CODEX_MCP_TOOL_TIMEOUT_FLOOR, } }, - "tool_output_token_limit": CODEX_TOOL_OUTPUT_TOKEN_LIMIT, + "tool_output_token_limit": CODEX_HISTORY_RETENTION_TOKEN_LIMIT, "model_auto_compact_token_limit": CODEX_AUTO_COMPACT_LIMIT, } ), diff --git a/tests/cli/test_doctor_runtime.py b/tests/cli/test_doctor_runtime.py index e1a6b466a2..679695b8a2 100644 --- a/tests/cli/test_doctor_runtime.py +++ b/tests/cli/test_doctor_runtime.py @@ -97,7 +97,7 @@ def test_codex_limits_verified_warns_when_cli_newer_than_pin( ) result = mod._check_codex_limits_verified(backend=CodexBackend()) assert result.severity == Severity.WARNING - assert "CODEX_TOOL_OUTPUT_TOKEN_LIMIT" in result.message + assert "CODEX_HISTORY_RETENTION_TOKEN_LIMIT" in result.message assert "CODEX_AUTO_COMPACT_LIMIT" in result.message def test_codex_limits_verified_ok_at_pinned_version( diff --git a/tests/contracts/test_delivery_bound_fitness.py b/tests/contracts/test_delivery_bound_fitness.py index 396ca2201e..6efbbff705 100644 --- a/tests/contracts/test_delivery_bound_fitness.py +++ b/tests/contracts/test_delivery_bound_fitness.py @@ -15,26 +15,34 @@ import json from pathlib import Path -from typing import cast +from types import SimpleNamespace +from typing import TYPE_CHECKING, cast import pytest +if TYPE_CHECKING: + from autoskillit.pipeline import ToolContext + from autoskillit.config import OutputBudgetConfig from autoskillit.core import ( RESPONSE_BACKSTOP_EXEMPTION_REGISTRY, BackendCapabilities, - resolve_effective_delivery_bound, + resolve_general_output_token_limit, +) +from autoskillit.execution.backends import BACKEND_REGISTRY, CODEX_HISTORY_RETENTION_TOKEN_LIMIT +from autoskillit.recipe import ( + all_validated_recipe_names, + find_recipe_by_name, + load_and_validate, + load_recipe, ) -from autoskillit.execution.backends import BACKEND_REGISTRY, CODEX_TOOL_OUTPUT_TOKEN_LIMIT -from autoskillit.recipe import all_validated_recipe_names, load_and_validate +from autoskillit.server._recipe_delivery import build_recipe_envelope, persist_recipe_artifact from autoskillit.server._response_budget import ( RESPONSE_SPILL_METADATA_KEY, enforce_response_budget, ) from autoskillit.server.tools._serve_helpers import ( build_open_kitchen_recipe_payload, - build_recipe_envelope, - extract_step_skeleton, ) pytestmark = [pytest.mark.layer("contracts"), pytest.mark.small] @@ -81,7 +89,10 @@ def _full_open_kitchen_payload(recipe_name: str) -> dict[str, object]: @pytest.mark.parametrize("recipe_name", _recipe_names(), ids=lambda n: n) @pytest.mark.parametrize("backend_name", sorted(_backend_capabilities().keys()), ids=lambda n: n) def test_bundled_recipe_open_kitchen_envelope_fits_per_backend( - recipe_name: str, backend_name: str, tmp_path: Path + recipe_name: str, + backend_name: str, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, ) -> None: """System-level fitness contract (issue #4304 Part B REQ-B-T7): the bounded envelope built unconditionally for every bundled recipe via @@ -96,37 +107,48 @@ def test_bundled_recipe_open_kitchen_envelope_fits_per_backend( envelope-fit-by-construction guarantee is the post-#4304-Part-B invariant that this file exists to defend. """ + from autoskillit.recipe import _api_cache + from autoskillit.recipe._api_cache import LoadCache + + monkeypatch.setattr(_api_cache, "_LOAD_CACHE", LoadCache()) payload = _full_open_kitchen_payload(recipe_name) - step_names = [ - str(n) - for n in cast(list[object], payload.get("post_prune_step_names") or []) - if isinstance(n, str) - ] - skeleton = extract_step_skeleton( - step_names, - routing_edges_by_step={}, - step_summaries={name: f"summary-{name}" for name in step_names}, + generation = persist_recipe_artifact( + tmp_path, + kitchen_id="fitness-kitchen", + producer_tool="open_kitchen", + recipe_name=recipe_name, + payload=payload, ) - artifact_path = tmp_path / f"{recipe_name}.log" - sha256 = "0" * 64 caps = _backend_capabilities()[backend_name] - bound_tokens = resolve_effective_delivery_bound(caps) + bound_tokens = resolve_general_output_token_limit(caps) bound_bytes = _effective_bound_bytes(bound_tokens) + recipe_info = find_recipe_by_name(recipe_name, _PROJECT_ROOT) + assert recipe_info is not None + recipe = load_recipe(recipe_info.path) envelope = build_recipe_envelope( payload, recipe_name=recipe_name, - artifact_path=str(artifact_path), - artifact_sha256=sha256, - skeleton=skeleton, + generation=generation, + skeleton_source=cast( + "ToolContext", + SimpleNamespace(recipe_name=recipe_name, active_recipe_steps=recipe.steps), + ), bound_bytes=bound_bytes, ) - serialized = json.dumps(envelope, ensure_ascii=False) + serialized = json.dumps(envelope, ensure_ascii=False, separators=(",", ":")) assert len(serialized.encode("utf-8")) <= bound_bytes, ( f"{backend_name}: envelope for {recipe_name} exceeds " f"{bound_bytes} bytes (effective delivery bound)" ) assert envelope["recipe_pull"]["pull_tool"] == "get_recipe_section" + skeleton_steps = envelope["step_flow_skeleton"]["steps"] + expected_step_names = set(payload.get("post_prune_step_names") or []) + assert {step["name"] for step in skeleton_steps} == expected_step_names + if expected_step_names: + assert any(step.get("summary") or step["edges"] for step in skeleton_steps) + else: + assert not recipe.steps, "step-bearing recipes must expose production skeleton metadata" @pytest.mark.parametrize("recipe_name", _recipe_names(), ids=lambda n: n) @@ -151,7 +173,7 @@ def test_bundled_recipe_open_kitchen_raw_spill_projection_fits_per_backend( serialized = json.dumps(payload) serialized_bytes = len(serialized.encode("utf-8")) for backend_name, caps in _backend_capabilities().items(): - bound_tokens = resolve_effective_delivery_bound(caps) + bound_tokens = resolve_general_output_token_limit(caps) bound_bytes = _effective_bound_bytes(bound_tokens) if serialized_bytes <= bound_bytes: continue @@ -160,7 +182,7 @@ def test_bundled_recipe_open_kitchen_raw_spill_projection_fits_per_backend( tool_name="open_kitchen", artifact_dir=tmp_path / backend_name, config=OutputBudgetConfig(), - effective_delivery_token_limit=bound_tokens, + selected_result_token_limit=bound_tokens, ) assert isinstance(result, str), ( f"{backend_name}: expected str result for {recipe_name} payload" @@ -183,7 +205,7 @@ def test_non_exempted_oversized_payload_spills_within_delivery_bound(tmp_path) - serialized = json.dumps(payload) config = OutputBudgetConfig() for backend_name, caps in _backend_capabilities().items(): - bound_tokens = resolve_effective_delivery_bound(caps) + bound_tokens = resolve_general_output_token_limit(caps) bound_bytes = _effective_bound_bytes(bound_tokens) assert len(serialized.encode("utf-8")) > bound_bytes, ( f"{backend_name}: payload does not exceed bound ({bound_bytes} bytes)" @@ -193,7 +215,7 @@ def test_non_exempted_oversized_payload_spills_within_delivery_bound(tmp_path) - tool_name="run_skill", artifact_dir=tmp_path / backend_name, config=config, - effective_delivery_token_limit=bound_tokens, + selected_result_token_limit=bound_tokens, ) assert isinstance(result, str) assert len(result.encode("utf-8")) <= bound_bytes, ( @@ -244,7 +266,7 @@ def test_delivery_bound_summary_carries_all_step_names( str(name) for name in cast(list[object], step_names_raw or []) if isinstance(name, str) ] for backend_name, caps in _backend_capabilities().items(): - bound_tokens = resolve_effective_delivery_bound(caps) + bound_tokens = resolve_general_output_token_limit(caps) bound_bytes = _effective_bound_bytes(bound_tokens) if serialized_bytes <= bound_bytes: continue @@ -253,7 +275,7 @@ def test_delivery_bound_summary_carries_all_step_names( tool_name="open_kitchen", artifact_dir=tmp_path / backend_name, config=OutputBudgetConfig(), - effective_delivery_token_limit=bound_tokens, + selected_result_token_limit=bound_tokens, ) assert isinstance(result, str), ( f"{backend_name}: expected str result for {recipe_name} payload" @@ -322,43 +344,46 @@ def test_delivery_bound_summary_carries_all_step_names( ) -def test_codex_configured_limit_not_less_than_effective_delivery_bound() -> None: - """Cross-relational invariant: the configured Codex tool-output token - limit must not drift below the operative delivery bound, and must not - drift below the upstream Codex code-mode default output bound (~10,000 - tokens, per issue #4300).""" +def test_codex_history_retention_not_less_than_unnegotiated_result_limit() -> None: + """History retention must accommodate the ordinary outer-result default. + + This comparison is a storage-safety relation only; it does not authorize + the retention value as the selected outer result. + """ codex_caps = BACKEND_REGISTRY["codex"]().capabilities - codex_effective = resolve_effective_delivery_bound(codex_caps) - assert codex_effective > 0 - assert CODEX_TOOL_OUTPUT_TOKEN_LIMIT >= codex_effective, ( - f"CODEX_TOOL_OUTPUT_TOKEN_LIMIT ({CODEX_TOOL_OUTPUT_TOKEN_LIMIT}) is " - f"below Codex effective delivery bound ({codex_effective}); config " + codex_unnegotiated = resolve_general_output_token_limit(codex_caps) + assert codex_unnegotiated > 0 + assert CODEX_HISTORY_RETENTION_TOKEN_LIMIT >= codex_unnegotiated, ( + "CODEX_HISTORY_RETENTION_TOKEN_LIMIT " + f"({CODEX_HISTORY_RETENTION_TOKEN_LIMIT}) is below Codex unnegotiated " + f"result limit ({codex_unnegotiated}); config " f"and capability field have drifted apart" ) # Upstream floor: Codex code-mode default ~10,000 tokens. - assert CODEX_TOOL_OUTPUT_TOKEN_LIMIT >= 10_000, ( - f"CODEX_TOOL_OUTPUT_TOKEN_LIMIT ({CODEX_TOOL_OUTPUT_TOKEN_LIMIT}) is " + assert CODEX_HISTORY_RETENTION_TOKEN_LIMIT >= 10_000, ( + "CODEX_HISTORY_RETENTION_TOKEN_LIMIT " + f"({CODEX_HISTORY_RETENTION_TOKEN_LIMIT}) is " f"below the upstream Codex code-mode default output bound (10,000); " f"config value must not drift below this floor" ) def test_capability_default_uses_conservative_bound() -> None: - """BackendCapabilities() with no effective_delivery_token_limit set must + """BackendCapabilities() with no unnegotiated_tool_result_token_limit set must default to a conservative worst-case bound (the smallest registered backend bound). The historical 0-sentinel silently disabled delivery bounding for any future backend that omitted the field, leading to unbounded transport delivery.""" caps = BackendCapabilities() - bound = caps.effective_delivery_token_limit + bound = caps.unnegotiated_tool_result_token_limit assert bound > 0, ( - f"BackendCapabilities() default effective_delivery_token_limit must " + f"BackendCapabilities() default unnegotiated_tool_result_token_limit must " f"be conservative (non-zero); got {bound}" ) # The default must be at most the smallest registered backend bound, # so the worst-case delivery is bounded to the strictest transport. min_registered = min( - resolve_effective_delivery_bound(cls().capabilities) for cls in BACKEND_REGISTRY.values() + resolve_general_output_token_limit(cls().capabilities) for cls in BACKEND_REGISTRY.values() ) assert bound <= min_registered, ( f"BackendCapabilities() default ({bound}) must be at most the " @@ -389,7 +414,7 @@ def test_non_exempted_delivery_bound_preserves_result_field(tmp_path: Path) -> N tool_name="run_skill", artifact_dir=tmp_path, config=OutputBudgetConfig(), - effective_delivery_token_limit=bound_tokens, + selected_result_token_limit=bound_tokens, ) assert isinstance(result, str) assert len(result.encode("utf-8")) <= bound_bytes, ( @@ -420,7 +445,7 @@ def test_non_exempted_delivery_bound_preserves_non_string_result(tmp_path: Path) tool_name="run_skill", artifact_dir=tmp_path, config=OutputBudgetConfig(), - effective_delivery_token_limit=bound_tokens, + selected_result_token_limit=bound_tokens, ) assert isinstance(result, str) assert len(result.encode("utf-8")) <= bound_tokens * 4 @@ -437,7 +462,7 @@ def test_non_exempted_delivery_bound_finds_multibyte_result_prefix(tmp_path: Pat tool_name="run_skill", artifact_dir=tmp_path, config=OutputBudgetConfig(), - effective_delivery_token_limit=bound_tokens, + selected_result_token_limit=bound_tokens, ) assert isinstance(result, str) assert len(result.encode("utf-8")) <= bound_tokens * 4 diff --git a/tests/contracts/test_output_budget_discipline.py b/tests/contracts/test_output_budget_discipline.py index 8024935645..30835de64b 100644 --- a/tests/contracts/test_output_budget_discipline.py +++ b/tests/contracts/test_output_budget_discipline.py @@ -106,18 +106,70 @@ def test_intake_digest_is_safe_for_agent_toml_multiline_literal() -> None: assert "'''" not in CODEX_INTAKE_DISCIPLINE_DIGEST -def test_intake_digest_numeric_rule_matches_codex_effective_delivery_bound() -> None: - """The intake digest's ``max_output_tokens`` numeric rule must be at least - the Codex backend's effective delivery bound — the operative bound the - server-side response backstop applies. If the bound changes, the digest - text must be updated to reflect it (this is the regression guard).""" +def test_intake_digest_numeric_rule_matches_codex_unnegotiated_result_limit() -> None: + """The ordinary intake rule must name Codex's omitted outer-result limit. + + History retention is deliberately a separate domain and must not select + the outer result delivered by Code Mode. + """ from autoskillit.execution.backends import BACKEND_REGISTRY codex_caps = BACKEND_REGISTRY["codex"]().capabilities - codex_effective_bound = codex_caps.effective_delivery_token_limit - assert codex_effective_bound > 0 - assert f"max_output_tokens above {codex_effective_bound}" in CODEX_INTAKE_DISCIPLINE_DIGEST, ( + codex_unnegotiated_limit = codex_caps.unnegotiated_tool_result_token_limit + assert codex_unnegotiated_limit > 0 + assert ( + f"max_output_tokens above {codex_unnegotiated_limit}" in CODEX_INTAKE_DISCIPLINE_DIGEST + ), ( "CODEX_INTAKE_DISCIPLINE_DIGEST numeric rule must reference the Codex " - f"effective_delivery_token_limit ({codex_effective_bound}); update " + "unnegotiated_tool_result_token_limit " + f"({codex_unnegotiated_limit}); update " "the digest if the bound changed." ) + + +def test_codex_recipe_delivery_contract_is_generated_from_static_budget() -> None: + from autoskillit.execution import ( + CODEX_RECIPE_DELIVERY_BUDGET, + CODEX_RECIPE_DELIVERY_CALLING_CONTRACT, + ) + + budget = CODEX_RECIPE_DELIVERY_BUDGET + contract = CODEX_RECIPE_DELIVERY_CALLING_CONTRACT + assert ( + f'// @exec: {{"max_output_tokens": ' + f"{budget.authoritative_attested_recipe_result_token_limit}" + "}" + ) in contract + assert f"contract_version={budget.contract_version}" in contract + assert f"contract_digest={budget.contract_digest}" in contract + assert ( + f"caller_requested_outer_tokens={budget.authoritative_attested_recipe_result_token_limit}" + ) in contract + for field in ( + "audience", + "delivery_call_id", + "contract_version", + "contract_digest", + "caller_requested_outer_tokens", + "code_digest", + ): + assert field in contract + assert "Never synthesize, infer, alter, or replay" in contract + assert "bounded recipe_pull path" in contract + + +def test_codex_recipe_delivery_contract_reaches_all_prompt_families() -> None: + from autoskillit.cli._prompts import ( + _build_open_kitchen_prompt, + _build_orchestrator_prompt, + ) + from autoskillit.execution import ( + CODEX_RECIPE_DELIVERY_CALLING_CONTRACT, + codex_recipe_delivery_calling_contract, + ) + from autoskillit.execution.backends._claude_prompt import codex_discipline_suffix + + assert CODEX_RECIPE_DELIVERY_CALLING_CONTRACT in codex_discipline_suffix() + prefix = "mcp__autoskillit__" + expected = codex_recipe_delivery_calling_contract(mcp_prefix=prefix) + assert expected in _build_orchestrator_prompt("remediation", prefix) + assert expected in _build_open_kitchen_prompt(prefix) diff --git a/tests/contracts/test_package_gateways.py b/tests/contracts/test_package_gateways.py index 013c534825..717a602fe4 100644 --- a/tests/contracts/test_package_gateways.py +++ b/tests/contracts/test_package_gateways.py @@ -398,6 +398,7 @@ def test_root_module_allowlist() -> None: "__main__.py", "_llm_triage.py", "_probe_canary.py", + "_recipe_delivery_framing.py", "_test_filter.py", "hook_registry.py", "version.py", diff --git a/tests/core/AGENTS.md b/tests/core/AGENTS.md index f1ce2570d1..06685ee395 100644 --- a/tests/core/AGENTS.md +++ b/tests/core/AGENTS.md @@ -46,6 +46,7 @@ Core layer (IL-0) unit tests — paths, IO, types, feature flags. | `test_input_spec_type.py` | InputSpec type system tests — closed type vocabulary validation | | `test_parse_plan_paths.py` | parse_plan_paths unit tests — comma/newline splitter used by file_path_list gate | | `test_backend_capabilities.py` | Tests for BackendCapabilities frozen invariants and CLAUDE_CODE_CAPABILITIES field values | +| `test_recipe_delivery_contract.py` | Typed recipe budget, request, attestation, and static decision resolver contracts | | `test_backend_event_kind.py` | Tests for BackendEventKind StrEnum — member exhaustiveness, values, importability | | `test_backend_dataclasses.py` | Tests for CmdSpec, SkillSessionConfig, ClaudeEventData, CodexEventData, SessionEvent, AgentSessionResult — frozen invariants, field types, defaults | | `test_backend_protocols.py` | Tests for StreamParser, ResultParser, EnvPolicy, SessionLocator, CodingAgentBackend — @runtime_checkable, isinstance conformance, IL-0 compliance | diff --git a/tests/core/test_backend_capabilities.py b/tests/core/test_backend_capabilities.py index 0c6e8cdba9..96917237ac 100644 --- a/tests/core/test_backend_capabilities.py +++ b/tests/core/test_backend_capabilities.py @@ -103,6 +103,7 @@ def test_backend_capabilities_field_count(): "session_dir_persistent", "supports_model_invocation_gating", "github_api_callable", + "protected_recipe_delivery_capable", } assert frozenset_fields == { "completion_record_types", @@ -177,7 +178,9 @@ def test_backend_capabilities_field_names_locked(): "session_dir_persistent", "supports_model_invocation_gating", "github_api_callable", - "effective_delivery_token_limit", + "unnegotiated_tool_result_token_limit", + "protected_recipe_delivery_capable", + "recipe_delivery_budget", } actual = {f.name for f in dataclasses.fields(BackendCapabilities)} assert actual == expected diff --git a/tests/core/test_recipe_delivery_contract.py b/tests/core/test_recipe_delivery_contract.py new file mode 100644 index 0000000000..bf97ae734b --- /dev/null +++ b/tests/core/test_recipe_delivery_contract.py @@ -0,0 +1,363 @@ +"""Typed budget, provenance, and static recipe-delivery decision contracts.""" + +from __future__ import annotations + +import dataclasses + +import pytest + +from autoskillit.core import ( + AGENT_BACKEND_CLAUDE_CODE, + AGENT_BACKEND_CODEX, + RECIPE_DELIVERY_ATTESTATION_AUDIENCE, + BackendCapabilities, + RecipeDeliveryAttestation, + RecipeDeliveryBudgetDef, + RecipeDeliveryDecision, + RecipeDeliveryEvidenceDef, + RecipeDeliveryMode, + RecipeDeliveryRequest, + recipe_delivery_request_digest, + resolve_general_output_token_limit, + resolve_recipe_delivery_decision, +) + +pytestmark = [pytest.mark.layer("core"), pytest.mark.small] + +_PAYLOAD_SHA256 = f"sha256:{'a' * 64}" +_CONTRACT_DIGEST = f"sha256:{'f' * 64}" +_BUDGET = RecipeDeliveryBudgetDef( + ordinary_omitted_result_token_limit=10_000, + authoritative_attested_recipe_result_token_limit=56_750, + history_retention_token_limit=56_750, + measured_recipe_exemption_max_utf8_bytes=195_000, + headroom_tokens=8_000, + contract_version=1, + parser_version=1, + evidence_version=1, + contract_digest=_CONTRACT_DIGEST, +) +_CODEX_CAPABILITIES = BackendCapabilities( + unnegotiated_tool_result_token_limit=10_000, + protected_recipe_delivery_capable=True, + recipe_delivery_budget=_BUDGET, +) + + +def _request(*, requested: int | None = None) -> RecipeDeliveryRequest: + budget = _BUDGET + request = RecipeDeliveryRequest( + audience=RECIPE_DELIVERY_ATTESTATION_AUDIENCE, + delivery_call_id="delivery-1", + contract_version=budget.contract_version, + contract_digest=budget.contract_digest, + caller_requested_outer_tokens=( + budget.authoritative_attested_recipe_result_token_limit + if requested is None + else requested + ), + code_digest=f"sha256:{'b' * 64}", + ) + return request + + +def _attestation( + request: RecipeDeliveryRequest, + *, + observed: int | None = None, + evidence_identity: str = "protected-test-host-v1", +) -> RecipeDeliveryAttestation: + budget = _BUDGET + attestation = RecipeDeliveryAttestation( + audience=request.audience, + thread_id="thread-1", + turn_id="turn-1", + outer_call_id="outer-call-1", + code_mode_cell_id="cell-1", + delivery_call_id=request.delivery_call_id, + host_observed_requested_outer_tokens=( + request.caller_requested_outer_tokens if observed is None else observed + ), + selected_result_token_limit=budget.authoritative_attested_recipe_result_token_limit, + code_digest=request.code_digest, + request_digest=recipe_delivery_request_digest(request), + nonce="nonce-1", + expires_at_unix=2_000_000_000, + contract_version=request.contract_version, + contract_digest=request.contract_digest, + parser_version=budget.parser_version, + evidence_version=budget.evidence_version, + evidence_identity=evidence_identity, + ) + return attestation + + +def _protected_evidence() -> RecipeDeliveryEvidenceDef: + budget = _BUDGET + return RecipeDeliveryEvidenceDef( + identity="protected-test-host-v1", + host_channel="test-only-process-isolated-host", + evidence_schema_version=budget.evidence_version, + parser_version=budget.parser_version, + cli_identity="codex-test-cli", + selected_limit_derivation="protected_resolved_outer_limit", + selected_result_token_limit=budget.authoritative_attested_recipe_result_token_limit, + contract_digest=budget.contract_digest, + ) + + +def _resolve( + *, + backend_name: str = "codex", + required: int, + request: RecipeDeliveryRequest | None = None, + attestation: RecipeDeliveryAttestation | None = None, + evidence: RecipeDeliveryEvidenceDef | None = None, + budget: RecipeDeliveryBudgetDef | None = _BUDGET, +) -> RecipeDeliveryDecision: + capabilities = ( + _CODEX_CAPABILITIES + if backend_name == AGENT_BACKEND_CODEX + else BackendCapabilities(unnegotiated_tool_result_token_limit=46_500) + ) + return resolve_recipe_delivery_decision( + capabilities=capabilities, + required_serialized_tokens=required, + budget=budget, + producer="open_kitchen", + payload_sha256=_PAYLOAD_SHA256, + request=request, + attestation=attestation, + supported_evidence=evidence, + now_unix=1_900_000_000, + ) + + +def test_recipe_delivery_type_fields_keep_provenance_domains_distinct() -> None: + assert RecipeDeliveryBudgetDef._fields == ( + "ordinary_omitted_result_token_limit", + "authoritative_attested_recipe_result_token_limit", + "history_retention_token_limit", + "measured_recipe_exemption_max_utf8_bytes", + "headroom_tokens", + "contract_version", + "parser_version", + "evidence_version", + "contract_digest", + ) + assert {field.name for field in dataclasses.fields(RecipeDeliveryRequest)} == { + "delivery_call_id", + "audience", + "contract_version", + "contract_digest", + "caller_requested_outer_tokens", + "code_digest", + } + assert {field.name for field in dataclasses.fields(RecipeDeliveryAttestation)} == { + "audience", + "thread_id", + "turn_id", + "outer_call_id", + "code_mode_cell_id", + "delivery_call_id", + "host_observed_requested_outer_tokens", + "selected_result_token_limit", + "code_digest", + "request_digest", + "nonce", + "expires_at_unix", + "contract_version", + "contract_digest", + "parser_version", + "evidence_version", + "evidence_identity", + } + assert {field.name for field in dataclasses.fields(RecipeDeliveryDecision)} == { + "mode", + "caller_requested_outer_tokens", + "host_observed_requested_outer_tokens", + "required_outer_tokens", + "unnegotiated_tool_result_token_limit", + "selected_result_token_limit", + "contract_digest", + "evidence_identity", + "reason", + "producer", + "payload_sha256", + "receipt_status", + } + + +@pytest.mark.parametrize( + "contract_type", + [RecipeDeliveryRequest, RecipeDeliveryAttestation, RecipeDeliveryDecision], +) +def test_recipe_delivery_dataclasses_have_no_defaults( + contract_type: type[RecipeDeliveryRequest] + | type[RecipeDeliveryAttestation] + | type[RecipeDeliveryDecision], +) -> None: + for field in dataclasses.fields(contract_type): + assert field.default is dataclasses.MISSING, field.name + assert field.default_factory is dataclasses.MISSING, field.name + + +def test_static_budget_is_required_and_keeps_history_separate() -> None: + budget = _BUDGET + assert RecipeDeliveryBudgetDef._field_defaults == {} + assert budget.ordinary_omitted_result_token_limit == 10_000 + assert budget.measured_recipe_exemption_max_utf8_bytes == 195_000 + assert ( + budget.authoritative_attested_recipe_result_token_limit + == ((budget.measured_recipe_exemption_max_utf8_bytes + 3) // 4) + budget.headroom_tokens + ) + assert budget.history_retention_token_limit == 56_750 + + +def test_general_output_resolver_reads_only_backend_unnegotiated_limit() -> None: + caps = BackendCapabilities(unnegotiated_tool_result_token_limit=321) + assert resolve_general_output_token_limit(caps) == 321 + + +@pytest.mark.parametrize("required", [1, 10_000]) +def test_codex_ordinary_boundary_needs_no_attestation(required: int) -> None: + decision = _resolve(required=required) + assert decision.mode is RecipeDeliveryMode.ORDINARY_INLINE + assert decision.selected_result_token_limit == 10_000 + assert decision.caller_requested_outer_tokens is None + assert decision.host_observed_requested_outer_tokens is None + + +def test_exact_supported_codex_high_boundary_is_attested_inline() -> None: + budget = _BUDGET + request = _request() + decision = _resolve( + required=budget.authoritative_attested_recipe_result_token_limit, + request=request, + attestation=_attestation(request), + evidence=_protected_evidence(), + ) + assert decision.mode is RecipeDeliveryMode.ATTESTED_INLINE + assert decision.caller_requested_outer_tokens == ( + budget.authoritative_attested_recipe_result_token_limit + ) + assert decision.host_observed_requested_outer_tokens == ( + budget.authoritative_attested_recipe_result_token_limit + ) + assert decision.selected_result_token_limit == ( + budget.authoritative_attested_recipe_result_token_limit + ) + + +def test_over_authoritative_ceiling_is_envelope() -> None: + budget = _BUDGET + request = _request() + decision = _resolve( + required=budget.authoritative_attested_recipe_result_token_limit + 1, + request=request, + attestation=_attestation(request), + evidence=_protected_evidence(), + ) + assert decision.mode is RecipeDeliveryMode.ENVELOPE + assert decision.selected_result_token_limit == budget.ordinary_omitted_result_token_limit + + +def test_protected_backend_without_selected_budget_fails_closed() -> None: + decision = _resolve(required=10_001, budget=None) + + assert decision.mode is RecipeDeliveryMode.ENVELOPE + assert decision.reason == "protected_delivery_budget_unavailable" + assert decision.contract_digest == "" + + +@pytest.mark.parametrize("missing", ["request", "attestation", "evidence"]) +def test_missing_authority_is_envelope(missing: str) -> None: + request = _request() + decision = _resolve( + required=10_001, + request=None if missing == "request" else request, + attestation=None if missing == "attestation" else _attestation(request), + evidence=None if missing == "evidence" else _protected_evidence(), + ) + assert decision.mode is RecipeDeliveryMode.ENVELOPE + + +def test_mismatched_observation_and_unsupported_identity_are_envelope() -> None: + request = _request() + mismatched = _resolve( + required=10_001, + request=request, + attestation=_attestation(request, observed=10_000), + evidence=_protected_evidence(), + ) + unsupported = _resolve( + required=10_001, + request=request, + attestation=_attestation(request, evidence_identity="unsigned-rollout"), + evidence=_protected_evidence(), + ) + assert mismatched.mode is RecipeDeliveryMode.ENVELOPE + assert unsupported.mode is RecipeDeliveryMode.ENVELOPE + + +@pytest.mark.parametrize( + ("field_name", "invalid_value", "expected_reason"), + [ + ("selected_result_token_limit", 10_000, "host_selected_limit_mismatch"), + ("code_digest", f"sha256:{'d' * 64}", "code_digest_mismatch"), + ("request_digest", f"sha256:{'e' * 64}", "request_digest_mismatch"), + ("expires_at_unix", 1_900_000_000, "attestation_expired"), + ("nonce", "", "attestation_identity_incomplete"), + ], +) +def test_attested_inline_requires_complete_fresh_host_binding( + field_name: str, invalid_value: str | int, expected_reason: str +) -> None: + request = _request() + attestation = dataclasses.replace( + _attestation(request), + **{field_name: invalid_value}, + ) + decision = _resolve( + required=10_001, + request=request, + attestation=attestation, + evidence=_protected_evidence(), + ) + assert decision.mode is RecipeDeliveryMode.ENVELOPE + assert decision.reason == expected_reason + + +def test_non_codex_request_cannot_upgrade_claude() -> None: + request = _request() + decision = _resolve( + backend_name=AGENT_BACKEND_CLAUDE_CODE, + required=46_501, + request=request, + attestation=_attestation(request), + evidence=_protected_evidence(), + ) + assert decision.mode is RecipeDeliveryMode.ENVELOPE + assert decision.selected_result_token_limit == 46_500 + + +def test_history_retention_never_becomes_selected_outer_result() -> None: + budget = _BUDGET._replace(history_retention_token_limit=999_999) + request = _request() + evidence = _protected_evidence()._replace( + selected_result_token_limit=budget.authoritative_attested_recipe_result_token_limit, + ) + decision = _resolve( + required=10_001, + request=request, + attestation=_attestation(request), + evidence=evidence, + budget=budget, + ) + assert decision.mode is RecipeDeliveryMode.ATTESTED_INLINE + assert decision.selected_result_token_limit != budget.history_retention_token_limit + + +def test_negative_required_tokens_fail_closed() -> None: + decision = _resolve(required=-1) + assert decision.mode is RecipeDeliveryMode.ENVELOPE diff --git a/tests/core/test_type_constants.py b/tests/core/test_type_constants.py index c77b01a317..eef41988e8 100644 --- a/tests/core/test_type_constants.py +++ b/tests/core/test_type_constants.py @@ -4,6 +4,7 @@ import hashlib import json +import operator import pytest @@ -42,6 +43,12 @@ def test_response_backstop_exemption_registry_is_closed_and_pinned() -> None: measurement_id="bundled-recipes-all-modes-2026-07-22/open-kitchen", ), } + with pytest.raises(TypeError): + operator.setitem( + RESPONSE_BACKSTOP_EXEMPTION_REGISTRY, + "mutated", + RESPONSE_BACKSTOP_EXEMPTION_REGISTRY["load_recipe"], + ) def test_response_backstop_exemption_registry_digest_is_canonical() -> None: diff --git a/tests/docs/test_output_budget_protocol_decision.py b/tests/docs/test_output_budget_protocol_decision.py index aa3d5e4852..e8bc068f94 100644 --- a/tests/docs/test_output_budget_protocol_decision.py +++ b/tests/docs/test_output_budget_protocol_decision.py @@ -29,7 +29,7 @@ def test_output_budget_decision_is_indexed(decision_text: str) -> None: def test_decision_owns_independent_backend_limits(decision_text: str) -> None: assert "MAX_MCP_OUTPUT_TOKENS" in decision_text - assert "CODEX_TOOL_OUTPUT_TOKEN_LIMIT" in decision_text + assert "CODEX_HISTORY_RETENTION_TOKEN_LIMIT" in decision_text assert "no shared source of truth" in decision_text @@ -49,14 +49,12 @@ def test_decision_names_all_four_layers(decision_text: str, required: str) -> No @pytest.mark.parametrize( "required", [ - "max_chars = 188_000", - "max_utf8_bytes = 188_000", - "186,621 characters and UTF-8 bytes", - "max_chars = 188_000", - "max_utf8_bytes = 188_000", - "186,680 characters and UTF-8 bytes", - "((188_000 + 3) // 4) + 8_000", - "CODEX_TOOL_OUTPUT_TOKEN_LIMIT = 55_000", + "max_chars = 195_000", + "max_utf8_bytes = 195_000", + "ordinary_omitted_result_token_limit = 10_000", + "((195_000 + 3) // 4) + 8_000", + "authoritative_attested_recipe_result_token_limit = 56_750", + "CODEX_HISTORY_RETENTION_TOKEN_LIMIT = 56_750", "CODEX_AUTO_COMPACT_LIMIT = 999_999_999", "inline_max_chars = 5_000", "response_max_bytes = 90_000", @@ -72,14 +70,14 @@ def test_decision_pins_ceiling_backstop_pair(decision_text: str) -> None: assert "44918ea10c0f99151c6710411b4322c2f5c96bea" in decision_text assert "codex-rs/utils/string/src/truncate.rs" in decision_text assert "one-token-per-four-UTF-8-bytes" in decision_text - assert "response_max_bytes // 3 < CODEX_TOOL_OUTPUT_TOKEN_LIMIT" in decision_text + assert "response_max_bytes // 3 < ordinary_omitted_result_token_limit" in decision_text assert "open_kitchen" in decision_text assert "load_recipe" in decision_text assert "live large-output probe" in decision_text assert "RESPONSE_BACKSTOP_EXEMPTION_REGISTRY" in decision_text assert "canonical digest" in decision_text - assert "bundled-recipes-all-modes-2026-07-21/load-recipe" in decision_text - assert "bundled-recipes-all-modes-2026-07-21/open-kitchen" in decision_text + assert "bundled-recipes-all-modes-2026-07-22/load-recipe" in decision_text + assert "bundled-recipes-all-modes-2026-07-22/open-kitchen" in decision_text def test_decision_records_both_corrections(decision_text: str) -> None: @@ -119,18 +117,18 @@ def test_decision_limits_operational_signal_claims(decision_text: str) -> None: "measured exemption use", "spill failures grouped by bounded cause code", "must never contain artifact paths, hashes, or output content", - "artifact quota", - "artifact cleanup", - "cumulative reserve accounting", + "recipe delivery decision mode", + "receipt reservation outcomes", + "one-high-insertion receipt", ]: assert required in signals def test_adr_0005_contains_per_repo_ceiling_and_upgrade_tracking(decision_text: str) -> None: for required in [ - "codex -c tool_output_token_limit=10000", - "CODEX_HOME", - "40 KB", + "CODEX_HISTORY_RETENTION_TOKEN_LIMIT", + "ordinary_omitted_result_token_limit", + "history-retained tokens", "372,000", "CODEX_LIMITS_LAST_VERIFIED_VERSION", "codex_limits_verified", diff --git a/tests/execution/AGENTS.md b/tests/execution/AGENTS.md index 7f30383741..42f1ed1566 100644 --- a/tests/execution/AGENTS.md +++ b/tests/execution/AGENTS.md @@ -159,6 +159,9 @@ Subprocess integration, headless session, process lifecycle, and session result | `test_codex_stream_parser.py` | Full test suite for CodexStreamParser: happy-path, item parsing, degradation, fixture-driven integration, protocol conformance | | `test_codex_result_parser.py` | Tests for CodexResultParser | | `test_codex_config.py` | Tests for TOML read/write primitives, _is_autoskillit_registered, and ensure_codex_mcp_registered | +| `test_codex_recipe_delivery.py` | Protected Codex recipe-delivery attestation and durable receipt-ledger tests | +| `test_codex_recipe_delivery_fixtures.py` | Protected and diagnostic Codex recipe-delivery fixture ratchets | +| `test_codex_recipe_delivery_conformance.py` | Dedicated recipe-delivery conformance matrix plus credentialed Code Mode envelope/pull retention probe | | `test_codex_deterministic_conformance.py` | Sealed-enum vocabulary, hook event format, and config.toml schema template conformance tests with --update-fixtures review gate | | `test_cmd_builder.py` | CmdBuilder ordering invariant and CmdSpec origin tests | | `test_coding_agent_backend_conformance.py` | Parametrized conformance tests for all CodingAgentBackend implementations via BackendContractBase | diff --git a/tests/execution/backends/test_backend_registry.py b/tests/execution/backends/test_backend_registry.py index 39d7459c2c..01e6cfbbd2 100644 --- a/tests/execution/backends/test_backend_registry.py +++ b/tests/execution/backends/test_backend_registry.py @@ -46,8 +46,11 @@ def test_all_exports_complete(self) -> None: "CODEX_MCP_REQUIRED_KEYS", "CODEX_MCP_STARTUP_TIMEOUT_SEC", "CODEX_MCP_TOOL_TIMEOUT_FLOOR", - "CODEX_TOOL_OUTPUT_TOKEN_LIMIT", + "CODEX_HISTORY_RETENTION_TOKEN_LIMIT", "CODEX_TOP_LEVEL_ONLY_FLAGS", + "CODEX_RECIPE_DELIVERY_BUDGET", + "CODEX_RECIPE_DELIVERY_CALLING_CONTRACT", + "CODEX_RECIPE_DELIVERY_CALLING_CONTRACT_DIGEST", "CompositeSessionLocator", "ClaudeCodeBackend", "ClaudeEnvPolicy", @@ -55,6 +58,9 @@ def test_all_exports_complete(self) -> None: "ClaudeSessionLocator", "ClaudeStreamParser", "CodexBackend", + "CodexAttestationResult", + "CodexHostCorrelation", + "CodexOuterBudgetAttestor", "CodexEnvPolicy", "CodexFlags", "CodexResultParser", @@ -62,16 +68,27 @@ def test_all_exports_complete(self) -> None: "CodexSessionLocator", "CodexStreamParser", "NON_VARIADIC_CODEX_FLAGS", + "NullProtectedHostAttestationProvider", + "ProtectedHostAttestationProvider", + "ProtectedStoreAuthority", + "RecipeDeliveryReceiptLedger", + "RecipeReceiptHandle", + "RecipeReservationResult", + "SUPPORTED_CODEX_RECIPE_EVIDENCE_REGISTRY", "VARIADIC_CODEX_FLAGS", "_is_autoskillit_hook_entry", "_is_autoskillit_registered", "_read_codex_config", "_serialize_toml", "_write_codex_config", + "codex_recipe_delivery_calling_contract", "ensure_codex_mcp_registered", + "enumerate_fresh_codex_marker_ids", "generate_codex_hooks_config", "get_backend", "make_codex_scenario_player", + "read_rollout_thread_id", + "resolve_unique_codex_host_correlation", "sync_hooks_to_codex_config", } assert set(all_exports) == expected diff --git a/tests/execution/backends/test_cli_conformance_probes.py b/tests/execution/backends/test_cli_conformance_probes.py index 2454015b4e..9987fdfd88 100644 --- a/tests/execution/backends/test_cli_conformance_probes.py +++ b/tests/execution/backends/test_cli_conformance_probes.py @@ -33,7 +33,7 @@ pkg_root, ) from autoskillit.execution.backends._codex_config import ( - CODEX_TOOL_OUTPUT_TOKEN_LIMIT, + CODEX_HISTORY_RETENTION_TOKEN_LIMIT, ensure_codex_mcp_registered, ) from autoskillit.execution.backends._codex_hooks import sync_hooks_to_codex_config @@ -837,7 +837,7 @@ def test_hook_fires_and_command_is_rewritten(self, tmp_path: Path) -> None: _SOURCE_SPILL_THRESHOLD = OutputBudgetConfig().inline_max_chars -_CODEX_HEURISTIC_BYTES = CODEX_TOOL_OUTPUT_TOKEN_LIMIT * 4 +_CODEX_HEURISTIC_BYTES = CODEX_HISTORY_RETENTION_TOKEN_LIMIT * 4 _SERIALIZED_ENVELOPE_SLACK_BYTES = 4096 _LARGE_OUTPUT_CASE_BYTES = tuple( sorted( @@ -1038,7 +1038,7 @@ def _assert_large_output_probe(output: _LargeOutputProbe) -> None: spill_by_size[size] = spilled assert_inline_within_byte_budget( json.dumps(entry), - CODEX_TOOL_OUTPUT_TOKEN_LIMIT * 4, + CODEX_HISTORY_RETENTION_TOKEN_LIMIT * 4, envelope_slack_bytes=_SERIALIZED_ENVELOPE_SLACK_BYTES, ) if spilled: diff --git a/tests/execution/backends/test_codex_backend.py b/tests/execution/backends/test_codex_backend.py index dabe1e3294..155ccb7992 100644 --- a/tests/execution/backends/test_codex_backend.py +++ b/tests/execution/backends/test_codex_backend.py @@ -832,7 +832,7 @@ def test_bare_resume_produces_resume_subcommand_without_session_id(self) -> None def test_system_prompt_with_no_resume_appends_config_override(self) -> None: import tomllib - from autoskillit.core import CODEX_INTAKE_DISCIPLINE_DIGEST, OUTPUT_DISCIPLINE_DIGEST + from autoskillit.execution.backends._claude_prompt import codex_discipline_suffix spec = CodexBackend().build_interactive_cmd(system_prompt="foo") overrides = [ @@ -844,10 +844,7 @@ def test_system_prompt_with_no_resume_appends_config_override(self) -> None: if value.startswith("developer_instructions=") ) parsed = tomllib.loads(f"developer_instructions = {rendered}") - assert ( - parsed["developer_instructions"] - == f"foo\n\n{OUTPUT_DISCIPLINE_DIGEST}\n\n{CODEX_INTAKE_DISCIPLINE_DIGEST}" - ) + assert parsed["developer_instructions"] == f"foo\n\n{codex_discipline_suffix()}" assert "features.image_generation=false" in overrides def test_system_prompt_with_named_resume_does_not_append_config_override(self) -> None: @@ -1846,13 +1843,12 @@ def test_agent_toml_required_fields_present_and_nonempty(self) -> None: assert data["developer_instructions"], ( f"{toml_path.name}: developer_instructions empty" ) - from autoskillit.core import CODEX_INTAKE_DISCIPLINE_DIGEST, OUTPUT_DISCIPLINE_DIGEST + from autoskillit.execution.backends._claude_prompt import codex_discipline_suffix - assert OUTPUT_DISCIPLINE_DIGEST in data["developer_instructions"] assert ( data["developer_instructions"] .rstrip() - .endswith(CODEX_INTAKE_DISCIPLINE_DIGEST.rstrip()) + .endswith(codex_discipline_suffix().rstrip()) ) assert data["sandbox_mode"] == "workspace-write", ( f"{toml_path.name}: wrong sandbox_mode" diff --git a/tests/execution/backends/test_codex_config.py b/tests/execution/backends/test_codex_config.py index 5400178309..61cb710a6c 100644 --- a/tests/execution/backends/test_codex_config.py +++ b/tests/execution/backends/test_codex_config.py @@ -9,15 +9,22 @@ from autoskillit.core import ( CODEX_MCP_ENV_FORWARD_VARS, HEADLESS_AUTO_GATE_ENV_VAR, + RECIPE_DELIVERY_SURFACE_REGISTRY, RESPONSE_BACKSTOP_EXEMPTION_REGISTRY, ) -from autoskillit.execution.backends import ensure_codex_mcp_registered +from autoskillit.execution.backends import ( + CODEX_RECIPE_DELIVERY_BUDGET, + SUPPORTED_CODEX_RECIPE_EVIDENCE_REGISTRY, + CodexBackend, + codex_recipe_delivery_calling_contract, + ensure_codex_mcp_registered, +) from autoskillit.execution.backends._codex_config import ( CODEX_AUTO_COMPACT_LIMIT, + CODEX_HISTORY_RETENTION_TOKEN_LIMIT, CODEX_MCP_REQUIRED_KEYS, CODEX_MCP_STARTUP_TIMEOUT_SEC, CODEX_MCP_TOOL_TIMEOUT_FLOOR, - CODEX_TOOL_OUTPUT_TOKEN_LIMIT, _is_autoskillit_registered, _read_codex_config, _serialize_toml, @@ -27,7 +34,7 @@ pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] -def test_codex_tool_output_limit_is_derived_from_largest_measured_exemption() -> None: +def test_codex_history_retention_limit_is_derived_from_largest_measured_exemption() -> None: """The four-byte relation is Codex's coarse truncation heuristic, not tokenization. This ceiling is a blast-radius damage bound; producer guards and output @@ -37,16 +44,47 @@ def test_codex_tool_output_limit_is_derived_from_largest_measured_exemption() -> definition.max_utf8_bytes for definition in RESPONSE_BACKSTOP_EXEMPTION_REGISTRY.values() ) assert max_bytes == 195_000 - assert CODEX_TOOL_OUTPUT_TOKEN_LIMIT == ((max_bytes + 3) // 4) + 8_000 - assert CODEX_TOOL_OUTPUT_TOKEN_LIMIT == 56_750 + assert CODEX_HISTORY_RETENTION_TOKEN_LIMIT == ((max_bytes + 3) // 4) + 8_000 + assert CODEX_HISTORY_RETENTION_TOKEN_LIMIT == 56_750 + + +def test_codex_recipe_delivery_authorities_start_fail_closed() -> None: + budget = CODEX_RECIPE_DELIVERY_BUDGET + assert budget.ordinary_omitted_result_token_limit == 10_000 + assert ( + CodexBackend().capabilities.unnegotiated_tool_result_token_limit + == budget.ordinary_omitted_result_token_limit + ) + assert budget.authoritative_attested_recipe_result_token_limit == 56_750 + assert budget.history_retention_token_limit == CODEX_HISTORY_RETENTION_TOKEN_LIMIT + assert budget.contract_digest.startswith("sha256:") + assert SUPPORTED_CODEX_RECIPE_EVIDENCE_REGISTRY == {} + + +def test_codex_recipe_calling_contract_derives_eligible_producers_from_registry() -> None: + expected = sorted( + { + definition.producer_tool + for definition in RECIPE_DELIVERY_SURFACE_REGISTRY.values() + if definition.negotiation_eligible + } + ) + + contract = codex_recipe_delivery_calling_contract(mcp_prefix="mcp__autoskillit__") + + assert ( + "- The sole exception is a full recipe call to " + + ", ".join(f"mcp__autoskillit__{producer}" for producer in expected) + + " with a protected host-provided delivery request." + ) in contract -def test_response_backstop_fires_below_codex_transport_ceiling() -> None: - assert OutputBudgetConfig().response_max_bytes // 3 < CODEX_TOOL_OUTPUT_TOKEN_LIMIT +def test_response_backstop_fires_below_codex_history_retention_limit() -> None: + assert OutputBudgetConfig().response_max_bytes // 3 < CODEX_HISTORY_RETENTION_TOKEN_LIMIT -def test_resolve_effective_delivery_bound_per_backend() -> None: - from autoskillit.core import BackendCapabilities, resolve_effective_delivery_bound +def test_resolve_general_output_token_limit_per_backend() -> None: + from autoskillit.core import BackendCapabilities, resolve_general_output_token_limit from autoskillit.execution.backends import BACKEND_REGISTRY expected = {"codex": 10_000, "claude-code": 46_500} @@ -54,7 +92,7 @@ def test_resolve_effective_delivery_bound_per_backend() -> None: for name, cls in BACKEND_REGISTRY.items(): caps = cls().capabilities assert isinstance(caps, BackendCapabilities) - observed[name] = resolve_effective_delivery_bound(caps) + observed[name] = resolve_general_output_token_limit(caps) assert observed == expected @@ -266,7 +304,7 @@ def test_headless_auto_gate_false_ignores_auto_gate_env(self): "tool_timeout_sec": CODEX_MCP_TOOL_TIMEOUT_FLOOR, } }, - "tool_output_token_limit": CODEX_TOOL_OUTPUT_TOKEN_LIMIT, + "tool_output_token_limit": CODEX_HISTORY_RETENTION_TOKEN_LIMIT, "model_auto_compact_token_limit": CODEX_AUTO_COMPACT_LIMIT, } assert _is_autoskillit_registered(config, headless_auto_gate=False) is True @@ -281,7 +319,7 @@ def test_all_checks_pass_with_auto_gate(self): "tool_timeout_sec": CODEX_MCP_TOOL_TIMEOUT_FLOOR, } }, - "tool_output_token_limit": CODEX_TOOL_OUTPUT_TOKEN_LIMIT, + "tool_output_token_limit": CODEX_HISTORY_RETENTION_TOKEN_LIMIT, "model_auto_compact_token_limit": CODEX_AUTO_COMPACT_LIMIT, } assert _is_autoskillit_registered(config, headless_auto_gate=True) is True @@ -297,7 +335,7 @@ def test_extra_keys_tolerated(self): "extra_field": 42, } }, - "tool_output_token_limit": CODEX_TOOL_OUTPUT_TOKEN_LIMIT, + "tool_output_token_limit": CODEX_HISTORY_RETENTION_TOKEN_LIMIT, "model_auto_compact_token_limit": CODEX_AUTO_COMPACT_LIMIT, } assert _is_autoskillit_registered(config, headless_auto_gate=False) is True @@ -369,7 +407,7 @@ def test_low_tool_output_token_limit_returns_false(self): "tool_timeout_sec": CODEX_MCP_TOOL_TIMEOUT_FLOOR, } }, - "tool_output_token_limit": CODEX_TOOL_OUTPUT_TOKEN_LIMIT - 1, + "tool_output_token_limit": CODEX_HISTORY_RETENTION_TOKEN_LIMIT - 1, } assert _is_autoskillit_registered(config, headless_auto_gate=True) is False @@ -384,7 +422,7 @@ def test_high_tool_output_token_limit_returns_false(self): "tool_timeout_sec": CODEX_MCP_TOOL_TIMEOUT_FLOOR, } }, - "tool_output_token_limit": CODEX_TOOL_OUTPUT_TOKEN_LIMIT + 1, + "tool_output_token_limit": CODEX_HISTORY_RETENTION_TOKEN_LIMIT + 1, "model_auto_compact_token_limit": CODEX_AUTO_COMPACT_LIMIT, } assert _is_autoskillit_registered(config, headless_auto_gate=True) is False @@ -400,7 +438,7 @@ def test_missing_auto_compact_limit_returns_false(self): "tool_timeout_sec": CODEX_MCP_TOOL_TIMEOUT_FLOOR, } }, - "tool_output_token_limit": CODEX_TOOL_OUTPUT_TOKEN_LIMIT, + "tool_output_token_limit": CODEX_HISTORY_RETENTION_TOKEN_LIMIT, } assert _is_autoskillit_registered(config, headless_auto_gate=True) is False @@ -415,7 +453,7 @@ def test_low_auto_compact_limit_returns_false(self): "tool_timeout_sec": CODEX_MCP_TOOL_TIMEOUT_FLOOR, } }, - "tool_output_token_limit": CODEX_TOOL_OUTPUT_TOKEN_LIMIT, + "tool_output_token_limit": CODEX_HISTORY_RETENTION_TOKEN_LIMIT, "model_auto_compact_token_limit": 100_000, } assert _is_autoskillit_registered(config, headless_auto_gate=True) is False @@ -431,7 +469,7 @@ def test_auto_compact_limit_at_sentinel_returns_true(self): "tool_timeout_sec": CODEX_MCP_TOOL_TIMEOUT_FLOOR, } }, - "tool_output_token_limit": CODEX_TOOL_OUTPUT_TOKEN_LIMIT, + "tool_output_token_limit": CODEX_HISTORY_RETENTION_TOKEN_LIMIT, "model_auto_compact_token_limit": CODEX_AUTO_COMPACT_LIMIT, } assert _is_autoskillit_registered(config, headless_auto_gate=True) is True @@ -455,7 +493,7 @@ def test_tool_output_token_limit_written_to_top_level(self, tmp_path): ensure_codex_mcp_registered(config_path=p) config = _read_codex_config(p).data assert "tool_output_token_limit" in config - assert config["tool_output_token_limit"] == CODEX_TOOL_OUTPUT_TOKEN_LIMIT + assert config["tool_output_token_limit"] == CODEX_HISTORY_RETENTION_TOKEN_LIMIT assert "tool_output_token_limit" not in config["mcp_servers"]["autoskillit"], ( "tool_output_token_limit is a Codex global key, not a server-entry key" ) @@ -486,13 +524,13 @@ def test_parseable_config_rewrites_tool_limit_without_lowering_compact_minimum(s ensure_codex_mcp_registered(config_path=p) higher_compact_limit = CODEX_AUTO_COMPACT_LIMIT + 1 text = p.read_text(encoding="utf-8") - text = text.replace(str(CODEX_TOOL_OUTPUT_TOKEN_LIMIT), "50000") + text = text.replace(str(CODEX_HISTORY_RETENTION_TOKEN_LIMIT), "50000") text = text.replace(str(CODEX_AUTO_COMPACT_LIMIT), str(higher_compact_limit)) p.write_text(text, encoding="utf-8") assert ensure_codex_mcp_registered(config_path=p) is True config = _read_codex_config(p).data - assert config["tool_output_token_limit"] == CODEX_TOOL_OUTPUT_TOKEN_LIMIT + assert config["tool_output_token_limit"] == CODEX_HISTORY_RETENTION_TOKEN_LIMIT assert config["model_auto_compact_token_limit"] == higher_compact_limit def test_headless_auto_gate_true_includes_auto_gate_env(self, tmp_path): @@ -657,7 +695,7 @@ def test_corrupt_file_path_writes_tool_output_token_limit(self, tmp_path): p.write_text("not valid toml = =", encoding="utf-8") ensure_codex_mcp_registered(config_path=p) content = p.read_text(encoding="utf-8") - assert f"tool_output_token_limit = {CODEX_TOOL_OUTPUT_TOKEN_LIMIT}" in content + assert f"tool_output_token_limit = {CODEX_HISTORY_RETENTION_TOKEN_LIMIT}" in content assert "[mcp_servers.autoskillit]" in content def test_corrupt_file_rewrites_existing_tool_output_limit_exactly(self, tmp_path): @@ -670,7 +708,7 @@ def test_corrupt_file_rewrites_existing_tool_output_limit_exactly(self, tmp_path ensure_codex_mcp_registered(config_path=p) content = p.read_text(encoding="utf-8") - assert f"tool_output_token_limit = {CODEX_TOOL_OUTPUT_TOKEN_LIMIT}" in content + assert f"tool_output_token_limit = {CODEX_HISTORY_RETENTION_TOKEN_LIMIT}" in content assert "tool_output_token_limit = 50_000" not in content assert content.count("tool_output_token_limit =") == 1 @@ -802,7 +840,7 @@ def test_env_vars_format_returns_true(self): "tool_timeout_sec": CODEX_MCP_TOOL_TIMEOUT_FLOOR, } }, - "tool_output_token_limit": CODEX_TOOL_OUTPUT_TOKEN_LIMIT, + "tool_output_token_limit": CODEX_HISTORY_RETENTION_TOKEN_LIMIT, "model_auto_compact_token_limit": CODEX_AUTO_COMPACT_LIMIT, } assert _is_autoskillit_registered(config, headless_auto_gate=True) is True @@ -955,7 +993,7 @@ def _full_config(self, env_vars: list[str]) -> dict: "tool_timeout_sec": CODEX_MCP_TOOL_TIMEOUT_FLOOR, } }, - "tool_output_token_limit": CODEX_TOOL_OUTPUT_TOKEN_LIMIT, + "tool_output_token_limit": CODEX_HISTORY_RETENTION_TOKEN_LIMIT, "model_auto_compact_token_limit": CODEX_AUTO_COMPACT_LIMIT, } diff --git a/tests/execution/backends/test_codex_deterministic_conformance.py b/tests/execution/backends/test_codex_deterministic_conformance.py index 599adf8259..98f83997a8 100644 --- a/tests/execution/backends/test_codex_deterministic_conformance.py +++ b/tests/execution/backends/test_codex_deterministic_conformance.py @@ -20,8 +20,8 @@ from autoskillit.core.types._type_enums import CodexEventType, CodexItemType from autoskillit.execution.backends._codex_config import ( CODEX_AUTO_COMPACT_LIMIT, + CODEX_HISTORY_RETENTION_TOKEN_LIMIT, CODEX_MCP_REQUIRED_KEYS, - CODEX_TOOL_OUTPUT_TOKEN_LIMIT, ) from autoskillit.execution.backends._codex_hooks import generate_codex_hooks_config from autoskillit.hook_registry import HOOK_REGISTRY_HASH, HOOKS_DIR @@ -86,7 +86,7 @@ def _generate_config_template() -> dict: "tool_output_token_limit": { "expected_type": "int", "constraint": "exact", - "expected_value": CODEX_TOOL_OUTPUT_TOKEN_LIMIT, + "expected_value": CODEX_HISTORY_RETENTION_TOKEN_LIMIT, }, "model_auto_compact_token_limit": { "expected_type": "int", @@ -214,7 +214,7 @@ def test_generator_preserves_distinct_top_level_constraint_semantics(self) -> No assert top["tool_output_token_limit"] == { "constraint": "exact", "expected_type": "int", - "expected_value": CODEX_TOOL_OUTPUT_TOKEN_LIMIT, + "expected_value": CODEX_HISTORY_RETENTION_TOKEN_LIMIT, } assert top["model_auto_compact_token_limit"] == { "constraint": "minimum", @@ -238,7 +238,7 @@ def test_pinned_constants_match(self) -> None: assert top["tool_output_token_limit"] == { "constraint": "exact", "expected_type": "int", - "expected_value": CODEX_TOOL_OUTPUT_TOKEN_LIMIT, + "expected_value": CODEX_HISTORY_RETENTION_TOKEN_LIMIT, } assert top["model_auto_compact_token_limit"]["floor_value"] == CODEX_AUTO_COMPACT_LIMIT, ( f"CODEX_AUTO_COMPACT_LIMIT drift: " @@ -268,7 +268,7 @@ def test_update_fixtures_regenerates_template( assert set(reloaded["_codex_mcp_required_keys"]) == CODEX_MCP_REQUIRED_KEYS assert ( reloaded["top_level_keys"]["tool_output_token_limit"]["expected_value"] - == CODEX_TOOL_OUTPUT_TOKEN_LIMIT + == CODEX_HISTORY_RETENTION_TOKEN_LIMIT ) assert ( reloaded["top_level_keys"]["model_auto_compact_token_limit"]["floor_value"] diff --git a/tests/execution/backends/test_codex_interactive.py b/tests/execution/backends/test_codex_interactive.py index d237cd4d47..ec5054e6a8 100644 --- a/tests/execution/backends/test_codex_interactive.py +++ b/tests/execution/backends/test_codex_interactive.py @@ -10,14 +10,8 @@ import pytest -from autoskillit.core import ( - CODEX_INTAKE_DISCIPLINE_DIGEST, - OUTPUT_DISCIPLINE_DIGEST, - BareResume, - CmdSpec, - NamedResume, - NoResume, -) +from autoskillit.core import BareResume, CmdSpec, NamedResume, NoResume +from autoskillit.execution.backends._claude_prompt import codex_discipline_suffix from autoskillit.execution.backends.codex import CodexBackend, CodexFlags pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] @@ -98,10 +92,7 @@ def test_system_prompt_with_no_resume_produces_config_override(self) -> None: system_prompt="do stuff", resume_spec=NoResume(), ) - assert ( - _developer_instructions(spec) - == f"do stuff\n\n{OUTPUT_DISCIPLINE_DIGEST}\n\n{CODEX_INTAKE_DISCIPLINE_DIGEST}" - ) + assert _developer_instructions(spec) == f"do stuff\n\n{codex_discipline_suffix()}" overrides = [ spec.cmd[i + 1] for i, v in enumerate(spec.cmd[:-1]) if v == CodexFlags.CONFIG_OVERRIDE ] @@ -134,10 +125,7 @@ def test_no_system_prompt_with_no_resume_excludes_config_override(self) -> None: overrides = [ spec.cmd[i + 1] for i, v in enumerate(spec.cmd[:-1]) if v == CodexFlags.CONFIG_OVERRIDE ] - assert ( - _developer_instructions(spec) - == f"{OUTPUT_DISCIPLINE_DIGEST}\n\n{CODEX_INTAKE_DISCIPLINE_DIGEST}" - ) + assert _developer_instructions(spec) == codex_discipline_suffix() assert "features.image_generation=false" in overrides def test_system_prompt_override_is_valid_toml_with_quotes_and_newlines(self) -> None: @@ -146,9 +134,7 @@ def test_system_prompt_override_is_valid_toml_with_quotes_and_newlines(self) -> system_prompt=caller_prompt, resume_spec=NoResume(), ) - assert _developer_instructions(spec) == ( - f"{caller_prompt}\n\n{OUTPUT_DISCIPLINE_DIGEST}\n\n{CODEX_INTAKE_DISCIPLINE_DIGEST}" - ) + assert _developer_instructions(spec) == (f"{caller_prompt}\n\n{codex_discipline_suffix()}") def test_installed_codex_parses_exact_fresh_config_overrides(self, tmp_path: Path) -> None: binary = shutil.which("codex") diff --git a/tests/execution/backends/test_codex_recipe_delivery.py b/tests/execution/backends/test_codex_recipe_delivery.py new file mode 100644 index 0000000000..b1c856cc7b --- /dev/null +++ b/tests/execution/backends/test_codex_recipe_delivery.py @@ -0,0 +1,699 @@ +"""Protected Codex attestation and durable receipt-store contracts.""" + +from __future__ import annotations + +import copy +import json +import sqlite3 +from dataclasses import dataclass, replace +from datetime import UTC, datetime +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock + +import pytest + +import autoskillit.execution.backends._codex_recipe_delivery as recipe_delivery +from autoskillit.core import ( + RecipeDeliveryAttestation, + RecipeDeliveryEvidenceDef, + RecipeDeliveryMode, + RecipeDeliveryRequest, + recipe_delivery_request_digest, + resolve_recipe_delivery_decision, +) +from autoskillit.execution.backends import BACKEND_REGISTRY, CODEX_RECIPE_DELIVERY_BUDGET +from autoskillit.execution.backends._codex_recipe_delivery import ( + _MARKER_MAX_CANDIDATES, + CodexHostCorrelation, + CodexOuterBudgetAttestor, + NullProtectedHostAttestationProvider, + ProtectedStoreAuthority, + RecipeDeliveryReceiptLedger, + _fresh_marker_session_id, + enumerate_fresh_codex_marker_ids, + read_rollout_thread_id, + resolve_unique_codex_host_correlation, +) +from tests.fixtures.codex_recipe_diagnostic import ( + UNSIGNED_TRACE_V1, + WRITABLE_ROLLOUT_V1, +) +from tests.fixtures.codex_recipe_diagnostic import ( + fixture_path as diagnostic_fixture_path, +) +from tests.fixtures.codex_recipe_protected import ( + PROTECTED_FUNCTIONS_EXEC_V1, +) +from tests.fixtures.codex_recipe_protected import ( + fixture_path as protected_fixture_path, +) + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + +_NOW = 1_800_000_000 +_HIGH = 56_750 +_FIXTURE_DIGEST = "sha256:" + ("d" * 64) +_PAYLOAD_SHA = "sha256:" + ("a" * 64) + + +def _protected_record() -> dict[str, object]: + return json.loads(protected_fixture_path(PROTECTED_FUNCTIONS_EXEC_V1).read_text()) + + +def _request() -> RecipeDeliveryRequest: + return RecipeDeliveryRequest( + audience="autoskillit.recipe-delivery", + delivery_call_id="delivery-001", + contract_version=1, + contract_digest=_FIXTURE_DIGEST, + caller_requested_outer_tokens=_HIGH, + code_digest="sha256:" + ("b" * 64), + ) + + +def _evidence() -> RecipeDeliveryEvidenceDef: + return RecipeDeliveryEvidenceDef( + identity="protected-test-host-v1", + host_channel="test-only-process-isolated-host", + evidence_schema_version=1, + parser_version=1, + cli_identity="codex-test-cli", + selected_limit_derivation="protected_resolved_outer_limit", + selected_result_token_limit=_HIGH, + contract_digest=_FIXTURE_DIGEST, + ) + + +@dataclass(frozen=True, slots=True) +class _FixtureProtectedProvider: + record: dict[str, object] + authority: ProtectedStoreAuthority | None = None + unavailable: bool = False + + def attest( + self, + *, + request: RecipeDeliveryRequest, + correlation: CodexHostCorrelation, + now_unix: int, + ) -> RecipeDeliveryAttestation | None: + if self.unavailable: + raise OSError("protected provider unavailable") + if len(json.dumps(self.record).encode("utf-8")) > 256 * 1024: + return None + if ( + self.record.get("schema_version") != 1 + or self.record.get("authenticated") is not True + or self.record.get("caller_writable") is not False + or self.record.get("complete") is not True + or self.record.get("event_sequence") != ["outer_call_selected", "nested_call_started"] + ): + return None + event = self.record.get("event") + if not isinstance(event, dict): + return None + pragma = event.get("pragma") + if not isinstance(pragma, str) or not pragma.startswith("// @exec: "): + return None + try: + pragma_maximum = json.loads(pragma.removeprefix("// @exec: "))["max_output_tokens"] + except (json.JSONDecodeError, KeyError, TypeError): + return None + required = { + "audience": request.audience, + "thread_id": correlation.thread_id, + "turn_id": "turn-001", + "outer_call_id": "functions-exec-001", + "code_mode_cell_id": "cell-001", + "delivery_call_id": request.delivery_call_id, + "nested_call_id": request.delivery_call_id, + "host_observed_requested_outer_tokens": _HIGH, + "selected_result_token_limit": _HIGH, + "code_digest": request.code_digest, + "request_digest": recipe_delivery_request_digest(request), + "contract_version": request.contract_version, + "contract_digest": request.contract_digest, + "parser_version": 1, + "evidence_version": 1, + "evidence_identity": "protected-test-host-v1", + } + if pragma_maximum != _HIGH or any( + event.get(key) != value for key, value in required.items() + ): + return None + if event.get("expires_at_unix", 0) < now_unix or not event.get("nonce"): + return None + return RecipeDeliveryAttestation( + audience=request.audience, + thread_id=correlation.thread_id, + turn_id=str(event["turn_id"]), + outer_call_id=str(event["outer_call_id"]), + code_mode_cell_id=str(event["code_mode_cell_id"]), + delivery_call_id=request.delivery_call_id, + host_observed_requested_outer_tokens=_HIGH, + selected_result_token_limit=_HIGH, + code_digest=request.code_digest, + request_digest=str(event["request_digest"]), + nonce=str(event["nonce"]), + expires_at_unix=int(event["expires_at_unix"]), + contract_version=request.contract_version, + contract_digest=request.contract_digest, + parser_version=1, + evidence_version=1, + evidence_identity="protected-test-host-v1", + ) + + def store_authority(self, *, thread_id: str) -> ProtectedStoreAuthority | None: + del thread_id + return self.authority + + +def _write_marker_and_rollout(project_dir: Path, thread_id: str) -> Path: + state_dir = project_dir / ".autoskillit" / "temp" / "kitchen_state" + state_dir.mkdir(parents=True, exist_ok=True) + opened_at = datetime.fromtimestamp(_NOW - 10, tz=UTC).isoformat() + (state_dir / f"{thread_id}.json").write_text( + json.dumps({"session_id": thread_id, "opened_at": opened_at}), + encoding="utf-8", + ) + rollout = project_dir / f"rollout-{thread_id}.jsonl" + rollout.write_text( + json.dumps({"type": "thread.started", "thread_id": thread_id}) + "\n", + encoding="utf-8", + ) + return rollout + + +def test_marker_validation_uses_one_bounded_file_descriptor(tmp_path: Path) -> None: + _write_marker_and_rollout(tmp_path, "thread-single-descriptor") + marker = tmp_path / ".autoskillit" / "temp" / "kitchen_state" / "thread-single-descriptor.json" + open_count = 0 + + class _GuardedMarkerPath(type(marker)): + def open(self, *args: Any, **kwargs: Any) -> Any: + nonlocal open_count + open_count += 1 + if open_count > 1: + raise AssertionError("marker validation opened the marker more than once") + return super().open(*args, **kwargs) + + def stat(self, *_args: Any, **_kwargs: Any) -> Any: + raise AssertionError("marker validation performed a separate stat") + + def is_file(self) -> bool: + raise AssertionError("marker validation performed a separate file check") + + def read_text(self, *args: Any, **kwargs: Any) -> str: + raise AssertionError("marker validation performed an unbounded path read") + + assert ( + _fresh_marker_session_id( + _GuardedMarkerPath(marker), + now_unix=_NOW, + ttl_seconds=24 * 60 * 60, + ) + == "thread-single-descriptor" + ) + assert open_count == 1 + + +def test_oversized_marker_fails_closed_without_unbounded_read(tmp_path: Path) -> None: + _write_marker_and_rollout(tmp_path, "thread-oversized-marker") + marker = tmp_path / ".autoskillit" / "temp" / "kitchen_state" / "thread-oversized-marker.json" + marker.write_bytes(b"x" * ((64 * 1024) + 1)) + + assert enumerate_fresh_codex_marker_ids(tmp_path, now_unix=_NOW) == () + + +def test_marker_enumeration_fails_closed_above_candidate_ceiling(tmp_path: Path) -> None: + state_dir = tmp_path / ".autoskillit" / "temp" / "kitchen_state" + state_dir.mkdir(parents=True) + opened_at = datetime.fromtimestamp(_NOW - 10, tz=UTC).isoformat() + for index in range(_MARKER_MAX_CANDIDATES + 1): + session_id = f"thread-candidate-{index:04d}" + (state_dir / f"{session_id}.json").write_text( + json.dumps({"session_id": session_id, "opened_at": opened_at}), + encoding="utf-8", + ) + + assert enumerate_fresh_codex_marker_ids(tmp_path, now_unix=_NOW) == () + + +@pytest.mark.parametrize("campaign_id", [".", ".."]) +def test_marker_enumeration_rejects_dot_campaign_components( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, campaign_id: str +) -> None: + base = tmp_path / ".autoskillit" / "temp" / "kitchen_state" + marker_dir = base if campaign_id == "." else base.parent + marker_dir.mkdir(parents=True) + (marker_dir / "thread-outside-campaign.json").write_text( + json.dumps( + { + "session_id": "thread-outside-campaign", + "opened_at": datetime.fromtimestamp(_NOW - 10, tz=UTC).isoformat(), + } + ), + encoding="utf-8", + ) + monkeypatch.setenv("AUTOSKILLIT_CAMPAIGN_ID", campaign_id) + + assert enumerate_fresh_codex_marker_ids(tmp_path, now_unix=_NOW) == () + + +def _attestor( + project_dir: Path, + provider: _FixtureProtectedProvider | NullProtectedHostAttestationProvider, + rollouts: dict[str, Path], +) -> CodexOuterBudgetAttestor: + del project_dir + return CodexOuterBudgetAttestor( + provider=provider, + locate_rollout=rollouts.get, + supported_evidence={_evidence().identity: _evidence()}, + ) + + +def _delivery_mode(result) -> RecipeDeliveryMode: + caps = replace( + BACKEND_REGISTRY["codex"]().capabilities, + protected_recipe_delivery_capable=True, + ) + decision = resolve_recipe_delivery_decision( + capabilities=caps, + required_serialized_tokens=10_001, + budget=CODEX_RECIPE_DELIVERY_BUDGET._replace(contract_digest=_FIXTURE_DIGEST), + producer="open_kitchen", + payload_sha256=_PAYLOAD_SHA, + request=_request(), + attestation=result.attestation, + supported_evidence=result.evidence, + now_unix=_NOW, + ) + return decision.mode + + +def test_valid_protected_fixture_attests_and_selects_high_inline(tmp_path: Path) -> None: + rollout = _write_marker_and_rollout(tmp_path, "thread-protected-001") + result = _attestor( + tmp_path, + _FixtureProtectedProvider(_protected_record()), + {"thread-protected-001": rollout}, + ).attest(request=_request(), project_dir=tmp_path, now_unix=_NOW) + assert result.reason == "attested" + assert _delivery_mode(result) is RecipeDeliveryMode.ATTESTED_INLINE + + +@pytest.mark.parametrize( + ("target", "key", "value"), + [ + ("root", "complete", False), + ("root", "event_sequence", ["nested_call_started", "outer_call_selected"]), + ("event", "thread_id", "wrong-thread"), + ("event", "turn_id", "wrong-turn"), + ("event", "outer_call_id", "wrong-call"), + ("event", "code_mode_cell_id", "wrong-cell"), + ("event", "nested_call_id", "wrong-nested"), + ("event", "pragma", "// @exec: not-json"), + ("event", "pragma", "// @exec: {}"), + ("event", "pragma", '// @exec: {"max_output_tokens": 10000}'), + ("event", "pragma", '// @exec: {"max_output_tokens": 56751}'), + ("event", "host_observed_requested_outer_tokens", None), + ("event", "host_observed_requested_outer_tokens", 10_000), + ("event", "host_observed_requested_outer_tokens", _HIGH + 1), + ("root", "schema_version", 2), + ("event", "parser_version", 2), + ("event", "evidence_version", 2), + ], +) +def test_protected_evidence_negative_matrix_is_envelope( + tmp_path: Path, + target: str, + key: str, + value: object, +) -> None: + record = copy.deepcopy(_protected_record()) + container = record if target == "root" else record["event"] + assert isinstance(container, dict) + if value is None: + container.pop(key) + else: + container[key] = value + rollout = _write_marker_and_rollout(tmp_path, "thread-protected-001") + result = _attestor( + tmp_path, + _FixtureProtectedProvider(record), + {"thread-protected-001": rollout}, + ).attest(request=_request(), project_dir=tmp_path, now_unix=_NOW) + assert result.attestation is None + assert _delivery_mode(result) is RecipeDeliveryMode.ENVELOPE + + +def test_oversized_record_and_unavailable_provider_are_envelope(tmp_path: Path) -> None: + rollout = _write_marker_and_rollout(tmp_path, "thread-protected-001") + oversized = _protected_record() + oversized["padding"] = "x" * (256 * 1024) + for provider in ( + _FixtureProtectedProvider(oversized), + _FixtureProtectedProvider(_protected_record(), unavailable=True), + ): + result = _attestor( + tmp_path, + provider, + {"thread-protected-001": rollout}, + ).attest(request=_request(), project_dir=tmp_path, now_unix=_NOW) + assert result.attestation is None + assert _delivery_mode(result) is RecipeDeliveryMode.ENVELOPE + + +def test_unavailable_provider_preserves_protected_failure_diagnostics( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + rollout = _write_marker_and_rollout(tmp_path, "thread-protected-001") + logger = MagicMock() + monkeypatch.setattr(recipe_delivery, "get_logger", lambda _name: logger) + + result = _attestor( + tmp_path, + _FixtureProtectedProvider(_protected_record(), unavailable=True), + {"thread-protected-001": rollout}, + ).attest(request=_request(), project_dir=tmp_path, now_unix=_NOW) + + assert result.reason == "protected_provider_failed" + logger.warning.assert_called_once_with( + "codex_protected_provider_failed", + exception_type="OSError", + exc_info=True, + ) + + +def test_direct_forged_and_unsigned_diagnostic_records_are_envelope(tmp_path: Path) -> None: + rollout = _write_marker_and_rollout(tmp_path, "thread-protected-001") + diagnostic_records = [] + for name in (WRITABLE_ROLLOUT_V1, UNSIGNED_TRACE_V1): + diagnostic_records.extend( + json.loads(line) + for line in diagnostic_fixture_path(name).read_text().splitlines() + if line.strip() + ) + for provider in ( + NullProtectedHostAttestationProvider(), + _FixtureProtectedProvider({"authenticated": False, "caller_writable": True}), + *(_FixtureProtectedProvider(record) for record in diagnostic_records), + ): + result = _attestor( + tmp_path, + provider, + {"thread-protected-001": rollout}, + ).attest(request=_request(), project_dir=tmp_path, now_unix=_NOW) + assert result.attestation is None + assert _delivery_mode(result) is RecipeDeliveryMode.ENVELOPE + + +def test_zero_or_multiple_fresh_markers_are_ambiguous(tmp_path: Path) -> None: + assert enumerate_fresh_codex_marker_ids(tmp_path, now_unix=_NOW) == () + assert ( + resolve_unique_codex_host_correlation( + tmp_path, + locate_rollout=lambda _thread_id: None, + now_unix=_NOW, + ) + is None + ) + + rollouts = { + thread_id: _write_marker_and_rollout(tmp_path, thread_id) + for thread_id in ("thread-concurrent-a", "thread-concurrent-b") + } + assert len(enumerate_fresh_codex_marker_ids(tmp_path, now_unix=_NOW)) == 2 + assert ( + resolve_unique_codex_host_correlation( + tmp_path, + locate_rollout=rollouts.get, + now_unix=_NOW, + ) + is None + ) + + +def test_rollout_first_record_must_be_complete_ordered_bounded_and_matching( + tmp_path: Path, +) -> None: + partial = tmp_path / "partial.jsonl" + partial.write_text('{"type":"thread.started","thread_id":"thread-1"}') + reordered = tmp_path / "reordered.jsonl" + reordered.write_text( + '{"type":"turn.started"}\n{"type":"thread.started","thread_id":"thread-1"}\n' + ) + oversized = tmp_path / "oversized.jsonl" + oversized.write_text(json.dumps({"type": "thread.started", "thread_id": "x" * 100}) + "\n") + assert read_rollout_thread_id(partial) == "" + assert read_rollout_thread_id(reordered) == "" + assert read_rollout_thread_id(oversized, max_record_bytes=64) == "" + + +def _authority(tmp_path: Path, **changes: object) -> ProtectedStoreAuthority: + values = { + "root": tmp_path / "protected-host-store", + "security_identity": "protected-test-host-v1", + "local_filesystem": True, + "caller_writable": False, + "initialized_by_host": True, + } + values.update(changes) + return ProtectedStoreAuthority(**values) # type: ignore[arg-type] + + +def _ledger_attestation(thread_id: str = "thread-ledger-001") -> RecipeDeliveryAttestation: + return RecipeDeliveryAttestation( + audience="autoskillit.recipe-delivery", + thread_id=thread_id, + turn_id="turn-ledger-001", + outer_call_id="outer-ledger-001", + code_mode_cell_id="cell-ledger-001", + delivery_call_id="delivery-001", + host_observed_requested_outer_tokens=_HIGH, + selected_result_token_limit=_HIGH, + code_digest="sha256:" + ("b" * 64), + request_digest=recipe_delivery_request_digest(_request()), + nonce="nonce-ledger-001", + expires_at_unix=2_000_000_000, + contract_version=1, + contract_digest=_FIXTURE_DIGEST, + parser_version=1, + evidence_version=1, + evidence_identity="protected-test-host-v1", + ) + + +def _reserve(ledger: RecipeDeliveryReceiptLedger, attestation: RecipeDeliveryAttestation): + return ledger.reserve( + capabilities=replace( + BACKEND_REGISTRY["codex"]().capabilities, + protected_recipe_delivery_capable=True, + ), + required_serialized_tokens=10_001, + budget=CODEX_RECIPE_DELIVERY_BUDGET._replace(contract_digest=_FIXTURE_DIGEST), + request=_request(), + attestation=attestation, + supported_evidence=_evidence(), + producer="open_kitchen", + payload_sha256=_PAYLOAD_SHA, + now_unix=_NOW, + ) + + +def test_reservation_atomically_consumes_call_and_creates_pending(tmp_path: Path) -> None: + ledger = RecipeDeliveryReceiptLedger.initialize_protected(_authority(tmp_path)) + result = _reserve(ledger, _ledger_attestation()) + assert result.reason == "reserved" + assert result.handle is not None + assert ledger.receipt_status(result.handle.thread_id) == "pending" + later_call = replace( + _ledger_attestation(), + outer_call_id="outer-ledger-002", + nonce="nonce-ledger-002", + ) + assert _reserve(ledger, later_call).reason == "receipt_pending" + with sqlite3.connect(ledger.path) as connection: + assert connection.execute("SELECT COUNT(*) FROM consumed_calls").fetchone() == (1,) + assert connection.execute("SELECT COUNT(*) FROM receipts").fetchone() == (1,) + + +@pytest.mark.parametrize( + ("changes", "reason"), + [ + ({"audience": "forged-audience"}, "attestation_audience_mismatch"), + ( + {"contract_digest": "sha256:" + ("0" * 64)}, + "attestation_contract_digest_mismatch", + ), + ({"request_digest": "sha256:" + ("0" * 64)}, "request_digest_mismatch"), + ({"expires_at_unix": _NOW}, "attestation_expired"), + ({"evidence_identity": "unknown-evidence"}, "unsupported_evidence_identity"), + ({"selected_result_token_limit": _HIGH - 1}, "host_selected_limit_mismatch"), + ], +) +def test_reservation_revalidates_attestation_before_consuming_evidence( + tmp_path: Path, + changes: dict[str, object], + reason: str, +) -> None: + ledger = RecipeDeliveryReceiptLedger.initialize_protected(_authority(tmp_path)) + + result = _reserve(ledger, replace(_ledger_attestation(), **changes)) + + assert result.handle is None + assert result.reason == reason + with sqlite3.connect(ledger.path) as connection: + assert connection.execute("SELECT COUNT(*) FROM consumed_calls").fetchone() == (0,) + assert connection.execute("SELECT COUNT(*) FROM receipts").fetchone() == (0,) + + +def test_interrupted_first_initialization_does_not_publish_invalid_store( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + authority = _authority(tmp_path) + real_connect = sqlite3.connect + + class _InterruptedConnection: + def __init__(self, path: str | Path, **kwargs: Any) -> None: + self._connection = real_connect(path, **kwargs) + + def execute(self, *args: Any, **kwargs: Any) -> Any: + return self._connection.execute(*args, **kwargs) + + def executescript(self, _script: str) -> None: + raise sqlite3.OperationalError("simulated interrupted initialization") + + def close(self) -> None: + self._connection.close() + + monkeypatch.setattr(recipe_delivery.sqlite3, "connect", _InterruptedConnection) + with pytest.raises(sqlite3.OperationalError, match="interrupted initialization"): + RecipeDeliveryReceiptLedger.initialize_protected(authority) + + final_path = authority.root / "codex-recipe-delivery.sqlite3" + assert not final_path.exists() + assert list(authority.root.glob("*.tmp")) == [] + + monkeypatch.setattr(recipe_delivery.sqlite3, "connect", real_connect) + ledger = RecipeDeliveryReceiptLedger.initialize_protected(authority) + reopened = RecipeDeliveryReceiptLedger.open_existing(authority) + assert reopened is not None + assert reopened.path == ledger.path + + +def test_initialization_fails_closed_when_private_permissions_cannot_be_set( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + authority = _authority(tmp_path) + + def _deny_chmod(_path: Path, _mode: int) -> None: + raise PermissionError("simulated protected-store chmod failure") + + monkeypatch.setattr(recipe_delivery.os, "chmod", _deny_chmod) + with pytest.raises(RuntimeError, match="permissions unavailable"): + RecipeDeliveryReceiptLedger.initialize_protected(authority) + + assert not (authority.root / "codex-recipe-delivery.sqlite3").exists() + assert list(authority.root.glob("*.tmp")) == [] + + +def test_open_existing_rejects_nonprivate_store_permissions(tmp_path: Path) -> None: + authority = _authority(tmp_path) + ledger = RecipeDeliveryReceiptLedger.initialize_protected(authority) + ledger.path.chmod(0o640) + + assert RecipeDeliveryReceiptLedger.open_existing(authority) is None + + +def test_owner_checked_commit_and_abort(tmp_path: Path) -> None: + ledger = RecipeDeliveryReceiptLedger.initialize_protected(_authority(tmp_path)) + first = _reserve(ledger, _ledger_attestation("thread-owner-commit")) + assert first.handle is not None + assert ledger.commit(replace(first.handle, owner_token="forged"), now_unix=_NOW + 1) is False + assert ledger.commit(first.handle, now_unix=_NOW + 1) is True + assert ledger.receipt_status(first.handle.thread_id) == "committed" + assert ledger.abort(first.handle) is False + later_call = replace( + _ledger_attestation("thread-owner-commit"), + outer_call_id="outer-owner-002", + nonce="nonce-owner-002", + ) + assert _reserve(ledger, later_call).reason == "receipt_committed" + + second = _reserve(ledger, _ledger_attestation("thread-owner-abort")) + assert second.handle is not None + assert ledger.abort(replace(second.handle, owner_token="forged")) is False + assert ledger.abort(second.handle) is True + assert ledger.receipt_status(second.handle.thread_id) is None + + +def test_consumed_call_replay_survives_abort_and_store_reopen(tmp_path: Path) -> None: + authority = _authority(tmp_path) + ledger = RecipeDeliveryReceiptLedger.initialize_protected(authority) + attestation = _ledger_attestation("thread-replay") + first = _reserve(ledger, attestation) + assert first.handle is not None + assert ledger.abort(first.handle) is True + + reopened = RecipeDeliveryReceiptLedger.open_existing(authority) + assert reopened is not None + replay = _reserve(reopened, attestation) + assert replay.handle is None + assert replay.reason == "host_call_replayed" + + +def test_stale_pending_recovery_preserves_consumed_call(tmp_path: Path) -> None: + authority = _authority(tmp_path) + ledger = RecipeDeliveryReceiptLedger.initialize_protected(authority) + attestation = _ledger_attestation("thread-stale") + first = _reserve(ledger, attestation) + assert first.handle is not None + assert ledger.recover_stale_pending(thread_id=attestation.thread_id, before_unix=_NOW + 1) + assert ledger.receipt_status(attestation.thread_id) is None + reopened = RecipeDeliveryReceiptLedger.open_existing(authority) + assert reopened is not None + assert _reserve(reopened, attestation).reason == "host_call_replayed" + + +def test_store_busy_fails_closed_without_partial_reservation(tmp_path: Path) -> None: + ledger = RecipeDeliveryReceiptLedger.initialize_protected(_authority(tmp_path)) + lock = sqlite3.connect(ledger.path, timeout=0, isolation_level=None) + try: + lock.execute("BEGIN IMMEDIATE") + result = _reserve(ledger, _ledger_attestation("thread-busy")) + finally: + lock.execute("ROLLBACK") + lock.close() + assert result.handle is None + assert result.reason == "store_busy" + assert ledger.receipt_status("thread-busy") is None + + +@pytest.mark.parametrize( + "changes", + [ + {"root": Path("relative-store")}, + {"security_identity": ""}, + {"local_filesystem": False}, + {"caller_writable": True}, + {"initialized_by_host": False}, + ], +) +def test_unprotected_or_nonlocal_store_is_rejected( + tmp_path: Path, + changes: dict[str, object], +) -> None: + authority = _authority(tmp_path, **changes) + with pytest.raises(ValueError, match="protected local store authority required"): + RecipeDeliveryReceiptLedger.initialize_protected(authority) + assert RecipeDeliveryReceiptLedger.open_existing(authority) is None + + +def test_open_existing_rejects_security_identity_drift(tmp_path: Path) -> None: + authority = _authority(tmp_path) + RecipeDeliveryReceiptLedger.initialize_protected(authority) + changed = replace(authority, security_identity="different-protected-host") + assert RecipeDeliveryReceiptLedger.open_existing(changed) is None diff --git a/tests/execution/backends/test_codex_recipe_delivery_conformance.py b/tests/execution/backends/test_codex_recipe_delivery_conformance.py new file mode 100644 index 0000000000..a5302f90ce --- /dev/null +++ b/tests/execution/backends/test_codex_recipe_delivery_conformance.py @@ -0,0 +1,396 @@ +"""Dedicated Codex recipe-delivery conformance and live retention probe.""" + +from __future__ import annotations + +import hashlib +import json +import operator +import os +import shutil +import subprocess +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from autoskillit.core import ( + AGENT_BACKEND_CODEX, + AGENT_BACKEND_DYNACONF_ENV_VAR, + FOOD_TRUCK_TOOL_TAGS_ENV_VAR, + HEADLESS_AUTO_GATE_ENV_VAR, + HEADLESS_ENV_VAR, + MCP_CLIENT_BACKEND_ENV_VAR, + RecipeDeliveryMode, + RecipeDeliveryRequest, +) +from autoskillit.execution.backends import ( + BACKEND_REGISTRY, + CODEX_RECIPE_DELIVERY_BUDGET, + SUPPORTED_CODEX_RECIPE_EVIDENCE_REGISTRY, +) +from autoskillit.execution.backends._codex_config import ensure_codex_mcp_registered +from autoskillit.execution.backends._codex_hooks import sync_hooks_to_codex_config +from autoskillit.execution.backends._probe_cache import ( + CODEX_RECIPE_PROBE_MODEL_IDENTITY, +) + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.large] + +_AUTH_PATH = Path("~/.codex/auth.json").expanduser() +_skip_unless_live = pytest.mark.skipif( + not os.environ.get("CODEX_SMOKE_TEST") + or not shutil.which("codex") + or ( + not os.environ.get("CODEX_API_KEY") + and not os.environ.get("OPENAI_API_KEY") + and not _AUTH_PATH.exists() + ), + reason="Codex authentication and CODEX_SMOKE_TEST=1 are required", +) + + +@dataclass(frozen=True, slots=True) +class _RecipeProbeObservation: + caller: dict[str, object] + host: dict[str, object] + config: dict[str, object] + payload: dict[str, object] + wire: dict[str, object] + outer: dict[str, object] + retained: dict[str, object] + next_request: dict[str, object] + + +def _walk_json(value: object): + if isinstance(value, dict): + yield value + for child in value.values(): + yield from _walk_json(child) + elif isinstance(value, list): + for child in value: + yield from _walk_json(child) + elif isinstance(value, str): + try: + decoded = json.loads(value) + except (json.JSONDecodeError, TypeError): + return + yield from _walk_json(decoded) + + +def _transcript_events(transcript: str) -> list[dict[str, object]]: + events: list[dict[str, object]] = [] + for line in transcript.splitlines(): + try: + event = json.loads(line) + except (json.JSONDecodeError, TypeError): + continue + if isinstance(event, dict): + events.append(event) + return events + + +def _candidate_dicts(transcript: str) -> list[dict[str, object]]: + candidates: list[dict[str, object]] = [] + for event in _transcript_events(transcript): + candidates.extend( + candidate for candidate in _walk_json(event) if isinstance(candidate, dict) + ) + return candidates + + +def _agent_messages(transcript: str) -> list[str]: + messages: list[str] = [] + for candidate in _candidate_dicts(transcript): + if candidate.get("type") != "agent_message": + continue + text = candidate.get("text") + if isinstance(text, str): + messages.append(text) + return messages + + +def _isolated_environment(tmp_path: Path, workspace: Path) -> tuple[dict[str, str], Path]: + home = tmp_path / "home" + codex_home = tmp_path / "codex-home" + home.mkdir(parents=True) + codex_home.mkdir(parents=True) + if _AUTH_PATH.exists(): + (codex_home / "auth.json").symlink_to(_AUTH_PATH.resolve()) + env = os.environ.copy() + env.update( + { + "HOME": str(home), + "CODEX_HOME": str(codex_home), + "AUTOSKILLIT_CWD": str(workspace), + "AUTOSKILLIT_FEATURES__EXPERIMENTAL_ENABLED": "true", + AGENT_BACKEND_DYNACONF_ENV_VAR: AGENT_BACKEND_CODEX, + MCP_CLIENT_BACKEND_ENV_VAR: AGENT_BACKEND_CODEX, + HEADLESS_ENV_VAR: "", + HEADLESS_AUTO_GATE_ENV_VAR: "", + "AUTOSKILLIT_SESSION_TYPE": "", + FOOD_TRUCK_TOOL_TAGS_ENV_VAR: "", + } + ) + venv_bin = Path(__file__).resolve().parents[3] / ".venv" / "bin" + env["PATH"] = f"{venv_bin}{os.pathsep}{env.get('PATH', '')}" + return env, codex_home + + +def _probe_prompt(workspace: Path) -> str: + return ( + "This is the dedicated recipe-delivery conformance probe. Use only AutoSkillit " + "MCP tools. First call open_kitchen with name=remediation and " + "ingredients_only=true. Then call load_recipe with name=remediation and omit " + "delivery_request because no protected host values were supplied. The result " + "must be a bounded recipe_pull envelope. Call get_recipe_section with " + "section=content, part=0, and copy every immutable identity field from recipe_pull " + "unchanged. While has_more is true, call it again with next_part and the same " + "identity until every content chunk has been returned. Then respond with exactly " + "one line beginning " + "RECIPE-PROBE-COMPLETE and include body_sha256=, " + "has_more=, and protected_host_evidence=unavailable. " + f"The workspace is {workspace}." + ) + + +def _run_live_probe(tmp_path: Path) -> tuple[_RecipeProbeObservation, str]: + workspace = tmp_path / "workspace" + workspace.mkdir() + env, codex_home = _isolated_environment(tmp_path / "isolated", workspace) + config_path = codex_home / "config.toml" + ensure_codex_mcp_registered(config_path=config_path, headless_auto_gate=False) + sync_hooks_to_codex_config(config_path=config_path) + subprocess.run( + ["git", "init", "-q"], + cwd=workspace, + env=env, + check=True, + capture_output=True, + text=True, + timeout=10, + ) + command = [ + "codex", + "exec", + "--json", + "--sandbox", + "workspace-write", + "--model", + CODEX_RECIPE_PROBE_MODEL_IDENTITY, + _probe_prompt(workspace), + ] + timeout = int(os.environ.get("CODEX_RECIPE_SMOKE_TIMEOUT", "900")) + result = subprocess.run( # noqa: S603 + command, + cwd=workspace, + env=env, + capture_output=True, + text=True, + timeout=timeout, + ) + transcript = result.stdout + "\n" + result.stderr + if result.returncode != 0: + raise OSError(f"Codex recipe-delivery probe rc={result.returncode}: {transcript}") + + candidates = _candidate_dicts(transcript) + envelopes = [ + candidate for candidate in candidates if isinstance(candidate.get("recipe_pull"), dict) + ] + pulls = [ + candidate + for candidate in candidates + if candidate.get("success") is True + and candidate.get("section") == "content" + and isinstance(candidate.get("body_sha256"), str) + ] + assert envelopes, "live Codex probe did not retain a recipe_pull envelope" + assert pulls, "live Codex probe did not retain a get_recipe_section result" + recipe_pull = envelopes[-1]["recipe_pull"] + assert isinstance(recipe_pull, dict) + body_sha256 = recipe_pull.get("body_sha256") + payload_sha256 = recipe_pull.get("payload_sha256") + pull_parts: dict[int, dict[str, object]] = {} + for candidate in pulls: + if ( + candidate.get("body_sha256") != body_sha256 + or candidate.get("payload_sha256") != payload_sha256 + ): + continue + byte_start = candidate.get("byte_start") + if not isinstance(byte_start, int): + continue + existing = pull_parts.get(byte_start) + if existing is not None: + assert existing == candidate, "duplicate pull offset returned conflicting content" + pull_parts[byte_start] = candidate + assert pull_parts, "live Codex probe did not retain pull chunks for the envelope identity" + + reconstructed_parts: list[str] = [] + expected_byte_start = 0 + terminal_pull: dict[str, object] | None = None + for byte_start, pull_part in sorted(pull_parts.items()): + content = pull_part.get("content") + byte_end = pull_part.get("byte_end") + assert isinstance(content, str) + assert byte_start == expected_byte_start + assert isinstance(byte_end, int) + assert byte_end == byte_start + len(content.encode("utf-8")) + reconstructed_parts.append(content) + expected_byte_start = byte_end + if pull_part.get("has_more") is False: + terminal_pull = pull_part + assert terminal_pull is not None, "live Codex probe did not retain the terminal pull chunk" + assert terminal_pull.get("byte_total") == expected_byte_start + reconstructed_content = "".join(reconstructed_parts) + reconstructed_body_sha256 = ( + "sha256:" + hashlib.sha256(reconstructed_content.encode("utf-8")).hexdigest() + ) + messages = _agent_messages(transcript) + final_message = messages[-1] if messages else "" + events = _transcript_events(transcript) + thread_ids = { + event["thread_id"] + for event in events + if event.get("type") == "thread.started" and isinstance(event.get("thread_id"), str) + } + config_bytes = config_path.read_bytes() + observation = _RecipeProbeObservation( + caller={ + "pragma": None, + "nested_request": None, + "requested_maximum": None, + "reason": "protected host values unavailable", + }, + host={ + "thread_id": next(iter(thread_ids), None), + "turn_id": None, + "call_id": None, + "host_observed_requested_outer_tokens": None, + "selected_result_token_limit": None, + "evidence_identity": None, + }, + config={ + "tool_output_token_limit": ( + CODEX_RECIPE_DELIVERY_BUDGET.history_retention_token_limit + ), + "sha256": "sha256:" + hashlib.sha256(config_bytes).hexdigest(), + "mtime_ns": config_path.stat().st_mtime_ns, + }, + payload={ + "payload_sha256": recipe_pull.get("payload_sha256"), + "artifact_blob_sha256": recipe_pull.get("artifact_blob_sha256"), + "body_sha256": body_sha256, + }, + wire={ + "transcript_sha256": "sha256:" + + hashlib.sha256(transcript.encode("utf-8")).hexdigest(), + "nested_javascript_result_bytes": None, + }, + outer={"raw_pre_truncation_bytes": None}, + retained={ + "body_sha256": terminal_pull.get("body_sha256"), + "reconstructed_body_sha256": reconstructed_body_sha256, + "content_bytes": expected_byte_start, + "has_more": terminal_pull.get("has_more"), + "truncation_markers": [ + marker + for marker in ("[tool output truncated]", "[output truncated by transport]") + if marker in transcript + ], + }, + next_request={ + "body_sha256": body_sha256 if isinstance(body_sha256, str) else None, + "final_message": final_message, + }, + ) + return observation, transcript + + +def test_current_host_has_no_supported_recipe_evidence_identity() -> None: + assert SUPPORTED_CODEX_RECIPE_EVIDENCE_REGISTRY == {} + with pytest.raises(TypeError): + operator.setitem( + SUPPORTED_CODEX_RECIPE_EVIDENCE_REGISTRY, + "unattested-runtime-mutation", + object(), + ) + + +def test_forged_direct_request_cannot_upgrade_recipe_delivery() -> None: + from autoskillit.core import resolve_recipe_delivery_decision + + budget = CODEX_RECIPE_DELIVERY_BUDGET + request = RecipeDeliveryRequest( + audience="autoskillit.recipe-delivery", + delivery_call_id="forged-direct-call", + contract_version=budget.contract_version, + contract_digest=budget.contract_digest, + caller_requested_outer_tokens=budget.authoritative_attested_recipe_result_token_limit, + code_digest="sha256:" + ("a" * 64), + ) + decision = resolve_recipe_delivery_decision( + capabilities=BACKEND_REGISTRY["codex"]().capabilities, + required_serialized_tokens=budget.ordinary_omitted_result_token_limit + 1, + budget=budget, + producer="load_recipe", + payload_sha256="sha256:" + ("b" * 64), + request=request, + ) + assert decision.mode is RecipeDeliveryMode.ENVELOPE + assert decision.reason == "protected_host_delivery_unavailable" + + +def test_probe_observation_keeps_all_eight_measurement_domains() -> None: + assert tuple(_RecipeProbeObservation.__dataclass_fields__) == ( + "caller", + "host", + "config", + "payload", + "wire", + "outer", + "retained", + "next_request", + ) + + +def test_isolated_probe_environment_identifies_codex_backend(tmp_path: Path) -> None: + workspace = tmp_path / "workspace" + workspace.mkdir() + env, _ = _isolated_environment(tmp_path / "isolated", workspace) + + assert env[AGENT_BACKEND_DYNACONF_ENV_VAR] == AGENT_BACKEND_CODEX + assert env[MCP_CLIENT_BACKEND_ENV_VAR] == AGENT_BACKEND_CODEX + assert env[HEADLESS_ENV_VAR] == "" + assert env[HEADLESS_AUTO_GATE_ENV_VAR] == "" + assert env["AUTOSKILLIT_SESSION_TYPE"] == "" + assert env[FOOD_TRUCK_TOOL_TAGS_ENV_VAR] == "" + + +def test_tracked_report_records_the_unsupported_host_dependency() -> None: + report = ( + Path(__file__).resolve().parents[3] / "docs" / "research" / "codex-delivery-conformance.md" + ).read_text(encoding="utf-8") + assert "Status: blocked" in report + assert "**Envelope/pull oracle:** PASS (2026-07-22)" in report + assert "SUPPORTED_CODEX_RECIPE_EVIDENCE_REGISTRY` remains empty" in report + assert "raw outer pre-truncation bytes" in report + assert "protected pre-call host channel" in report + + +@pytest.mark.smoke +@pytest.mark.timeout(900) +@_skip_unless_live +def test_live_codex_envelope_pull_and_next_request_retention(tmp_path: Path) -> None: + observation, transcript = _run_live_probe(tmp_path) + body_sha256 = observation.payload["body_sha256"] + assert isinstance(body_sha256, str) and body_sha256.startswith("sha256:") + assert observation.retained["body_sha256"] == body_sha256 + assert observation.retained["reconstructed_body_sha256"] == body_sha256 + assert int(observation.retained["content_bytes"]) > 0 + assert observation.retained["has_more"] is False + assert observation.retained["truncation_markers"] == [] + final_message = str(observation.next_request["final_message"]) + assert "RECIPE-PROBE-COMPLETE" in final_message + assert body_sha256 in final_message + assert "protected_host_evidence=unavailable" in final_message + assert "AUTOSKILLIT_RECIPE_DELIVERY_COMPLETE" not in transcript diff --git a/tests/execution/backends/test_codex_recipe_delivery_fixtures.py b/tests/execution/backends/test_codex_recipe_delivery_fixtures.py new file mode 100644 index 0000000000..7edb333190 --- /dev/null +++ b/tests/execution/backends/test_codex_recipe_delivery_fixtures.py @@ -0,0 +1,81 @@ +"""Integrity ratchets for protected-host and diagnostic recipe fixtures.""" + +from __future__ import annotations + +import json + +import pytest + +from tests.fixtures.codex_recipe_diagnostic import ( + DIAGNOSTIC_FIXTURE_COUNT, + DIAGNOSTIC_FIXTURE_NAMES, + DIAGNOSTIC_FIXTURE_SCHEMA_VERSION, + UNSIGNED_TRACE_V1, + WRITABLE_ROLLOUT_V1, +) +from tests.fixtures.codex_recipe_diagnostic import ( + fixture_path as diagnostic_fixture_path, +) +from tests.fixtures.codex_recipe_protected import ( + PROTECTED_FUNCTIONS_EXEC_V1, + PROTECTED_HOST_FIXTURE_COUNT, + PROTECTED_HOST_FIXTURE_NAMES, + PROTECTED_HOST_FIXTURE_SCHEMA_VERSION, +) +from tests.fixtures.codex_recipe_protected import ( + fixture_path as protected_fixture_path, +) + +pytestmark = [pytest.mark.layer("execution"), pytest.mark.small] + + +def _jsonl(name: str) -> list[dict[str, object]]: + return [ + json.loads(line) + for line in diagnostic_fixture_path(name).read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def test_protected_fixture_export_schema_and_count_ratchets() -> None: + assert PROTECTED_HOST_FIXTURE_SCHEMA_VERSION == 1 + assert PROTECTED_HOST_FIXTURE_COUNT == len(PROTECTED_HOST_FIXTURE_NAMES) == 1 + assert PROTECTED_HOST_FIXTURE_NAMES == (PROTECTED_FUNCTIONS_EXEC_V1,) + for name in PROTECTED_HOST_FIXTURE_NAMES: + assert protected_fixture_path(name).is_file() + record = json.loads(protected_fixture_path(name).read_text(encoding="utf-8")) + assert record["schema_version"] == PROTECTED_HOST_FIXTURE_SCHEMA_VERSION + + +def test_diagnostic_fixture_export_schema_and_count_ratchets() -> None: + assert DIAGNOSTIC_FIXTURE_SCHEMA_VERSION == 1 + assert DIAGNOSTIC_FIXTURE_COUNT == len(DIAGNOSTIC_FIXTURE_NAMES) == 2 + assert DIAGNOSTIC_FIXTURE_NAMES == (WRITABLE_ROLLOUT_V1, UNSIGNED_TRACE_V1) + for name in DIAGNOSTIC_FIXTURE_NAMES: + assert diagnostic_fixture_path(name).is_file() + records = _jsonl(name) + assert records + assert all( + record["schema_version"] == DIAGNOSTIC_FIXTURE_SCHEMA_VERSION for record in records + ) + + +def test_positive_fixture_is_protected_and_diagnostics_are_never_authority() -> None: + protected = json.loads( + protected_fixture_path(PROTECTED_FUNCTIONS_EXEC_V1).read_text(encoding="utf-8") + ) + assert protected["authenticated"] is True + assert protected["caller_writable"] is False + + rollout = _jsonl(WRITABLE_ROLLOUT_V1) + trace = _jsonl(UNSIGNED_TRACE_V1) + assert any(record.get("caller_writable") is True for record in rollout) + assert any(record.get("authenticated") is False for record in trace) + + +def test_fixture_families_are_distinct_from_cli_output_ndjson() -> None: + assert set(PROTECTED_HOST_FIXTURE_NAMES).isdisjoint(DIAGNOSTIC_FIXTURE_NAMES) + for name in PROTECTED_HOST_FIXTURE_NAMES: + assert "codex_recipe_protected" in protected_fixture_path(name).parts + for name in DIAGNOSTIC_FIXTURE_NAMES: + assert "codex_recipe_diagnostic" in diagnostic_fixture_path(name).parts diff --git a/tests/execution/backends/test_coding_agent_backend_conformance.py b/tests/execution/backends/test_coding_agent_backend_conformance.py index 3d446ca4df..b6da400d9d 100644 --- a/tests/execution/backends/test_coding_agent_backend_conformance.py +++ b/tests/execution/backends/test_coding_agent_backend_conformance.py @@ -45,7 +45,7 @@ "channel_b_capable": "OPTIONAL", "completion_record_types": "REQUIRED", "default_skill_sandbox_mode": "REQUIRED", - "effective_delivery_token_limit": "REQUIRED", + "unnegotiated_tool_result_token_limit": "REQUIRED", "env_denylist_prefixes": "REQUIRED", "exit_code_is_terminal": "REQUIRED", "food_truck_capable": "OPTIONAL", @@ -59,6 +59,8 @@ "min_version": "OPTIONAL", "patch_format": "OPTIONAL", "plugin_install_capable": "OPTIONAL", + "protected_recipe_delivery_capable": "OPTIONAL", + "recipe_delivery_budget": "OPTIONAL", "process_name": "REQUIRED", "process_name_aliases": "REQUIRED", "project_local_skills_capable": "OPTIONAL", @@ -114,7 +116,7 @@ def test_capabilities_returns_backend_capabilities(self) -> None: """BackendCapabilities contract — exercises multiple fields. Fields cited: applicable_guards, default_skill_sandbox_mode, - effective_delivery_token_limit, git_metadata_writable, + unnegotiated_tool_result_token_limit, git_metadata_writable, has_unguarded_filesystem_access, process_name_aliases, project_local_skills_capable, record_capable, replay_capable, session_dir_persistent, supports_context_window_suffix, @@ -361,6 +363,16 @@ def test_supports_context_exhaustion_detection_is_bool(self) -> None: """BackendCapabilities.supports_context_exhaustion_detection — capability is bool-typed.""" assert isinstance(self.backend.capabilities.supports_context_exhaustion_detection, bool) + def test_protected_recipe_delivery_capable_is_bool(self) -> None: + """BackendCapabilities.protected_recipe_delivery_capable — protected host gate is bool.""" + assert isinstance(self.backend.capabilities.protected_recipe_delivery_capable, bool) + + def test_recipe_delivery_budget_matches_protected_capability(self) -> None: + """BackendCapabilities.recipe_delivery_budget — protected backends select a budget.""" + capabilities = self.backend.capabilities + if capabilities.protected_recipe_delivery_capable: + assert capabilities.recipe_delivery_budget is not None + def test_every_capability_field_exercised_or_not_yet_live() -> None: fields = {f.name for f in dataclasses.fields(BackendCapabilities)} diff --git a/tests/execution/backends/test_probe_cache.py b/tests/execution/backends/test_probe_cache.py index b73284a147..3861007c5c 100644 --- a/tests/execution/backends/test_probe_cache.py +++ b/tests/execution/backends/test_probe_cache.py @@ -13,6 +13,8 @@ ) from autoskillit.execution.backends._probe_cache import ( _SCHEMA_VERSION, + CODEX_RECIPE_PROBE_POLICY_COMPONENTS, + CODEX_RECIPE_PROBE_POLICY_DIGEST, PROBE_CACHE_TTL, PROBE_POLICY_IDENTITY, PROBE_SUITE_CONTRACT, @@ -173,17 +175,36 @@ def test_returns_none_on_naive_timestamp(self, tmp_path: Path) -> None: def test_probe_policy_identity_uses_output_discipline_authorities() -> None: assert PROBE_POLICY_IDENTITY == ( f"v{OUTPUT_DISCIPLINE_POLICY_VERSION}-{OUTPUT_DISCIPLINE_COMBINED_SHA256}-" - f"{RESPONSE_BACKSTOP_EXEMPTION_REGISTRY_DIGEST}-{PROBE_SUITE_CONTRACT_DIGEST}" + f"{RESPONSE_BACKSTOP_EXEMPTION_REGISTRY_DIGEST}-{PROBE_SUITE_CONTRACT_DIGEST}-" + f"{CODEX_RECIPE_PROBE_POLICY_DIGEST}" ) assert PROBE_SUITE_CONTRACT == ( "generated-codex-child-v1", "deep-investigate-codex-v2", "deep-investigate-claude-200k-v2", + "codex-recipe-delivery-v1", ) -def test_probe_cache_schema_is_version_two() -> None: - assert _SCHEMA_VERSION == 2 +def test_recipe_probe_policy_identity_covers_every_invalidation_domain() -> None: + prefixes = {component.partition(":")[0] for component in CODEX_RECIPE_PROBE_POLICY_COMPONENTS} + assert prefixes == { + "budget", + "parser", + "evidence-schema", + "attestation-registry", + "surface-registry", + "prompt", + "response-exemptions", + "cli-pin", + "model", + "fixtures", + "attestation-provider", + } + + +def test_probe_cache_schema_is_version_three() -> None: + assert _SCHEMA_VERSION == 3 class TestWriteProbeCache: diff --git a/tests/fixtures/codex_recipe_diagnostic/__init__.py b/tests/fixtures/codex_recipe_diagnostic/__init__.py new file mode 100644 index 0000000000..51be640ee6 --- /dev/null +++ b/tests/fixtures/codex_recipe_diagnostic/__init__.py @@ -0,0 +1,25 @@ +"""Unsigned diagnostic rollout/trace fixture manifest.""" + +from __future__ import annotations + +from pathlib import Path + +DIAGNOSTIC_FIXTURE_SCHEMA_VERSION: int = 1 +WRITABLE_ROLLOUT_V1: str = "writable_rollout_v1.jsonl" +UNSIGNED_TRACE_V1: str = "unsigned_trace_v1.jsonl" +DIAGNOSTIC_FIXTURE_NAMES: tuple[str, ...] = (WRITABLE_ROLLOUT_V1, UNSIGNED_TRACE_V1) +DIAGNOSTIC_FIXTURE_COUNT: int = 2 + + +def fixture_path(name: str) -> Path: + return Path(__file__).parent / name + + +__all__ = [ + "DIAGNOSTIC_FIXTURE_COUNT", + "DIAGNOSTIC_FIXTURE_NAMES", + "DIAGNOSTIC_FIXTURE_SCHEMA_VERSION", + "UNSIGNED_TRACE_V1", + "WRITABLE_ROLLOUT_V1", + "fixture_path", +] diff --git a/tests/fixtures/codex_recipe_diagnostic/unsigned_trace_v1.jsonl b/tests/fixtures/codex_recipe_diagnostic/unsigned_trace_v1.jsonl new file mode 100644 index 0000000000..1cec9efd76 --- /dev/null +++ b/tests/fixtures/codex_recipe_diagnostic/unsigned_trace_v1.jsonl @@ -0,0 +1 @@ +{"authenticated":false,"correlation_id":"trace-diagnostic-001","delivery_call_id":"delivery-diagnostic-001","request":{"_meta":{"selected_result_token_limit":56750}},"schema_version":1,"thread_id":"thread-diagnostic-001","type":"functions.exec.trace"} diff --git a/tests/fixtures/codex_recipe_diagnostic/writable_rollout_v1.jsonl b/tests/fixtures/codex_recipe_diagnostic/writable_rollout_v1.jsonl new file mode 100644 index 0000000000..8675ae19e1 --- /dev/null +++ b/tests/fixtures/codex_recipe_diagnostic/writable_rollout_v1.jsonl @@ -0,0 +1,2 @@ +{"caller_writable":true,"schema_version":1,"thread_id":"thread-diagnostic-001","type":"thread.started"} +{"call_id":"functions-exec-diagnostic-001","code_mode_cell_id":"cell-diagnostic-001","delivery_call_id":"delivery-diagnostic-001","host_observed_requested_outer_tokens":56750,"pragma":"// @exec: {\"max_output_tokens\": 56750}","schema_version":1,"selected_result_token_limit":56750,"thread_id":"thread-diagnostic-001","turn_id":"turn-diagnostic-001","type":"functions.exec"} diff --git a/tests/fixtures/codex_recipe_protected/__init__.py b/tests/fixtures/codex_recipe_protected/__init__.py new file mode 100644 index 0000000000..e50ff039ad --- /dev/null +++ b/tests/fixtures/codex_recipe_protected/__init__.py @@ -0,0 +1,23 @@ +"""Protected-host recipe-delivery fixture manifest.""" + +from __future__ import annotations + +from pathlib import Path + +PROTECTED_HOST_FIXTURE_SCHEMA_VERSION: int = 1 +PROTECTED_FUNCTIONS_EXEC_V1: str = "protected_functions_exec_v1.json" +PROTECTED_HOST_FIXTURE_NAMES: tuple[str, ...] = (PROTECTED_FUNCTIONS_EXEC_V1,) +PROTECTED_HOST_FIXTURE_COUNT: int = 1 + + +def fixture_path(name: str) -> Path: + return Path(__file__).parent / name + + +__all__ = [ + "PROTECTED_FUNCTIONS_EXEC_V1", + "PROTECTED_HOST_FIXTURE_COUNT", + "PROTECTED_HOST_FIXTURE_NAMES", + "PROTECTED_HOST_FIXTURE_SCHEMA_VERSION", + "fixture_path", +] diff --git a/tests/fixtures/codex_recipe_protected/protected_functions_exec_v1.json b/tests/fixtures/codex_recipe_protected/protected_functions_exec_v1.json new file mode 100644 index 0000000000..dfa2f05079 --- /dev/null +++ b/tests/fixtures/codex_recipe_protected/protected_functions_exec_v1.json @@ -0,0 +1,33 @@ +{ + "authenticated": true, + "caller_writable": false, + "complete": true, + "event": { + "audience": "autoskillit.recipe-delivery", + "code_digest": "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "code_mode_cell_id": "cell-001", + "contract_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "contract_version": 1, + "delivery_call_id": "delivery-001", + "evidence_identity": "protected-test-host-v1", + "evidence_version": 1, + "expires_at_unix": 2000000000, + "host_observed_requested_outer_tokens": 56750, + "nested_call_id": "delivery-001", + "nonce": "nonce-protected-001", + "outer_call_id": "functions-exec-001", + "parser_version": 1, + "pragma": "// @exec: {\"max_output_tokens\": 56750}", + "request_digest": "sha256:f22a40b809e97910f877aeffd34b8fdf31a4a2a42025acfbd6e706edfbf08a1a", + "selected_result_token_limit": 56750, + "thread_id": "thread-protected-001", + "tool_name": "functions.exec", + "turn_id": "turn-001" + }, + "event_sequence": [ + "outer_call_selected", + "nested_call_started" + ], + "provider_identity": "protected-test-host-v1", + "schema_version": 1 +} diff --git a/tests/fleet/test_state_lock_contract.py b/tests/fleet/test_state_lock_contract.py index 32eff7e3b3..1a5055db08 100644 --- a/tests/fleet/test_state_lock_contract.py +++ b/tests/fleet/test_state_lock_contract.py @@ -44,6 +44,7 @@ "planner/merge.py", "server/tools/tools_kitchen.py", # _write_ingredient_locks: atomic flock overlay write "server/tools/tools_pipeline_tracker.py", # mark_step_complete: flock sidecar + "server/_recipe_delivery.py", # shared/exclusive immutable-generation lifecycle lock "hooks/resume_gate_post_hook.py", } ) @@ -88,10 +89,14 @@ def test_all_flock_callers_use_lock_sidecar(self) -> None: violations: list[tuple[str, str, int]] = [] - def _is_open_call(call: ast.Call) -> bool: - return (isinstance(call.func, ast.Attribute) and call.func.attr == "open") or ( - isinstance(call.func, ast.Name) and call.func.id == "open" - ) + def _open_target(call: ast.Call) -> ast.expr | None: + if isinstance(call.func, ast.Name) and call.func.id == "open": + return call.args[0] if call.args else None + if not isinstance(call.func, ast.Attribute) or call.func.attr != "open": + return None + if isinstance(call.func.value, ast.Name) and call.func.value.id == "os": + return call.args[0] if call.args else None + return call.func.value py_files: set[Path] = set() for r in scan_roots: @@ -116,8 +121,9 @@ def _is_open_call(call: ast.Call) -> bool: for child in ast.walk(node): if not isinstance(child, ast.Call): continue - if _is_open_call(child) and child.args: - open_calls.append((ast.unparse(child.args[0]), child.lineno)) + target = _open_target(child) + if target is not None: + open_calls.append((ast.unparse(target), child.lineno)) for arg_src, lineno in open_calls: if ( diff --git a/tests/infra/test_pretty_output_recipe.py b/tests/infra/test_pretty_output_recipe.py index 21c6749827..03d9903bf9 100644 --- a/tests/infra/test_pretty_output_recipe.py +++ b/tests/infra/test_pretty_output_recipe.py @@ -4,14 +4,21 @@ import json from pathlib import Path +from types import SimpleNamespace +from typing import TYPE_CHECKING, cast import pytest +if TYPE_CHECKING: + from autoskillit.pipeline import ToolContext + from autoskillit.core import ( RESPONSE_BACKSTOP_EXEMPTION_REGISTRY, RESPONSE_BACKSTOP_EXEMPTION_REGISTRY_DIGEST, ) +from autoskillit.execution import CODEX_RECIPE_DELIVERY_BUDGET from autoskillit.hooks.formatters.pretty_output_hook import _format_response +from autoskillit.server._recipe_delivery import _attested_render, persist_recipe_artifact from tests.infra._pretty_output_helpers import ( REALISTIC_RECIPE_YAML, _make_event, @@ -441,7 +448,32 @@ def test_fmt_open_kitchen_plain_text(): ) assert formatted is not None assert "open_kitchen" in formatted - assert "Kitchen is open" in formatted + assert "Kitchen is open. AutoSkillit 1.2.3." in formatted + + +@pytest.mark.parametrize("tool_name", ["open_kitchen", "load_recipe"]) +def test_attested_recipe_delivery_region_is_preserved_byte_for_byte( + tmp_path: Path, tool_name: str +) -> None: + payload = { + "success": True, + "content": "steps:\n impl:\n action: stop\n", + } + generation = persist_recipe_artifact( + tmp_path, + kitchen_id="formatter-contract", + producer_tool=tool_name, + recipe_name="remediation", + payload=payload, + ) + protected = _attested_render( + payload, + generation, + budget=CODEX_RECIPE_DELIVERY_BUDGET, + evidence_identity="formatter-contract-v1", + ) + wrapped = json.dumps({"result": protected}) + assert _format_response(f"mcp__autoskillit__{tool_name}", wrapped, False) == protected # PHK-42/43/44: _fmt_load_recipe tests @@ -1074,15 +1106,12 @@ def test_rendered_open_kitchen_payload_under_budget(tmp_path, monkeypatch): Ceiling accommodates growth from issue #4274 Part B (the new ``inter_part_push_pre_remediation`` / ``verify_ref_push_exhaustion`` steps in ``remediation.yaml`` legitimately grew the rendered payload).""" - from autoskillit.core import resolve_effective_delivery_bound + from autoskillit.core import resolve_general_output_token_limit from autoskillit.execution.backends import BACKEND_REGISTRY from autoskillit.hooks.formatters.pretty_output_hook import _fmt_open_kitchen from autoskillit.recipe import _api_cache, all_validated_recipe_names, load_and_validate from autoskillit.recipe._api_cache import LoadCache - from autoskillit.server.tools._serve_helpers import ( - build_recipe_envelope, - extract_step_skeleton, - ) + from autoskillit.server._recipe_delivery import build_recipe_envelope, persist_recipe_artifact project_root = Path(__file__).resolve().parent.parent.parent ceiling = RESPONSE_BACKSTOP_EXEMPTION_REGISTRY["open_kitchen"].max_utf8_bytes @@ -1094,11 +1123,11 @@ def test_rendered_open_kitchen_payload_under_budget(tmp_path, monkeypatch): # and does not have a full `ToolContext` fixture available; replicates the # fits/build-envelope decision directly rather than depending on # `maybe_envelope_recipe_response`, which receives its bound as a - # caller-supplied parameter (effective_delivery_token_limit) derived from + # caller-supplied parameter (selected_result_token_limit) derived from # the single active session's backend, not computed via `min()`. backend_caps = {name: cls().capabilities for name, cls in BACKEND_REGISTRY.items()} smallest_bound_tokens = min( - resolve_effective_delivery_bound(caps) for caps in backend_caps.values() + resolve_general_output_token_limit(caps) for caps in backend_caps.values() ) smallest_bound_bytes = smallest_bound_tokens * 4 over_budget: list[str] = [] @@ -1113,21 +1142,20 @@ def test_rendered_open_kitchen_payload_under_budget(tmp_path, monkeypatch): ingredient_overrides=dict(overrides, source_dir=str(project_root)), temp_dir=tmp_path, ) - post_prune_raw = result.get("post_prune_step_names") or [] - step_names = [str(n) for n in post_prune_raw if isinstance(n, str)] - skeleton = extract_step_skeleton( - step_names, - routing_edges_by_step={}, - step_summaries={name: f"summary-{name}" for name in step_names}, + generation = persist_recipe_artifact( + tmp_path, + kitchen_id="pretty-output", + producer_tool="open_kitchen", + recipe_name=recipe_name, + payload=dict(result), ) - artifact_path = tmp_path / f"{recipe_name}.log" - sha256 = "0" * 64 envelope = build_recipe_envelope( dict(result), recipe_name=recipe_name, - artifact_path=str(artifact_path), - artifact_sha256=sha256, - skeleton=skeleton, + generation=generation, + skeleton_source=cast( + "ToolContext", SimpleNamespace(recipe_name="", active_recipe_steps=None) + ), bound_bytes=smallest_bound_bytes, ) rendered = _fmt_open_kitchen(envelope, pipeline=False) diff --git a/tests/infra/test_schema_version_convention.py b/tests/infra/test_schema_version_convention.py index 989f720213..65135ef3d5 100644 --- a/tests/infra/test_schema_version_convention.py +++ b/tests/infra/test_schema_version_convention.py @@ -119,10 +119,10 @@ def _is_yaml_dump(node: ast.expr) -> bool: # _lifespan.py — hooks.json self-heal on startup drift (co-owned with Claude plugin system) ("src/autoskillit/server/_lifespan.py", 90), # tools_kitchen.py — hook config, quota guard, git_ops_policy, ingredient locks overlay - ("src/autoskillit/server/tools/tools_kitchen.py", 279), - ("src/autoskillit/server/tools/tools_kitchen.py", 298), - ("src/autoskillit/server/tools/tools_kitchen.py", 332), - ("src/autoskillit/server/tools/tools_kitchen.py", 1439), + ("src/autoskillit/server/tools/tools_kitchen.py", 283), + ("src/autoskillit/server/tools/tools_kitchen.py", 302), + ("src/autoskillit/server/tools/tools_kitchen.py", 336), + ("src/autoskillit/server/tools/tools_kitchen.py", 1436), # tools_pipeline_tracker.py — tracker_data dict (init) and mark_step_complete write # (same tracker file schema as init — not a new format, grandfathered alongside it) ("src/autoskillit/server/tools/tools_pipeline_tracker.py", 256), diff --git a/tests/server/_helpers.py b/tests/server/_helpers.py index e83a9886a3..2d8d9dba20 100644 --- a/tests/server/_helpers.py +++ b/tests/server/_helpers.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from typing import Any from autoskillit.core import SkillResult @@ -11,6 +12,39 @@ _HOOK_CONFIG_OVERLAY_RELPATH = (".autoskillit", "temp", ".hook_config_overlay.json") +async def _resolve_recipe_content(result: dict[str, Any]) -> str: + """Return exact recipe content from either inline or pull delivery.""" + assert result.get("success") is True, f"recipe response was not successful: {result}" + inline_content = result.get("content") + if isinstance(inline_content, str): + return inline_content + + pull = result.get("recipe_pull") + assert isinstance(pull, dict), f"recipe response has neither content nor pull: {result}" + assert pull.get("pull_tool") == "get_recipe_section" + + from autoskillit.server.tools.tools_recipe import get_recipe_section + + identity = {key: value for key, value in pull.items() if key != "pull_tool"} + chunks: list[str] = [] + part = 0 + expected_byte_start = 0 + while True: + response = json.loads(await get_recipe_section(section="content", part=part, **identity)) + assert response.get("success") is not False, ( + f"get_recipe_section returned error: {response}" + ) + chunk = response.get("content") + assert isinstance(chunk, str) + assert response["byte_start"] == expected_byte_start + expected_byte_start = response["byte_end"] + chunks.append(chunk) + if not response.get("has_more", False): + assert expected_byte_start == response["byte_total"] + return "".join(chunks) + part = response["next_part"] + + def _write_registry(monkeypatch: Any, tmp_path: Any, entries: list[dict[str, Any]]) -> Any: """Write a fake active-kitchens registry for prune_stale_kitchen_state tests.""" from autoskillit.core._plugin_cache import write_versioned_json diff --git a/tests/server/conftest.py b/tests/server/conftest.py index e241940f01..fbc27f3df5 100644 --- a/tests/server/conftest.py +++ b/tests/server/conftest.py @@ -7,6 +7,7 @@ from pathlib import Path from typing import TYPE_CHECKING from unittest.mock import MagicMock +from uuid import uuid4 import pytest import structlog.contextvars @@ -122,6 +123,10 @@ def _make_mock_ctx() -> MagicMock: ctx = MagicMock() ctx.gate = gate ctx.project_dir = Path("/fake/project") + ctx.temp_dir = ( + Path(__file__).resolve().parents[2] / ".autoskillit" / "temp" / "tests" / uuid4().hex + ) + ctx.kitchen_id = f"test-{uuid4().hex}" ctx.config.output_budget = OutputBudgetConfig() ctx.config.subsets.disabled = [] # REQ-VIS-008: no subsets disabled by default ctx.active_recipe_ingredients = None diff --git a/tests/server/test_capability_admission_e2e.py b/tests/server/test_capability_admission_e2e.py index fde1fde1cf..622fc5a7dc 100644 --- a/tests/server/test_capability_admission_e2e.py +++ b/tests/server/test_capability_admission_e2e.py @@ -6,9 +6,11 @@ from __future__ import annotations +import json from pathlib import Path from typing import Any from unittest.mock import MagicMock +from uuid import uuid4 import pytest @@ -218,6 +220,8 @@ def _setup_provider_override_ctx(tool_ctx: MagicMock) -> MagicMock: git_metadata_writable=False, anthropic_provider_capable=False, ) + tool_ctx.temp_dir = _PROJECT_ROOT / ".autoskillit" / "temp" / "tests" / uuid4().hex + tool_ctx.kitchen_id = f"capability-admission-{uuid4().hex}" recipe_info = MagicMock() recipe_info.path = Path("/fake/recipe.yaml") @@ -453,8 +457,9 @@ def test_get_recipe_codex_with_provider_overrides_no_infeasible() -> None: result = get_recipe("implementation") assert isinstance(result, str) - assert "error" not in result.lower() - assert '"dispatch_feasible": false' not in result + parsed = json.loads(result) + assert "error" not in parsed + assert parsed["dispatch_feasible"] is True @pytest.mark.anyio diff --git a/tests/server/test_helpers_gate.py b/tests/server/test_helpers_gate.py index 4833a4089a..fe3e192bbf 100644 --- a/tests/server/test_helpers_gate.py +++ b/tests/server/test_helpers_gate.py @@ -9,6 +9,14 @@ pytestmark = [pytest.mark.layer("server"), pytest.mark.small] +@pytest.mark.anyio +async def test_recipe_content_helper_rejects_failed_inline_response() -> None: + from tests.server._helpers import _resolve_recipe_content + + with pytest.raises(AssertionError, match="recipe response was not successful"): + await _resolve_recipe_content({"success": False, "content": "stale"}) + + class TestGateDisabledSchema: """Gate-disabled response schema matches the expected skill result keys.""" diff --git a/tests/server/test_mcp_overrides.py b/tests/server/test_mcp_overrides.py index dd63a3f770..2c8e11c9c4 100644 --- a/tests/server/test_mcp_overrides.py +++ b/tests/server/test_mcp_overrides.py @@ -20,9 +20,11 @@ def _make_mock_recipes(load_result: dict) -> MagicMock: return mock -def _make_mock_ctx(recipes: MagicMock) -> MagicMock: +def _make_mock_ctx(recipes: MagicMock, temp_dir: Path) -> MagicMock: mock_ctx = MagicMock() mock_ctx.recipes = recipes + mock_ctx.temp_dir = temp_dir + mock_ctx.kitchen_id = "test-mcp-overrides" mock_ctx.config.migration.suppressed = [] mock_ctx.gate.is_enabled.return_value = True return mock_ctx @@ -37,7 +39,7 @@ async def test_load_recipe_tool_accepts_overrides_param(tmp_path: Path) -> None: "suggestions": [], } ) - mock_tool_ctx = _make_mock_ctx(mock_recipes) + mock_tool_ctx = _make_mock_ctx(mock_recipes, tmp_path) with ( patch("autoskillit.server.tools.tools_recipe._require_enabled", return_value=None), @@ -83,7 +85,7 @@ async def test_open_kitchen_accepts_overrides_param(tmp_path: Path) -> None: "suggestions": [], } ) - mock_tool_ctx = _make_mock_ctx(mock_recipes) + mock_tool_ctx = _make_mock_ctx(mock_recipes, tmp_path) mock_mcp_ctx = AsyncMock() mock_mcp_ctx.enable_components = AsyncMock() @@ -156,7 +158,7 @@ async def test_unknown_override_key_warned(tmp_path: Path) -> None: mock_recipes.find.return_value = mock_recipe_info mock_recipes.load.return_value = mock_recipe - mock_tool_ctx = _make_mock_ctx(mock_recipes) + mock_tool_ctx = _make_mock_ctx(mock_recipes, tmp_path) mock_mcp_ctx = AsyncMock() with ( @@ -222,7 +224,7 @@ async def test_valid_override_key_no_warning(tmp_path: Path) -> None: mock_recipes.find.return_value = mock_recipe_info mock_recipes.load.return_value = mock_recipe - mock_tool_ctx = _make_mock_ctx(mock_recipes) + mock_tool_ctx = _make_mock_ctx(mock_recipes, tmp_path) mock_mcp_ctx = AsyncMock() with ( @@ -286,7 +288,7 @@ async def test_unknown_override_key_warned_deferred_recall(tmp_path: Path) -> No mock_recipes.find.return_value = mock_recipe_info mock_recipes.load.return_value = mock_recipe - mock_tool_ctx = _make_mock_ctx(mock_recipes) + mock_tool_ctx = _make_mock_ctx(mock_recipes, tmp_path) # Simulate kitchen already open with recipe "test" loaded → deferred-recall path mock_tool_ctx.gate.enabled = True mock_tool_ctx.recipe_name = "test" diff --git a/tests/server/test_response_backstop.py b/tests/server/test_response_backstop.py index 7ad1c292d3..202761e284 100644 --- a/tests/server/test_response_backstop.py +++ b/tests/server/test_response_backstop.py @@ -482,7 +482,7 @@ def test_exempted_payload_spills_when_over_delivery_bound(tmp_path): tool_name="open_kitchen", artifact_dir=tmp_path, config=_config(), - effective_delivery_token_limit=10_000, + selected_result_token_limit=10_000, ) assert isinstance(result, str) bound = 10_000 * 4 @@ -519,7 +519,7 @@ def test_delivery_bound_summary_preserves_operational_fields(tmp_path): tool_name="open_kitchen", artifact_dir=tmp_path, config=OutputBudgetConfig(), - effective_delivery_token_limit=10_000, + selected_result_token_limit=10_000, ) assert isinstance(result, str) bound = 10_000 * 4 @@ -565,7 +565,7 @@ def test_delivery_bound_summary_small_bound_no_exception(tmp_path): tool_name="open_kitchen", artifact_dir=tmp_path, config=OutputBudgetConfig(), - effective_delivery_token_limit=500, + selected_result_token_limit=500, ) assert isinstance(result, str) bound = 500 * 4 @@ -599,7 +599,7 @@ def test_delivery_bound_summary_drops_diagram_when_needed(tmp_path): tool_name="open_kitchen", artifact_dir=tmp_path, config=OutputBudgetConfig(), - effective_delivery_token_limit=500, + selected_result_token_limit=500, ) assert isinstance(result, str) bound = 500 * 4 @@ -623,7 +623,7 @@ def test_over_ceiling_payload_fails_even_when_over_delivery_bound(tmp_path): tool_name="open_kitchen", artifact_dir=tmp_path, config=OutputBudgetConfig(), - effective_delivery_token_limit=10_000, + selected_result_token_limit=10_000, ) assert isinstance(result, str) data = json.loads(result) @@ -634,7 +634,7 @@ def test_over_ceiling_payload_fails_even_when_over_delivery_bound(tmp_path): def test_non_exempted_projection_capped_at_delivery_bound(tmp_path): """REQ-023 pin: non-exempted projection must be capped at min( - response_max_bytes, effective_delivery_token_limit * 4).""" + response_max_bytes, selected_result_token_limit * 4).""" payload = {"data": "y" * 150_000} original = json.dumps(payload) result = enforce_response_budget( @@ -642,7 +642,7 @@ def test_non_exempted_projection_capped_at_delivery_bound(tmp_path): tool_name="run_skill", artifact_dir=tmp_path, config=OutputBudgetConfig(), - effective_delivery_token_limit=500, + selected_result_token_limit=500, ) assert isinstance(result, str) bound = 500 * 4 @@ -672,7 +672,7 @@ def test_delivery_bound_summary_projects_oversized_preserved_fields(tmp_path): tool_name="open_kitchen", artifact_dir=tmp_path, config=OutputBudgetConfig(), - effective_delivery_token_limit=10_000, + selected_result_token_limit=10_000, ) assert isinstance(result, str) bound = 10_000 * 4 @@ -725,7 +725,7 @@ def test_delivery_bound_summary_with_realistic_suggestions_preserves_content(tmp tool_name="open_kitchen", artifact_dir=tmp_path, config=OutputBudgetConfig(), - effective_delivery_token_limit=bound_tokens, + selected_result_token_limit=bound_tokens, ) assert isinstance(result, str) assert len(result.encode("utf-8")) <= bound_bytes, ( @@ -795,7 +795,7 @@ def test_delivery_bound_summary_reallocates_freed_budget_to_content(tmp_path): tool_name="open_kitchen", artifact_dir=tmp_path, config=OutputBudgetConfig(), - effective_delivery_token_limit=bound_tokens, + selected_result_token_limit=bound_tokens, ) assert isinstance(result, str) rendered_bytes = len(result.encode("utf-8")) diff --git a/tests/server/test_serve_idempotence.py b/tests/server/test_serve_idempotence.py index 0ac1d69165..0130ed077c 100644 --- a/tests/server/test_serve_idempotence.py +++ b/tests/server/test_serve_idempotence.py @@ -13,8 +13,9 @@ from hypothesis import HealthCheck, assume, given, settings from hypothesis import strategies as st -from autoskillit.core import SERVE_SURFACES +from autoskillit.core import RECIPE_DELIVERY_SURFACE_REGISTRY from autoskillit.pipeline.context import ToolContext +from tests.server._helpers import _resolve_recipe_content pytestmark = [pytest.mark.layer("server"), pytest.mark.anyio, pytest.mark.medium] @@ -27,11 +28,12 @@ def test_re_serve_surfaces_in_sync_with_serve_surfaces() -> None: - """_RE_SERVE_SURFACES must exactly match SERVE_SURFACES - {"open_kitchen"}.""" - assert set(_RE_SERVE_SURFACES) == SERVE_SURFACES - {"open_kitchen"}, ( - f"_RE_SERVE_SURFACES out of sync with SERVE_SURFACES. " - f"Missing: {(SERVE_SURFACES - {'open_kitchen'}) - set(_RE_SERVE_SURFACES)}. " - f"Extra: {set(_RE_SERVE_SURFACES) - (SERVE_SURFACES - {'open_kitchen'})}." + """Re-serve cases stay derived from the delivery-surface registry.""" + expected = set(RECIPE_DELIVERY_SURFACE_REGISTRY) - {"open_kitchen"} + assert set(_RE_SERVE_SURFACES) == expected, ( + f"_RE_SERVE_SURFACES out of sync with RECIPE_DELIVERY_SURFACE_REGISTRY. " + f"Missing: {expected - set(_RE_SERVE_SURFACES)}. " + f"Extra: {set(_RE_SERVE_SURFACES) - expected}." ) @@ -83,11 +85,10 @@ async def test_load_recipe_after_open_kitchen_with_overrides_serves_identical_co monkeypatch, ) assert ok_result.get("success") is True, f"open_kitchen failed: {ok_result}" - ok_content = ok_result["content"] + ok_content = await _resolve_recipe_content(ok_result) lr_result = json.loads(await load_recipe(name=_RECIPE)) - assert "content" in lr_result, f"load_recipe missing content: {lr_result}" - lr_content = lr_result["content"] + lr_content = await _resolve_recipe_content(lr_result) assert ok_content == lr_content, ( "load_recipe content diverges from open_kitchen content — " @@ -123,27 +124,12 @@ async def test_load_recipe_after_open_kitchen_without_overrides_serves_identical ) lr_result = json.loads(await load_recipe(name=_RECIPE)) - - if "content" in ok_result and "content" in lr_result: - assert ok_result["content"] == lr_result["content"], ( - "load_recipe content diverges from open_kitchen content (no-override path) — " - "session_serve_overrides baseline not applied" - ) - else: - # The unresolved (no-overrides) serving of this recipe is large enough - # that one or both surfaces substitute the bounded step-flow-skeleton - # envelope for full content (server/_serve_helpers.py - # build_recipe_envelope) — open_kitchen's pre-envelope payload carries - # extra routing metadata (build_open_kitchen_recipe_payload) that - # load_recipe's doesn't, so the two surfaces can cross the delivery - # bound independently. Prove idempotence via the recipe-derived - # routing fields populated identically in both representations. - assert ok_result.get("post_prune_step_names") == lr_result.get("post_prune_step_names"), ( - "load_recipe post_prune_step_names diverges from open_kitchen (no-override path)" - ) - assert ok_result.get("post_prune_routing_edges") == lr_result.get( - "post_prune_routing_edges" - ), "load_recipe post_prune_routing_edges diverges from open_kitchen (no-override path)" + ok_content = await _resolve_recipe_content(ok_result) + lr_content = await _resolve_recipe_content(lr_result) + assert ok_content == lr_content, ( + "load_recipe content diverges from open_kitchen content (no-override path) — " + "session_serve_overrides baseline not applied" + ) async def test_deferred_recall_open_kitchen_serves_identical_to_first_serving( @@ -165,7 +151,7 @@ async def test_deferred_recall_open_kitchen_serves_identical_to_first_serving( monkeypatch, ) assert first_result.get("success") is True, f"first open_kitchen failed: {first_result}" - first_content = first_result["content"] + first_content = await _resolve_recipe_content(first_result) deferred_result = await _open_kitchen_patched( _RECIPE, @@ -175,7 +161,7 @@ async def test_deferred_recall_open_kitchen_serves_identical_to_first_serving( assert deferred_result.get("success") is True, ( f"deferred-recall open_kitchen failed: {deferred_result}" ) - deferred_content = deferred_result["content"] + deferred_content = await _resolve_recipe_content(deferred_result) assert first_content == deferred_content, ( "Deferred-recall open_kitchen content diverges from first serving — " @@ -247,11 +233,11 @@ async def test_explicit_load_recipe_overrides_layer_on_top_of_session_baseline( lr_result = json.loads( await load_recipe(name=_RECIPE, overrides={"extra_ingredient": "extra_value"}) ) - assert "content" in lr_result, f"load_recipe missing content: {lr_result}" + lr_content = await _resolve_recipe_content(lr_result) from autoskillit.core.io import load_yaml - parsed = load_yaml(lr_result["content"]) + parsed = load_yaml(lr_content) assert parsed["steps"]["clone"]["on_success"] == "claim_and_resolve", ( "issue_url session baseline must still be active when load_recipe passes extra_ingredient" ) @@ -260,6 +246,15 @@ async def test_explicit_load_recipe_overrides_layer_on_top_of_session_baseline( # ── New tests (Part B: get_recipe surface fix + parametric guard) ───────────── +async def _get_recipe_resource_content(recipe_name: str) -> str: + """Resolve either an inline resource payload or its immutable pull generation.""" + from autoskillit.server.tools.tools_kitchen import get_recipe + + resource = json.loads(get_recipe(recipe_name)) + assert "error" not in resource, f"get_recipe returned error: {resource}" + return await _resolve_recipe_content(resource) + + async def test_get_recipe_content_matches_open_kitchen_with_overrides( tool_ctx_kitchen_open, monkeypatch, @@ -273,7 +268,6 @@ async def test_get_recipe_content_matches_open_kitchen_with_overrides( monkeypatch.chdir(tmp_path) from autoskillit.recipe import _api_cache from autoskillit.recipe._api_cache import LoadCache - from autoskillit.server.tools.tools_kitchen import get_recipe monkeypatch.setattr(_api_cache, "_LOAD_CACHE", LoadCache()) @@ -283,14 +277,9 @@ async def test_get_recipe_content_matches_open_kitchen_with_overrides( monkeypatch, ) assert ok_result.get("success") is True, f"open_kitchen failed: {ok_result}" - ok_content = ok_result["content"] + ok_content = await _resolve_recipe_content(ok_result) - gr_content = get_recipe(_RECIPE) - try: - _err = json.loads(gr_content) - pytest.fail(f"get_recipe returned error: {_err}") - except json.JSONDecodeError: - pass + gr_content = await _get_recipe_resource_content(_RECIPE) assert ok_content == gr_content, ( "get_recipe content diverges from open_kitchen content — " @@ -321,22 +310,13 @@ async def _call_re_serve_surface( from autoskillit.server.tools.tools_recipe import load_recipe result = json.loads(await load_recipe(name=recipe_name)) - assert "content" in result, f"load_recipe returned no content: {result}" - return result["content"] + return await _resolve_recipe_content(result) elif surface == "get_recipe": - from autoskillit.server.tools.tools_kitchen import get_recipe - - content = get_recipe(recipe_name) - try: - _err = json.loads(content) - pytest.fail(f"get_recipe returned error: {_err}") - except json.JSONDecodeError: - pass - return content + return await _get_recipe_resource_content(recipe_name) elif surface == "open_kitchen_deferred_recall": result = await _open_kitchen_patched(recipe_name, None, monkeypatch) assert result.get("success") is True, f"deferred-recall failed: {result}" - return result["content"] + return await _resolve_recipe_content(result) else: raise ValueError(f"Unknown surface: {surface!r}") @@ -352,7 +332,7 @@ async def test_serve_surfaces_parametric_content_identity( Parametrized across _RE_SERVE_SURFACES (load_recipe, get_recipe, open_kitchen_deferred_recall). Collection-time guard at module level ensures - _RE_SERVE_SURFACES stays in sync with SERVE_SURFACES. + _RE_SERVE_SURFACES stays in sync with RECIPE_DELIVERY_SURFACE_REGISTRY. """ monkeypatch.chdir(tmp_path) # type: ignore[arg-type] from autoskillit.recipe import _api_cache @@ -366,7 +346,7 @@ async def test_serve_surfaces_parametric_content_identity( monkeypatch, ) assert ok_result.get("success") is True, f"open_kitchen failed: {ok_result}" - ok_content = ok_result["content"] + ok_content = await _resolve_recipe_content(ok_result) re_served_content = await _call_re_serve_surface(surface, _RECIPE, monkeypatch) @@ -386,7 +366,6 @@ async def test_get_recipe_snapshot_lifecycle( from autoskillit.core.io import load_yaml from autoskillit.recipe import _api_cache from autoskillit.recipe._api_cache import LoadCache - from autoskillit.server.tools.tools_kitchen import get_recipe monkeypatch.setattr(_api_cache, "_LOAD_CACHE", LoadCache()) @@ -396,14 +375,9 @@ async def test_get_recipe_snapshot_lifecycle( monkeypatch, ) assert ok_result.get("success") is True, f"open_kitchen failed: {ok_result}" - ok_content = ok_result["content"] + ok_content = await _resolve_recipe_content(ok_result) - gr_content = get_recipe(_RECIPE) - try: - _err = json.loads(gr_content) - pytest.fail(f"get_recipe returned error: {_err}") - except json.JSONDecodeError: - pass + gr_content = await _get_recipe_resource_content(_RECIPE) parsed_ok = load_yaml(ok_content) parsed_gr = load_yaml(gr_content) @@ -477,25 +451,11 @@ async def test_load_recipe_routing_matches_open_kitchen_for_arbitrary_overrides( assume(False) # discard: open_kitchen failed (e.g. invalid combos) lr_result = json.loads(await load_recipe(name=_RECIPE)) - if "content" not in lr_result: + if lr_result.get("success") is False: assume(False) # discard: load_recipe failed - - if "content" in ok_result: - assert lr_result["content"] == ok_result["content"], ( - f"Routing divergence for overrides={overrides!r}: " - "load_recipe content must match open_kitchen content" - ) - else: - # open_kitchen's pre-envelope payload carries extra routing metadata - # (build_open_kitchen_recipe_payload) that load_recipe's doesn't, so - # for a recipe near the delivery bound the two surfaces can cross - # independently and open_kitchen may substitute the bounded - # step-flow-skeleton envelope while load_recipe still serves full - # content. Prove idempotence via the recipe-derived routing fields - # populated identically in both representations. - assert ok_result.get("post_prune_step_names") == lr_result.get("post_prune_step_names"), ( - f"Routing divergence for overrides={overrides!r}: post_prune_step_names mismatch" - ) - assert ok_result.get("post_prune_routing_edges") == lr_result.get( - "post_prune_routing_edges" - ), f"Routing divergence for overrides={overrides!r}: post_prune_routing_edges mismatch" + ok_content = await _resolve_recipe_content(ok_result) + lr_content = await _resolve_recipe_content(lr_result) + assert lr_content == ok_content, ( + f"Routing divergence for overrides={overrides!r}: " + "load_recipe content must match open_kitchen content" + ) diff --git a/tests/server/test_tools_kitchen_cache_poison.py b/tests/server/test_tools_kitchen_cache_poison.py index 3fa5e2009f..d0b6b1fc89 100644 --- a/tests/server/test_tools_kitchen_cache_poison.py +++ b/tests/server/test_tools_kitchen_cache_poison.py @@ -9,6 +9,8 @@ import pytest +from tests.server._helpers import _resolve_recipe_content + pytestmark = [pytest.mark.layer("server"), pytest.mark.anyio, pytest.mark.medium] @@ -45,9 +47,5 @@ async def test_open_kitchen_ingredients_only_does_not_poison_load_recipe( ) lr_result = json.loads(await load_recipe(name="implementation")) - assert "content" in lr_result, ( - "load_recipe must return 'content' after open_kitchen(ingredients_only=True); " - "cache was poisoned by the ingredients_only pop" - ) - assert isinstance(lr_result["content"], str) - assert len(lr_result["content"]) > 0 + content = await _resolve_recipe_content(lr_result) + assert content, "load_recipe must provide pullable content after ingredients-only serving" diff --git a/tests/server/test_tools_kitchen_gate_features.py b/tests/server/test_tools_kitchen_gate_features.py index 604c8b9d84..1a3e38ba64 100644 --- a/tests/server/test_tools_kitchen_gate_features.py +++ b/tests/server/test_tools_kitchen_gate_features.py @@ -427,9 +427,12 @@ def test_recipe_resource_returns_composed_content(): backend_capabilities_map={}, backend_origin_map={}, ) - assert result == ("name: test-recipe\nsteps:\n stop:\n action: stop\n message: done\n") - assert "optional: true" not in result - assert "skip_when_false:" not in result + parsed = json.loads(result) + assert parsed["content"] == ( + "name: test-recipe\nsteps:\n stop:\n action: stop\n message: done\n" + ) + assert "optional: true" not in parsed["content"] + assert "skip_when_false:" not in parsed["content"] # 1h: get_recipe resource returns error for invalid recipe diff --git a/tests/server/test_tools_load_recipe.py b/tests/server/test_tools_load_recipe.py index e5fcf32029..3fd740c3a6 100644 --- a/tests/server/test_tools_load_recipe.py +++ b/tests/server/test_tools_load_recipe.py @@ -13,7 +13,7 @@ from autoskillit.core.types import RetryReason from autoskillit.pipeline.gate import DefaultGateState from autoskillit.server.tools.tools_recipe import load_recipe -from tests.server._helpers import _MINIMAL_SCRIPT_YAML +from tests.server._helpers import _MINIMAL_SCRIPT_YAML, _resolve_recipe_content pytestmark = [pytest.mark.layer("server"), pytest.mark.small] @@ -114,8 +114,7 @@ async def test_load_recipe_mcp_returns_builtin_recipe( monkeypatch.chdir(tmp_path) result = json.loads(await load_recipe(name="implementation")) assert "error" not in result, f"Unexpected error: {result.get('error')}" - assert "content" in result - assert len(result["content"]) > 0 + assert await _resolve_recipe_content(result) @pytest.mark.anyio async def test_load_recipe_parse_failure_is_logged_and_surfaced( diff --git a/tests/server/test_tools_recipe.py b/tests/server/test_tools_recipe.py index 2b5b80907f..e63d4f8815 100644 --- a/tests/server/test_tools_recipe.py +++ b/tests/server/test_tools_recipe.py @@ -292,6 +292,18 @@ async def _get_tools(self) -> dict: tools = await client.list_tools() return {t.name: t for t in tools} + @pytest.mark.anyio + async def test_recipe_tool_schemas_include_generated_codex_delivery_contract( + self, kitchen_enabled + ): + from autoskillit.execution import CODEX_RECIPE_DELIVERY_CALLING_CONTRACT + + tools = await self._get_tools() + for tool_name in ("open_kitchen", "load_recipe"): + description = tools[tool_name].description or "" + assert CODEX_RECIPE_DELIVERY_CALLING_CONTRACT in description + assert "delivery_request" in description + @pytest.mark.anyio async def test_load_recipe_action_protocol_routes_through_skill(self, kitchen_enabled): """After loading section must route modifications through write-recipe.""" diff --git a/tests/server/test_tools_recipe_pull.py b/tests/server/test_tools_recipe_pull.py index 36b4189561..ca02e60081 100644 --- a/tests/server/test_tools_recipe_pull.py +++ b/tests/server/test_tools_recipe_pull.py @@ -1,1262 +1,869 @@ -"""Part B (issue #4304) — bounded envelope + pull-access architecture. - -Regression guards for the recipe delivery-bound re-architecture: every -bundled recipe's envelope fits the smallest backend delivery bound by -construction, the pull tool returns bounded step content, the artifact -path is deterministic across calls, and a missing artifact is -re-created via the same serve pipeline that built it originally. - -Covers the "always fits" invariant in `build_recipe_envelope`, -the `get_recipe_section` MCP tool's chunked-content contract, and the -recipe-prompt discipline enforcement in `cli/_prompts_kitchen.py`. -""" +"""Immutable recipe generations, finalization, and bounded pull contracts.""" from __future__ import annotations import hashlib import json +import shutil +from concurrent.futures import ThreadPoolExecutor +from dataclasses import replace from pathlib import Path -from typing import cast -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import MagicMock import pytest +from autoskillit.config import OutputBudgetConfig from autoskillit.core import ( - BackendCapabilities, - fast_dumps, + RECIPE_DELIVERY_SURFACE_REGISTRY, + RecipeDeliveryAttestation, + RecipeDeliveryEvidenceDef, + RecipeDeliveryMode, + RecipeDeliveryRequest, load_yaml, - resolve_effective_delivery_bound, + recipe_delivery_request_digest, ) -from autoskillit.execution.backends import BACKEND_REGISTRY, CODEX_TOOL_OUTPUT_TOKEN_LIMIT -from autoskillit.recipe import ( - _extract_routing_edges, - all_validated_recipe_names, - find_recipe_by_name, - load_and_validate, - load_recipe, +from autoskillit.execution import ( + CODEX_RECIPE_DELIVERY_BUDGET, + ProtectedStoreAuthority, + RecipeDeliveryReceiptLedger, ) -from autoskillit.server._response_budget import _recipe_artifact_path -from autoskillit.server.tools import _serve_helpers -from autoskillit.server.tools._serve_helpers import ( - _compute_step_byte_ranges, +from autoskillit.execution.backends import CodexBackend +from autoskillit.server._recipe_delivery import ( + RECIPE_ARTIFACT_MAX_BLOB_BYTES, + RECIPE_BODY_END, + RECIPE_BODY_START, + RECIPE_COMPLETION_SENTINEL, + RecipeArtifactError, + RecipeArtifactGeneration, + _generation_dir, build_recipe_envelope, - build_routing_edges_by_step, - build_step_summaries, - extract_step_skeleton, - maybe_envelope_recipe_response, + complete_finalized_recipe_response, + finalize_recipe_delivery, + load_recipe_artifact, persist_recipe_artifact, + recipe_pull_producers, + recipe_recreation_producers, + retire_recipe_artifacts, ) -from autoskillit.server.tools.tools_recipe import ( - _bounded_recipe_section_response, - _extract_step_body_from_persisted, - _RecipeSectionError, -) - -pytestmark = [pytest.mark.layer("server"), pytest.mark.small] +from autoskillit.server._response_budget import enforce_response_budget +from autoskillit.server.tools.tools_recipe import get_recipe_section +pytestmark = [pytest.mark.layer("server"), pytest.mark.medium] -_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent +_NOW = 1_800_000_000 -def _recipe_names() -> list[str]: - return sorted(all_validated_recipe_names(_PROJECT_ROOT)) - - -def _backend_capabilities() -> dict[str, BackendCapabilities]: - return {name: cls().capabilities for name, cls in BACKEND_REGISTRY.items()} +def _payload( + content: str = "name: remediation\nsteps:\n first:\n action: stop\n", +) -> dict[str, object]: + return { + "success": True, + "valid": True, + "content": content, + "post_prune_step_names": ["first"], + "orchestration_rules": "follow the graph", + "ingredients_table": {"task": {"required": True}}, + } -def _smallest_bound_tokens() -> int: - """Smallest backend delivery token bound across the registry.""" - return min(resolve_effective_delivery_bound(caps) for caps in _backend_capabilities().values()) +def _persist( + tmp_path: Path, + payload: dict[str, object] | None = None, + *, + producer: str = "open_kitchen", +) -> RecipeArtifactGeneration: + return persist_recipe_artifact( + tmp_path, + kitchen_id="kitchen-test", + producer_tool=producer, + recipe_name="remediation", + payload=dict(payload or _payload()), + ) -def _bound_bytes(bound_tokens: int) -> int: - return bound_tokens * 4 +def test_artifact_namespace_encodes_colliding_kitchen_ids_injectively(tmp_path: Path) -> None: + first = persist_recipe_artifact( + tmp_path, + kitchen_id="a/b", + producer_tool="open_kitchen", + recipe_name="remediation", + payload=_payload(), + ) + second = persist_recipe_artifact( + tmp_path, + kitchen_id="a?b", + producer_tool="open_kitchen", + recipe_name="remediation", + payload=_payload(), + ) + assert retire_recipe_artifacts(tmp_path, kitchen_id="a/b") is True + with pytest.raises(RecipeArtifactError): + load_recipe_artifact(tmp_path, kitchen_id="a/b", identity=first) + assert load_recipe_artifact(tmp_path, kitchen_id="a?b", identity=second) == _payload() -def _artifact_sha256(path: Path) -> str: - return hashlib.sha256(path.read_bytes()).hexdigest() +def test_artifact_persistence_rejects_blob_above_read_ceiling(tmp_path: Path) -> None: + oversized = _payload("x" * (RECIPE_ARTIFACT_MAX_BLOB_BYTES + 1)) -def test_section_chunks_fit_serialized_utf8_bound() -> None: - content = ('\u96ea"\\\n\t' * 2_000) + "tail" - bound_bytes = 512 - chunks: list[str] = [] - part = 0 - while True: - rendered = _bounded_recipe_section_response( - "step", content, part=part, bound_bytes=bound_bytes - ) - assert len(rendered.encode("utf-8")) <= bound_bytes - response = json.loads(rendered) - assert response["success"] is True - chunks.append(response["content"]) - if not response["has_more"]: - break - part = response["next_part"] - assert "".join(chunks) == content + with pytest.raises(RecipeArtifactError, match="exceeds persistence limit"): + _persist(tmp_path, oversized) -def test_step_extraction_distinguishes_artifact_and_serialization_failures( +def test_shared_producer_surfaces_require_identical_pull_policies( monkeypatch: pytest.MonkeyPatch, ) -> None: - with pytest.raises(_RecipeSectionError) as parse_error: - _extract_step_body_from_persisted({"content": "steps: ["}, "step") - assert parse_error.value.code == "recipe_artifact_parse_failed" - - import autoskillit.server.tools.tools_recipe as tools_recipe - - def _broken_dumps(_value: object) -> str: - raise TypeError("cannot serialize") - - monkeypatch.setattr(tools_recipe, "fast_dumps", _broken_dumps) - with pytest.raises(_RecipeSectionError) as serialization_error: - _extract_step_body_from_persisted( - {"content": "steps:\n step:\n action: stop\n"}, "step" - ) - assert serialization_error.value.code == "recipe_section_serialization_failed" - - -def _full_open_kitchen_payload(recipe_name: str) -> dict[str, object]: - """Build the production-shape open_kitchen payload for *recipe_name*.""" - result = load_and_validate( - recipe_name, - project_dir=_PROJECT_ROOT, - ingredient_overrides={ - "task": "test task", - "issue_url": "https://github.com/test/test/issues/1", - "source_dir": str(_PROJECT_ROOT), - }, + conflicting = dict(RECIPE_DELIVERY_SURFACE_REGISTRY) + conflicting["conflicting_open_kitchen"] = conflicting["open_kitchen"]._replace( + pull_eligible=False + ) + monkeypatch.setattr( + "autoskillit.server._recipe_delivery.RECIPE_DELIVERY_SURFACE_REGISTRY", + conflicting, ) - from autoskillit.server.tools._serve_helpers import build_open_kitchen_recipe_payload - return build_open_kitchen_recipe_payload(dict(result), version="0.0.0") + with pytest.raises(RecipeArtifactError, match="must share pull policies"): + recipe_pull_producers() + with pytest.raises(RecipeArtifactError, match="must share pull policies"): + recipe_recreation_producers() -@pytest.mark.parametrize("recipe_name", _recipe_names(), ids=lambda n: n) -@pytest.mark.parametrize("backend_name", sorted(_backend_capabilities().keys()), ids=lambda n: n) -def test_envelope_fits_every_backend_by_construction( - recipe_name: str, backend_name: str, tmp_path: Path -) -> None: - """The production envelope path preserves its complete pull contract.""" - payload = _full_open_kitchen_payload(recipe_name) - post_prune_raw = cast(list[object], payload.get("post_prune_step_names") or []) - step_names = [str(n) for n in post_prune_raw if isinstance(n, str)] - - caps = _backend_capabilities()[backend_name] - bound_tokens = resolve_effective_delivery_bound(caps) - bound_bytes = _bound_bytes(bound_tokens) - content = cast(str, payload.get("content") or "") - padding = "delivery-fit-padding" * (bound_bytes // len("delivery-fit-padding") + 1) - payload["content"] = content + "\n# " + padding - - info = find_recipe_by_name(recipe_name, _PROJECT_ROOT) - assert info is not None - recipe = load_recipe(info.path) - ctx = _make_minimal_ctx(tmp_path) - ctx.recipe_name = recipe_name - ctx.active_recipe_steps = recipe.steps - - envelope = maybe_envelope_recipe_response( - payload, - tool_name="open_kitchen", - recipe_name=recipe_name, - tool_ctx=ctx, - effective_delivery_token_limit=bound_tokens, - ) - serialized = json.dumps(envelope, ensure_ascii=False, separators=(",", ":")) - assert len(serialized.encode("utf-8")) <= bound_bytes, ( - f"{backend_name}: envelope for {recipe_name} exceeds " - f"{bound_bytes} bytes (effective delivery bound)" - ) +def _remove_persisted_namespace(temp_dir: Path, *, kitchen_id: str) -> None: + shutil.rmtree(temp_dir / "recipe-delivery" / kitchen_id) - artifact_path = _recipe_artifact_path(ctx.temp_dir, "open_kitchen", recipe_name) - assert envelope["recipe_pull"] == { - "recipe_name": recipe_name, - "producer_tool": "open_kitchen", - "artifact_path": str(artifact_path), - "sha256": _artifact_sha256(artifact_path), - "pull_tool": "get_recipe_section", - } - expected_skeleton = extract_step_skeleton( - step_names, - build_routing_edges_by_step(recipe.steps), - build_step_summaries(recipe.steps), - byte_ranges=_compute_step_byte_ranges(cast(str, payload["content"])), - ) - actual_skeleton = envelope["step_flow_skeleton"] - assert actual_skeleton["step_count"] == expected_skeleton["step_count"] - actual_steps = actual_skeleton["steps"] - expected_steps = expected_skeleton["steps"] - assert [step["name"] for step in actual_steps] == [step["name"] for step in expected_steps] - for actual_step, expected_step in zip(actual_steps, expected_steps, strict=True): - assert actual_step["edges"] == expected_step["edges"] - assert actual_step.get("byte_range") == expected_step.get("byte_range") - if expected_step.get("summary"): - assert expected_step["summary"].startswith(actual_step.get("summary", "")) - - -def test_envelope_carries_priority_fields_verbatim(tmp_path: Path) -> None: - """orchestration_rules, stop_step_semantics, and ingredients_table are - passed through unchanged so the orchestrator can route without pulling.""" - payload = { - "success": True, - "kitchen": "open", - "version": "0.0.0", - "orchestration_rules": "ORCH: steps route strictly on success/failure.", - "stop_step_semantics": "STOP means stop — never auto-recover.", - "errors": ["warn-A", "warn-B"], - "ingredients_table": {"task": {"type": "string", "required": True}}, - } + +def _build_envelope( + tmp_path: Path, payload: dict[str, object], *, bound_bytes: int +) -> tuple[dict[str, object], RecipeArtifactGeneration]: + generation = _persist(tmp_path, payload) + skeleton_source = MagicMock() + skeleton_source.recipe_name = "remediation" + skeleton_source.active_recipe_steps = {} envelope = build_recipe_envelope( payload, - recipe_name="test-recipe", - artifact_path=str(tmp_path / "x.log"), - artifact_sha256="0" * 64, - skeleton=extract_step_skeleton([], {}), - bound_bytes=64_000, - ) - assert envelope["orchestration_rules"] == payload["orchestration_rules"] - assert envelope["stop_step_semantics"] == payload["stop_step_semantics"] - assert envelope["ingredients_table"] == payload["ingredients_table"] - assert envelope["errors"] == ["warn-A", "warn-B"] - - -def test_extract_step_skeleton_preserves_routing_edges() -> None: - """Step skeleton includes outgoing routing edges (edge_type + target) - so the orchestrator can reason about flow without pulling bodies.""" - skeleton = extract_step_skeleton( - ["a", "b", "c"], - routing_edges_by_step={ - "a": [("success", "b"), ("failure", "c")], - "b": [("success", "c")], - "c": [], - }, - step_summaries={"a": "step a", "b": "step b", "c": "step c"}, - ) - assert skeleton["step_count"] == 3 - by_name = {step["name"]: step for step in skeleton["steps"]} - assert [edge["type"] for edge in by_name["a"]["edges"]] == ["success", "failure"] - assert [edge["target"] for edge in by_name["a"]["edges"]] == ["b", "c"] - assert by_name["b"]["edges"] == [{"type": "success", "target": "c"}] - assert by_name["c"]["edges"] == [] - assert by_name["a"]["summary"] == "step a" - - -def test_recipe_artifact_path_is_deterministic(tmp_path: Path) -> None: - """_recipe_artifact_path returns the same path for the same (tool, - recipe_name) on repeated calls — no UUID suffix. - - The pull tool relies on this determinism to reconstruct the path from - ``tool_ctx.recipe_name`` without needing it threaded through every - surface. This is the regression guard for the original UUID-suffixed - path that broke pull access (#4304 related issue #2).""" - p1 = _recipe_artifact_path(tmp_path, "open_kitchen", "remediation") - p2 = _recipe_artifact_path(tmp_path, "open_kitchen", "remediation") - p3 = _recipe_artifact_path(tmp_path, "load_recipe", "remediation") - assert p1 == p2, "deterministic path must not vary between calls" - assert p1 != p3, "different tool must produce different path" - assert "open_kitchen" in str(p1) and "remediation" in str(p1) - - -def test_persist_recipe_artifact_overwrites_idempotently(tmp_path: Path) -> None: - """Re-persisting the same recipe overwrites the artifact in place — - pull access can rely on a stable path and fresh content.""" - path, sha1 = persist_recipe_artifact( - tmp_path, - tool_name="open_kitchen", recipe_name="remediation", - payload={"success": True, "content": "v1"}, - ) - size1 = Path(path).stat().st_size - path2, sha2 = persist_recipe_artifact( - tmp_path, - tool_name="open_kitchen", - recipe_name="remediation", - payload={"success": True, "content": "v2 is longer than v1"}, - ) - assert path == path2 - assert sha1 != sha2 - size2 = Path(path).stat().st_size - assert size2 != size1 - assert json.loads(Path(path).read_text())["content"].startswith("v2") - - -@pytest.mark.parametrize("recipe_name", _recipe_names(), ids=lambda n: n) -def test_maybe_envelope_returns_envelope_when_payload_oversized( - recipe_name: str, tmp_path: Path -) -> None: - """When the payload's estimated token count exceeds - effective_delivery_token_limit, maybe_envelope_recipe_response - returns the bounded envelope (not the full payload).""" - ctx = _make_minimal_ctx(tmp_path) - ctx.recipe_name = recipe_name - - payload = _full_open_kitchen_payload(recipe_name) - bound_tokens = _smallest_bound_tokens() - payload_size_tokens = (len(json.dumps(payload, ensure_ascii=False).encode("utf-8")) + 3) // 4 - result = maybe_envelope_recipe_response( - payload, - tool_name="open_kitchen", - recipe_name=recipe_name, - tool_ctx=ctx, - effective_delivery_token_limit=bound_tokens, - ) - serialized = json.dumps(result, ensure_ascii=False) - assert len(serialized.encode("utf-8")) <= _bound_bytes(bound_tokens), ( - f"{recipe_name}: envelope exceeds {bound_tokens}-token bound" - ) - if payload_size_tokens <= bound_tokens: - # Recipe payload fits the bound; no spill triggered. The result - # is the unchanged payload — which is the correct behavior. - assert result.get("delivery_bound_spill") is not True - return - assert len(json.dumps(result).encode("utf-8")) <= bound_tokens * 4 - assert result.get("recipe_pull", {}).get("pull_tool") == "get_recipe_section" - - -def test_maybe_envelope_passthrough_when_payload_fits(tmp_path: Path) -> None: - """When the payload fits the bound, maybe_envelope returns the - payload unchanged AND unconditionally persists the artifact so the - pull tool always has a backing store (Part A REQ-B02).""" - ctx = _make_minimal_ctx(tmp_path) - ctx.recipe_name = "any_recipe" - - payload = {"success": True, "content": "short"} - bound_tokens = 10_000 # 40KB — plenty for "short" - result = maybe_envelope_recipe_response( - payload, - tool_name="open_kitchen", - recipe_name="any_recipe", - tool_ctx=ctx, - effective_delivery_token_limit=bound_tokens, - ) - assert result is payload - artifact_path = _recipe_artifact_path(ctx.temp_dir, "open_kitchen", "any_recipe") - assert Path(artifact_path).exists() - assert Path(artifact_path).read_text(encoding="utf-8") == json.dumps( - payload, ensure_ascii=False + generation=generation, + skeleton_source=skeleton_source, + bound_bytes=bound_bytes, ) + return envelope, generation -def test_maybe_envelope_persists_artifact_even_when_payload_fits(tmp_path: Path) -> None: - """When the payload fits, the returned value is the original payload - unchanged AND the artifact file is created at the deterministic path - with content matching ``json.dumps(payload, ensure_ascii=False)``. - Persistence is unconditional — gated only by ``temp_dir`` availability.""" - ctx = _make_minimal_ctx(tmp_path) - ctx.recipe_name = "any_recipe" +def test_same_payload_is_idempotent_and_changed_payload_is_immutable(tmp_path: Path) -> None: + first = _persist(tmp_path) + same = _persist(tmp_path) + changed = _persist(tmp_path, _payload("name: remediation\nsteps: {}\n")) - payload = {"success": True, "content": "short"} - bound_tokens = 10_000 # 40KB — plenty for "short" - result = maybe_envelope_recipe_response( - payload, - tool_name="open_kitchen", - recipe_name="any_recipe", - tool_ctx=ctx, - effective_delivery_token_limit=bound_tokens, - ) - assert result is payload - artifact_path = _recipe_artifact_path(ctx.temp_dir, "open_kitchen", "any_recipe") - assert artifact_path.exists(), ( - "artifact file must exist at the deterministic path even when the " - "payload fits the bound (Part A REQ-B02 unconditional persistence)" - ) - assert Path(artifact_path).read_text(encoding="utf-8") == json.dumps( - payload, ensure_ascii=False + assert same == first + assert changed.payload_sha256 != first.payload_sha256 + assert load_recipe_artifact(tmp_path, kitchen_id="kitchen-test", identity=first) == _payload() + assert load_recipe_artifact(tmp_path, kitchen_id="kitchen-test", identity=changed) == _payload( + "name: remediation\nsteps: {}\n" ) -def test_maybe_envelope_persists_exactly_once_when_oversized( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch +@pytest.mark.parametrize( + "version_name", + ["RECIPE_ARTIFACT_DESCRIPTOR_VERSION", "RECIPE_ARTIFACT_SCHEMA_VERSION"], +) +def test_generation_path_includes_version_domain( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, version_name: str ) -> None: - """When the payload is oversized, ``persist_recipe_artifact`` is - invoked exactly once — guards against the regression where the - oversized branch contained a second persistence call alongside - the unconditional early-call (Part A REQ-B02 single-call invariant).""" - - real_persist = _serve_helpers.persist_recipe_artifact - calls: list[tuple[str, str]] = [] - - def spy_persist(artifact_dir, *, tool_name, recipe_name, payload): - calls.append((tool_name, recipe_name)) - return real_persist( - artifact_dir, tool_name=tool_name, recipe_name=recipe_name, payload=payload - ) + first = _persist(tmp_path) + monkeypatch.setattr(f"autoskillit.server._recipe_delivery.{version_name}", 2) - monkeypatch.setattr(_serve_helpers, "persist_recipe_artifact", spy_persist) + second = _persist(tmp_path) - ctx = _make_minimal_ctx(tmp_path) - ctx.recipe_name = "any_recipe" + assert second.payload_sha256 == first.payload_sha256 + assert second.pull_identity() != first.pull_identity() + assert load_recipe_artifact(tmp_path, kitchen_id="kitchen-test", identity=first) == _payload() + assert load_recipe_artifact(tmp_path, kitchen_id="kitchen-test", identity=second) == _payload() - payload = {"success": True, "content": "a" * 10_000} - # Force oversized: 1-token bound → envelope branch - bound_tokens = 1 - result = maybe_envelope_recipe_response( - payload, - tool_name="open_kitchen", - recipe_name="any_recipe", - tool_ctx=ctx, - effective_delivery_token_limit=bound_tokens, - ) - assert len(json.dumps(result).encode("utf-8")) <= bound_tokens * 4 - assert len(calls) == 1, ( - f"persist_recipe_artifact must be called exactly once; got {len(calls)} calls" - ) - assert calls[0] == ("open_kitchen", "any_recipe") +def test_concurrent_writers_publish_one_exact_generation(tmp_path: Path) -> None: + with ThreadPoolExecutor(max_workers=8) as pool: + generations = list(pool.map(lambda _: _persist(tmp_path), range(24))) + assert generations == [generations[0]] * len(generations) + assert ( + load_recipe_artifact(tmp_path, kitchen_id="kitchen-test", identity=generations[0]) + == _payload() + ) -def test_maybe_envelope_passthrough_when_temp_dir_unset(tmp_path: Path) -> None: - """When tool_ctx.temp_dir is unset (not a Path), the helper returns - the payload unchanged rather than failing — track_response_size's - spill path remains the fallback.""" - ctx = _make_minimal_ctx(tmp_path) - # Replace temp_dir with a non-Path value to simulate the sentinel state - # without tripping ToolContext.__post_init__'s TypeError guard. - ctx.temp_dir = None # type: ignore[assignment] - ctx.recipe_name = "any_recipe" - payload = {"success": True, "content": "x" * 10_000_000} - result = maybe_envelope_recipe_response( - payload, - tool_name="open_kitchen", - recipe_name="any_recipe", - tool_ctx=ctx, - effective_delivery_token_limit=100, - ) - assert result is payload - - -def _make_minimal_ctx(tmp_path: Path): - """Build a minimal ToolContext for envelope integration tests.""" - from autoskillit.config import AutomationConfig - from autoskillit.core.types._type_plugin_source import DirectInstall - from autoskillit.pipeline.audit import DefaultAuditLog - from autoskillit.pipeline.context import ToolContext - from autoskillit.pipeline.gate import DefaultGateState - from autoskillit.pipeline.timings import DefaultTimingLog - from autoskillit.pipeline.tokens import DefaultTokenLog - - return ToolContext( - config=AutomationConfig(features={"fleet": True}), - audit=DefaultAuditLog(), - token_log=DefaultTokenLog(), - timing_log=DefaultTimingLog(), - gate=DefaultGateState(enabled=False), - plugin_source=DirectInstall(plugin_dir=tmp_path), - runner=None, - temp_dir=tmp_path / ".autoskillit" / "temp", - project_dir=tmp_path, - ) +@pytest.mark.parametrize( + ("filename", "error"), + [ + ("payload.json", "content-addressed payload collision"), + ("descriptor.json", "content-addressed descriptor collision"), + ], +) +def test_persistence_collision_checks_use_bounded_descriptor_reads( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + filename: str, + error: str, +) -> None: + _persist(tmp_path) + target = next((tmp_path / "recipe-delivery").rglob(filename)) + target.write_bytes(target.read_bytes() + b"x") + def _unbounded_read_forbidden(*_args, **_kwargs): + raise AssertionError("unbounded pathlib read used during collision check") -# --------------------------------------------------------------------------- -# Prompt contract tests (Part B Step 2.6) -# --------------------------------------------------------------------------- + monkeypatch.setattr(Path, "read_bytes", _unbounded_read_forbidden) + monkeypatch.setattr(Path, "read_text", _unbounded_read_forbidden) + with pytest.raises(RecipeArtifactError, match=error): + _persist(tmp_path) -def test_prompt_contract_describes_pull_protocol() -> None: - """cli/_prompts_kitchen.py must NOT promise inline recipe content on - open_kitchen; it must reference the pull tool by name.""" - prompts_path = _PROJECT_ROOT / "src" / "autoskillit" / "cli" / "_prompts_kitchen.py" - text = prompts_path.read_text(encoding="utf-8") - assert "to receive the full recipe content" not in text, ( - "cli/_prompts_kitchen.py must not promise inline 'full recipe content' " - "on open_kitchen — the envelope + pull protocol is the supported path." - ) - assert "get_recipe_section" in text, ( - "cli/_prompts_kitchen.py must reference the pull tool name so the " - "orchestrator knows how to retrieve step bodies." - ) - assert "artifact_sha256=recipe_pull.sha256" in text +@pytest.mark.parametrize( + ("field", "value"), + [ + ("producer_tool", "invalid"), + ("payload_sha256", "sha256:" + ("0" * 64)), + ("artifact_blob_sha256", "sha256:" + ("1" * 64)), + ("artifact_blob_size_bytes", 1), + ("body_sha256", "sha256:" + ("2" * 64)), + ("body_size_bytes", 1), + ], +) +def test_generation_identity_domains_are_independently_verified( + tmp_path: Path, field: str, value: str | int +) -> None: + generation = _persist(tmp_path) + with pytest.raises(RecipeArtifactError): + load_recipe_artifact( + tmp_path, + kitchen_id="kitchen-test", + identity=replace(generation, **{field: value}), + ) -def test_pull_tool_registered_in_gated_tools() -> None: - """get_recipe_section must be in GATED_TOOLS so _require_enabled guards it.""" - from autoskillit.core.types import GATED_TOOLS - assert "get_recipe_section" in GATED_TOOLS +def test_generation_descriptor_read_has_server_owned_ceiling(tmp_path: Path) -> None: + generation = _persist(tmp_path) + descriptor_path = next((tmp_path / "recipe-delivery").rglob("descriptor.json")) + descriptor_path.write_bytes(b"x" * 20_000) + with pytest.raises(RecipeArtifactError, match="descriptor exceeds read limit"): + load_recipe_artifact(tmp_path, kitchen_id="kitchen-test", identity=generation) -def test_pull_tool_registered_in_tool_subset_tags() -> None: - """get_recipe_section must be in TOOL_SUBSET_TAGS so kitchen-core - visibility resolves correctly.""" - from autoskillit.core.types import TOOL_SUBSET_TAGS - assert "get_recipe_section" in TOOL_SUBSET_TAGS - assert "kitchen-core" in TOOL_SUBSET_TAGS["get_recipe_section"] +def test_non_utf8_payload_is_normalized_to_recipe_artifact_error(tmp_path: Path) -> None: + blob = b"\xff" + qualified_blob_sha = f"sha256:{hashlib.sha256(blob).hexdigest()}" + payload_sha = "sha256:" + hashlib.sha256(b"autoskillit.recipe-payload.v1\0" + blob).hexdigest() + empty_body_sha = f"sha256:{hashlib.sha256(b'').hexdigest()}" + generation = RecipeArtifactGeneration( + producer_tool="open_kitchen", + recipe_name="remediation", + descriptor_version=1, + schema_version=1, + payload_sha256=payload_sha, + artifact_blob_sha256=qualified_blob_sha, + artifact_blob_size_bytes=len(blob), + body_sha256=empty_body_sha, + body_size_bytes=0, + ) + directory = _generation_dir( + tmp_path, + kitchen_id="kitchen-test", + producer_tool=generation.producer_tool, + recipe_name=generation.recipe_name, + descriptor_version=generation.descriptor_version, + schema_version=generation.schema_version, + payload_sha256=generation.payload_sha256, + ) + directory.mkdir(parents=True) + (directory / "payload.json").write_bytes(blob) + (directory / "descriptor.json").write_text( + json.dumps(generation.pull_identity(), sort_keys=True, separators=(",", ":")), + encoding="utf-8", + ) + + with pytest.raises(RecipeArtifactError, match="not valid JSON"): + load_recipe_artifact(tmp_path, kitchen_id="kitchen-test", identity=generation) + + +def test_generation_descriptor_has_no_caller_selected_path(tmp_path: Path) -> None: + pull = _persist(tmp_path).pull_identity() + assert set(pull) == { + "producer_tool", + "recipe_name", + "descriptor_version", + "schema_version", + "payload_sha256", + "artifact_blob_sha256", + "artifact_blob_size_bytes", + "body_sha256", + "body_size_bytes", + "pull_tool", + } + assert not {"artifact_path", "path", "sha256"} & set(pull) + assert recipe_pull_producers() == {"open_kitchen", "load_recipe", "get_recipe"} + assert recipe_recreation_producers() == {"open_kitchen", "get_recipe"} -def test_pull_tool_in_unformatted_or_formatters() -> None: - """New MCP tools must appear in _UNFORMATTED_TOOLS or _FORMATTERS so - the dispatcher does not silently drop them.""" - formatter_path = ( - _PROJECT_ROOT / "src" / "autoskillit" / "hooks" / "formatters" / "pretty_output_hook.py" +def test_kitchen_retirement_removes_only_that_namespace(tmp_path: Path) -> None: + first = _persist(tmp_path) + second = persist_recipe_artifact( + tmp_path, + kitchen_id="other-kitchen", + producer_tool="open_kitchen", + recipe_name="remediation", + payload=_payload(), + ) + assert retire_recipe_artifacts(tmp_path, kitchen_id="kitchen-test") is True + with pytest.raises(RecipeArtifactError): + load_recipe_artifact(tmp_path, kitchen_id="kitchen-test", identity=first) + with pytest.raises(RecipeArtifactError, match="namespace is retired"): + _persist(tmp_path) + assert ( + load_recipe_artifact(tmp_path, kitchen_id="other-kitchen", identity=second) == _payload() ) - text = formatter_path.read_text(encoding="utf-8") - assert "get_recipe_section" in text -def test_codex_token_limit_unchanged_part_b() -> None: - """Part B does not change the exemption ceilings — it adds the pull - pathway. The CODEX_TOOL_OUTPUT_TOKEN_LIMIT remains driven by the - registry's max, with the envelope fitting within that bound by - construction. This test pins the current value so future changes - requiring ADR-0005 amendment trip this guard.""" - assert CODEX_TOOL_OUTPUT_TOKEN_LIMIT >= 10_000 +@pytest.mark.parametrize("kitchen_id", [".", ".."]) +def test_kitchen_retirement_rejects_dot_path_components(tmp_path: Path, kitchen_id: str) -> None: + sentinel = tmp_path / "unrelated-temp-data" + sentinel.write_text("preserve me", encoding="utf-8") + assert retire_recipe_artifacts(tmp_path, kitchen_id=kitchen_id) is False + assert sentinel.read_text(encoding="utf-8") == "preserve me" -def test_build_step_summaries_handles_missing_or_malformed() -> None: - """build_step_summaries returns {} when active_recipe_steps is None - or not a dict (defensive against pre-recipe-open contexts).""" - assert build_step_summaries(None) == {} - assert build_step_summaries({}) == {} - assert build_step_summaries({"a": object()}) == {"a": ""} +def test_codex_without_supported_host_evidence_uses_bounded_envelope(tool_ctx) -> None: + tool_ctx.backend = CodexBackend() + tool_ctx.kitchen_id = "codex-envelope" + payload = _payload("x" * 50_000) -def test_build_routing_edges_handles_missing_extractor() -> None: - """build_routing_edges_by_step with edge_extractor=None maps each step - to an empty list (not omitted) — callers can rely on .get(name) returning - a list, never KeyError.""" - assert build_routing_edges_by_step({"a": object()}, edge_extractor=None) == {"a": []} - assert build_routing_edges_by_step(None, edge_extractor=None) == {} + finalized = finalize_recipe_delivery( + payload, + surface="open_kitchen", + recipe_name="remediation", + tool_ctx=tool_ctx, + ) + assert finalized.decision.mode is RecipeDeliveryMode.ENVELOPE + envelope = json.loads(finalized.rendered) + assert "content" not in envelope + assert envelope["recipe_pull"]["payload_sha256"].startswith("sha256:") + assert len(finalized.rendered.encode("utf-8")) <= 40_000 -def test_build_routing_edges_propagates_extractor_failure() -> None: - def _broken_extractor(_step: object) -> list[object]: - raise ValueError("invalid route") - with pytest.raises(ValueError, match="invalid route"): - build_routing_edges_by_step({"a": object()}, edge_extractor=_broken_extractor) +def test_token_dense_payload_does_not_use_four_byte_ordinary_estimate(tool_ctx) -> None: + tool_ctx.backend = CodexBackend() + tool_ctx.kitchen_id = "codex-token-dense" + finalized = finalize_recipe_delivery( + _payload("!" * 20_000), + surface="open_kitchen", + recipe_name="remediation", + tool_ctx=tool_ctx, + ) -# --------------------------------------------------------------------------- -# Part A (REQ-B02 / REQ-B03) gap-closure regression guards -# --------------------------------------------------------------------------- + assert finalized.decision.mode is RecipeDeliveryMode.ENVELOPE + assert len(finalized.rendered.encode("utf-8")) <= ( + CODEX_RECIPE_DELIVERY_BUDGET.ordinary_omitted_result_token_limit + ) -def test_build_recipe_envelope_requires_recipe_name(tmp_path: Path) -> None: - """build_recipe_envelope requires ``recipe_name`` as a keyword-only - argument (no positional default). Mirrors the sibling-function - convention used by ``persist_recipe_artifact`` and - ``maybe_envelope_recipe_response`` (Part A REQ-B03).""" - skeleton = extract_step_skeleton([], {}) - with pytest.raises(TypeError, match="recipe_name"): - build_recipe_envelope( # type: ignore[call-arg] - {"success": True}, - artifact_path=str(tmp_path / "x.log"), - artifact_sha256="0" * 64, - skeleton=skeleton, - bound_bytes=1024, - ) +def test_envelope_priority_fields_share_multibyte_budget_safely(tmp_path: Path) -> None: + payload = _payload() + payload["orchestration_rules"] = "雪" * 1_000 + payload["stop_step_semantics"] = "界" * 1_000 + bound_bytes = 4_000 + envelope, _generation = _build_envelope(tmp_path, payload, bound_bytes=bound_bytes) + rendered = json.dumps(envelope, ensure_ascii=False, separators=(",", ":")) -def _envelope_overheads(tmp_path: Path, recipe_name: str) -> tuple[int, int, int]: - """Compute (skeleton_overhead, pull_overhead, envelope_bytes) for an - empty-skeleton, success=True envelope with the given recipe_name, so - budget-tight envelope tests can derive exact bound_bytes values.""" - skeleton = extract_step_skeleton([], {}) - artifact_path = str(tmp_path / "x.log") - artifact_sha256 = "0" * 64 - skeleton_overhead = len( - json.dumps({"step_flow_skeleton": skeleton}, ensure_ascii=False).encode("utf-8") - ) - pull_overhead = len( - json.dumps( - { - "recipe_pull": { - "artifact_path": artifact_path, - "producer_tool": "open_kitchen", - "sha256": artifact_sha256, - "pull_tool": "get_recipe_section", - "recipe_name": recipe_name, - }, - "delivery_bound_spill": True, - }, - ensure_ascii=False, - ).encode("utf-8") - ) - envelope_bytes = len(json.dumps(True, ensure_ascii=False).encode("utf-8")) - return skeleton_overhead, pull_overhead, envelope_bytes - - -def test_envelope_priority_fields_share_floor_under_tight_budget(tmp_path: Path) -> None: - """Both ``orchestration_rules`` and ``stop_step_semantics`` receive - a fair, non-zero share of the content budget under tight budgets. - The naive sequential allocator lets the first key exhaust the pool; - the two-phase water-filling allocator splits evenly.""" - orch = "ORCH_RULES: " + ("X" * 600) - stop = "STOP_STEP_SEMANTICS: " + ("Y" * 600) - payload = { - "success": True, - "orchestration_rules": orch, - "stop_step_semantics": stop, + assert len(rendered.encode("utf-8")) <= bound_bytes + for key in ("orchestration_rules", "stop_step_semantics"): + projected = envelope[key] + assert isinstance(projected, str) + assert projected + assert len(projected.encode("utf-8")) < len(str(payload[key]).encode("utf-8")) + assert projected.encode("utf-8").decode("utf-8") == projected + + +def test_envelope_fallbacks_follow_tight_and_extreme_bounds(tmp_path: Path) -> None: + payload = _payload() + payload["post_prune_step_names"] = [f"step-{index:03d}" for index in range(100)] + generation = _persist(tmp_path, payload) + pull_fallback = { + "success": False, + "error": "recipe_envelope_exceeds_delivery_bound", + "recipe_pull": generation.pull_identity(), + } + error_fallback = { + "success": False, + "error": "recipe_envelope_exceeds_delivery_bound", } - skeleton = extract_step_skeleton([], {}) - artifact_path = str(tmp_path / "x.log") - artifact_sha256 = "0" * 64 - - sk, pl, env = _envelope_overheads(tmp_path, "test-recipe") - # remaining ≈ 500 bytes (well above combined overhead ~54 bytes, - # well below union of full field lengths) — forces a content split. - # The +64 compensates for the 64-byte envelope serialization - # structural-overhead reserve (``build_recipe_envelope`` reserves - # 64 bytes for outer-key braces / separators / UTF-8 quoting the - # per-field allocator cannot account for). - bound_bytes = sk + pl + env + 500 + 64 - envelope = build_recipe_envelope( - payload, - recipe_name="test-recipe", - artifact_path=artifact_path, - artifact_sha256=artifact_sha256, - skeleton=skeleton, - bound_bytes=bound_bytes, + pull_bound = len( + json.dumps(pull_fallback, ensure_ascii=False, separators=(",", ":")).encode("utf-8") ) - - assert "orchestration_rules" in envelope - assert "stop_step_semantics" in envelope - orch_truncated = envelope["orchestration_rules"] - stop_truncated = envelope["stop_step_semantics"] - assert orch_truncated, "orchestration_rules must be non-empty under tight budget" - assert stop_truncated, "stop_step_semantics must be non-empty under tight budget" - # Roughly even split: neither is allowed to dominate (old bug: first - # key would claim ~all; new allocator splits ~half/half). - shorter, longer = sorted((len(orch_truncated), len(stop_truncated))) - assert longer - shorter <= max(1, shorter // 2), ( - f"priority fields must share roughly evenly under tight budget; " - f"orch={len(orch_truncated)}, stop={len(stop_truncated)}" + error_bound = len( + json.dumps(error_fallback, ensure_ascii=False, separators=(",", ":")).encode("utf-8") ) + assert _build_envelope(tmp_path, payload, bound_bytes=pull_bound)[0] == pull_fallback + assert _build_envelope(tmp_path, payload, bound_bytes=error_bound)[0] == error_fallback + assert _build_envelope(tmp_path, payload, bound_bytes=2)[0] == {} + with pytest.raises(ValueError, match="too small for a JSON object"): + _build_envelope(tmp_path, payload, bound_bytes=1) -def test_envelope_priority_fields_omitted_not_emptied_under_extreme_budget( - tmp_path: Path, -) -> None: - """When the content budget falls below both keys' combined - JSON-key overhead, at least one priority field must be OMITTED - from the envelope — never emitted as an empty string.""" - payload = { - "success": True, - "orchestration_rules": "ORCH: " + ("X" * 500), - "stop_step_semantics": "STOP: " + ("Y" * 500), - } - skeleton = extract_step_skeleton([], {}) - artifact_path = str(tmp_path / "x.log") - artifact_sha256 = "0" * 64 - - sk, pl, env = _envelope_overheads(tmp_path, "test-recipe") - # remaining = 30 bytes → well below combined overhead (~53 bytes) - bound_bytes = sk + pl + env + 30 - envelope = build_recipe_envelope( - payload, - recipe_name="test-recipe", - artifact_path=artifact_path, - artifact_sha256=artifact_sha256, - skeleton=skeleton, - bound_bytes=bound_bytes, +def test_finalizer_uses_backend_selected_recipe_budget(tool_ctx) -> None: + selected_budget = CODEX_RECIPE_DELIVERY_BUDGET._replace(contract_digest="sha256:" + ("d" * 64)) + backend = MagicMock() + backend.capabilities = replace( + CodexBackend().capabilities, + recipe_delivery_budget=selected_budget, ) + tool_ctx.backend = backend + tool_ctx.kitchen_id = "selected-budget" - # At least one priority key MUST be omitted (or absent); none may be "". - for key in ("orchestration_rules", "stop_step_semantics"): - value = envelope.get(key) - if key in envelope: - assert value != "", ( - f"{key!r} must not be emitted as an empty string even under " - f"extreme budget pressure; either present-and-non-empty or omitted" - ) - omitted = [k for k in ("orchestration_rules", "stop_step_semantics") if k not in envelope] - assert len(omitted) >= 1, ( - f"at least one priority key must be omitted under extreme budget; " - f"present={set(envelope) & {'orchestration_rules', 'stop_step_semantics'}}" + finalized = finalize_recipe_delivery( + _payload(), + surface="open_kitchen", + recipe_name="remediation", + tool_ctx=tool_ctx, + ) + + assert finalized.decision.mode is RecipeDeliveryMode.ORDINARY_INLINE + assert finalized.decision.contract_digest == selected_budget.contract_digest + + +def _request() -> RecipeDeliveryRequest: + budget = CODEX_RECIPE_DELIVERY_BUDGET + return RecipeDeliveryRequest( + audience="autoskillit.recipe-delivery", + delivery_call_id="delivery-finalizer-001", + contract_version=budget.contract_version, + contract_digest=budget.contract_digest, + caller_requested_outer_tokens=(budget.authoritative_attested_recipe_result_token_limit), + code_digest="sha256:" + ("b" * 64), + ) + + +def _evidence() -> RecipeDeliveryEvidenceDef: + budget = CODEX_RECIPE_DELIVERY_BUDGET + return RecipeDeliveryEvidenceDef( + identity="protected-finalizer-test-v1", + host_channel="test-process-isolated-host", + evidence_schema_version=budget.evidence_version, + parser_version=budget.parser_version, + cli_identity="codex-test-cli", + selected_limit_derivation="protected-resolved-outer-limit", + selected_result_token_limit=(budget.authoritative_attested_recipe_result_token_limit), + contract_digest=budget.contract_digest, + ) + + +def _attestation(thread_id: str = "thread-finalizer") -> RecipeDeliveryAttestation: + request = _request() + budget = CODEX_RECIPE_DELIVERY_BUDGET + return RecipeDeliveryAttestation( + audience=request.audience, + thread_id=thread_id, + turn_id="turn-finalizer-001", + outer_call_id="outer-finalizer-001", + code_mode_cell_id="cell-finalizer-001", + delivery_call_id=request.delivery_call_id, + host_observed_requested_outer_tokens=request.caller_requested_outer_tokens, + selected_result_token_limit=(budget.authoritative_attested_recipe_result_token_limit), + code_digest=request.code_digest, + request_digest=recipe_delivery_request_digest(request), + nonce="nonce-finalizer-001", + expires_at_unix=2_000_000_000, + contract_version=budget.contract_version, + contract_digest=budget.contract_digest, + parser_version=budget.parser_version, + evidence_version=budget.evidence_version, + evidence_identity=_evidence().identity, + ) + + +def _ledger(tmp_path: Path) -> RecipeDeliveryReceiptLedger: + return RecipeDeliveryReceiptLedger.initialize_protected( + ProtectedStoreAuthority( + root=tmp_path / "protected-receipts", + security_identity="protected-finalizer-test-v1", + local_filesystem=True, + caller_writable=False, + initialized_by_host=True, + ) ) -def test_envelope_fails_closed_when_fixed_fields_exceed_bound(tmp_path: Path) -> None: - skeleton = { - "step_count": 1, - "steps": [{"name": "x", "summary": "\u96ea" * 2_000, "routing_edges": []}], - } - bound_bytes = 256 - envelope = build_recipe_envelope( - {"success": True, "errors": ["x" * 2_000]}, - recipe_name="test-recipe", - artifact_path=str(tmp_path / "x.log"), - artifact_sha256="0" * 64, - skeleton=skeleton, - bound_bytes=bound_bytes, - ) - assert envelope["success"] is False - assert envelope["error"] == "recipe_envelope_exceeds_delivery_bound" - assert len(json.dumps(envelope, ensure_ascii=False).encode("utf-8")) <= bound_bytes - - -def test_envelope_priority_field_truncation_handles_multibyte_utf8(tmp_path: Path) -> None: - """Truncation of ``orchestration_rules`` does not split a multi-byte - UTF-8 codepoint mid-sequence. Constructs ``38 ASCII + 4-byte emoji`` - (42 bytes total) with allocation landing at byte 40 (the third byte - of the emoji, a continuation byte) — the naive byte-trim would - raise ``UnicodeDecodeError``; the new path backs off to the 38-byte - ASCII prefix via ``_safe_utf8_truncate``.""" - prefix = "a" * 38 - emoji = "\U0001f600" # 😀 — 4 UTF-8 bytes (0xF0 0x9F 0x98 0x80) - orchestration_rules = prefix + emoji - assert len(orchestration_rules.encode("utf-8")) == 42, "test invariant" - - payload = {"success": True, "orchestration_rules": orchestration_rules} - skeleton = extract_step_skeleton([], {}) - artifact_path = str(tmp_path / "x.log") - artifact_sha256 = "0" * 64 - - sk, pl, env = _envelope_overheads(tmp_path, "test-recipe") - # remaining = 65 bytes → pool = 40 → take = 40 → byte 40 lands in - # the middle of the 4-byte emoji (continuation byte at index 40). - # The +64 compensates for the 64-byte envelope serialization - # structural-overhead reserve (``build_recipe_envelope`` reserves - # 64 bytes for outer-key braces / separators / UTF-8 quoting the - # per-field allocator cannot account for). - bound_bytes = sk + pl + env + 65 + 64 - - envelope = build_recipe_envelope( - payload, - recipe_name="test-recipe", - artifact_path=artifact_path, - artifact_sha256=artifact_sha256, - skeleton=skeleton, - bound_bytes=bound_bytes, +def _protected_codex_backend() -> MagicMock: + backend = MagicMock() + backend.name = "codex" + backend.capabilities = replace( + CodexBackend().capabilities, + protected_recipe_delivery_capable=True, ) + return backend - # No UnicodeDecodeError raised means _safe_utf8_truncate backed off - # to the 38-byte ASCII prefix; the field must be present (take > 0) - # and a valid prefix of the original. - assert "orchestration_rules" in envelope - truncated = envelope["orchestration_rules"] - assert truncated.encode("utf-8") == prefix.encode("utf-8"), ( - f"truncation must back off to 38-byte ASCII prefix; " - f"got {len(truncated.encode('utf-8'))} bytes" - ) - assert orchestration_rules.startswith(truncated) - - -# --------------------------------------------------------------------------- -# Part B gap-closure regression guards (REQ-B04, REQ-B-T2/-T4/-T5) -# --------------------------------------------------------------------------- - - -def test_build_routing_edges_by_step_uses_canonical_extractor() -> None: - """The envelope's edge extraction must reuse the canonical - ``_extract_routing_edges`` from ``recipe/_analysis_graph.py`` rather - than the now-deleted local duplicate. Asserted independently: - ``build_routing_edges_by_step`` (which defaults to the canonical - extractor after the Step 1 change) and a direct call to - ``autoskillit.recipe._extract_routing_edges`` on each RecipeStep - must produce identical ``(edge_type, target)`` tuples. - - The previous local copy had two silent behavioral divergences - (a spurious falsy-target guard, and independent-vs-elif handling of - ``on_result.conditions``/``on_result.routes``); this test would - have caught both.""" - recipe_names = _recipe_names() - assert recipe_names, "no bundled recipes available for the equivalence test" - - for recipe_name in recipe_names: - info = find_recipe_by_name(recipe_name, _PROJECT_ROOT) - assert info is not None - recipe = load_recipe(info.path) - active_recipe_steps = recipe.steps - - default_edges = build_routing_edges_by_step( - active_recipe_steps, - edge_extractor=_extract_routing_edges, - ) - for step_name, step in active_recipe_steps.items(): - canonical = [ - (edge.edge_type, edge.target) - for edge in _extract_routing_edges(step) - if edge.target - ] - assert default_edges.get(step_name) == canonical, ( - f"{recipe_name}.{step_name}: build_routing_edges_by_step " - f"default={default_edges.get(step_name)!r} " - f"differs from canonical _extract_routing_edges={canonical!r}" - ) - - -def test_extract_step_skeleton_includes_byte_ranges() -> None: - """``extract_step_skeleton`` gains an optional ``byte_ranges`` keyword; - a step present in the map gets a ``byte_range`` field carrying the - ``[start, end]`` pair, while a step absent from the map gets no - such key (fail-open, matching the existing edges/summary fallback).""" - byte_ranges = {"step_a": (10, 42)} - skeleton = extract_step_skeleton( - ["step_a", "step_b"], - routing_edges_by_step={}, - step_summaries={}, - byte_ranges=byte_ranges, - ) - by_name = {step["name"]: step for step in skeleton["steps"]} - assert by_name["step_a"].get("byte_range") == [10, 42], ( - "step present in byte_ranges must carry the byte_range field as a JSON list" +def test_attested_finalization_commits_only_after_exact_enforcement( + tmp_path: Path, tool_ctx +) -> None: + tool_ctx.backend = _protected_codex_backend() + tool_ctx.kitchen_id = "codex-attested" + ledger = _ledger(tmp_path) + finalized = finalize_recipe_delivery( + _payload("x" * 50_000), + surface="open_kitchen", + recipe_name="remediation", + tool_ctx=tool_ctx, + delivery_request=_request(), + attestation=_attestation(), + supported_evidence=_evidence(), + receipt_ledger=ledger, + now_unix=_NOW, + ) + + assert finalized.decision.mode is RecipeDeliveryMode.ATTESTED_INLINE + assert finalized.receipt_handle is not None + assert RECIPE_BODY_START in finalized.rendered + assert RECIPE_BODY_END in finalized.rendered + assert RECIPE_COMPLETION_SENTINEL in finalized.rendered + assert ledger.receipt_status("thread-finalizer") == "pending" + + enforced = enforce_response_budget( + finalized.rendered, + tool_name="open_kitchen", + artifact_dir=tmp_path / "responses", + config=OutputBudgetConfig(), + selected_result_token_limit=finalized.decision.selected_result_token_limit, ) - assert "byte_range" not in by_name["step_b"], ( - "step absent from byte_ranges must get no byte_range key (fail-open)" + assert ( + complete_finalized_recipe_response(finalized, enforced, now_unix=_NOW) + == finalized.rendered ) + assert ledger.receipt_status("thread-finalizer") == "committed" -def test_compute_step_byte_ranges_matches_step_boundaries() -> None: - """Each returned ``(start, end)`` slice covers the original step's - key + body in the source text. We assert substring containment - (rather than exact-equality) because YAML mark ranges commonly - include trailing whitespace up to the next sibling key.""" - content = ( - 'version: "1"\n' - "steps:\n" - " alpha:\n" - " tool: run_cmd\n" - " with_args:\n" - ' cmd: "echo alpha-body-marker"\n' - " beta:\n" - " tool: run_cmd\n" - " with_args:\n" - ' cmd: "echo beta-body-marker"\n' - ) - ranges = _compute_step_byte_ranges(content) - assert set(ranges.keys()) == {"alpha", "beta"}, ( - f"expected exactly the two top-level steps; got {sorted(ranges.keys())}" +def test_transformed_attested_response_aborts_pending_receipt(tmp_path: Path, tool_ctx) -> None: + tool_ctx.backend = _protected_codex_backend() + tool_ctx.kitchen_id = "codex-abort" + ledger = _ledger(tmp_path) + finalized = finalize_recipe_delivery( + _payload("x" * 50_000), + surface="load_recipe", + recipe_name="remediation", + tool_ctx=tool_ctx, + delivery_request=_request(), + attestation=_attestation("thread-abort"), + supported_evidence=_evidence(), + receipt_ledger=ledger, + now_unix=_NOW, ) - encoded = content.encode("utf-8") - for step_name, expected_marker in ( - ("alpha", "alpha-body-marker"), - ("beta", "beta-body-marker"), - ): - start, end = ranges[step_name] - slice_text = encoded[start:end].decode("utf-8", errors="replace") - assert expected_marker in slice_text, ( - f"{step_name}: marker {expected_marker!r} not in utf-8 decode of " - f"content[{start}:{end}] = {slice_text!r}" - ) + assert finalized.decision.mode is RecipeDeliveryMode.ATTESTED_INLINE + transformed = "bounded replacement" + assert complete_finalized_recipe_response(finalized, transformed) == transformed + assert ledger.receipt_status("thread-abort") is None -@pytest.mark.parametrize( - "malformed", - [ - "- a\n- b\n", # bare YAML sequence at root - "steps: oops\n", # steps: whose value is a scalar, not a mapping - ], - ids=["bare-sequence", "scalar-steps-value"], -) -def test_compute_step_byte_ranges_returns_empty_on_non_mapping_document( - malformed: str, -) -> None: - """``_compute_step_byte_ranges`` must fail open on any malformed or - non-mapping YAML document, returning ``{}`` without raising. The - guard is ``isinstance(..., yaml.MappingNode)`` rather than a bare - ``except yaml.YAMLError`` because a document that parses - successfully but isn't a mapping raises ``TypeError`` / - ``ValueError`` from tuple-unpacking (not ``YAMLError``).""" - assert _compute_step_byte_ranges(malformed) == {} - - -# --------------------------------------------------------------------------- -# Part B pull-tool end-to-end tests (REQ-B-T2, REQ-B-T4, REQ-B-T5) -# --------------------------------------------------------------------------- - - -_RECIPE_FOR_PULL = "remediation" - - -def _mock_fmcp_ctx() -> MagicMock: - """Return a minimal FastMCP ``Context`` mock with async component methods.""" - ctx = MagicMock() - ctx.enable_components = AsyncMock() - ctx.disable_components = AsyncMock() - return ctx - - -async def _open_kitchen_patched(name: str, overrides: dict | None, monkeypatch) -> dict: - """Call ``open_kitchen`` with all infrastructure side-effects patched out. - - Mirrors the helper in ``tests/server/test_serve_idempotence.py``: - patches ``_prime_quota_cache`` / ``_write_hook_config`` / - ``create_background_task`` / ``resolve_kitchen_id`` (open_kitchen's - background + identity side effects) and resets the recipe - ``_LOAD_CACHE`` so a fresh load + validate runs for every test. - """ - from autoskillit.recipe import _api_cache - from autoskillit.recipe._api_cache import LoadCache - from autoskillit.server.tools.tools_kitchen import open_kitchen - - monkeypatch.setattr(_api_cache, "_LOAD_CACHE", LoadCache()) - fmcp_ctx = _mock_fmcp_ctx() - with patch("autoskillit.server.tools.tools_kitchen._prime_quota_cache", new=AsyncMock()): - with patch("autoskillit.server.tools.tools_kitchen._write_hook_config"): - with patch("autoskillit.server.tools.tools_kitchen.create_background_task"): - with patch( - "autoskillit.server.tools.tools_kitchen.resolve_kitchen_id", - return_value="test-kitchen", - ): - return json.loads( - await open_kitchen(name=name, overrides=overrides, ctx=fmcp_ctx) - ) - - -@pytest.mark.anyio -@pytest.mark.medium -async def test_load_recipe_envelope_pulls_from_its_own_artifact( - tool_ctx_kitchen_open, monkeypatch, tmp_path: Path +def test_failed_receipt_abort_is_reported( + tmp_path: Path, tool_ctx, monkeypatch: pytest.MonkeyPatch ) -> None: - from autoskillit.server.tools.tools_recipe import get_recipe_section - from autoskillit.server.tools.tools_recipe import load_recipe as load_recipe_tool - - monkeypatch.chdir(tmp_path) - opened = await _open_kitchen_patched(_RECIPE_FOR_PULL, None, monkeypatch) - assert opened.get("success") is True - assert tool_ctx_kitchen_open.recipe_name == _RECIPE_FOR_PULL - - tool_ctx_kitchen_open.backend = BACKEND_REGISTRY["codex"]() - - # Force the load_recipe payload above the production 10,000-token - # bound so ``maybe_envelope_recipe_response`` actually builds an - # envelope (its recipe_pull + step_flow_skeleton + delivery_bound_spill - # trio is the artifact the test below pulls from). When the payload - # fits, ``maybe_envelope_recipe_response`` short-circuits to raw - # JSON and ``loaded["recipe_pull"]`` never exists — defeating the - # *envelope-pulls-from-its-own-artifact* contract. - _tight_backend = MagicMock() - _tight_backend.capabilities = BackendCapabilities( - effective_delivery_token_limit=1_000, - ) - _tight_backend.name = "codex" - tool_ctx_kitchen_open.backend = _tight_backend - - loaded = json.loads(await load_recipe_tool(name="implementation")) - pull = loaded["recipe_pull"] - assert pull["recipe_name"] == "implementation" - assert pull["producer_tool"] == "load_recipe" - assert tool_ctx_kitchen_open.recipe_name == _RECIPE_FOR_PULL - - # When the bound is tight enough to trigger delivery-bound spill, the - # envelope may drop ``step_flow_skeleton`` / ``artifact_path`` from - # ``recipe_pull`` to fit. Reconstruct the artifact path the same way - # the pull tool does (deterministic from temp_dir + producer_tool + - # recipe_name) and look up the post-prune step list from the - # persisted payload. - artifact_path_str = pull.get("artifact_path") or str( - _recipe_artifact_path( - tool_ctx_kitchen_open.temp_dir, pull["producer_tool"], pull["recipe_name"] - ) - ) - skeleton = loaded.get("step_flow_skeleton") - if skeleton is not None: - step_name = skeleton["steps"][0]["name"] - else: - persisted = json.loads(Path(artifact_path_str).read_text(encoding="utf-8")) - post_prune_raw = cast(list[object], persisted.get("post_prune_step_names") or []) - step_name = next(str(name) for name in post_prune_raw if isinstance(name, str)) - - response = json.loads( - await get_recipe_section( - section=step_name, - recipe_name=pull["recipe_name"], - producer_tool=pull["producer_tool"], - artifact_path=artifact_path_str, - artifact_sha256=pull.get("sha256"), - ) + tool_ctx.backend = _protected_codex_backend() + tool_ctx.kitchen_id = "codex-abort-failure" + ledger = _ledger(tmp_path) + finalized = finalize_recipe_delivery( + _payload("x" * 50_000), + surface="load_recipe", + recipe_name="remediation", + tool_ctx=tool_ctx, + delivery_request=_request(), + attestation=_attestation("thread-abort-failure"), + supported_evidence=_evidence(), + receipt_ledger=ledger, + now_unix=_NOW, ) - assert response.get("success") is True, response - assert response["content"] + assert finalized.decision.mode is RecipeDeliveryMode.ATTESTED_INLINE + monkeypatch.setattr(RecipeDeliveryReceiptLedger, "abort", lambda *_args: False) + completed = complete_finalized_recipe_response(finalized, "bounded replacement") -@pytest.mark.anyio -@pytest.mark.medium -async def test_pull_tool_returns_bounded_step_content( - tool_ctx_kitchen_open, monkeypatch, tmp_path: Path -) -> None: - """Live ``get_recipe_section`` end-to-end against the persisted - artifact (REQ-B-T2): every post-prune step name returns the bounded - step body matching the canonical serializer, the response itself - fits the smallest backend bound, and an unknown section name - produces ``{"success": False, "error": "section_not_found"}``.""" - from autoskillit.server.tools.tools_recipe import get_recipe_section - - monkeypatch.chdir(tmp_path) - - ok_result = await _open_kitchen_patched( - _RECIPE_FOR_PULL, - None, - monkeypatch, - ) - assert ok_result.get("success") is True, f"open_kitchen failed: {ok_result}" - - persisted = json.loads( - ( - artifact_path := _recipe_artifact_path( - tool_ctx_kitchen_open.temp_dir, "open_kitchen", _RECIPE_FOR_PULL - ) - ).read_text(encoding="utf-8") - ) - artifact_sha256 = _artifact_sha256(artifact_path) - persisted_yaml = persisted.get("content", "") or "" - parsed = load_yaml(persisted_yaml) - assert isinstance(parsed, dict), "persisted content must parse as a mapping" - parsed_steps = parsed.get("steps", {}) - assert isinstance(parsed_steps, dict) - - post_prune_raw = cast(list[object], persisted.get("post_prune_step_names") or []) - step_names = [str(n) for n in post_prune_raw if isinstance(n, str)] - assert step_names, "recipe must expose at least one post-prune step" - - bound_tokens = _smallest_bound_tokens() - bound_bytes_for_response = bound_tokens * 4 - missing_identity = json.loads(await get_recipe_section(section=step_names[0])) - assert missing_identity == { + assert json.loads(completed) == { "success": False, - "error": "recipe_artifact_identity_required", + "error": "recipe_delivery_receipt_abort_failed", } - for step_name in step_names: - response = json.loads( - await get_recipe_section( - section=step_name, - recipe_name=_RECIPE_FOR_PULL, - producer_tool="open_kitchen", - artifact_sha256=artifact_sha256, - ) - ) - assert response.get("success") is True, ( - f"get_recipe_section({step_name!r}) failed: {response}" - ) - expected = fast_dumps({step_name: parsed_steps[step_name]}) - assert response["section"] == step_name - assert response["content"] == expected, ( - f"step {step_name!r}: pull content diverges from " - f"fast_dumps({{step: parsed['steps'][step_name]}})" + assert ledger.receipt_status("thread-abort-failure") == "pending" + + +async def test_pull_tool_reads_exact_generation_and_reports_byte_offsets( + tool_ctx_kitchen_open, +) -> None: + tool_ctx_kitchen_open.backend = CodexBackend() + tool_ctx_kitchen_open.kitchen_id = "pull-kitchen" + expected_content = "héllo\n" * 12_000 + generation = persist_recipe_artifact( + tool_ctx_kitchen_open.temp_dir, + kitchen_id=tool_ctx_kitchen_open.kitchen_id, + producer_tool="open_kitchen", + recipe_name="remediation", + payload=_payload(expected_content), + ) + kwargs = generation.pull_identity() + kwargs.pop("pull_tool") + chunks: list[str] = [] + expected_byte_start = 0 + part = 0 + while True: + rendered = await get_recipe_section(section="content", part=part, **kwargs) + assert len(rendered.encode("utf-8")) <= ( + CODEX_RECIPE_DELIVERY_BUDGET.ordinary_omitted_result_token_limit ) - serialized = json.dumps(response, ensure_ascii=False) - assert len(serialized.encode("utf-8")) <= bound_bytes_for_response, ( - f"step {step_name!r}: response exceeds the backend bound " - f"({bound_bytes_for_response} bytes); got {len(serialized.encode('utf-8'))}" + response = json.loads(rendered) + assert response["success"] is True + assert response["byte_start"] == expected_byte_start + assert response["byte_end"] == response["byte_start"] + len( + response["content"].encode("utf-8") ) + assert response["byte_end"] <= response["byte_total"] + assert response["payload_sha256"] == generation.payload_sha256 + assert response["body_sha256"] == generation.body_sha256 + chunks.append(response["content"]) + expected_byte_start = response["byte_end"] + if not response["has_more"]: + assert "next_part" not in response + break + assert response["next_part"] == part + 1 + part = response["next_part"] - unknown = json.loads( - await get_recipe_section( - section="not_a_real_step", - recipe_name=_RECIPE_FOR_PULL, - producer_tool="open_kitchen", - artifact_sha256=artifact_sha256, - ) + assert part > 0 + assert expected_byte_start == response["byte_total"] + assert "".join(chunks) == expected_content + + +async def test_pull_tool_returns_named_step_and_rejects_unknown_section( + tool_ctx_kitchen_open, +) -> None: + tool_ctx_kitchen_open.backend = CodexBackend() + tool_ctx_kitchen_open.kitchen_id = "pull-named-step" + generation = persist_recipe_artifact( + tool_ctx_kitchen_open.temp_dir, + kitchen_id=tool_ctx_kitchen_open.kitchen_id, + producer_tool="open_kitchen", + recipe_name="remediation", + payload=_payload(), ) + kwargs = generation.pull_identity() + kwargs.pop("pull_tool") + + named = json.loads(await get_recipe_section(section="first", **kwargs)) + assert named["success"] is True + assert named["section"] == "first" + assert load_yaml(named["content"]) == {"first": {"action": "stop"}} + + unknown = json.loads(await get_recipe_section(section="not_a_real_step", **kwargs)) assert unknown == { "success": False, "error": "section_not_found", "section": "not_a_real_step", } - persisted["post_prune_step_names"] = [] - _, artifact_sha256 = persist_recipe_artifact( + +async def test_pull_tool_reports_malformed_named_step_yaml(tool_ctx_kitchen_open) -> None: + tool_ctx_kitchen_open.kitchen_id = "pull-malformed-step" + malformed = _payload("steps: [") + generation = persist_recipe_artifact( tool_ctx_kitchen_open.temp_dir, - tool_name="open_kitchen", - recipe_name=_RECIPE_FOR_PULL, - payload=persisted, + kitchen_id=tool_ctx_kitchen_open.kitchen_id, + producer_tool="open_kitchen", + recipe_name="remediation", + payload=malformed, + ) + kwargs = generation.pull_identity() + kwargs.pop("pull_tool") + + response = json.loads(await get_recipe_section(section="first", **kwargs)) + + assert response["success"] is False + assert response["error"] == "recipe_artifact_parse_failed" + + +async def test_oversized_named_step_round_trips_through_continuation( + tool_ctx_kitchen_open, +) -> None: + tool_ctx_kitchen_open.backend = CodexBackend() + tool_ctx_kitchen_open.kitchen_id = "pull-oversized-step" + content = "steps:\n giant_step:\n note: " + ("X" * 80_000) + "\n" + payload = _payload(content) + payload["post_prune_step_names"] = ["giant_step"] + generation = persist_recipe_artifact( + tool_ctx_kitchen_open.temp_dir, + kitchen_id=tool_ctx_kitchen_open.kitchen_id, + producer_tool="open_kitchen", + recipe_name="remediation", + payload=payload, ) - pruned = json.loads( - await get_recipe_section( - section=step_names[0], - recipe_name=_RECIPE_FOR_PULL, - producer_tool="open_kitchen", - artifact_sha256=artifact_sha256, + kwargs = generation.pull_identity() + kwargs.pop("pull_tool") + chunks: list[str] = [] + part = 0 + + while True: + rendered = await get_recipe_section(section="giant_step", part=part, **kwargs) + assert len(rendered.encode("utf-8")) <= ( + CODEX_RECIPE_DELIVERY_BUDGET.ordinary_omitted_result_token_limit ) + response = json.loads(rendered) + assert response["success"] is True + chunks.append(response["content"]) + if response["has_more"] is False: + break + part = response["next_part"] + + assert part > 0 + reconstructed = load_yaml("".join(chunks)) + parsed = load_yaml(content) + assert reconstructed == {"giant_step": parsed["steps"]["giant_step"]} + + +async def test_pull_tool_rejects_wrong_generation_identity(tool_ctx_kitchen_open) -> None: + tool_ctx_kitchen_open.kitchen_id = "pull-wrong" + generation = persist_recipe_artifact( + tool_ctx_kitchen_open.temp_dir, + kitchen_id=tool_ctx_kitchen_open.kitchen_id, + producer_tool="open_kitchen", + recipe_name="remediation", + payload=_payload(), ) - assert pruned == { - "success": False, - "error": "section_not_found", - "section": step_names[0], - } + kwargs = generation.pull_identity() + kwargs.pop("pull_tool") + kwargs["artifact_blob_sha256"] = "sha256:" + ("0" * 64) + response = json.loads(await get_recipe_section(section="content", **kwargs)) + assert response == {"success": False, "error": "invalid_recipe_artifact_identity"} + + +@pytest.mark.parametrize("field", ["artifact_blob_size_bytes", "body_size_bytes"]) +async def test_pull_tool_rejects_forged_unbounded_identity_sizes( + tool_ctx_kitchen_open, field: str +) -> None: + tool_ctx_kitchen_open.kitchen_id = "pull-unbounded-identity" + generation = persist_recipe_artifact( + tool_ctx_kitchen_open.temp_dir, + kitchen_id=tool_ctx_kitchen_open.kitchen_id, + producer_tool="open_kitchen", + recipe_name="remediation", + payload=_payload(), + ) + kwargs = generation.pull_identity() + kwargs.pop("pull_tool") + kwargs[field] = 1_000_000_000 + response = json.loads(await get_recipe_section(section="content", **kwargs)) -@pytest.mark.anyio -@pytest.mark.medium -async def test_artifact_recreation_from_parsed_recipe( - tool_ctx_kitchen_open, monkeypatch, tmp_path: Path + assert response == {"success": False, "error": "invalid_recipe_artifact_identity"} + + +async def test_pull_tool_recreates_missing_exact_generation( + tool_ctx_kitchen_open, monkeypatch: pytest.MonkeyPatch ) -> None: - """When the persisted artifact is deleted, ``get_recipe_section`` - re-creates it via the same serve pipeline that built it originally - (REQ-B-T4). Digest binding rejects a recreation that does not match - the exact original artifact. When the recreation pipeline returns - an invalid recipe, the pull tool surfaces a structured - ``recipe_artifact_unavailable`` envelope (driven here via a - ``serve_recipe`` monkeypatch, since the literal "no active recipe" - condition is checked earlier and produces a different error).""" - from autoskillit.server.tools.tools_recipe import get_recipe_section - - monkeypatch.chdir(tmp_path) - - ok_result = await _open_kitchen_patched(_RECIPE_FOR_PULL, None, monkeypatch) - assert ok_result.get("success") is True, f"open_kitchen failed: {ok_result}" - - artifact_path = _recipe_artifact_path( - tool_ctx_kitchen_open.temp_dir, "open_kitchen", _RECIPE_FOR_PULL + import autoskillit.server.tools.tools_recipe as tools_recipe + + tool_ctx_kitchen_open.kitchen_id = "pull-recreate" + generation = persist_recipe_artifact( + tool_ctx_kitchen_open.temp_dir, + kitchen_id=tool_ctx_kitchen_open.kitchen_id, + producer_tool="open_kitchen", + recipe_name="remediation", + payload=_payload(), ) - artifact_sha256 = _artifact_sha256(artifact_path) - artifact_path.unlink() - assert not artifact_path.exists(), "precondition: artifact file deleted" - - post_prune_raw = cast(list[object], ok_result.get("post_prune_step_names") or []) - step_name = next( - (str(n) for n in post_prune_raw if isinstance(n, str)), - None, + _remove_persisted_namespace( + tool_ctx_kitchen_open.temp_dir, kitchen_id=tool_ctx_kitchen_open.kitchen_id ) - assert step_name is not None, "recipe must expose at least one post-prune step" - - response = json.loads( - await get_recipe_section( - section=step_name, - recipe_name=_RECIPE_FOR_PULL, - producer_tool="open_kitchen", - artifact_sha256=artifact_sha256, - ) + monkeypatch.setattr(tools_recipe, "serve_recipe", lambda *_args, **_kwargs: _payload()) + monkeypatch.setattr( + tools_recipe, + "build_open_kitchen_recipe_payload", + lambda data, *, version: data, ) - assert response == { - "success": False, - "error": "invalid_recipe_artifact_identity", - } - assert artifact_path.exists(), "artifact must be rewritten by the recreation path on miss" - - # Now drive the invalid-recreation branch via a monkeypatched - # ``serve_recipe`` — get_recipe_section imports it into its own - # module namespace, so we must override the attribute on the - # importing module (``tools_recipe``) for the patch to take - # effect. The pulled result must be a structured - # ``recipe_artifact_unavailable`` envelope rather than a generic - # error. - artifact_path.unlink() - import autoskillit.server.tools.tools_recipe as _tools_recipe_mod # noqa: PLC0415 - - def _fake_serve_recipe(*args, **kwargs) -> dict: - # Reference args/kwargs so linters see them as "used" — the patch - # accepts any signature, we just need to return invalid-recipe. - del args, kwargs - return {"valid": False, "content": "x"} - - monkeypatch.setattr(_tools_recipe_mod, "serve_recipe", _fake_serve_recipe) - failed = json.loads( - await get_recipe_section( - section=step_name, - recipe_name=_RECIPE_FOR_PULL, - producer_tool="open_kitchen", - artifact_sha256=artifact_sha256, + + kwargs = generation.pull_identity() + kwargs.pop("pull_tool") + response = json.loads(await get_recipe_section(section="content", **kwargs)) + + assert response["success"] is True + assert response["content"] == _payload()["content"] + assert ( + load_recipe_artifact( + tool_ctx_kitchen_open.temp_dir, + kitchen_id=tool_ctx_kitchen_open.kitchen_id, + identity=generation, ) + == _payload() ) - assert failed.get("success") is False - assert failed["error"] == "recipe_artifact_unavailable" - assert failed["detail"] == "recreation returned invalid recipe" -@pytest.mark.anyio -@pytest.mark.medium -async def test_pull_tool_rejects_non_mapping_artifact( - tool_ctx_kitchen_open, monkeypatch, tmp_path: Path +async def test_pull_tool_reports_invalid_missing_generation_recreation( + tool_ctx_kitchen_open, monkeypatch: pytest.MonkeyPatch ) -> None: - """Valid JSON with the wrong top-level shape is artifact corruption.""" - from autoskillit.server.tools.tools_recipe import get_recipe_section - - monkeypatch.chdir(tmp_path) - ok_result = await _open_kitchen_patched(_RECIPE_FOR_PULL, None, monkeypatch) - assert ok_result.get("success") is True + import autoskillit.server.tools.tools_recipe as tools_recipe - artifact_path = _recipe_artifact_path( - tool_ctx_kitchen_open.temp_dir, "open_kitchen", _RECIPE_FOR_PULL + tool_ctx_kitchen_open.kitchen_id = "pull-recreate-invalid" + generation = persist_recipe_artifact( + tool_ctx_kitchen_open.temp_dir, + kitchen_id=tool_ctx_kitchen_open.kitchen_id, + producer_tool="open_kitchen", + recipe_name="remediation", + payload=_payload(), ) - artifact_path.write_text('["not", "a", "mapping"]', encoding="utf-8") - response = json.loads( - await get_recipe_section( - section="content", - recipe_name=_RECIPE_FOR_PULL, - producer_tool="open_kitchen", - artifact_sha256=_artifact_sha256(artifact_path), - ) + _remove_persisted_namespace( + tool_ctx_kitchen_open.temp_dir, kitchen_id=tool_ctx_kitchen_open.kitchen_id + ) + monkeypatch.setattr( + tools_recipe, + "serve_recipe", + lambda *_args, **_kwargs: {"valid": False, "content": "invalid"}, ) + kwargs = generation.pull_identity() + kwargs.pop("pull_tool") + response = json.loads(await get_recipe_section(section="content", **kwargs)) + assert response == { "success": False, "error": "recipe_artifact_unavailable", - "detail": "persisted recipe artifact is not a mapping", + "detail": "recreation returned invalid recipe", } -@pytest.mark.anyio -@pytest.mark.medium -async def test_large_step_chunked_via_continuation( - tool_ctx_kitchen_open, monkeypatch, tmp_path: Path +async def test_pull_tool_rejects_changed_recreated_generation( + tool_ctx_kitchen_open, monkeypatch: pytest.MonkeyPatch ) -> None: - """Persist a synthetic oversized artifact whose single ``giant_step`` - body exceeds the smallest backend's bound; driving - ``get_recipe_section`` against ``backend = None`` (which forces - the 10,000-token / 40,000-byte fallback) produces a chunked - response with ``has_more=True`` and a ``next_part`` cursor; - following that cursor concatenates the chunks back into the - canonical serializer's output (REQ-B-T5).""" - from autoskillit.server.tools.tools_recipe import get_recipe_section - - monkeypatch.chdir(tmp_path) - - # Open the kitchen first so the tool_ctx has recipes wired and - # ``recipe_name`` is set — the giant-step payload is written - # directly to the artifact path to bypass the size guard. - ok_result = await _open_kitchen_patched(_RECIPE_FOR_PULL, None, monkeypatch) - assert ok_result.get("success") is True - - artifact_path = _recipe_artifact_path( - tool_ctx_kitchen_open.temp_dir, "open_kitchen", _RECIPE_FOR_PULL - ) - oversized_field = "X" * 80_000 # ~80KB >> 40KB Codex bound - persisted_payload = { - "success": True, - "content": (f'version: "1"\nsteps:\n giant_step:\n note: {oversized_field}\n'), - "post_prune_step_names": ["giant_step"], - } - _, artifact_sha256 = persist_recipe_artifact( + import autoskillit.server.tools.tools_recipe as tools_recipe + + tool_ctx_kitchen_open.kitchen_id = "pull-recreate-changed" + generation = persist_recipe_artifact( tool_ctx_kitchen_open.temp_dir, - tool_name="open_kitchen", - recipe_name=_RECIPE_FOR_PULL, - payload=persisted_payload, + kitchen_id=tool_ctx_kitchen_open.kitchen_id, + producer_tool="open_kitchen", + recipe_name="remediation", + payload=_payload(), ) - assert artifact_path.exists() - - # Force the 40KB fallback bound (no backend → bound_tokens=10_000). - tool_ctx_kitchen_open.backend = None # type: ignore[assignment] - tool_ctx_kitchen_open.recipe_name = _RECIPE_FOR_PULL - - expected = fast_dumps( - {"giant_step": load_yaml(persisted_payload["content"])["steps"]["giant_step"]} + _remove_persisted_namespace( + tool_ctx_kitchen_open.temp_dir, kitchen_id=tool_ctx_kitchen_open.kitchen_id + ) + monkeypatch.setattr( + tools_recipe, + "serve_recipe", + lambda *_args, **_kwargs: _payload("name: remediation\nsteps: {}\n"), + ) + monkeypatch.setattr( + tools_recipe, + "build_open_kitchen_recipe_payload", + lambda data, *, version: data, ) - chunks: list[str] = [] - part = 0 - has_more = True - while has_more: - response = json.loads( - await get_recipe_section( - section="giant_step", - part=part, - recipe_name=_RECIPE_FOR_PULL, - producer_tool="open_kitchen", - artifact_sha256=artifact_sha256, - ) - ) - assert response.get("success") is True, f"chunk {part} failed: {response}" - assert response["section"] == "giant_step" - chunks.append(response["content"]) - has_more = bool(response.get("has_more")) - if has_more: - assert "next_part" in response, ( - f"chunk {part} reports has_more=True without next_part cursor" - ) - part = int(response["next_part"]) - else: - assert "next_part" not in response + kwargs = generation.pull_identity() + kwargs.pop("pull_tool") + response = json.loads(await get_recipe_section(section="content", **kwargs)) - assert "".join(chunks) == expected, ( - "chunked pull content does not round-trip to the canonical serializer's output" - ) + assert response == {"success": False, "error": "invalid_recipe_artifact_identity"} diff --git a/tests/server/test_track_response_size.py b/tests/server/test_track_response_size.py index 84d8282c7a..b75f5aa091 100644 --- a/tests/server/test_track_response_size.py +++ b/tests/server/test_track_response_size.py @@ -14,6 +14,31 @@ pytestmark = [pytest.mark.layer("server"), pytest.mark.small] +def _finalized_recipe_response(): + from autoskillit.server._recipe_delivery import FinalizedRecipeResponse + + ledger = MagicMock() + ledger.commit.return_value = True + ledger.abort.return_value = True + finalized = FinalizedRecipeResponse( + rendered="attested recipe response", + decision=MagicMock(selected_result_token_limit=56_750), + receipt_handle=MagicMock(), + receipt_ledger=ledger, + ) + return finalized, ledger + + +def _tracking_ctx(): + return MagicMock( + response_log=MagicMock(record=MagicMock(return_value=False)), + config=MagicMock( + mcp_response=MagicMock(alert_threshold_tokens=0), + output_budget=OutputBudgetConfig(), + ), + ) + + class TestTrackResponseSize: @pytest.mark.anyio async def test_decorator_records_str_response(self): @@ -244,3 +269,91 @@ async def fake_handler(): ) assert event.kwargs["cause"] == "internal_invariant_failed" assert event.kwargs["original_utf8_bytes"] == len(b"small") + + @pytest.mark.anyio + async def test_finalized_recipe_exact_response_commits_through_decorator(self): + from autoskillit.server._notify import track_response_size + + finalized, ledger = _finalized_recipe_response() + + @track_response_size("open_kitchen") + async def fake_handler(): + return finalized + + with ( + patch( + "autoskillit.server._notify._get_ctx_or_none", + return_value=_tracking_ctx(), + ), + patch( + "autoskillit.server._notify.enforce_response_budget", + return_value=finalized.rendered, + ) as enforce, + ): + result = await fake_handler() + + assert result == finalized.rendered + assert enforce.call_args.kwargs["selected_result_token_limit"] == 56_750 + ledger.commit.assert_called_once() + ledger.abort.assert_not_called() + + @pytest.mark.parametrize( + "enforced", + ["transformed response", {"success": True, "artifact_path": "response.txt"}], + ) + @pytest.mark.anyio + async def test_finalized_recipe_transformation_aborts_through_decorator(self, enforced): + from autoskillit.server._notify import track_response_size + + finalized, ledger = _finalized_recipe_response() + + @track_response_size("load_recipe") + async def fake_handler(): + return finalized + + with ( + patch( + "autoskillit.server._notify._get_ctx_or_none", + return_value=_tracking_ctx(), + ), + patch( + "autoskillit.server._notify.enforce_response_budget", + return_value=enforced, + ), + ): + result = await fake_handler() + + assert result == enforced + ledger.commit.assert_not_called() + ledger.abort.assert_called_once_with(finalized.receipt_handle) + + @pytest.mark.anyio + async def test_finalized_recipe_enforcement_failure_aborts_through_decorator(self): + from autoskillit.server._notify import track_response_size + + finalized, ledger = _finalized_recipe_response() + bounded = {"success": False, "error": "response_budget_exceeded"} + + @track_response_size("open_kitchen") + async def fake_handler(): + return finalized + + with ( + patch( + "autoskillit.server._notify._get_ctx_or_none", + return_value=_tracking_ctx(), + ), + patch( + "autoskillit.server._notify.enforce_response_budget", + side_effect=RuntimeError("enforcement failed"), + ), + patch( + "autoskillit.server._notify.bounded_response_budget_failure", + return_value=bounded, + ), + ): + result = await fake_handler() + + assert result == bounded + ledger.commit.assert_not_called() + ledger.abort.assert_called_once_with(finalized.receipt_handle) diff --git a/tests/test_test_filter_core_cascade.py b/tests/test_test_filter_core_cascade.py index a3f4413a0f..ebc8ec9394 100644 --- a/tests/test_test_filter_core_cascade.py +++ b/tests/test_test_filter_core_cascade.py @@ -98,6 +98,7 @@ def test_all_entries_present(self) -> None: "_type_results", "_type_results_execution", "_type_backend", + "_type_recipe_delivery", "_type_dispatch_identity", "_type_figure_spec", "_type_session_env", @@ -170,6 +171,11 @@ def test_type_backend_cascade(self) -> None: {"core", "execution", "cli", "recipe", "server", "workspace"} ) + def test_type_recipe_delivery_cascade(self) -> None: + assert MODULE_CASCADE_CORE["_type_recipe_delivery"] == frozenset( + {"core", "execution", "server"} + ) + def test_type_dispatch_identity_cascade(self) -> None: assert MODULE_CASCADE_CORE["_type_dispatch_identity"] == frozenset( {"core", "fleet", "execution"}