From 621d54eebd131629ce2ca16ede0dd8cb8619e368 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 20 Jul 2026 09:08:31 -0700 Subject: [PATCH 1/5] feat(server): make enforce_response_budget backend-aware via BackendCapabilities.effective_delivery_token_limit Resolves #4300. Code-mode models (gpt-5.6-sol) bypass the tool_output_token_limit ceiling via model-declared max_output_tokens, so the server-side response backstop must apply each backend's worst-case operative bound at response time rather than rely on the static exemption ceiling. Changes: - Add effective_delivery_token_limit field to BackendCapabilities (46_500 for Claude Code, 10_000 for Codex). - Add resolve_effective_delivery_bound() canonical accessor in execution/backends/_delivery_bounds.py. - Add effective_delivery_token_limit parameter to enforce_response_budget() that gates both the exempted-within-ceiling and non-exempted under-budget early-return paths. Spill-and-project produces a bounded projection with reason="delivery_bound" via the existing _autoskillit_response_spill envelope. - Wire delivery-bound extraction through both call sites: track_response_size decorator and shape_execution_response. - Update ADR-0005: per-repo ceiling guidance now documents that the server-side backstop enforces the bound for code-mode, not the advisory digest. Move code-mode bypass from Accepted Gaps to Resolved. Tests: - Backend-aware delivery bound resolution across BACKEND_REGISTRY. - Exempted payload above Codex code-mode bound spills, not passed through. - Bundled recipe payload fits or spills per backend. - Non-exempted oversized payload within response_max_bytes but above delivery bound routes to spill-and-project. - CODEX_INTAKE_DISCIPLINE_DIGEST numeric max_output_tokens rule must match the Codex effective delivery bound (regression guard). Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/decisions/0005-output-budget-protocol.md | 19 +- src/autoskillit/core/types/_type_backend.py | 7 + src/autoskillit/execution/backends/AGENTS.md | 1 + .../execution/backends/__init__.py | 2 + .../execution/backends/_delivery_bounds.py | 19 ++ src/autoskillit/execution/backends/codex.py | 1 + src/autoskillit/server/_notify.py | 10 +- src/autoskillit/server/_response_budget.py | 187 +++++++++++++++++- .../server/tools/_execution_helpers.py | 9 + tests/contracts/AGENTS.md | 1 + .../contracts/test_delivery_bound_fitness.py | 137 +++++++++++++ .../test_output_budget_discipline.py | 17 ++ tests/execution/backends/test_codex_config.py | 16 ++ .../test_coding_agent_backend_conformance.py | 1 + tests/server/test_response_backstop.py | 22 +++ 15 files changed, 443 insertions(+), 6 deletions(-) create mode 100644 src/autoskillit/execution/backends/_delivery_bounds.py create mode 100644 tests/contracts/test_delivery_bound_fitness.py diff --git a/docs/decisions/0005-output-budget-protocol.md b/docs/decisions/0005-output-budget-protocol.md index 6da0e38c7..8af2d38c9 100644 --- a/docs/decisions/0005-output-budget-protocol.md +++ b/docs/decisions/0005-output-budget-protocol.md @@ -107,8 +107,13 @@ non-autoskillit repos, two lower-ceiling launch paths cap casual reads at ~40 KB The ceiling clamps regular-path exec/tool output via `min(model request, tool_output_token_limit)`, but code-mode models (gpt-5.6-sol) honor model-declared `max_output_tokens` unclamped — for those sessions the ceiling -is not a hard cap and the intake discipline digest's numeric rule -(`max_output_tokens` <= 10000) is the operative bound. +is not a hard cap and the operative bound is enforced server-side by +`BackendCapabilities.effective_delivery_token_limit` (Codex: 10,000 tokens). The +response backstop reads that capability at spill time and spills any payload whose +estimated token count exceeds it, with the artifact path surfaced via the existing +`_autoskillit_response_spill` envelope (`reason="delivery_bound"`). The intake +discipline digest's numeric rule (`max_output_tokens` <= 10000) remains as the +prompt-level reinforcement. ## Corrections of Record @@ -146,6 +151,16 @@ recorded explicitly in the issue body. bounded slices enter worker memory). Still open for `run_skill` and `test_check`, whose adjudication requires the full text. +## Resolved + +- **Code-mode bypass (`gpt-5.6-sol` honoring `max_output_tokens` unclamped)**: prior to + this revision, the `tool_output_token_limit` ceiling did not constrain code-mode + responses and the intake discipline digest's numeric rule was the only operative + bound — prompt-level and unenforced. The server-side response backstop now reads + `BackendCapabilities.effective_delivery_token_limit` (Codex: 10,000 tokens; + Claude Code: 46,500 tokens) and spills payloads that exceed it. This makes the + delivery bound authoritative on every backend, not advisory. + ## Operational Signals Output-budget instrumentation uses low-cardinality structured counters or events for: diff --git a/src/autoskillit/core/types/_type_backend.py b/src/autoskillit/core/types/_type_backend.py index 4d9929ffd..3f231f062 100644 --- a/src/autoskillit/core/types/_type_backend.py +++ b/src/autoskillit/core/types/_type_backend.py @@ -162,6 +162,12 @@ class BackendCapabilities: # directory rather than written with gating frontmatter that the backend # would ignore. supports_model_invocation_gating: bool = True + # Worst-case delivery bound in tokens: the lowest operative bound that + # could apply to this backend's tool-output transport. Distinct from any + # config-file ceiling (e.g. `tool_output_token_limit`), which code-mode + # models may bypass. Used by `enforce_response_budget` to spill payloads + # the downstream transport cannot deliver at full size. + effective_delivery_token_limit: int = 0 ALL_PROJECT_LOCAL_SKILL_SEARCH_DIRS: tuple[str, ...] = ( @@ -291,6 +297,7 @@ def model_class(model: str) -> str: skill_sigil="/", session_dir_persistent=False, supports_model_invocation_gating=True, + effective_delivery_token_limit=46_500, ) diff --git a/src/autoskillit/execution/backends/AGENTS.md b/src/autoskillit/execution/backends/AGENTS.md index ba41e09b7..b7ab735de 100644 --- a/src/autoskillit/execution/backends/AGENTS.md +++ b/src/autoskillit/execution/backends/AGENTS.md @@ -9,6 +9,7 @@ IL-1 backend abstraction layer — concrete `CodingAgentBackend` implementations | `__init__.py` | `BACKEND_REGISTRY`, `get_backend()` factory, re-exports | | `_backend_cmd_builder_base.py` | `BackendCmdBuilderBase` ABC (frozen dataclass), `FlagVocabulary` NamedTuple, `SHARED_BASELINE_ENV` — canonical location for shared env-assembly keys | | `_composite_locator.py` | `CompositeSessionLocator` — single dispatch point iterating BACKEND_REGISTRY for session location | +| `_delivery_bounds.py` | `resolve_effective_delivery_bound()` — canonical accessor for per-backend effective delivery token limits | | `_probe_cache.py` | `ProbeResult`, `PROBE_CACHE_TTL`, `read_probe_cache`, `write_probe_cache` — versioned probe result cache | | `claude.py` | `ClaudeCodeBackend` (incl. `validate_session_layout`), `ClaudeEnvPolicy`, `ClaudeSessionLocator`, `ClaudeStreamParser`, `ClaudeResultParser` (prompt utilities moved to `_claude_prompt.py`) | | `_claude_prompt.py` | Prompt injection utilities, session constants (`_ensure_skill_prefix`, `_inject_completion_directive`, `_compose_resume_prompt`, etc.), shared by claude + codex + commands | diff --git a/src/autoskillit/execution/backends/__init__.py b/src/autoskillit/execution/backends/__init__.py index d7ba36891..a6b6238d3 100644 --- a/src/autoskillit/execution/backends/__init__.py +++ b/src/autoskillit/execution/backends/__init__.py @@ -22,6 +22,7 @@ ) from ._codex_parse import CodexResultParser, CodexStreamParser from ._composite_locator import CompositeSessionLocator +from ._delivery_bounds import resolve_effective_delivery_bound from .claude import ( ClaudeCodeBackend, ClaudeEnvPolicy, @@ -95,4 +96,5 @@ def get_backend(name: str) -> CodingAgentBackend: "ensure_codex_mcp_registered", "get_backend", "make_codex_scenario_player", + "resolve_effective_delivery_bound", ] diff --git a/src/autoskillit/execution/backends/_delivery_bounds.py b/src/autoskillit/execution/backends/_delivery_bounds.py new file mode 100644 index 000000000..f9c9ce4ce --- /dev/null +++ b/src/autoskillit/execution/backends/_delivery_bounds.py @@ -0,0 +1,19 @@ +"""Per-backend effective delivery-bound resolution. + +The configured ``tool_output_token_limit`` written to Codex ``config.toml`` +is a config-file ceiling. Code-mode models may bypass that ceiling when +they emit ``max_output_tokens`` without an upper bound, so the operative +bound for those paths is the harness default (~10K tokens). The +``BackendCapabilities.effective_delivery_token_limit`` field encodes this +worst-case operative bound; ``resolve_effective_delivery_bound`` is the +canonical accessor. +""" + +from __future__ import annotations + +from autoskillit.core import BackendCapabilities + + +def resolve_effective_delivery_bound(caps: BackendCapabilities) -> int: + """Return the worst-case operative token bound for ``caps``'s backend transport.""" + return caps.effective_delivery_token_limit diff --git a/src/autoskillit/execution/backends/codex.py b/src/autoskillit/execution/backends/codex.py index a6d3f91ae..54106f4c0 100644 --- a/src/autoskillit/execution/backends/codex.py +++ b/src/autoskillit/execution/backends/codex.py @@ -639,6 +639,7 @@ def capabilities(self) -> BackendCapabilities: skill_sigil="$", session_dir_persistent=True, supports_model_invocation_gating=False, + effective_delivery_token_limit=10_000, ) @property diff --git a/src/autoskillit/server/_notify.py b/src/autoskillit/server/_notify.py index 7100fb602..3be541ce9 100644 --- a/src/autoskillit/server/_notify.py +++ b/src/autoskillit/server/_notify.py @@ -12,7 +12,8 @@ from anyio import ClosedResourceError as _ClosedResource from autoskillit.config import OutputBudgetConfig -from autoskillit.core import RESERVED_LOG_RECORD_KEYS, get_logger +from autoskillit.core import RESERVED_LOG_RECORD_KEYS, BackendCapabilities, get_logger +from autoskillit.execution.backends import resolve_effective_delivery_bound from autoskillit.server._response_budget import ( bounded_response_budget_failure, enforce_response_budget, @@ -137,12 +138,19 @@ async def wrapper(*args: Any, **kwargs: Any) -> Any: artifact_dir = ( temp_dir / "responses" / tool_name if isinstance(temp_dir, Path) else None ) + effective_delivery_token_limit: int | None = None + if ctx is not None: + backend = getattr(ctx, "backend", None) + caps = getattr(backend, "capabilities", None) if backend is not None else None + if isinstance(caps, BackendCapabilities): + effective_delivery_token_limit = resolve_effective_delivery_bound(caps) try: result = enforce_response_budget( result, tool_name=tool_name, artifact_dir=artifact_dir, config=budget, + effective_delivery_token_limit=effective_delivery_token_limit, ) except Exception: logger.error("track_response_size_enforcement_failed", tool_name=tool_name) diff --git a/src/autoskillit/server/_response_budget.py b/src/autoskillit/server/_response_budget.py index 11f28e12b..5fb37f2cf 100644 --- a/src/autoskillit/server/_response_budget.py +++ b/src/autoskillit/server/_response_budget.py @@ -34,7 +34,9 @@ "reason", } ) -RESPONSE_SPILL_REASONS = frozenset({"oversized_values", "minimal_projection", "plain_text"}) +RESPONSE_SPILL_REASONS = frozenset( + {"oversized_values", "minimal_projection", "plain_text", "delivery_bound"} +) RESPONSE_SPILL_SCHEMA_DIGEST = hashlib.sha256( json.dumps( { @@ -70,6 +72,16 @@ def _canonical_json(value: Any) -> str: return json.dumps(value, ensure_ascii=False, separators=(",", ":")) +def _estimated_tokens(original_size: int) -> int: + """Estimate tokens via the four-UTF-8-byte ceiling-division heuristic. + + Matches the ``CODEX_TOOL_OUTPUT_TOKEN_LIMIT`` derivation: a coarse + transport-layer estimate, not a tokenizer count. Used to compare + payload size against ``effective_delivery_token_limit``. + """ + return (original_size + 3) // 4 + + def _serialized(value: Any) -> str: if isinstance(value, str): return value @@ -345,6 +357,143 @@ def _plain_spill_envelope( preview_limit //= 2 +def _spill_for_delivery_bound( + result: Any, + *, + tool_name: str, + config: OutputBudgetConfig, + artifact_dir: Path | None, + original: str, + original_size: int, + effective_delivery_token_limit: int, +) -> Any: + """Persist ``original`` and return a bounded projection honoring the delivery bound. + + Used when an exempted or under-byte-budget payload still exceeds the + downstream backend's effective delivery token limit. Mirrors the + non-exempted spill machinery (atomic_write, _artifact_path, + _project_json_object) so the caller sees the same envelope shape, with + ``reason="delivery_bound"`` so downstream formatters distinguish it. + """ + if artifact_dir is None: + return bounded_response_budget_failure( + result, + cause="context_unavailable", + tool_name=tool_name, + max_bytes=config.response_max_bytes, + original_utf8_bytes=original_size, + ) + path = _artifact_path(artifact_dir, tool_name) + try: + atomic_write(path, original) + except OSError: + return bounded_response_budget_failure( + result, + cause="artifact_publication_failed", + tool_name=tool_name, + max_bytes=config.response_max_bytes, + original_utf8_bytes=original_size, + ) + published = str(path.resolve()) + metadata: dict[str, Any] = { + "artifact_path": published, + "sha256": hashlib.sha256(original.encode("utf-8")).hexdigest(), + "original_utf8_bytes": original_size, + "reason": "delivery_bound", + } + parsed: Any + if isinstance(result, str): + try: + parsed = json.loads(result) + except (TypeError, ValueError, RecursionError): + parsed = None + else: + parsed = result + rendered: str | None + try: + if isinstance(parsed, dict): + rendered = _project_json_object( + parsed, + metadata=metadata, + max_bytes=config.response_max_bytes, + inline_chars=config.inline_max_chars, + ) + else: + rendered = _plain_spill_envelope( + original, + metadata=metadata, + max_bytes=config.response_max_bytes, + inline_chars=config.inline_max_chars, + ) + except _ProjectionNonconvergentError as exc: + _emit_response_budget_event( + "response_budget_projection_nonconvergent", + tool_name=_bounded_tool_name(tool_name), + detail=str(exc), + ) + return bounded_response_budget_failure( + "", + cause="projection_nonconvergent", + tool_name=tool_name, + max_bytes=config.response_max_bytes, + original_utf8_bytes=original_size, + artifact_path=published, + ) + except RecursionError: + return bounded_response_budget_failure( + "", + cause="irreducible_shape", + tool_name=tool_name, + max_bytes=config.response_max_bytes, + original_utf8_bytes=original_size, + artifact_path=published, + ) + if rendered is None: + return bounded_response_budget_failure( + "", + cause="irreducible_shape", + tool_name=tool_name, + max_bytes=config.response_max_bytes, + original_utf8_bytes=original_size, + artifact_path=published, + ) + bound = effective_delivery_token_limit * 4 + if len(rendered.encode("utf-8")) > bound: + truncated_metadata = dict(metadata) + truncated_metadata["reason"] = "delivery_bound" + truncated_metadata["delivery_bound_utf8_bytes"] = bound + truncated_metadata["projected_utf8_bytes"] = len(rendered.encode("utf-8")) + truncated_metadata["omitted_chars"] = max(0, len(rendered.encode("utf-8")) - bound) + preview_limit = min(config.inline_max_chars, bound) + preview = original[:preview_limit] + truncated = _canonical_json( + { + "delivery_bound_spill": True, + "preview": preview, + RESPONSE_SPILL_METADATA_KEY: truncated_metadata, + } + ) + truncated = _finalize_envelope({**truncated_metadata, "preview": preview}, max_bytes=bound) + if isinstance(result, str): + return truncated + try: + return json.loads(truncated) + except (ValueError, RecursionError): + return {"success": False, "error": "response_budget_projection_invalid"} + _emit_response_budget_event( + "response_budget_spill", + tool_name=_bounded_tool_name(tool_name), + original_utf8_bytes=original_size, + projected_utf8_bytes=len(rendered.encode("utf-8")), + ) + if isinstance(result, str): + return rendered + try: + return json.loads(rendered) + except (ValueError, RecursionError): + return {"success": False, "error": "response_budget_projection_invalid"} + + def enforce_response_budget( result: Any, *, @@ -352,12 +501,19 @@ def enforce_response_budget( artifact_dir: Path | None, config: OutputBudgetConfig, force_spill: bool = False, + effective_delivery_token_limit: int | None = None, ) -> Any: """Return a bounded response of the same handler type. Oversized content is atomically persisted before a projection is returned. Artifact failure and missing-context cases fail closed without echoing the original payload. + + ``effective_delivery_token_limit`` is the worst-case operative bound on the + downstream transport (e.g. Codex code-mode default ~10K). When set, payloads + whose estimated token count exceeds it are spilled even if they pass the + server-side exemption or response-byte ceilings, because the transport + cannot deliver them at full size. """ try: original = _serialized(result) @@ -371,9 +527,17 @@ def enforce_response_budget( ) original_bytes = original.encode("utf-8") original_size = len(original_bytes) + over_delivery_bound = ( + effective_delivery_token_limit is not None + and _estimated_tokens(original_size) > effective_delivery_token_limit + ) exemption = RESPONSE_BACKSTOP_EXEMPTION_REGISTRY.get(tool_name) if exemption is not None: - if len(original) <= exemption.max_chars and original_size <= exemption.max_utf8_bytes: + if ( + not over_delivery_bound + and len(original) <= exemption.max_chars + and original_size <= exemption.max_utf8_bytes + ): _emit_response_budget_event( "response_budget_exemption", tool_name=_bounded_tool_name(tool_name), @@ -384,6 +548,17 @@ def enforce_response_budget( max_utf8_bytes=exemption.max_utf8_bytes, ) return result + if over_delivery_bound: + assert effective_delivery_token_limit is not None # narrowed by over_delivery_bound + return _spill_for_delivery_bound( + result, + tool_name=tool_name, + config=config, + artifact_dir=artifact_dir, + original=original, + original_size=original_size, + effective_delivery_token_limit=effective_delivery_token_limit, + ) return bounded_response_budget_failure( result, cause="exemption_ceiling_exceeded", @@ -391,7 +566,11 @@ def enforce_response_budget( max_bytes=config.response_max_bytes, original_utf8_bytes=original_size, ) - if not force_spill and len(original_bytes) <= config.response_max_bytes: + if ( + not force_spill + and not over_delivery_bound + and len(original_bytes) <= config.response_max_bytes + ): return result if artifact_dir is None: return bounded_response_budget_failure( @@ -501,6 +680,7 @@ def shape_json_response( tool_name: str, artifact_dir: Path, config: OutputBudgetConfig, + effective_delivery_token_limit: int | None = None, ) -> str: """Serialize a tool dict, spilling once it crosses the source threshold.""" rendered = json.dumps(payload) @@ -512,6 +692,7 @@ def shape_json_response( artifact_dir=artifact_dir, config=config, force_spill=True, + effective_delivery_token_limit=effective_delivery_token_limit, ) return shaped if isinstance(shaped, str) else json.dumps(shaped) diff --git a/src/autoskillit/server/tools/_execution_helpers.py b/src/autoskillit/server/tools/_execution_helpers.py index 4f620f9f9..1fe931382 100644 --- a/src/autoskillit/server/tools/_execution_helpers.py +++ b/src/autoskillit/server/tools/_execution_helpers.py @@ -14,6 +14,7 @@ from autoskillit.core import ( RUN_PYTHON_SENTINEL_KEYS, + BackendCapabilities, CapturedStream, SpillSpec, SubprocessResult, @@ -22,6 +23,7 @@ spill_output, ) from autoskillit.execution import CaptureReadError, summarize_capture +from autoskillit.execution.backends import resolve_effective_delivery_bound from autoskillit.server._misc import _hook_config_overlay_path from autoskillit.server._response_budget import shape_json_response @@ -201,11 +203,18 @@ def shape_execution_response( if work_dir and Path(work_dir).is_absolute() else tool_ctx.temp_dir / tool_name ) + effective_delivery_token_limit: int | None = None + backend = getattr(tool_ctx, "backend", None) + caps = getattr(backend, "capabilities", None) if backend is not None else None + + if isinstance(caps, BackendCapabilities): + effective_delivery_token_limit = resolve_effective_delivery_bound(caps) return shape_json_response( payload, tool_name=tool_name, artifact_dir=artifact_root, config=tool_ctx.config.output_budget, + effective_delivery_token_limit=effective_delivery_token_limit, ) diff --git a/tests/contracts/AGENTS.md b/tests/contracts/AGENTS.md index 2a55d46d1..8ae024b79 100644 --- a/tests/contracts/AGENTS.md +++ b/tests/contracts/AGENTS.md @@ -51,6 +51,7 @@ Protocol satisfaction, package gateway, and skill contract compliance tests. | `test_mermaid_palette_contracts.py` | Contract: any SKILL.md that generates mermaid diagrams must embed the canonical 9-class palette | | `test_no_pagination_file_read.py` | Contract tests for no-pagination file read instruction in high-turn SKILL.md files | | `test_output_budget_discipline.py` | Cross-skill contract: canonical output-discipline policy identity, cohort parity, and marker ownership | +| `test_delivery_bound_fitness.py` | Contract: bundled-recipe and non-exempted payload fitness against per-backend effective delivery bounds | | `test_package_gateways.py` | Tests for Package Gateway API (groupC) — REQ-GWAY-001 through REQ-GWAY-008 | | `test_plan_experiment_contracts.py` | Contract tests for plan-experiment SKILL.md — data provenance lifecycle | | `test_plan_visualization_contracts.py` | Contract tests: select-vis-lenses Tier B experiment-type canonical names match registry | diff --git a/tests/contracts/test_delivery_bound_fitness.py b/tests/contracts/test_delivery_bound_fitness.py new file mode 100644 index 000000000..857c1edf0 --- /dev/null +++ b/tests/contracts/test_delivery_bound_fitness.py @@ -0,0 +1,137 @@ +"""Bundled-recipe and non-exempted payload fitness against per-backend delivery bounds. + +Regression guard: every bundled recipe's ``open_kitchen`` payload must either +fit each backend's effective delivery bound, or be properly spilled by the +server-side response backstop. Likewise, non-exempted payloads that fit +``response_max_bytes`` but exceed a backend's effective delivery bound must +route to spill-and-project and produce a projection under the bound. + +Covers plan Step 1.3 (bundled recipe fitness) and Step 1.4 (non-exempted spill +projection size). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from autoskillit.config import OutputBudgetConfig +from autoskillit.core import RESPONSE_BACKSTOP_EXEMPTION_REGISTRY +from autoskillit.execution.backends import BACKEND_REGISTRY +from autoskillit.execution.backends._delivery_bounds import resolve_effective_delivery_bound +from autoskillit.recipe import all_validated_recipe_names +from autoskillit.server._response_budget import ( + RESPONSE_SPILL_METADATA_KEY, + enforce_response_budget, +) + +pytestmark = [pytest.mark.layer("contracts"), pytest.mark.small] + + +_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent + + +def _recipe_names() -> list[str]: + return sorted(all_validated_recipe_names(_PROJECT_ROOT)) + + +def _backend_capabilities(): + return {name: cls().capabilities for name, cls in BACKEND_REGISTRY.items()} + + +def _effective_bound_bytes(bound_tokens: int) -> int: + """Convert effective delivery token bound to UTF-8 byte ceiling (4 bytes/token).""" + return bound_tokens * 4 + + +def _full_open_kitchen_payload(recipe_name: str) -> dict[str, object]: + """Build the full ``open_kitchen`` payload for ``recipe_name``. + + The shape mirrors what ``open_kitchen`` returns at runtime: a routing + envelope plus a ``content`` body. Recipe body content is sized to the + exemption ceiling so this test exercises the spill path for oversized + payloads. + """ + return { + "success": True, + "kitchen": f"test-kitchen-{recipe_name}", + "version": "0.0.0", + "ingredients_table": {"recipe_name": recipe_name}, + "orchestration_rules": ["rule-1", "rule-2"], + "stop_step_semantics": {"on_success": "stop"}, + "content": "x" * 200_000, + "diagram": "graph TD; A-->B", + "suggestions": [], + } + + +@pytest.mark.parametrize("recipe_name", _recipe_names(), ids=lambda n: n) +def test_bundled_recipe_open_kitchen_fits_or_spills_per_backend( + recipe_name: str, tmp_path: Path +) -> None: + """For each backend, the ``open_kitchen`` payload either fits the + effective delivery bound or spills to a projection that fits.""" + payload = _full_open_kitchen_payload(recipe_name) + serialized = json.dumps(payload) + serialized_bytes = len(serialized.encode("utf-8")) + for backend_name, caps in _backend_capabilities().items(): + bound_tokens = resolve_effective_delivery_bound(caps) + bound_bytes = _effective_bound_bytes(bound_tokens) + if serialized_bytes <= bound_bytes: + continue + result = enforce_response_budget( + serialized, + tool_name="open_kitchen", + artifact_dir=tmp_path / backend_name, + config=OutputBudgetConfig(), + effective_delivery_token_limit=bound_tokens, + ) + assert isinstance(result, str), ( + f"{backend_name}: expected str result for {recipe_name} payload" + ) + assert len(result.encode("utf-8")) <= bound_bytes, ( + f"{backend_name}: projection for {recipe_name} exceeds " + f"{bound_bytes} bytes (effective delivery bound)" + ) + data = json.loads(result) + assert data[RESPONSE_SPILL_METADATA_KEY]["reason"] in { + "delivery_bound", + "oversized_values", + "minimal_projection", + } + + +def test_non_exempted_oversized_payload_spills_within_delivery_bound(tmp_path) -> None: + """A non-exempted payload sized between ``response_max_bytes`` and a + backend's effective delivery bound must spill and produce a projection + that fits the bound.""" + payload = {"data": "y" * 30_000} + serialized = json.dumps(payload) + config = OutputBudgetConfig() + for backend_name, caps in _backend_capabilities().items(): + bound_tokens = resolve_effective_delivery_bound(caps) + bound_bytes = _effective_bound_bytes(bound_tokens) + result = enforce_response_budget( + serialized, + tool_name="run_skill", + artifact_dir=tmp_path / backend_name, + config=config, + effective_delivery_token_limit=bound_tokens, + ) + assert isinstance(result, str) + assert len(result.encode("utf-8")) <= bound_bytes, ( + f"{backend_name}: projection exceeds {bound_bytes} bytes" + ) + + +def test_exemption_registry_tools_have_measured_ceiling() -> None: + """Every tool in the exemption registry has a measured ceiling; this + sanity check guards against future additions of exempted tools without a + measured identity.""" + assert RESPONSE_BACKSTOP_EXEMPTION_REGISTRY, "registry must not be empty" + for tool_name, definition in RESPONSE_BACKSTOP_EXEMPTION_REGISTRY.items(): + assert definition.max_chars > 0 + assert definition.max_utf8_bytes > 0 + assert definition.measurement_id, f"{tool_name} has no measurement_id" diff --git a/tests/contracts/test_output_budget_discipline.py b/tests/contracts/test_output_budget_discipline.py index 2ba23f265..802493564 100644 --- a/tests/contracts/test_output_budget_discipline.py +++ b/tests/contracts/test_output_budget_discipline.py @@ -104,3 +104,20 @@ def test_intake_digest_pins_numeric_rules() -> None: def test_intake_digest_is_safe_for_agent_toml_multiline_literal() -> None: assert "'''" not in CODEX_INTAKE_DISCIPLINE_DIGEST + + +def test_intake_digest_numeric_rule_matches_codex_effective_delivery_bound() -> None: + """The intake digest's ``max_output_tokens`` numeric rule must be at least + the Codex backend's effective delivery bound — the operative bound the + server-side response backstop applies. If the bound changes, the digest + text must be updated to reflect it (this is the regression guard).""" + from autoskillit.execution.backends import BACKEND_REGISTRY + + codex_caps = BACKEND_REGISTRY["codex"]().capabilities + codex_effective_bound = codex_caps.effective_delivery_token_limit + assert codex_effective_bound > 0 + assert f"max_output_tokens above {codex_effective_bound}" in CODEX_INTAKE_DISCIPLINE_DIGEST, ( + "CODEX_INTAKE_DISCIPLINE_DIGEST numeric rule must reference the Codex " + f"effective_delivery_token_limit ({codex_effective_bound}); update " + "the digest if the bound changed." + ) diff --git a/tests/execution/backends/test_codex_config.py b/tests/execution/backends/test_codex_config.py index d46f5e1ca..9164f53ec 100644 --- a/tests/execution/backends/test_codex_config.py +++ b/tests/execution/backends/test_codex_config.py @@ -45,6 +45,22 @@ def test_response_backstop_fires_below_codex_transport_ceiling() -> None: assert OutputBudgetConfig().response_max_bytes // 3 < CODEX_TOOL_OUTPUT_TOKEN_LIMIT +def test_resolve_effective_delivery_bound_per_backend() -> None: + from autoskillit.core import BackendCapabilities + from autoskillit.execution.backends import BACKEND_REGISTRY + from autoskillit.execution.backends._delivery_bounds import ( + resolve_effective_delivery_bound, + ) + + expected = {"codex": 10_000, "claude-code": 46_500} + observed: dict[str, int] = {} + for name, cls in BACKEND_REGISTRY.items(): + caps = cls().capabilities + assert isinstance(caps, BackendCapabilities) + observed[name] = resolve_effective_delivery_bound(caps) + assert observed == expected + + class TestReadCodexConfig: def test_returns_empty_dict_when_path_missing(self, tmp_path): result = _read_codex_config(tmp_path / "nonexistent.toml") diff --git a/tests/execution/backends/test_coding_agent_backend_conformance.py b/tests/execution/backends/test_coding_agent_backend_conformance.py index b5430e482..e992c1cee 100644 --- a/tests/execution/backends/test_coding_agent_backend_conformance.py +++ b/tests/execution/backends/test_coding_agent_backend_conformance.py @@ -45,6 +45,7 @@ "channel_b_capable": "OPTIONAL", "completion_record_types": "REQUIRED", "default_skill_sandbox_mode": "REQUIRED", + "effective_delivery_token_limit": "REQUIRED", "env_denylist_prefixes": "REQUIRED", "exit_code_is_terminal": "REQUIRED", "food_truck_capable": "OPTIONAL", diff --git a/tests/server/test_response_backstop.py b/tests/server/test_response_backstop.py index 777c72055..1521390d2 100644 --- a/tests/server/test_response_backstop.py +++ b/tests/server/test_response_backstop.py @@ -469,3 +469,25 @@ def test_plain_text_irreducible_shape_returns_failure(tmp_path): assert isinstance(result, str) data = json.loads(result) assert data.get("success") is False + + +def test_exempted_payload_spills_when_over_delivery_bound(tmp_path): + """An exempted payload that fits the exemption byte ceiling but exceeds the + backend's effective delivery token limit must be spilled, not passed through.""" + payload = {"success": True, "data": "x" * 80_000} + original = json.dumps(payload) + result = enforce_response_budget( + original, + tool_name="open_kitchen", + artifact_dir=tmp_path, + config=_config(), + effective_delivery_token_limit=10_000, + ) + assert isinstance(result, str) + bound = 10_000 * 4 + assert len(result.encode("utf-8")) <= bound + data = json.loads(result) + metadata = data[RESPONSE_SPILL_METADATA_KEY] + artifact_path = metadata["artifact_path"] + assert Path(artifact_path).read_text() == original + assert metadata["reason"] in {"delivery_bound", "oversized_values", "minimal_projection"} From 4399e08bebfdbf0e145944581afdcc352942ab97 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 20 Jul 2026 09:19:37 -0700 Subject: [PATCH 2/5] fix(server): relocate resolve_effective_delivery_bound to core/ and treat zero as no-op MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Move src/autoskillit/execution/backends/_delivery_bounds.py to src/autoskillit/core/_delivery_bounds.py — the function operates on BackendCapabilities (IL-0), and server-layer consumers cannot import from execution/backends per REQ-IMP-007 / REQ-ARCH-001. - Re-export through autoskillit.core.__init__.pyi. - Treat effective_delivery_token_limit <= 0 as "not configured" so pre-existing BackendCapabilities instances with the field default do not trigger the spill path (test mocks rely on this). - Mirror the new "delivery_bound" reason in the hook-formatter _RESPONSE_SPILL_REASONS frozenset. - Register _delivery_bounds in MODULE_CASCADE_CORE; bump core/ file budget to 25; add the field to the locked-name test set; add a docstring citation for the conformance coverage test. Co-Authored-By: Claude Opus 4.6 (1M context) --- src/autoskillit/core/AGENTS.md | 1 + src/autoskillit/core/__init__.pyi | 1 + .../{execution/backends => core}/_delivery_bounds.py | 2 +- src/autoskillit/execution/backends/AGENTS.md | 1 - src/autoskillit/execution/backends/__init__.py | 2 -- src/autoskillit/hooks/formatters/_fmt_response_spill.py | 4 +++- src/autoskillit/server/_notify.py | 8 ++++++-- src/autoskillit/server/_response_budget.py | 1 + src/autoskillit/server/tools/_execution_helpers.py | 2 +- tests/_test_filter.py | 1 + tests/arch/test_subpackage_isolation.py | 2 +- tests/contracts/test_delivery_bound_fitness.py | 3 +-- tests/core/test_backend_capabilities.py | 1 + tests/execution/backends/test_codex_config.py | 5 +---- .../backends/test_coding_agent_backend_conformance.py | 7 ++++--- 15 files changed, 23 insertions(+), 18 deletions(-) rename src/autoskillit/{execution/backends => core}/_delivery_bounds.py (93%) diff --git a/src/autoskillit/core/AGENTS.md b/src/autoskillit/core/AGENTS.md index 76525e6e8..8d02c3426 100644 --- a/src/autoskillit/core/AGENTS.md +++ b/src/autoskillit/core/AGENTS.md @@ -14,6 +14,7 @@ Sub-packages: types/ (see types/AGENTS.md) and runtime/ (see runtime/AGENTS.md). | `logging.py` | Logging configuration | | `paths.py` | `pkg_root()`, `is_git_worktree()`, `is_git_main_checkout()`, `is_in_git_repo()` | | `_claude_env.py` | IDE-scrubbing canonical env builder for agent subprocesses | +| `_delivery_bounds.py` | `resolve_effective_delivery_bound()` — canonical accessor for per-backend effective delivery token limits | | `_terminal_table.py` | IL-0 color-agnostic terminal table primitive | | `_version_snapshot.py` | Process-scoped version snapshot for session telemetry (`lru_cache`'d) | | `branch_guard.py` | Branch protection helpers | diff --git a/src/autoskillit/core/__init__.pyi b/src/autoskillit/core/__init__.pyi index ac3175e3c..9c0f39467 100644 --- a/src/autoskillit/core/__init__.pyi +++ b/src/autoskillit/core/__init__.pyi @@ -4,6 +4,7 @@ from ._cmd_runner import CmdRunner as CmdRunner from ._cmd_runner import default_cmd_runner as default_cmd_runner from ._cmd_runner import run_gh as run_gh from ._cmd_runner import run_git as run_git +from ._delivery_bounds import resolve_effective_delivery_bound as resolve_effective_delivery_bound from ._execution_marker import execution_marker as execution_marker from ._install_detect import DirectUrlInfo as DirectUrlInfo from ._install_detect import _is_release_tag as _is_release_tag diff --git a/src/autoskillit/execution/backends/_delivery_bounds.py b/src/autoskillit/core/_delivery_bounds.py similarity index 93% rename from src/autoskillit/execution/backends/_delivery_bounds.py rename to src/autoskillit/core/_delivery_bounds.py index f9c9ce4ce..ca7eb9c7c 100644 --- a/src/autoskillit/execution/backends/_delivery_bounds.py +++ b/src/autoskillit/core/_delivery_bounds.py @@ -11,7 +11,7 @@ from __future__ import annotations -from autoskillit.core import BackendCapabilities +from .types._type_backend import BackendCapabilities def resolve_effective_delivery_bound(caps: BackendCapabilities) -> int: diff --git a/src/autoskillit/execution/backends/AGENTS.md b/src/autoskillit/execution/backends/AGENTS.md index b7ab735de..ba41e09b7 100644 --- a/src/autoskillit/execution/backends/AGENTS.md +++ b/src/autoskillit/execution/backends/AGENTS.md @@ -9,7 +9,6 @@ IL-1 backend abstraction layer — concrete `CodingAgentBackend` implementations | `__init__.py` | `BACKEND_REGISTRY`, `get_backend()` factory, re-exports | | `_backend_cmd_builder_base.py` | `BackendCmdBuilderBase` ABC (frozen dataclass), `FlagVocabulary` NamedTuple, `SHARED_BASELINE_ENV` — canonical location for shared env-assembly keys | | `_composite_locator.py` | `CompositeSessionLocator` — single dispatch point iterating BACKEND_REGISTRY for session location | -| `_delivery_bounds.py` | `resolve_effective_delivery_bound()` — canonical accessor for per-backend effective delivery token limits | | `_probe_cache.py` | `ProbeResult`, `PROBE_CACHE_TTL`, `read_probe_cache`, `write_probe_cache` — versioned probe result cache | | `claude.py` | `ClaudeCodeBackend` (incl. `validate_session_layout`), `ClaudeEnvPolicy`, `ClaudeSessionLocator`, `ClaudeStreamParser`, `ClaudeResultParser` (prompt utilities moved to `_claude_prompt.py`) | | `_claude_prompt.py` | Prompt injection utilities, session constants (`_ensure_skill_prefix`, `_inject_completion_directive`, `_compose_resume_prompt`, etc.), shared by claude + codex + commands | diff --git a/src/autoskillit/execution/backends/__init__.py b/src/autoskillit/execution/backends/__init__.py index a6b6238d3..d7ba36891 100644 --- a/src/autoskillit/execution/backends/__init__.py +++ b/src/autoskillit/execution/backends/__init__.py @@ -22,7 +22,6 @@ ) from ._codex_parse import CodexResultParser, CodexStreamParser from ._composite_locator import CompositeSessionLocator -from ._delivery_bounds import resolve_effective_delivery_bound from .claude import ( ClaudeCodeBackend, ClaudeEnvPolicy, @@ -96,5 +95,4 @@ def get_backend(name: str) -> CodingAgentBackend: "ensure_codex_mcp_registered", "get_backend", "make_codex_scenario_player", - "resolve_effective_delivery_bound", ] diff --git a/src/autoskillit/hooks/formatters/_fmt_response_spill.py b/src/autoskillit/hooks/formatters/_fmt_response_spill.py index bb41aa770..05eb801a2 100644 --- a/src/autoskillit/hooks/formatters/_fmt_response_spill.py +++ b/src/autoskillit/hooks/formatters/_fmt_response_spill.py @@ -24,7 +24,9 @@ "reason", } ) -_RESPONSE_SPILL_REASONS = frozenset({"oversized_values", "minimal_projection", "plain_text"}) +_RESPONSE_SPILL_REASONS = frozenset( + {"oversized_values", "minimal_projection", "plain_text", "delivery_bound"} +) _RESPONSE_SPILL_SCHEMA_DIGEST = hashlib.sha256( json.dumps( { diff --git a/src/autoskillit/server/_notify.py b/src/autoskillit/server/_notify.py index 3be541ce9..d68b902bd 100644 --- a/src/autoskillit/server/_notify.py +++ b/src/autoskillit/server/_notify.py @@ -12,8 +12,12 @@ from anyio import ClosedResourceError as _ClosedResource from autoskillit.config import OutputBudgetConfig -from autoskillit.core import RESERVED_LOG_RECORD_KEYS, BackendCapabilities, get_logger -from autoskillit.execution.backends import resolve_effective_delivery_bound +from autoskillit.core import ( + RESERVED_LOG_RECORD_KEYS, + BackendCapabilities, + get_logger, + resolve_effective_delivery_bound, +) from autoskillit.server._response_budget import ( bounded_response_budget_failure, enforce_response_budget, diff --git a/src/autoskillit/server/_response_budget.py b/src/autoskillit/server/_response_budget.py index 5fb37f2cf..b792a55a3 100644 --- a/src/autoskillit/server/_response_budget.py +++ b/src/autoskillit/server/_response_budget.py @@ -529,6 +529,7 @@ def enforce_response_budget( original_size = len(original_bytes) over_delivery_bound = ( effective_delivery_token_limit is not None + and effective_delivery_token_limit > 0 and _estimated_tokens(original_size) > effective_delivery_token_limit ) exemption = RESPONSE_BACKSTOP_EXEMPTION_REGISTRY.get(tool_name) diff --git a/src/autoskillit/server/tools/_execution_helpers.py b/src/autoskillit/server/tools/_execution_helpers.py index 1fe931382..405587b17 100644 --- a/src/autoskillit/server/tools/_execution_helpers.py +++ b/src/autoskillit/server/tools/_execution_helpers.py @@ -19,11 +19,11 @@ SpillSpec, SubprocessResult, get_logger, + resolve_effective_delivery_bound, resolve_temp_dir, spill_output, ) from autoskillit.execution import CaptureReadError, summarize_capture -from autoskillit.execution.backends import resolve_effective_delivery_bound from autoskillit.server._misc import _hook_config_overlay_path from autoskillit.server._response_budget import shape_json_response diff --git a/tests/_test_filter.py b/tests/_test_filter.py index 3f4f32fb7..7a53d3eeb 100644 --- a/tests/_test_filter.py +++ b/tests/_test_filter.py @@ -265,6 +265,7 @@ class ImportContext(enum.StrEnum): "_step_context": frozenset({"core", "execution", "pipeline", "server"}), "_execution_marker": frozenset({"core", "execution", "fleet", "server"}), "bash_write_targets": frozenset({"core", "execution", "server"}), + "_delivery_bounds": frozenset({"core", "execution", "server"}), "_type_closure_report": frozenset({"core"}), "closure_hashing": frozenset({"core"}), "path_containment": frozenset({"core"}), diff --git a/tests/arch/test_subpackage_isolation.py b/tests/arch/test_subpackage_isolation.py index d6bfaf483..0c4542064 100644 --- a/tests/arch/test_subpackage_isolation.py +++ b/tests/arch/test_subpackage_isolation.py @@ -884,7 +884,7 @@ def test_no_subpackage_exceeds_10_files() -> None: "server": 14, "recipe": 42, # was 33; +9 from CI/graph/dataflow splits "execution": 18, - "core": 24, # +closure_hashing +path_containment +closure_verifier + "core": 25, # +_delivery_bounds (resolve_effective_delivery_bound) "core/types": 32, # +invariant_registry (INVARIANT_REGISTRY) +closure_report "cli": 21, "hooks": 18, # +recipe_confirmed_post_hook, +quota_guard_state_post_hook, +_policy_event, +shell_capture_hook (#4286), +_capture_cleanup.py # noqa: E501 diff --git a/tests/contracts/test_delivery_bound_fitness.py b/tests/contracts/test_delivery_bound_fitness.py index 857c1edf0..e0fb0e7f4 100644 --- a/tests/contracts/test_delivery_bound_fitness.py +++ b/tests/contracts/test_delivery_bound_fitness.py @@ -18,9 +18,8 @@ import pytest from autoskillit.config import OutputBudgetConfig -from autoskillit.core import RESPONSE_BACKSTOP_EXEMPTION_REGISTRY +from autoskillit.core import RESPONSE_BACKSTOP_EXEMPTION_REGISTRY, resolve_effective_delivery_bound from autoskillit.execution.backends import BACKEND_REGISTRY -from autoskillit.execution.backends._delivery_bounds import resolve_effective_delivery_bound from autoskillit.recipe import all_validated_recipe_names from autoskillit.server._response_budget import ( RESPONSE_SPILL_METADATA_KEY, diff --git a/tests/core/test_backend_capabilities.py b/tests/core/test_backend_capabilities.py index 1225d1e1b..0c6e8cdba 100644 --- a/tests/core/test_backend_capabilities.py +++ b/tests/core/test_backend_capabilities.py @@ -177,6 +177,7 @@ def test_backend_capabilities_field_names_locked(): "session_dir_persistent", "supports_model_invocation_gating", "github_api_callable", + "effective_delivery_token_limit", } actual = {f.name for f in dataclasses.fields(BackendCapabilities)} assert actual == expected diff --git a/tests/execution/backends/test_codex_config.py b/tests/execution/backends/test_codex_config.py index 9164f53ec..1b49a4100 100644 --- a/tests/execution/backends/test_codex_config.py +++ b/tests/execution/backends/test_codex_config.py @@ -46,11 +46,8 @@ def test_response_backstop_fires_below_codex_transport_ceiling() -> None: def test_resolve_effective_delivery_bound_per_backend() -> None: - from autoskillit.core import BackendCapabilities + from autoskillit.core import BackendCapabilities, resolve_effective_delivery_bound from autoskillit.execution.backends import BACKEND_REGISTRY - from autoskillit.execution.backends._delivery_bounds import ( - resolve_effective_delivery_bound, - ) expected = {"codex": 10_000, "claude-code": 46_500} observed: dict[str, int] = {} diff --git a/tests/execution/backends/test_coding_agent_backend_conformance.py b/tests/execution/backends/test_coding_agent_backend_conformance.py index e992c1cee..3d446ca4d 100644 --- a/tests/execution/backends/test_coding_agent_backend_conformance.py +++ b/tests/execution/backends/test_coding_agent_backend_conformance.py @@ -114,9 +114,10 @@ def test_capabilities_returns_backend_capabilities(self) -> None: """BackendCapabilities contract — exercises multiple fields. Fields cited: applicable_guards, default_skill_sandbox_mode, - git_metadata_writable, has_unguarded_filesystem_access, - process_name_aliases, project_local_skills_capable, record_capable, - replay_capable, session_dir_persistent, supports_context_window_suffix, + effective_delivery_token_limit, git_metadata_writable, + has_unguarded_filesystem_access, process_name_aliases, + project_local_skills_capable, record_capable, replay_capable, + session_dir_persistent, supports_context_window_suffix, supports_tool_list_changed, triage_capable, write_detection_strategy. """ assert isinstance(self.backend.capabilities, BackendCapabilities) From 7f6a12b3f9dac65ae23bea3c803da3b29f9a0f69 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 20 Jul 2026 09:23:25 -0700 Subject: [PATCH 3/5] test(cascade): register _delivery_bounds in core cascade expected-stems set Co-Authored-By: Claude Opus 4.6 (1M context) --- tests/test_test_filter_core_cascade.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_test_filter_core_cascade.py b/tests/test_test_filter_core_cascade.py index d41de6d03..a3f4413a0 100644 --- a/tests/test_test_filter_core_cascade.py +++ b/tests/test_test_filter_core_cascade.py @@ -119,6 +119,7 @@ def test_all_entries_present(self) -> None: "closure_hashing", "path_containment", "closure_verifier", + "_delivery_bounds", } assert set(MODULE_CASCADE_CORE.keys()) == expected_stems From a1d17d79ed81e330168393c4dd2dfd043e0fed82 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 20 Jul 2026 11:45:23 -0700 Subject: [PATCH 4/5] fix(server): bounded delivery-bound summary, restored exemption ordering, projection cap Replace the generic projection in _spill_for_delivery_bound with a bounded inline summary that preserves the operational fields (success, kitchen, version, ingredients_table, orchestration_rules, stop_step_semantics, errors, suggestions) verbatim when they fit, truncates content to fit the bound, drops diagram when content alone exceeds the bound, and uniformly projects oversized preserved values before reaching the minimal rung. Delete the broken truncation fallback (dead store + malformed _finalize_envelope call that flattened metadata and raised _ProjectionNonconvergentError). Add a reason kwarg to _plain_spill_envelope so delivery-bound plain-text spills are labeled "delivery_bound". Restore the exemption-ceiling-first ordering in enforce_response_budget: ceiling check dominates, delivery-bound spill applies only within the ceiling, exemption event is the pass-through. Cap the non-exempted projection target at min(response_max_bytes, effective_delivery_token_limit * 4) so the rendered projection is structurally guaranteed to fit the transport bound. Co-Authored-By: Claude Fable 5 --- src/autoskillit/server/_response_budget.py | 259 ++++++++++++++++----- 1 file changed, 200 insertions(+), 59 deletions(-) diff --git a/src/autoskillit/server/_response_budget.py b/src/autoskillit/server/_response_budget.py index b792a55a3..a67cc0d2c 100644 --- a/src/autoskillit/server/_response_budget.py +++ b/src/autoskillit/server/_response_budget.py @@ -336,6 +336,7 @@ def _plain_spill_envelope( metadata: dict[str, Any], max_bytes: int, inline_chars: int, + reason: str = "plain_text", ) -> str | None: preview_limit = max(0, min(inline_chars, max_bytes // 3)) while True: @@ -343,7 +344,7 @@ def _plain_spill_envelope( envelope: dict[str, Any] = { RESPONSE_SPILL_METADATA_KEY: _spill_metadata( metadata, - reason="plain_text", + reason=reason, omitted_chars=omitted_chars, omitted_items=0, ), @@ -357,6 +358,159 @@ def _plain_spill_envelope( preview_limit //= 2 +_DELIVERY_BOUND_PRESERVED_KEYS: tuple[str, ...] = ( + "success", + "kitchen", + "version", + "ingredients_table", + "orchestration_rules", + "stop_step_semantics", + "errors", + "suggestions", +) +_DELIVERY_BOUND_CONTENT_KEY = "content" +_DELIVERY_BOUND_DROPPABLE_KEYS: tuple[str, ...] = ("diagram",) + + +def _delivery_bound_summary( + parsed: dict[str, Any], + *, + metadata: dict[str, Any], + bound: int, +) -> str | None: + """Build a bounded inline summary honoring the delivery bound. + + Preserves ``success``, ``kitchen``, ``version``, ``ingredients_table``, + ``orchestration_rules``, ``stop_step_semantics``, ``errors``, + ``suggestions`` verbatim (when present and fitting); truncates ``content``; + drops ``diagram``; nests spill metadata under ``RESPONSE_SPILL_METADATA_KEY`` + with ``reason="delivery_bound"``; sets top-level ``delivery_bound_spill=True``. + + Returns the rendered envelope string when it fits ``bound``; otherwise + ``None`` so the caller fails closed. + """ + content_text = parsed.get(_DELIVERY_BOUND_CONTENT_KEY) + content_is_str = isinstance(content_text, str) + + precomputed_base_chars = 0 + precomputed_base_items = 0 + for key, value in parsed.items(): + if key in _DELIVERY_BOUND_PRESERVED_KEYS or key == _DELIVERY_BOUND_CONTENT_KEY: + continue + chars, items = _total_omissions(value) + precomputed_base_chars += chars + precomputed_base_items += items + + if not content_is_str and _DELIVERY_BOUND_CONTENT_KEY in parsed: + chars, items = _total_omissions(parsed[_DELIVERY_BOUND_CONTENT_KEY]) + precomputed_base_chars += chars + precomputed_base_items += items + + present_preserved = [key for key in _DELIVERY_BOUND_PRESERVED_KEYS if key in parsed] + + def _build(head_len: int, include_droppable: bool, value_projector=None) -> dict[str, Any]: + envelope: dict[str, Any] = {RESPONSE_SPILL_METADATA_KEY: dict(metadata)} + envelope["delivery_bound_spill"] = True + + base_chars = precomputed_base_chars + base_items = precomputed_base_items + + for key in present_preserved: + if value_projector is None: + envelope[key] = parsed[key] + else: + envelope[key] = value_projector(parsed[key]) + + if include_droppable: + for key in _DELIVERY_BOUND_DROPPABLE_KEYS: + if key in parsed: + envelope[key] = parsed[key] + else: + for key in _DELIVERY_BOUND_DROPPABLE_KEYS: + if key in parsed: + chars, items = _total_omissions(parsed[key]) + base_chars += chars + base_items += items + + if content_is_str: + text = content_text or "" + truncated = text[:head_len] + envelope[_DELIVERY_BOUND_CONTENT_KEY] = truncated + base_chars += max(0, len(text) - head_len) + elif _DELIVERY_BOUND_CONTENT_KEY in parsed: + envelope[_DELIVERY_BOUND_CONTENT_KEY] = parsed[_DELIVERY_BOUND_CONTENT_KEY] + + if value_projector is not None: + for key in present_preserved: + projected_value = envelope[key] + if ( + isinstance(projected_value, dict) + and RESPONSE_SPILL_METADATA_KEY in projected_value + ): + continue + chars, items = _total_omissions(parsed[key]) + base_chars += chars + base_items += items + + envelope[RESPONSE_SPILL_METADATA_KEY] = _spill_metadata( + envelope[RESPONSE_SPILL_METADATA_KEY], + reason="delivery_bound", + omitted_chars=base_chars, + omitted_items=base_items, + ) + return envelope + + def _fits(envelope: dict[str, Any]) -> str | None: + try: + rendered = _finalize_envelope(envelope, max_bytes=bound) + except _ProjectionNonconvergentError: + return None + if len(rendered.encode("utf-8")) <= bound: + return rendered + return None + + zero_envelope = _build(0, True) + base_rendered = _finalize_envelope(zero_envelope, max_bytes=bound) + base_bytes = len(base_rendered.encode("utf-8")) + head_limit = max(0, bound - base_bytes - 64) if content_is_str else 0 + + while head_limit > 0 and content_is_str: + rendered = _fits(_build(head_limit, True)) + if rendered is not None: + return rendered + head_limit //= 2 + + if content_is_str: + zero_head_envelope = _build(0, True) + rendered = _fits(zero_head_envelope) + if rendered is not None: + return rendered + + rendered = _fits(_build(0, False)) + if rendered is not None: + return rendered + + if present_preserved: + value_limit = max(16, bound // (len(present_preserved) + 2)) + while value_limit >= 16: + + def _project_with_limit(value: Any, _limit: int = value_limit) -> Any: + projected, _chars, _items = _project_value(value, _limit) + return projected + + rendered = _fits(_build(0, False, value_projector=_project_with_limit)) + if rendered is not None: + return rendered + value_limit //= 2 + + minimal_envelope = _build(0, False, value_projector=_minimal_same_type) + rendered = _fits(minimal_envelope) + if rendered is not None: + return rendered + + return None + + def _spill_for_delivery_bound( result: Any, *, @@ -409,21 +563,19 @@ def _spill_for_delivery_bound( parsed = None else: parsed = result + bound = effective_delivery_token_limit * 4 + floor_bytes = min(bound, config.response_max_bytes) rendered: str | None try: if isinstance(parsed, dict): - rendered = _project_json_object( - parsed, - metadata=metadata, - max_bytes=config.response_max_bytes, - inline_chars=config.inline_max_chars, - ) + rendered = _delivery_bound_summary(parsed, metadata=metadata, bound=bound) else: rendered = _plain_spill_envelope( original, metadata=metadata, - max_bytes=config.response_max_bytes, + max_bytes=floor_bytes, inline_chars=config.inline_max_chars, + reason="delivery_bound", ) except _ProjectionNonconvergentError as exc: _emit_response_budget_event( @@ -435,7 +587,7 @@ def _spill_for_delivery_bound( "", cause="projection_nonconvergent", tool_name=tool_name, - max_bytes=config.response_max_bytes, + max_bytes=floor_bytes, original_utf8_bytes=original_size, artifact_path=published, ) @@ -444,7 +596,7 @@ def _spill_for_delivery_bound( "", cause="irreducible_shape", tool_name=tool_name, - max_bytes=config.response_max_bytes, + max_bytes=floor_bytes, original_utf8_bytes=original_size, artifact_path=published, ) @@ -453,33 +605,10 @@ def _spill_for_delivery_bound( "", cause="irreducible_shape", tool_name=tool_name, - max_bytes=config.response_max_bytes, + max_bytes=floor_bytes, original_utf8_bytes=original_size, artifact_path=published, ) - bound = effective_delivery_token_limit * 4 - if len(rendered.encode("utf-8")) > bound: - truncated_metadata = dict(metadata) - truncated_metadata["reason"] = "delivery_bound" - truncated_metadata["delivery_bound_utf8_bytes"] = bound - truncated_metadata["projected_utf8_bytes"] = len(rendered.encode("utf-8")) - truncated_metadata["omitted_chars"] = max(0, len(rendered.encode("utf-8")) - bound) - preview_limit = min(config.inline_max_chars, bound) - preview = original[:preview_limit] - truncated = _canonical_json( - { - "delivery_bound_spill": True, - "preview": preview, - RESPONSE_SPILL_METADATA_KEY: truncated_metadata, - } - ) - truncated = _finalize_envelope({**truncated_metadata, "preview": preview}, max_bytes=bound) - if isinstance(result, str): - return truncated - try: - return json.loads(truncated) - except (ValueError, RecursionError): - return {"success": False, "error": "response_budget_projection_invalid"} _emit_response_budget_event( "response_budget_spill", tool_name=_bounded_tool_name(tool_name), @@ -534,21 +663,17 @@ def enforce_response_budget( ) exemption = RESPONSE_BACKSTOP_EXEMPTION_REGISTRY.get(tool_name) if exemption is not None: - if ( - not over_delivery_bound - and len(original) <= exemption.max_chars - and original_size <= exemption.max_utf8_bytes - ): - _emit_response_budget_event( - "response_budget_exemption", - tool_name=_bounded_tool_name(tool_name), - measurement_id=exemption.measurement_id, - original_chars=len(original), + within_ceiling = ( + len(original) <= exemption.max_chars and original_size <= exemption.max_utf8_bytes + ) + if not within_ceiling: + return bounded_response_budget_failure( + result, + cause="exemption_ceiling_exceeded", + tool_name=tool_name, + max_bytes=config.response_max_bytes, original_utf8_bytes=original_size, - max_chars=exemption.max_chars, - max_utf8_bytes=exemption.max_utf8_bytes, ) - return result if over_delivery_bound: assert effective_delivery_token_limit is not None # narrowed by over_delivery_bound return _spill_for_delivery_bound( @@ -560,13 +685,16 @@ def enforce_response_budget( original_size=original_size, effective_delivery_token_limit=effective_delivery_token_limit, ) - return bounded_response_budget_failure( - result, - cause="exemption_ceiling_exceeded", - tool_name=tool_name, - max_bytes=config.response_max_bytes, + _emit_response_budget_event( + "response_budget_exemption", + tool_name=_bounded_tool_name(tool_name), + measurement_id=exemption.measurement_id, + original_chars=len(original), original_utf8_bytes=original_size, + max_chars=exemption.max_chars, + max_utf8_bytes=exemption.max_utf8_bytes, ) + return result if ( not force_spill and not over_delivery_bound @@ -601,6 +729,18 @@ def enforce_response_budget( "original_utf8_bytes": len(original_bytes), } + delivery_bound_bytes = ( + effective_delivery_token_limit * 4 + if effective_delivery_token_limit is not None and effective_delivery_token_limit > 0 + else None + ) + projection_max_bytes = ( + min(config.response_max_bytes, delivery_bound_bytes) + if delivery_bound_bytes is not None + else config.response_max_bytes + ) + projection_inline_chars = min(config.inline_max_chars, projection_max_bytes) + parsed: Any = None if isinstance(result, str): try: @@ -616,15 +756,15 @@ def enforce_response_budget( rendered = _project_json_object( parsed, metadata=metadata, - max_bytes=config.response_max_bytes, - inline_chars=config.inline_max_chars, + max_bytes=projection_max_bytes, + inline_chars=projection_inline_chars, ) if rendered is None and not isinstance(parsed, dict): rendered = _plain_spill_envelope( original, metadata=metadata, - max_bytes=config.response_max_bytes, - inline_chars=config.inline_max_chars, + max_bytes=projection_max_bytes, + inline_chars=projection_inline_chars, ) except _ProjectionNonconvergentError as exc: _emit_response_budget_event( @@ -636,7 +776,7 @@ def enforce_response_budget( "", cause="projection_nonconvergent", tool_name=tool_name, - max_bytes=config.response_max_bytes, + max_bytes=projection_max_bytes, original_utf8_bytes=original_size, artifact_path=published, ) @@ -647,7 +787,7 @@ def enforce_response_budget( "", cause="irreducible_shape", tool_name=tool_name, - max_bytes=config.response_max_bytes, + max_bytes=projection_max_bytes, original_utf8_bytes=original_size, artifact_path=published, ) @@ -656,11 +796,12 @@ def enforce_response_budget( "", cause="irreducible_shape", tool_name=tool_name, - max_bytes=config.response_max_bytes, + max_bytes=projection_max_bytes, original_utf8_bytes=original_size, artifact_path=published, ) + assert isinstance(rendered, str) _emit_response_budget_event( "response_budget_spill", tool_name=_bounded_tool_name(tool_name), From 57116a448da6488fa40756aec6abac4cc5012c46 Mon Sep 17 00:00:00 2001 From: Trecek Date: Mon, 20 Jul 2026 11:49:24 -0700 Subject: [PATCH 5/5] test(server): regression coverage for delivery-bound summary, ordering, projection cap Tighten test_exempted_payload_spills_when_over_delivery_bound to assert delivery_bound_spill=True, success=True preserved, and reason strictly "delivery_bound" (not the 3-member set). Add six new tests in tests/server/test_response_backstop.py: - test_delivery_bound_summary_preserves_operational_fields (T1) - test_delivery_bound_summary_small_bound_no_exception (T2 REQ-026 guard) - test_delivery_bound_summary_drops_diagram_when_needed (T3) - test_over_ceiling_payload_fails_even_when_over_delivery_bound (T4 ordering) - test_non_exempted_projection_capped_at_delivery_bound (T5 REQ-023) - test_delivery_bound_summary_projects_oversized_preserved_fields (T9 rung-4) Rebuild _full_open_kitchen_payload in tests/contracts/test_delivery_bound_fitness.py on real bundled recipes via load_and_validate + build_open_kitchen_recipe_payload (the production routing-field injector) mirroring tests/recipe/test_bundled_recipes_all_truthy.py overrides; isolate via monkeypatch.chdir(tmp_path). Tighten the fit-or-spill assertion to require delivery_bound_spill=True and reason "delivery_bound". Resize the non-exempted overflow payload in Test 1.4 to 60 keys of 5,000 chars (~300 KB) so it exceeds every backend's bound and response_max_bytes; add the self-check that the payload exceeds the bound and that the spill envelope is produced. Co-Authored-By: Claude Fable 5 --- .../contracts/test_delivery_bound_fitness.py | 61 +++--- tests/server/test_response_backstop.py | 206 +++++++++++++++++- 2 files changed, 239 insertions(+), 28 deletions(-) diff --git a/tests/contracts/test_delivery_bound_fitness.py b/tests/contracts/test_delivery_bound_fitness.py index e0fb0e7f4..010681005 100644 --- a/tests/contracts/test_delivery_bound_fitness.py +++ b/tests/contracts/test_delivery_bound_fitness.py @@ -6,8 +6,9 @@ ``response_max_bytes`` but exceed a backend's effective delivery bound must route to spill-and-project and produce a projection under the bound. -Covers plan Step 1.3 (bundled recipe fitness) and Step 1.4 (non-exempted spill -projection size). +Covers plan Step 4 (real bundled-recipe payload fitness via +``load_and_validate`` + ``build_open_kitchen_recipe_payload``) and Step 5 +(non-exempted spill projection size). """ from __future__ import annotations @@ -20,11 +21,12 @@ from autoskillit.config import OutputBudgetConfig from autoskillit.core import RESPONSE_BACKSTOP_EXEMPTION_REGISTRY, resolve_effective_delivery_bound from autoskillit.execution.backends import BACKEND_REGISTRY -from autoskillit.recipe import all_validated_recipe_names +from autoskillit.recipe import all_validated_recipe_names, load_and_validate from autoskillit.server._response_budget import ( RESPONSE_SPILL_METADATA_KEY, enforce_response_budget, ) +from autoskillit.server.tools._serve_helpers import build_open_kitchen_recipe_payload pytestmark = [pytest.mark.layer("contracts"), pytest.mark.small] @@ -46,32 +48,34 @@ def _effective_bound_bytes(bound_tokens: int) -> int: def _full_open_kitchen_payload(recipe_name: str) -> dict[str, object]: - """Build the full ``open_kitchen`` payload for ``recipe_name``. - - The shape mirrors what ``open_kitchen`` returns at runtime: a routing - envelope plus a ``content`` body. Recipe body content is sized to the - exemption ceiling so this test exercises the spill path for oversized - payloads. + """Build the production-shape ``open_kitchen`` payload for ``recipe_name``. + + Uses ``load_and_validate`` + ``build_open_kitchen_recipe_payload`` so the + payload mirrors what ``open_kitchen`` returns at runtime: the routing + envelope injected by the production helper plus the full recipe body from + the bundled YAML. The fit-or-spill assertion below exercises the spill + path for any bundled recipe whose serialized payload outgrows a backend's + bound. """ - return { - "success": True, - "kitchen": f"test-kitchen-{recipe_name}", - "version": "0.0.0", - "ingredients_table": {"recipe_name": recipe_name}, - "orchestration_rules": ["rule-1", "rule-2"], - "stop_step_semantics": {"on_success": "stop"}, - "content": "x" * 200_000, - "diagram": "graph TD; A-->B", - "suggestions": [], - } + result = load_and_validate( + recipe_name, + project_dir=_PROJECT_ROOT, + ingredient_overrides={ + "task": "test task", + "issue_url": "https://github.com/test/test/issues/1", + "source_dir": str(_PROJECT_ROOT), + }, + ) + return build_open_kitchen_recipe_payload(dict(result), version="0.0.0") @pytest.mark.parametrize("recipe_name", _recipe_names(), ids=lambda n: n) def test_bundled_recipe_open_kitchen_fits_or_spills_per_backend( - recipe_name: str, tmp_path: Path + recipe_name: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """For each backend, the ``open_kitchen`` payload either fits the effective delivery bound or spills to a projection that fits.""" + monkeypatch.chdir(tmp_path) payload = _full_open_kitchen_payload(recipe_name) serialized = json.dumps(payload) serialized_bytes = len(serialized.encode("utf-8")) @@ -95,23 +99,24 @@ def test_bundled_recipe_open_kitchen_fits_or_spills_per_backend( f"{bound_bytes} bytes (effective delivery bound)" ) data = json.loads(result) - assert data[RESPONSE_SPILL_METADATA_KEY]["reason"] in { - "delivery_bound", - "oversized_values", - "minimal_projection", - } + assert data.get("delivery_bound_spill") is True + metadata = data[RESPONSE_SPILL_METADATA_KEY] + assert metadata["reason"] == "delivery_bound" def test_non_exempted_oversized_payload_spills_within_delivery_bound(tmp_path) -> None: """A non-exempted payload sized between ``response_max_bytes`` and a backend's effective delivery bound must spill and produce a projection that fits the bound.""" - payload = {"data": "y" * 30_000} + payload = {f"key_{index:03d}": "y" * 5_000 for index in range(60)} serialized = json.dumps(payload) config = OutputBudgetConfig() for backend_name, caps in _backend_capabilities().items(): bound_tokens = resolve_effective_delivery_bound(caps) bound_bytes = _effective_bound_bytes(bound_tokens) + assert len(serialized.encode("utf-8")) > bound_bytes, ( + f"{backend_name}: payload does not exceed bound ({bound_bytes} bytes)" + ) result = enforce_response_budget( serialized, tool_name="run_skill", @@ -123,6 +128,8 @@ def test_non_exempted_oversized_payload_spills_within_delivery_bound(tmp_path) - assert len(result.encode("utf-8")) <= bound_bytes, ( f"{backend_name}: projection exceeds {bound_bytes} bytes" ) + data = json.loads(result) + assert RESPONSE_SPILL_METADATA_KEY in data def test_exemption_registry_tools_have_measured_ceiling() -> None: diff --git a/tests/server/test_response_backstop.py b/tests/server/test_response_backstop.py index 1521390d2..44e2f1911 100644 --- a/tests/server/test_response_backstop.py +++ b/tests/server/test_response_backstop.py @@ -487,7 +487,211 @@ def test_exempted_payload_spills_when_over_delivery_bound(tmp_path): bound = 10_000 * 4 assert len(result.encode("utf-8")) <= bound data = json.loads(result) + assert data["success"] is True + assert data["delivery_bound_spill"] is True metadata = data[RESPONSE_SPILL_METADATA_KEY] artifact_path = metadata["artifact_path"] assert Path(artifact_path).read_text() == original - assert metadata["reason"] in {"delivery_bound", "oversized_values", "minimal_projection"} + assert metadata["reason"] == "delivery_bound" + + +def test_delivery_bound_summary_preserves_operational_fields(tmp_path): + """Bounded summary must preserve success/kitchen/version/ingredients_table/ + orchestration_rules/stop_step_semantics/errors/suggestions verbatim, + truncate content to fit, and nest spill metadata with reason='delivery_bound' + and top-level delivery_bound_spill=True.""" + payload = { + "success": True, + "kitchen": "open", + "version": "1.2.3", + "ingredients_table": "| a |", + "orchestration_rules": ["r1", "r2"], + "stop_step_semantics": {"on_success": "stop"}, + "errors": [], + "suggestions": [{"rule": "x"}], + "diagram": "graph TD; A-->B", + "content": "x" * 150_000, + } + original = json.dumps(payload) + result = enforce_response_budget( + original, + tool_name="open_kitchen", + artifact_dir=tmp_path, + config=OutputBudgetConfig(), + effective_delivery_token_limit=10_000, + ) + assert isinstance(result, str) + bound = 10_000 * 4 + assert len(result.encode("utf-8")) <= bound + data = json.loads(result) + assert data["delivery_bound_spill"] is True + assert data["success"] == payload["success"] + assert data["kitchen"] == payload["kitchen"] + assert data["version"] == payload["version"] + assert data["ingredients_table"] == payload["ingredients_table"] + assert data["orchestration_rules"] == payload["orchestration_rules"] + assert data["stop_step_semantics"] == payload["stop_step_semantics"] + assert data["errors"] == payload["errors"] + assert data["suggestions"] == payload["suggestions"] + assert data["diagram"] == payload["diagram"] + assert data["content"].startswith("x") + assert payload["content"].startswith(data["content"]) + metadata = data[RESPONSE_SPILL_METADATA_KEY] + assert metadata["reason"] == "delivery_bound" + assert set(metadata) == RESPONSE_SPILL_METADATA_KEYS + assert Path(metadata["artifact_path"]).read_text() == original + + +def test_delivery_bound_summary_small_bound_no_exception(tmp_path): + """Regression guard for the REQ-026 fallback branch: when the initial + projection lands between the bound and response_max_bytes, the bounded + summary must return a valid envelope rather than raise.""" + payload = { + "success": True, + "kitchen": "open", + "version": "1.2.3", + "ingredients_table": "| a |", + "orchestration_rules": ["r1", "r2"], + "stop_step_semantics": {"on_success": "stop"}, + "errors": [], + "suggestions": [{"rule": "x"}], + "diagram": "graph TD; A-->B", + "content": "x" * 30_000, + } + original = json.dumps(payload) + result = enforce_response_budget( + original, + tool_name="open_kitchen", + artifact_dir=tmp_path, + config=OutputBudgetConfig(), + effective_delivery_token_limit=500, + ) + assert isinstance(result, str) + bound = 500 * 4 + assert len(result.encode("utf-8")) <= bound + data = json.loads(result) + assert data["delivery_bound_spill"] is True + metadata = data[RESPONSE_SPILL_METADATA_KEY] + assert metadata["reason"] == "delivery_bound" + assert data["success"] is True + assert data["kitchen"] == payload["kitchen"] + + +def test_delivery_bound_summary_drops_diagram_when_needed(tmp_path): + """When the bound is too small for even an empty content + diagram, + the summary must drop diagram while preserving the operational fields.""" + payload = { + "success": True, + "kitchen": "open", + "version": "1.2.3", + "ingredients_table": "| a |", + "orchestration_rules": ["r1", "r2"], + "stop_step_semantics": {"on_success": "stop"}, + "errors": [], + "suggestions": [], + "diagram": "D" * 3_000, + "content": "x" * 50_000, + } + original = json.dumps(payload) + result = enforce_response_budget( + original, + tool_name="open_kitchen", + artifact_dir=tmp_path, + config=OutputBudgetConfig(), + effective_delivery_token_limit=500, + ) + assert isinstance(result, str) + bound = 500 * 4 + assert len(result.encode("utf-8")) <= bound + data = json.loads(result) + assert "diagram" not in data + assert data["delivery_bound_spill"] is True + assert data["success"] is True + assert data["kitchen"] == payload["kitchen"] + metadata = data[RESPONSE_SPILL_METADATA_KEY] + assert metadata["reason"] == "delivery_bound" + + +def test_over_ceiling_payload_fails_even_when_over_delivery_bound(tmp_path): + """Pin restored ordering: an over-ceiling exempted payload must fail with + exemption_ceiling_exceeded, not route to delivery-bound spill.""" + payload = {"success": True, "content": "x" * 190_000} + original = json.dumps(payload) + result = enforce_response_budget( + original, + tool_name="open_kitchen", + artifact_dir=tmp_path, + config=OutputBudgetConfig(), + effective_delivery_token_limit=10_000, + ) + assert isinstance(result, str) + data = json.loads(result) + assert data["success"] is False + assert data["error"] == "response_budget_exemption_ceiling_exceeded" + assert RESPONSE_SPILL_METADATA_KEY not in data + + +def test_non_exempted_projection_capped_at_delivery_bound(tmp_path): + """REQ-023 pin: non-exempted projection must be capped at min( + response_max_bytes, effective_delivery_token_limit * 4).""" + payload = {"data": "y" * 150_000} + original = json.dumps(payload) + result = enforce_response_budget( + original, + tool_name="run_skill", + artifact_dir=tmp_path, + config=OutputBudgetConfig(), + effective_delivery_token_limit=500, + ) + assert isinstance(result, str) + bound = 500 * 4 + assert len(result.encode("utf-8")) <= bound + data = json.loads(result) + assert RESPONSE_SPILL_METADATA_KEY in data + + +def test_delivery_bound_summary_projects_oversized_preserved_fields(tmp_path): + """REQ-026/REQ-027 rung-4 pin: preserved fields must stay present even + when they alone exceed the bound — the ladder projects values, never drops + keys.""" + payload = { + "success": True, + "kitchen": "open", + "version": "1.2.3", + "ingredients_table": "R" * 60_000, + "orchestration_rules": ["r1"], + "stop_step_semantics": {"on_success": "stop"}, + "errors": [], + "suggestions": [], + "content": "x" * 40_000, + } + original = json.dumps(payload) + result = enforce_response_budget( + original, + tool_name="open_kitchen", + artifact_dir=tmp_path, + config=OutputBudgetConfig(), + effective_delivery_token_limit=10_000, + ) + assert isinstance(result, str) + bound = 10_000 * 4 + assert len(result.encode("utf-8")) <= bound + data = json.loads(result) + assert data["delivery_bound_spill"] is True + for key in ( + "success", + "kitchen", + "version", + "ingredients_table", + "orchestration_rules", + "stop_step_semantics", + "errors", + "suggestions", + ): + assert key in data, f"preserved key {key!r} missing" + assert isinstance(data["ingredients_table"], str) + assert len(data["ingredients_table"]) < 60_000 + metadata = data[RESPONSE_SPILL_METADATA_KEY] + assert metadata["reason"] == "delivery_bound" + assert set(metadata) == RESPONSE_SPILL_METADATA_KEYS + assert Path(metadata["artifact_path"]).read_text() == original