diff --git a/docs/decisions/0005-output-budget-protocol.md b/docs/decisions/0005-output-budget-protocol.md index 6da0e38c7..8af2d38c9 100644 --- a/docs/decisions/0005-output-budget-protocol.md +++ b/docs/decisions/0005-output-budget-protocol.md @@ -107,8 +107,13 @@ non-autoskillit repos, two lower-ceiling launch paths cap casual reads at ~40 KB The ceiling clamps regular-path exec/tool output via `min(model request, tool_output_token_limit)`, but code-mode models (gpt-5.6-sol) honor model-declared `max_output_tokens` unclamped — for those sessions the ceiling -is not a hard cap and the intake discipline digest's numeric rule -(`max_output_tokens` <= 10000) is the operative bound. +is not a hard cap and the operative bound is enforced server-side by +`BackendCapabilities.effective_delivery_token_limit` (Codex: 10,000 tokens). The +response backstop reads that capability at spill time and spills any payload whose +estimated token count exceeds it, with the artifact path surfaced via the existing +`_autoskillit_response_spill` envelope (`reason="delivery_bound"`). The intake +discipline digest's numeric rule (`max_output_tokens` <= 10000) remains as the +prompt-level reinforcement. ## Corrections of Record @@ -146,6 +151,16 @@ recorded explicitly in the issue body. bounded slices enter worker memory). Still open for `run_skill` and `test_check`, whose adjudication requires the full text. +## Resolved + +- **Code-mode bypass (`gpt-5.6-sol` honoring `max_output_tokens` unclamped)**: prior to + this revision, the `tool_output_token_limit` ceiling did not constrain code-mode + responses and the intake discipline digest's numeric rule was the only operative + bound — prompt-level and unenforced. The server-side response backstop now reads + `BackendCapabilities.effective_delivery_token_limit` (Codex: 10,000 tokens; + Claude Code: 46,500 tokens) and spills payloads that exceed it. This makes the + delivery bound authoritative on every backend, not advisory. + ## Operational Signals Output-budget instrumentation uses low-cardinality structured counters or events for: diff --git a/src/autoskillit/core/AGENTS.md b/src/autoskillit/core/AGENTS.md index 76525e6e8..8d02c3426 100644 --- a/src/autoskillit/core/AGENTS.md +++ b/src/autoskillit/core/AGENTS.md @@ -14,6 +14,7 @@ Sub-packages: types/ (see types/AGENTS.md) and runtime/ (see runtime/AGENTS.md). | `logging.py` | Logging configuration | | `paths.py` | `pkg_root()`, `is_git_worktree()`, `is_git_main_checkout()`, `is_in_git_repo()` | | `_claude_env.py` | IDE-scrubbing canonical env builder for agent subprocesses | +| `_delivery_bounds.py` | `resolve_effective_delivery_bound()` — canonical accessor for per-backend effective delivery token limits | | `_terminal_table.py` | IL-0 color-agnostic terminal table primitive | | `_version_snapshot.py` | Process-scoped version snapshot for session telemetry (`lru_cache`'d) | | `branch_guard.py` | Branch protection helpers | diff --git a/src/autoskillit/core/__init__.pyi b/src/autoskillit/core/__init__.pyi index ac3175e3c..9c0f39467 100644 --- a/src/autoskillit/core/__init__.pyi +++ b/src/autoskillit/core/__init__.pyi @@ -4,6 +4,7 @@ from ._cmd_runner import CmdRunner as CmdRunner from ._cmd_runner import default_cmd_runner as default_cmd_runner from ._cmd_runner import run_gh as run_gh from ._cmd_runner import run_git as run_git +from ._delivery_bounds import resolve_effective_delivery_bound as resolve_effective_delivery_bound from ._execution_marker import execution_marker as execution_marker from ._install_detect import DirectUrlInfo as DirectUrlInfo from ._install_detect import _is_release_tag as _is_release_tag diff --git a/src/autoskillit/core/_delivery_bounds.py b/src/autoskillit/core/_delivery_bounds.py new file mode 100644 index 000000000..ca7eb9c7c --- /dev/null +++ b/src/autoskillit/core/_delivery_bounds.py @@ -0,0 +1,19 @@ +"""Per-backend effective delivery-bound resolution. + +The configured ``tool_output_token_limit`` written to Codex ``config.toml`` +is a config-file ceiling. Code-mode models may bypass that ceiling when +they emit ``max_output_tokens`` without an upper bound, so the operative +bound for those paths is the harness default (~10K tokens). The +``BackendCapabilities.effective_delivery_token_limit`` field encodes this +worst-case operative bound; ``resolve_effective_delivery_bound`` is the +canonical accessor. +""" + +from __future__ import annotations + +from .types._type_backend import BackendCapabilities + + +def resolve_effective_delivery_bound(caps: BackendCapabilities) -> int: + """Return the worst-case operative token bound for ``caps``'s backend transport.""" + return caps.effective_delivery_token_limit diff --git a/src/autoskillit/core/types/_type_backend.py b/src/autoskillit/core/types/_type_backend.py index 4d9929ffd..3f231f062 100644 --- a/src/autoskillit/core/types/_type_backend.py +++ b/src/autoskillit/core/types/_type_backend.py @@ -162,6 +162,12 @@ class BackendCapabilities: # directory rather than written with gating frontmatter that the backend # would ignore. supports_model_invocation_gating: bool = True + # Worst-case delivery bound in tokens: the lowest operative bound that + # could apply to this backend's tool-output transport. Distinct from any + # config-file ceiling (e.g. `tool_output_token_limit`), which code-mode + # models may bypass. Used by `enforce_response_budget` to spill payloads + # the downstream transport cannot deliver at full size. + effective_delivery_token_limit: int = 0 ALL_PROJECT_LOCAL_SKILL_SEARCH_DIRS: tuple[str, ...] = ( @@ -291,6 +297,7 @@ def model_class(model: str) -> str: skill_sigil="/", session_dir_persistent=False, supports_model_invocation_gating=True, + effective_delivery_token_limit=46_500, ) diff --git a/src/autoskillit/execution/backends/codex.py b/src/autoskillit/execution/backends/codex.py index a6d3f91ae..54106f4c0 100644 --- a/src/autoskillit/execution/backends/codex.py +++ b/src/autoskillit/execution/backends/codex.py @@ -639,6 +639,7 @@ def capabilities(self) -> BackendCapabilities: skill_sigil="$", session_dir_persistent=True, supports_model_invocation_gating=False, + effective_delivery_token_limit=10_000, ) @property diff --git a/src/autoskillit/hooks/formatters/_fmt_response_spill.py b/src/autoskillit/hooks/formatters/_fmt_response_spill.py index bb41aa770..05eb801a2 100644 --- a/src/autoskillit/hooks/formatters/_fmt_response_spill.py +++ b/src/autoskillit/hooks/formatters/_fmt_response_spill.py @@ -24,7 +24,9 @@ "reason", } ) -_RESPONSE_SPILL_REASONS = frozenset({"oversized_values", "minimal_projection", "plain_text"}) +_RESPONSE_SPILL_REASONS = frozenset( + {"oversized_values", "minimal_projection", "plain_text", "delivery_bound"} +) _RESPONSE_SPILL_SCHEMA_DIGEST = hashlib.sha256( json.dumps( { diff --git a/src/autoskillit/server/_notify.py b/src/autoskillit/server/_notify.py index 7100fb602..d68b902bd 100644 --- a/src/autoskillit/server/_notify.py +++ b/src/autoskillit/server/_notify.py @@ -12,7 +12,12 @@ from anyio import ClosedResourceError as _ClosedResource from autoskillit.config import OutputBudgetConfig -from autoskillit.core import RESERVED_LOG_RECORD_KEYS, get_logger +from autoskillit.core import ( + RESERVED_LOG_RECORD_KEYS, + BackendCapabilities, + get_logger, + resolve_effective_delivery_bound, +) from autoskillit.server._response_budget import ( bounded_response_budget_failure, enforce_response_budget, @@ -137,12 +142,19 @@ async def wrapper(*args: Any, **kwargs: Any) -> Any: artifact_dir = ( temp_dir / "responses" / tool_name if isinstance(temp_dir, Path) else None ) + effective_delivery_token_limit: int | None = None + if ctx is not None: + backend = getattr(ctx, "backend", None) + caps = getattr(backend, "capabilities", None) if backend is not None else None + if isinstance(caps, BackendCapabilities): + effective_delivery_token_limit = resolve_effective_delivery_bound(caps) try: result = enforce_response_budget( result, tool_name=tool_name, artifact_dir=artifact_dir, config=budget, + effective_delivery_token_limit=effective_delivery_token_limit, ) except Exception: logger.error("track_response_size_enforcement_failed", tool_name=tool_name) diff --git a/src/autoskillit/server/_response_budget.py b/src/autoskillit/server/_response_budget.py index 11f28e12b..a67cc0d2c 100644 --- a/src/autoskillit/server/_response_budget.py +++ b/src/autoskillit/server/_response_budget.py @@ -34,7 +34,9 @@ "reason", } ) -RESPONSE_SPILL_REASONS = frozenset({"oversized_values", "minimal_projection", "plain_text"}) +RESPONSE_SPILL_REASONS = frozenset( + {"oversized_values", "minimal_projection", "plain_text", "delivery_bound"} +) RESPONSE_SPILL_SCHEMA_DIGEST = hashlib.sha256( json.dumps( { @@ -70,6 +72,16 @@ def _canonical_json(value: Any) -> str: return json.dumps(value, ensure_ascii=False, separators=(",", ":")) +def _estimated_tokens(original_size: int) -> int: + """Estimate tokens via the four-UTF-8-byte ceiling-division heuristic. + + Matches the ``CODEX_TOOL_OUTPUT_TOKEN_LIMIT`` derivation: a coarse + transport-layer estimate, not a tokenizer count. Used to compare + payload size against ``effective_delivery_token_limit``. + """ + return (original_size + 3) // 4 + + def _serialized(value: Any) -> str: if isinstance(value, str): return value @@ -324,6 +336,7 @@ def _plain_spill_envelope( metadata: dict[str, Any], max_bytes: int, inline_chars: int, + reason: str = "plain_text", ) -> str | None: preview_limit = max(0, min(inline_chars, max_bytes // 3)) while True: @@ -331,7 +344,7 @@ def _plain_spill_envelope( envelope: dict[str, Any] = { RESPONSE_SPILL_METADATA_KEY: _spill_metadata( metadata, - reason="plain_text", + reason=reason, omitted_chars=omitted_chars, omitted_items=0, ), @@ -345,6 +358,271 @@ def _plain_spill_envelope( preview_limit //= 2 +_DELIVERY_BOUND_PRESERVED_KEYS: tuple[str, ...] = ( + "success", + "kitchen", + "version", + "ingredients_table", + "orchestration_rules", + "stop_step_semantics", + "errors", + "suggestions", +) +_DELIVERY_BOUND_CONTENT_KEY = "content" +_DELIVERY_BOUND_DROPPABLE_KEYS: tuple[str, ...] = ("diagram",) + + +def _delivery_bound_summary( + parsed: dict[str, Any], + *, + metadata: dict[str, Any], + bound: int, +) -> str | None: + """Build a bounded inline summary honoring the delivery bound. + + Preserves ``success``, ``kitchen``, ``version``, ``ingredients_table``, + ``orchestration_rules``, ``stop_step_semantics``, ``errors``, + ``suggestions`` verbatim (when present and fitting); truncates ``content``; + drops ``diagram``; nests spill metadata under ``RESPONSE_SPILL_METADATA_KEY`` + with ``reason="delivery_bound"``; sets top-level ``delivery_bound_spill=True``. + + Returns the rendered envelope string when it fits ``bound``; otherwise + ``None`` so the caller fails closed. + """ + content_text = parsed.get(_DELIVERY_BOUND_CONTENT_KEY) + content_is_str = isinstance(content_text, str) + + precomputed_base_chars = 0 + precomputed_base_items = 0 + for key, value in parsed.items(): + if key in _DELIVERY_BOUND_PRESERVED_KEYS or key == _DELIVERY_BOUND_CONTENT_KEY: + continue + chars, items = _total_omissions(value) + precomputed_base_chars += chars + precomputed_base_items += items + + if not content_is_str and _DELIVERY_BOUND_CONTENT_KEY in parsed: + chars, items = _total_omissions(parsed[_DELIVERY_BOUND_CONTENT_KEY]) + precomputed_base_chars += chars + precomputed_base_items += items + + present_preserved = [key for key in _DELIVERY_BOUND_PRESERVED_KEYS if key in parsed] + + def _build(head_len: int, include_droppable: bool, value_projector=None) -> dict[str, Any]: + envelope: dict[str, Any] = {RESPONSE_SPILL_METADATA_KEY: dict(metadata)} + envelope["delivery_bound_spill"] = True + + base_chars = precomputed_base_chars + base_items = precomputed_base_items + + for key in present_preserved: + if value_projector is None: + envelope[key] = parsed[key] + else: + envelope[key] = value_projector(parsed[key]) + + if include_droppable: + for key in _DELIVERY_BOUND_DROPPABLE_KEYS: + if key in parsed: + envelope[key] = parsed[key] + else: + for key in _DELIVERY_BOUND_DROPPABLE_KEYS: + if key in parsed: + chars, items = _total_omissions(parsed[key]) + base_chars += chars + base_items += items + + if content_is_str: + text = content_text or "" + truncated = text[:head_len] + envelope[_DELIVERY_BOUND_CONTENT_KEY] = truncated + base_chars += max(0, len(text) - head_len) + elif _DELIVERY_BOUND_CONTENT_KEY in parsed: + envelope[_DELIVERY_BOUND_CONTENT_KEY] = parsed[_DELIVERY_BOUND_CONTENT_KEY] + + if value_projector is not None: + for key in present_preserved: + projected_value = envelope[key] + if ( + isinstance(projected_value, dict) + and RESPONSE_SPILL_METADATA_KEY in projected_value + ): + continue + chars, items = _total_omissions(parsed[key]) + base_chars += chars + base_items += items + + envelope[RESPONSE_SPILL_METADATA_KEY] = _spill_metadata( + envelope[RESPONSE_SPILL_METADATA_KEY], + reason="delivery_bound", + omitted_chars=base_chars, + omitted_items=base_items, + ) + return envelope + + def _fits(envelope: dict[str, Any]) -> str | None: + try: + rendered = _finalize_envelope(envelope, max_bytes=bound) + except _ProjectionNonconvergentError: + return None + if len(rendered.encode("utf-8")) <= bound: + return rendered + return None + + zero_envelope = _build(0, True) + base_rendered = _finalize_envelope(zero_envelope, max_bytes=bound) + base_bytes = len(base_rendered.encode("utf-8")) + head_limit = max(0, bound - base_bytes - 64) if content_is_str else 0 + + while head_limit > 0 and content_is_str: + rendered = _fits(_build(head_limit, True)) + if rendered is not None: + return rendered + head_limit //= 2 + + if content_is_str: + zero_head_envelope = _build(0, True) + rendered = _fits(zero_head_envelope) + if rendered is not None: + return rendered + + rendered = _fits(_build(0, False)) + if rendered is not None: + return rendered + + if present_preserved: + value_limit = max(16, bound // (len(present_preserved) + 2)) + while value_limit >= 16: + + def _project_with_limit(value: Any, _limit: int = value_limit) -> Any: + projected, _chars, _items = _project_value(value, _limit) + return projected + + rendered = _fits(_build(0, False, value_projector=_project_with_limit)) + if rendered is not None: + return rendered + value_limit //= 2 + + minimal_envelope = _build(0, False, value_projector=_minimal_same_type) + rendered = _fits(minimal_envelope) + if rendered is not None: + return rendered + + return None + + +def _spill_for_delivery_bound( + result: Any, + *, + tool_name: str, + config: OutputBudgetConfig, + artifact_dir: Path | None, + original: str, + original_size: int, + effective_delivery_token_limit: int, +) -> Any: + """Persist ``original`` and return a bounded projection honoring the delivery bound. + + Used when an exempted or under-byte-budget payload still exceeds the + downstream backend's effective delivery token limit. Mirrors the + non-exempted spill machinery (atomic_write, _artifact_path, + _project_json_object) so the caller sees the same envelope shape, with + ``reason="delivery_bound"`` so downstream formatters distinguish it. + """ + if artifact_dir is None: + return bounded_response_budget_failure( + result, + cause="context_unavailable", + tool_name=tool_name, + max_bytes=config.response_max_bytes, + original_utf8_bytes=original_size, + ) + path = _artifact_path(artifact_dir, tool_name) + try: + atomic_write(path, original) + except OSError: + return bounded_response_budget_failure( + result, + cause="artifact_publication_failed", + tool_name=tool_name, + max_bytes=config.response_max_bytes, + original_utf8_bytes=original_size, + ) + published = str(path.resolve()) + metadata: dict[str, Any] = { + "artifact_path": published, + "sha256": hashlib.sha256(original.encode("utf-8")).hexdigest(), + "original_utf8_bytes": original_size, + "reason": "delivery_bound", + } + parsed: Any + if isinstance(result, str): + try: + parsed = json.loads(result) + except (TypeError, ValueError, RecursionError): + parsed = None + else: + parsed = result + bound = effective_delivery_token_limit * 4 + floor_bytes = min(bound, config.response_max_bytes) + rendered: str | None + try: + if isinstance(parsed, dict): + rendered = _delivery_bound_summary(parsed, metadata=metadata, bound=bound) + else: + rendered = _plain_spill_envelope( + original, + metadata=metadata, + max_bytes=floor_bytes, + inline_chars=config.inline_max_chars, + reason="delivery_bound", + ) + except _ProjectionNonconvergentError as exc: + _emit_response_budget_event( + "response_budget_projection_nonconvergent", + tool_name=_bounded_tool_name(tool_name), + detail=str(exc), + ) + return bounded_response_budget_failure( + "", + cause="projection_nonconvergent", + tool_name=tool_name, + max_bytes=floor_bytes, + original_utf8_bytes=original_size, + artifact_path=published, + ) + except RecursionError: + return bounded_response_budget_failure( + "", + cause="irreducible_shape", + tool_name=tool_name, + max_bytes=floor_bytes, + original_utf8_bytes=original_size, + artifact_path=published, + ) + if rendered is None: + return bounded_response_budget_failure( + "", + cause="irreducible_shape", + tool_name=tool_name, + max_bytes=floor_bytes, + original_utf8_bytes=original_size, + artifact_path=published, + ) + _emit_response_budget_event( + "response_budget_spill", + tool_name=_bounded_tool_name(tool_name), + original_utf8_bytes=original_size, + projected_utf8_bytes=len(rendered.encode("utf-8")), + ) + if isinstance(result, str): + return rendered + try: + return json.loads(rendered) + except (ValueError, RecursionError): + return {"success": False, "error": "response_budget_projection_invalid"} + + def enforce_response_budget( result: Any, *, @@ -352,12 +630,19 @@ def enforce_response_budget( artifact_dir: Path | None, config: OutputBudgetConfig, force_spill: bool = False, + effective_delivery_token_limit: int | None = None, ) -> Any: """Return a bounded response of the same handler type. Oversized content is atomically persisted before a projection is returned. Artifact failure and missing-context cases fail closed without echoing the original payload. + + ``effective_delivery_token_limit`` is the worst-case operative bound on the + downstream transport (e.g. Codex code-mode default ~10K). When set, payloads + whose estimated token count exceeds it are spilled even if they pass the + server-side exemption or response-byte ceilings, because the transport + cannot deliver them at full size. """ try: original = _serialized(result) @@ -371,27 +656,50 @@ def enforce_response_budget( ) original_bytes = original.encode("utf-8") original_size = len(original_bytes) + over_delivery_bound = ( + effective_delivery_token_limit is not None + and effective_delivery_token_limit > 0 + and _estimated_tokens(original_size) > effective_delivery_token_limit + ) exemption = RESPONSE_BACKSTOP_EXEMPTION_REGISTRY.get(tool_name) if exemption is not None: - if len(original) <= exemption.max_chars and original_size <= exemption.max_utf8_bytes: - _emit_response_budget_event( - "response_budget_exemption", - tool_name=_bounded_tool_name(tool_name), - measurement_id=exemption.measurement_id, - original_chars=len(original), + within_ceiling = ( + len(original) <= exemption.max_chars and original_size <= exemption.max_utf8_bytes + ) + if not within_ceiling: + return bounded_response_budget_failure( + result, + cause="exemption_ceiling_exceeded", + tool_name=tool_name, + max_bytes=config.response_max_bytes, original_utf8_bytes=original_size, - max_chars=exemption.max_chars, - max_utf8_bytes=exemption.max_utf8_bytes, ) - return result - return bounded_response_budget_failure( - result, - cause="exemption_ceiling_exceeded", - tool_name=tool_name, - max_bytes=config.response_max_bytes, + if over_delivery_bound: + assert effective_delivery_token_limit is not None # narrowed by over_delivery_bound + return _spill_for_delivery_bound( + result, + tool_name=tool_name, + config=config, + artifact_dir=artifact_dir, + original=original, + original_size=original_size, + effective_delivery_token_limit=effective_delivery_token_limit, + ) + _emit_response_budget_event( + "response_budget_exemption", + tool_name=_bounded_tool_name(tool_name), + measurement_id=exemption.measurement_id, + original_chars=len(original), original_utf8_bytes=original_size, + max_chars=exemption.max_chars, + max_utf8_bytes=exemption.max_utf8_bytes, ) - if not force_spill and len(original_bytes) <= config.response_max_bytes: + return result + if ( + not force_spill + and not over_delivery_bound + and len(original_bytes) <= config.response_max_bytes + ): return result if artifact_dir is None: return bounded_response_budget_failure( @@ -421,6 +729,18 @@ def enforce_response_budget( "original_utf8_bytes": len(original_bytes), } + delivery_bound_bytes = ( + effective_delivery_token_limit * 4 + if effective_delivery_token_limit is not None and effective_delivery_token_limit > 0 + else None + ) + projection_max_bytes = ( + min(config.response_max_bytes, delivery_bound_bytes) + if delivery_bound_bytes is not None + else config.response_max_bytes + ) + projection_inline_chars = min(config.inline_max_chars, projection_max_bytes) + parsed: Any = None if isinstance(result, str): try: @@ -436,15 +756,15 @@ def enforce_response_budget( rendered = _project_json_object( parsed, metadata=metadata, - max_bytes=config.response_max_bytes, - inline_chars=config.inline_max_chars, + max_bytes=projection_max_bytes, + inline_chars=projection_inline_chars, ) if rendered is None and not isinstance(parsed, dict): rendered = _plain_spill_envelope( original, metadata=metadata, - max_bytes=config.response_max_bytes, - inline_chars=config.inline_max_chars, + max_bytes=projection_max_bytes, + inline_chars=projection_inline_chars, ) except _ProjectionNonconvergentError as exc: _emit_response_budget_event( @@ -456,7 +776,7 @@ def enforce_response_budget( "", cause="projection_nonconvergent", tool_name=tool_name, - max_bytes=config.response_max_bytes, + max_bytes=projection_max_bytes, original_utf8_bytes=original_size, artifact_path=published, ) @@ -467,7 +787,7 @@ def enforce_response_budget( "", cause="irreducible_shape", tool_name=tool_name, - max_bytes=config.response_max_bytes, + max_bytes=projection_max_bytes, original_utf8_bytes=original_size, artifact_path=published, ) @@ -476,11 +796,12 @@ def enforce_response_budget( "", cause="irreducible_shape", tool_name=tool_name, - max_bytes=config.response_max_bytes, + max_bytes=projection_max_bytes, original_utf8_bytes=original_size, artifact_path=published, ) + assert isinstance(rendered, str) _emit_response_budget_event( "response_budget_spill", tool_name=_bounded_tool_name(tool_name), @@ -501,6 +822,7 @@ def shape_json_response( tool_name: str, artifact_dir: Path, config: OutputBudgetConfig, + effective_delivery_token_limit: int | None = None, ) -> str: """Serialize a tool dict, spilling once it crosses the source threshold.""" rendered = json.dumps(payload) @@ -512,6 +834,7 @@ def shape_json_response( artifact_dir=artifact_dir, config=config, force_spill=True, + effective_delivery_token_limit=effective_delivery_token_limit, ) return shaped if isinstance(shaped, str) else json.dumps(shaped) diff --git a/src/autoskillit/server/tools/_execution_helpers.py b/src/autoskillit/server/tools/_execution_helpers.py index 4f620f9f9..405587b17 100644 --- a/src/autoskillit/server/tools/_execution_helpers.py +++ b/src/autoskillit/server/tools/_execution_helpers.py @@ -14,10 +14,12 @@ from autoskillit.core import ( RUN_PYTHON_SENTINEL_KEYS, + BackendCapabilities, CapturedStream, SpillSpec, SubprocessResult, get_logger, + resolve_effective_delivery_bound, resolve_temp_dir, spill_output, ) @@ -201,11 +203,18 @@ def shape_execution_response( if work_dir and Path(work_dir).is_absolute() else tool_ctx.temp_dir / tool_name ) + effective_delivery_token_limit: int | None = None + backend = getattr(tool_ctx, "backend", None) + caps = getattr(backend, "capabilities", None) if backend is not None else None + + if isinstance(caps, BackendCapabilities): + effective_delivery_token_limit = resolve_effective_delivery_bound(caps) return shape_json_response( payload, tool_name=tool_name, artifact_dir=artifact_root, config=tool_ctx.config.output_budget, + effective_delivery_token_limit=effective_delivery_token_limit, ) diff --git a/tests/_test_filter.py b/tests/_test_filter.py index 3f4f32fb7..7a53d3eeb 100644 --- a/tests/_test_filter.py +++ b/tests/_test_filter.py @@ -265,6 +265,7 @@ class ImportContext(enum.StrEnum): "_step_context": frozenset({"core", "execution", "pipeline", "server"}), "_execution_marker": frozenset({"core", "execution", "fleet", "server"}), "bash_write_targets": frozenset({"core", "execution", "server"}), + "_delivery_bounds": frozenset({"core", "execution", "server"}), "_type_closure_report": frozenset({"core"}), "closure_hashing": frozenset({"core"}), "path_containment": frozenset({"core"}), diff --git a/tests/arch/test_subpackage_isolation.py b/tests/arch/test_subpackage_isolation.py index d6bfaf483..0c4542064 100644 --- a/tests/arch/test_subpackage_isolation.py +++ b/tests/arch/test_subpackage_isolation.py @@ -884,7 +884,7 @@ def test_no_subpackage_exceeds_10_files() -> None: "server": 14, "recipe": 42, # was 33; +9 from CI/graph/dataflow splits "execution": 18, - "core": 24, # +closure_hashing +path_containment +closure_verifier + "core": 25, # +_delivery_bounds (resolve_effective_delivery_bound) "core/types": 32, # +invariant_registry (INVARIANT_REGISTRY) +closure_report "cli": 21, "hooks": 18, # +recipe_confirmed_post_hook, +quota_guard_state_post_hook, +_policy_event, +shell_capture_hook (#4286), +_capture_cleanup.py # noqa: E501 diff --git a/tests/contracts/AGENTS.md b/tests/contracts/AGENTS.md index 2a55d46d1..8ae024b79 100644 --- a/tests/contracts/AGENTS.md +++ b/tests/contracts/AGENTS.md @@ -51,6 +51,7 @@ Protocol satisfaction, package gateway, and skill contract compliance tests. | `test_mermaid_palette_contracts.py` | Contract: any SKILL.md that generates mermaid diagrams must embed the canonical 9-class palette | | `test_no_pagination_file_read.py` | Contract tests for no-pagination file read instruction in high-turn SKILL.md files | | `test_output_budget_discipline.py` | Cross-skill contract: canonical output-discipline policy identity, cohort parity, and marker ownership | +| `test_delivery_bound_fitness.py` | Contract: bundled-recipe and non-exempted payload fitness against per-backend effective delivery bounds | | `test_package_gateways.py` | Tests for Package Gateway API (groupC) — REQ-GWAY-001 through REQ-GWAY-008 | | `test_plan_experiment_contracts.py` | Contract tests for plan-experiment SKILL.md — data provenance lifecycle | | `test_plan_visualization_contracts.py` | Contract tests: select-vis-lenses Tier B experiment-type canonical names match registry | diff --git a/tests/contracts/test_delivery_bound_fitness.py b/tests/contracts/test_delivery_bound_fitness.py new file mode 100644 index 000000000..010681005 --- /dev/null +++ b/tests/contracts/test_delivery_bound_fitness.py @@ -0,0 +1,143 @@ +"""Bundled-recipe and non-exempted payload fitness against per-backend delivery bounds. + +Regression guard: every bundled recipe's ``open_kitchen`` payload must either +fit each backend's effective delivery bound, or be properly spilled by the +server-side response backstop. Likewise, non-exempted payloads that fit +``response_max_bytes`` but exceed a backend's effective delivery bound must +route to spill-and-project and produce a projection under the bound. + +Covers plan Step 4 (real bundled-recipe payload fitness via +``load_and_validate`` + ``build_open_kitchen_recipe_payload``) and Step 5 +(non-exempted spill projection size). +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from autoskillit.config import OutputBudgetConfig +from autoskillit.core import RESPONSE_BACKSTOP_EXEMPTION_REGISTRY, resolve_effective_delivery_bound +from autoskillit.execution.backends import BACKEND_REGISTRY +from autoskillit.recipe import all_validated_recipe_names, load_and_validate +from autoskillit.server._response_budget import ( + RESPONSE_SPILL_METADATA_KEY, + enforce_response_budget, +) +from autoskillit.server.tools._serve_helpers import build_open_kitchen_recipe_payload + +pytestmark = [pytest.mark.layer("contracts"), pytest.mark.small] + + +_PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent + + +def _recipe_names() -> list[str]: + return sorted(all_validated_recipe_names(_PROJECT_ROOT)) + + +def _backend_capabilities(): + return {name: cls().capabilities for name, cls in BACKEND_REGISTRY.items()} + + +def _effective_bound_bytes(bound_tokens: int) -> int: + """Convert effective delivery token bound to UTF-8 byte ceiling (4 bytes/token).""" + return bound_tokens * 4 + + +def _full_open_kitchen_payload(recipe_name: str) -> dict[str, object]: + """Build the production-shape ``open_kitchen`` payload for ``recipe_name``. + + Uses ``load_and_validate`` + ``build_open_kitchen_recipe_payload`` so the + payload mirrors what ``open_kitchen`` returns at runtime: the routing + envelope injected by the production helper plus the full recipe body from + the bundled YAML. The fit-or-spill assertion below exercises the spill + path for any bundled recipe whose serialized payload outgrows a backend's + bound. + """ + result = load_and_validate( + recipe_name, + project_dir=_PROJECT_ROOT, + ingredient_overrides={ + "task": "test task", + "issue_url": "https://github.com/test/test/issues/1", + "source_dir": str(_PROJECT_ROOT), + }, + ) + return build_open_kitchen_recipe_payload(dict(result), version="0.0.0") + + +@pytest.mark.parametrize("recipe_name", _recipe_names(), ids=lambda n: n) +def test_bundled_recipe_open_kitchen_fits_or_spills_per_backend( + recipe_name: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """For each backend, the ``open_kitchen`` payload either fits the + effective delivery bound or spills to a projection that fits.""" + monkeypatch.chdir(tmp_path) + payload = _full_open_kitchen_payload(recipe_name) + serialized = json.dumps(payload) + serialized_bytes = len(serialized.encode("utf-8")) + for backend_name, caps in _backend_capabilities().items(): + bound_tokens = resolve_effective_delivery_bound(caps) + bound_bytes = _effective_bound_bytes(bound_tokens) + if serialized_bytes <= bound_bytes: + continue + result = enforce_response_budget( + serialized, + tool_name="open_kitchen", + artifact_dir=tmp_path / backend_name, + config=OutputBudgetConfig(), + effective_delivery_token_limit=bound_tokens, + ) + assert isinstance(result, str), ( + f"{backend_name}: expected str result for {recipe_name} payload" + ) + assert len(result.encode("utf-8")) <= bound_bytes, ( + f"{backend_name}: projection for {recipe_name} exceeds " + f"{bound_bytes} bytes (effective delivery bound)" + ) + data = json.loads(result) + assert data.get("delivery_bound_spill") is True + metadata = data[RESPONSE_SPILL_METADATA_KEY] + assert metadata["reason"] == "delivery_bound" + + +def test_non_exempted_oversized_payload_spills_within_delivery_bound(tmp_path) -> None: + """A non-exempted payload sized between ``response_max_bytes`` and a + backend's effective delivery bound must spill and produce a projection + that fits the bound.""" + payload = {f"key_{index:03d}": "y" * 5_000 for index in range(60)} + serialized = json.dumps(payload) + config = OutputBudgetConfig() + for backend_name, caps in _backend_capabilities().items(): + bound_tokens = resolve_effective_delivery_bound(caps) + bound_bytes = _effective_bound_bytes(bound_tokens) + assert len(serialized.encode("utf-8")) > bound_bytes, ( + f"{backend_name}: payload does not exceed bound ({bound_bytes} bytes)" + ) + result = enforce_response_budget( + serialized, + tool_name="run_skill", + artifact_dir=tmp_path / backend_name, + config=config, + effective_delivery_token_limit=bound_tokens, + ) + assert isinstance(result, str) + assert len(result.encode("utf-8")) <= bound_bytes, ( + f"{backend_name}: projection exceeds {bound_bytes} bytes" + ) + data = json.loads(result) + assert RESPONSE_SPILL_METADATA_KEY in data + + +def test_exemption_registry_tools_have_measured_ceiling() -> None: + """Every tool in the exemption registry has a measured ceiling; this + sanity check guards against future additions of exempted tools without a + measured identity.""" + assert RESPONSE_BACKSTOP_EXEMPTION_REGISTRY, "registry must not be empty" + for tool_name, definition in RESPONSE_BACKSTOP_EXEMPTION_REGISTRY.items(): + assert definition.max_chars > 0 + assert definition.max_utf8_bytes > 0 + assert definition.measurement_id, f"{tool_name} has no measurement_id" diff --git a/tests/contracts/test_output_budget_discipline.py b/tests/contracts/test_output_budget_discipline.py index 2ba23f265..802493564 100644 --- a/tests/contracts/test_output_budget_discipline.py +++ b/tests/contracts/test_output_budget_discipline.py @@ -104,3 +104,20 @@ def test_intake_digest_pins_numeric_rules() -> None: def test_intake_digest_is_safe_for_agent_toml_multiline_literal() -> None: assert "'''" not in CODEX_INTAKE_DISCIPLINE_DIGEST + + +def test_intake_digest_numeric_rule_matches_codex_effective_delivery_bound() -> None: + """The intake digest's ``max_output_tokens`` numeric rule must be at least + the Codex backend's effective delivery bound — the operative bound the + server-side response backstop applies. If the bound changes, the digest + text must be updated to reflect it (this is the regression guard).""" + from autoskillit.execution.backends import BACKEND_REGISTRY + + codex_caps = BACKEND_REGISTRY["codex"]().capabilities + codex_effective_bound = codex_caps.effective_delivery_token_limit + assert codex_effective_bound > 0 + assert f"max_output_tokens above {codex_effective_bound}" in CODEX_INTAKE_DISCIPLINE_DIGEST, ( + "CODEX_INTAKE_DISCIPLINE_DIGEST numeric rule must reference the Codex " + f"effective_delivery_token_limit ({codex_effective_bound}); update " + "the digest if the bound changed." + ) diff --git a/tests/core/test_backend_capabilities.py b/tests/core/test_backend_capabilities.py index 1225d1e1b..0c6e8cdba 100644 --- a/tests/core/test_backend_capabilities.py +++ b/tests/core/test_backend_capabilities.py @@ -177,6 +177,7 @@ def test_backend_capabilities_field_names_locked(): "session_dir_persistent", "supports_model_invocation_gating", "github_api_callable", + "effective_delivery_token_limit", } actual = {f.name for f in dataclasses.fields(BackendCapabilities)} assert actual == expected diff --git a/tests/execution/backends/test_codex_config.py b/tests/execution/backends/test_codex_config.py index d46f5e1ca..1b49a4100 100644 --- a/tests/execution/backends/test_codex_config.py +++ b/tests/execution/backends/test_codex_config.py @@ -45,6 +45,19 @@ def test_response_backstop_fires_below_codex_transport_ceiling() -> None: assert OutputBudgetConfig().response_max_bytes // 3 < CODEX_TOOL_OUTPUT_TOKEN_LIMIT +def test_resolve_effective_delivery_bound_per_backend() -> None: + from autoskillit.core import BackendCapabilities, resolve_effective_delivery_bound + from autoskillit.execution.backends import BACKEND_REGISTRY + + expected = {"codex": 10_000, "claude-code": 46_500} + observed: dict[str, int] = {} + for name, cls in BACKEND_REGISTRY.items(): + caps = cls().capabilities + assert isinstance(caps, BackendCapabilities) + observed[name] = resolve_effective_delivery_bound(caps) + assert observed == expected + + class TestReadCodexConfig: def test_returns_empty_dict_when_path_missing(self, tmp_path): result = _read_codex_config(tmp_path / "nonexistent.toml") diff --git a/tests/execution/backends/test_coding_agent_backend_conformance.py b/tests/execution/backends/test_coding_agent_backend_conformance.py index b5430e482..3d446ca4d 100644 --- a/tests/execution/backends/test_coding_agent_backend_conformance.py +++ b/tests/execution/backends/test_coding_agent_backend_conformance.py @@ -45,6 +45,7 @@ "channel_b_capable": "OPTIONAL", "completion_record_types": "REQUIRED", "default_skill_sandbox_mode": "REQUIRED", + "effective_delivery_token_limit": "REQUIRED", "env_denylist_prefixes": "REQUIRED", "exit_code_is_terminal": "REQUIRED", "food_truck_capable": "OPTIONAL", @@ -113,9 +114,10 @@ def test_capabilities_returns_backend_capabilities(self) -> None: """BackendCapabilities contract — exercises multiple fields. Fields cited: applicable_guards, default_skill_sandbox_mode, - git_metadata_writable, has_unguarded_filesystem_access, - process_name_aliases, project_local_skills_capable, record_capable, - replay_capable, session_dir_persistent, supports_context_window_suffix, + effective_delivery_token_limit, git_metadata_writable, + has_unguarded_filesystem_access, process_name_aliases, + project_local_skills_capable, record_capable, replay_capable, + session_dir_persistent, supports_context_window_suffix, supports_tool_list_changed, triage_capable, write_detection_strategy. """ assert isinstance(self.backend.capabilities, BackendCapabilities) diff --git a/tests/server/test_response_backstop.py b/tests/server/test_response_backstop.py index 777c72055..44e2f1911 100644 --- a/tests/server/test_response_backstop.py +++ b/tests/server/test_response_backstop.py @@ -469,3 +469,229 @@ def test_plain_text_irreducible_shape_returns_failure(tmp_path): assert isinstance(result, str) data = json.loads(result) assert data.get("success") is False + + +def test_exempted_payload_spills_when_over_delivery_bound(tmp_path): + """An exempted payload that fits the exemption byte ceiling but exceeds the + backend's effective delivery token limit must be spilled, not passed through.""" + payload = {"success": True, "data": "x" * 80_000} + original = json.dumps(payload) + result = enforce_response_budget( + original, + tool_name="open_kitchen", + artifact_dir=tmp_path, + config=_config(), + effective_delivery_token_limit=10_000, + ) + assert isinstance(result, str) + bound = 10_000 * 4 + assert len(result.encode("utf-8")) <= bound + data = json.loads(result) + assert data["success"] is True + assert data["delivery_bound_spill"] is True + metadata = data[RESPONSE_SPILL_METADATA_KEY] + artifact_path = metadata["artifact_path"] + assert Path(artifact_path).read_text() == original + assert metadata["reason"] == "delivery_bound" + + +def test_delivery_bound_summary_preserves_operational_fields(tmp_path): + """Bounded summary must preserve success/kitchen/version/ingredients_table/ + orchestration_rules/stop_step_semantics/errors/suggestions verbatim, + truncate content to fit, and nest spill metadata with reason='delivery_bound' + and top-level delivery_bound_spill=True.""" + payload = { + "success": True, + "kitchen": "open", + "version": "1.2.3", + "ingredients_table": "| a |", + "orchestration_rules": ["r1", "r2"], + "stop_step_semantics": {"on_success": "stop"}, + "errors": [], + "suggestions": [{"rule": "x"}], + "diagram": "graph TD; A-->B", + "content": "x" * 150_000, + } + original = json.dumps(payload) + result = enforce_response_budget( + original, + tool_name="open_kitchen", + artifact_dir=tmp_path, + config=OutputBudgetConfig(), + effective_delivery_token_limit=10_000, + ) + assert isinstance(result, str) + bound = 10_000 * 4 + assert len(result.encode("utf-8")) <= bound + data = json.loads(result) + assert data["delivery_bound_spill"] is True + assert data["success"] == payload["success"] + assert data["kitchen"] == payload["kitchen"] + assert data["version"] == payload["version"] + assert data["ingredients_table"] == payload["ingredients_table"] + assert data["orchestration_rules"] == payload["orchestration_rules"] + assert data["stop_step_semantics"] == payload["stop_step_semantics"] + assert data["errors"] == payload["errors"] + assert data["suggestions"] == payload["suggestions"] + assert data["diagram"] == payload["diagram"] + assert data["content"].startswith("x") + assert payload["content"].startswith(data["content"]) + metadata = data[RESPONSE_SPILL_METADATA_KEY] + assert metadata["reason"] == "delivery_bound" + assert set(metadata) == RESPONSE_SPILL_METADATA_KEYS + assert Path(metadata["artifact_path"]).read_text() == original + + +def test_delivery_bound_summary_small_bound_no_exception(tmp_path): + """Regression guard for the REQ-026 fallback branch: when the initial + projection lands between the bound and response_max_bytes, the bounded + summary must return a valid envelope rather than raise.""" + payload = { + "success": True, + "kitchen": "open", + "version": "1.2.3", + "ingredients_table": "| a |", + "orchestration_rules": ["r1", "r2"], + "stop_step_semantics": {"on_success": "stop"}, + "errors": [], + "suggestions": [{"rule": "x"}], + "diagram": "graph TD; A-->B", + "content": "x" * 30_000, + } + original = json.dumps(payload) + result = enforce_response_budget( + original, + tool_name="open_kitchen", + artifact_dir=tmp_path, + config=OutputBudgetConfig(), + effective_delivery_token_limit=500, + ) + assert isinstance(result, str) + bound = 500 * 4 + assert len(result.encode("utf-8")) <= bound + data = json.loads(result) + assert data["delivery_bound_spill"] is True + metadata = data[RESPONSE_SPILL_METADATA_KEY] + assert metadata["reason"] == "delivery_bound" + assert data["success"] is True + assert data["kitchen"] == payload["kitchen"] + + +def test_delivery_bound_summary_drops_diagram_when_needed(tmp_path): + """When the bound is too small for even an empty content + diagram, + the summary must drop diagram while preserving the operational fields.""" + payload = { + "success": True, + "kitchen": "open", + "version": "1.2.3", + "ingredients_table": "| a |", + "orchestration_rules": ["r1", "r2"], + "stop_step_semantics": {"on_success": "stop"}, + "errors": [], + "suggestions": [], + "diagram": "D" * 3_000, + "content": "x" * 50_000, + } + original = json.dumps(payload) + result = enforce_response_budget( + original, + tool_name="open_kitchen", + artifact_dir=tmp_path, + config=OutputBudgetConfig(), + effective_delivery_token_limit=500, + ) + assert isinstance(result, str) + bound = 500 * 4 + assert len(result.encode("utf-8")) <= bound + data = json.loads(result) + assert "diagram" not in data + assert data["delivery_bound_spill"] is True + assert data["success"] is True + assert data["kitchen"] == payload["kitchen"] + metadata = data[RESPONSE_SPILL_METADATA_KEY] + assert metadata["reason"] == "delivery_bound" + + +def test_over_ceiling_payload_fails_even_when_over_delivery_bound(tmp_path): + """Pin restored ordering: an over-ceiling exempted payload must fail with + exemption_ceiling_exceeded, not route to delivery-bound spill.""" + payload = {"success": True, "content": "x" * 190_000} + original = json.dumps(payload) + result = enforce_response_budget( + original, + tool_name="open_kitchen", + artifact_dir=tmp_path, + config=OutputBudgetConfig(), + effective_delivery_token_limit=10_000, + ) + assert isinstance(result, str) + data = json.loads(result) + assert data["success"] is False + assert data["error"] == "response_budget_exemption_ceiling_exceeded" + assert RESPONSE_SPILL_METADATA_KEY not in data + + +def test_non_exempted_projection_capped_at_delivery_bound(tmp_path): + """REQ-023 pin: non-exempted projection must be capped at min( + response_max_bytes, effective_delivery_token_limit * 4).""" + payload = {"data": "y" * 150_000} + original = json.dumps(payload) + result = enforce_response_budget( + original, + tool_name="run_skill", + artifact_dir=tmp_path, + config=OutputBudgetConfig(), + effective_delivery_token_limit=500, + ) + assert isinstance(result, str) + bound = 500 * 4 + assert len(result.encode("utf-8")) <= bound + data = json.loads(result) + assert RESPONSE_SPILL_METADATA_KEY in data + + +def test_delivery_bound_summary_projects_oversized_preserved_fields(tmp_path): + """REQ-026/REQ-027 rung-4 pin: preserved fields must stay present even + when they alone exceed the bound — the ladder projects values, never drops + keys.""" + payload = { + "success": True, + "kitchen": "open", + "version": "1.2.3", + "ingredients_table": "R" * 60_000, + "orchestration_rules": ["r1"], + "stop_step_semantics": {"on_success": "stop"}, + "errors": [], + "suggestions": [], + "content": "x" * 40_000, + } + original = json.dumps(payload) + result = enforce_response_budget( + original, + tool_name="open_kitchen", + artifact_dir=tmp_path, + config=OutputBudgetConfig(), + effective_delivery_token_limit=10_000, + ) + assert isinstance(result, str) + bound = 10_000 * 4 + assert len(result.encode("utf-8")) <= bound + data = json.loads(result) + assert data["delivery_bound_spill"] is True + for key in ( + "success", + "kitchen", + "version", + "ingredients_table", + "orchestration_rules", + "stop_step_semantics", + "errors", + "suggestions", + ): + assert key in data, f"preserved key {key!r} missing" + assert isinstance(data["ingredients_table"], str) + assert len(data["ingredients_table"]) < 60_000 + metadata = data[RESPONSE_SPILL_METADATA_KEY] + assert metadata["reason"] == "delivery_bound" + assert set(metadata) == RESPONSE_SPILL_METADATA_KEYS + assert Path(metadata["artifact_path"]).read_text() == original diff --git a/tests/test_test_filter_core_cascade.py b/tests/test_test_filter_core_cascade.py index d41de6d03..a3f4413a0 100644 --- a/tests/test_test_filter_core_cascade.py +++ b/tests/test_test_filter_core_cascade.py @@ -119,6 +119,7 @@ def test_all_entries_present(self) -> None: "closure_hashing", "path_containment", "closure_verifier", + "_delivery_bounds", } assert set(MODULE_CASCADE_CORE.keys()) == expected_stems