From be504866a30453826bb8af02231b68d43001ffd5 Mon Sep 17 00:00:00 2001 From: Shane Vitarana Date: Wed, 19 Aug 2026 12:48:08 -0400 Subject: [PATCH] Streamline and verify documentation --- AGENTS.md | 20 +- CONTRIBUTING.md | 9 + README.md | 6 +- UPGRADING.md | 1409 +--------------------- docs/README.md | 35 +- docs/architecture-overview.md | 31 - docs/architecture.md | 514 +------- docs/budgets.md | 84 -- docs/embedding.md | 112 +- docs/internal/README.md | 6 - docs/internal/architecture-principles.md | 247 ---- docs/internal/brand.md | 47 - docs/mcp-http.md | 149 --- docs/mcp-stdio.md | 162 --- docs/prompt-packs.md | 152 --- docs/provider-manifests.md | 257 ---- docs/providers.md | 77 ++ docs/quickstart.md | 28 + docs/recipes.md | 41 - docs/reference/prompt-packs.md | 44 + docs/reference/runner.md | 81 ++ docs/reference/scaffold.md | 38 + docs/reference/trace.md | 71 ++ docs/rlm-skills.md | 61 - docs/scaffold-manifest.md | 128 -- docs/trace-abi.md | 266 ---- docs/verifiers-harness.md | 70 -- examples/embedding.py | 43 + pyodide/README.md | 441 ++----- src/droste/execution/trace.py | 33 +- src/droste/testing/__init__.py | 4 +- tests/test_docs.py | 124 ++ 32 files changed, 771 insertions(+), 4019 deletions(-) delete mode 100644 docs/architecture-overview.md delete mode 100644 docs/budgets.md delete mode 100644 docs/internal/README.md delete mode 100644 docs/internal/architecture-principles.md delete mode 100644 docs/internal/brand.md delete mode 100644 docs/mcp-http.md delete mode 100644 docs/mcp-stdio.md delete mode 100644 docs/prompt-packs.md delete mode 100644 docs/provider-manifests.md create mode 100644 docs/providers.md delete mode 100644 docs/recipes.md create mode 100644 docs/reference/prompt-packs.md create mode 100644 docs/reference/runner.md create mode 100644 docs/reference/scaffold.md create mode 100644 docs/reference/trace.md delete mode 100644 docs/rlm-skills.md delete mode 100644 docs/scaffold-manifest.md delete mode 100644 docs/trace-abi.md delete mode 100644 docs/verifiers-harness.md create mode 100644 examples/embedding.py create mode 100644 tests/test_docs.py diff --git a/AGENTS.md b/AGENTS.md index 49a5ad7..11e3608 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -116,7 +116,7 @@ evidence with that status rather than leaving the model to interpret prefixes. ## Trace ABI -- Every structured event is a strict Trace ABI v7 value. Stamp it exactly once +- Every structured event is a strict Trace ABI v10 value. Stamp it exactly once through `ExecutionContext`; do not emit raw or partially enveloped event dictionaries at host boundaries. - Treat every envelope/body, classification, and ordering change as an ABI @@ -189,7 +189,7 @@ evidence with that status rather than leaving the model to interpret prefixes. - A strict published event vocabulary/body change requires a Trace ABI bump and, when embedded in runner output, an atomic runner-protocol bump. Do not expand an old strict version in place or add a compatibility decoder in the engine. -- Trace ABI v7 usage is `resolved` only when both root and subcall scopes have +- Trace ABI v10 usage is `resolved` only when both root and subcall scopes have complete provider usage. Missing or malformed usage preserves any known counts, marks that scope `complete=false`, and makes terminal usage `kind="partial"`; never report conservative reservations as provider usage. @@ -239,7 +239,7 @@ evidence with that status rather than leaving the model to interpret prefixes. partial evidence rather than invalidating an otherwise valid envelope. - `reasoning_tokens` is a non-negative breakdown inside `completion_tokens`. Preserve it through internal usage copies, folds, and root/subcall - `ExecutionStats`, but do not add it to totals or Trace ABI v7 events. + `ExecutionStats`, but do not add it to totals or Trace ABI v10 events. Observation basis and reasoning usage are internal callback/accounting facts; do not add them to the public usage projections. @@ -272,7 +272,7 @@ evidence with that status rather than leaving the model to interpret prefixes. partial and settle conservatively rather than falling back to the start estimate. - `ExecutionStats` folds those same cache classes separately for root and - subcall scopes. Trace ABI v7 exposes them in every durable usage breakdown; + subcall scopes. Trace ABI v10 exposes them in every durable usage breakdown; complete scopes require the disjoint cache classes to fit inside inclusive input tokens, while partial scopes preserve independently validated counts. ModelRelay names the classes `cache_read_input_tokens` and @@ -519,16 +519,8 @@ evidence with that status rather than leaving the model to interpret prefixes. optional revision, and explicit byte/line/section ranges. Cursor pagination must describe both the input cursor and output `next_cursor`. - MCP transport binding is shared through `McpToolTransport`, - `McpBindingPolicy`, and `bind_mcp_transport_source`; stdio, native HTTP, and - cross-language host bridges must not copy descriptor/result projection. -- Streamable HTTP source specs contain exact HTTPS allowlists and tenant-scoped - secret references only. Live `McpHttpHost` state owns resolved secrets, - private-network CIDR exceptions, TLS, DNS, and bounded raw debug sinks. - Redirects fail closed; every DNS answer is checked and the selected IP is - pinned through TLS. Startup and request timeouts are total budgets across - auth, retry, and SSE-resume hops; close uses only cached auth under one close - deadline. Generated code and durable traces receive none of the - URL/auth/session/raw protocol state. + `McpBindingPolicy`, and `bind_mcp_transport_source`; stdio and cross-language + host bridges must not copy descriptor/result projection. - Trusted runner hosts use `source_opener` for dynamic-manifest sources. The request cannot choose the hook; both run and preflight acquire through it and transfer lifecycle ownership to the ordinary registry/environment path. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 63f0da1..9a26687 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,6 +18,15 @@ Thanks for your interest. Ground rules, kept short: - **Benchmarks**: claims about accuracy, cost, or latency require checked-in immutable artifacts and a version-matched generated report. Unpublished runs must not support public comparative claims. +- **Documentation**: document shipped behavior, not roadmap work. Give each + concept one canonical page and link to it instead of copying contract prose. + Version history belongs in `UPGRADING.md`; implementation rationale belongs + in code, tests, issues, or pull requests. Add every maintained page to + `docs/README.md`. Runnable examples live under `examples/` and need tests. +- **Documentation contracts**: `tests/test_docs.py` checks local links, anchors, + orphaned pages, and version headings against runtime constants. A protocol or + schema bump updates its implementation, reference heading, fixtures, and + migration note in one change. Development: diff --git a/README.md b/README.md index 733fc77..008e187 100644 --- a/README.md +++ b/README.md @@ -61,8 +61,8 @@ useful flags. If the data already fits comfortably in a context window, or the task is open-ended agent work rather than a question about a corpus, use a general -agent instead. The [recipes](docs/recipes.md) cover logs, chat archives, and -SQLite. +agent instead. The [quickstart](docs/quickstart.md) includes examples for logs, +archives, and SQLite. ## Results @@ -89,7 +89,7 @@ implementers can use the [technical reference](docs/README.md#technical-referenc ## Development ```bash -uv sync --extra verifiers +uv sync uv run pytest uv build ``` diff --git a/UPGRADING.md b/UPGRADING.md index a58749d..a142916 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -1,1391 +1,52 @@ -# Upgrading droste +# Upgrading Droste -Embedder-facing notes for moving between engine versions: what changed at the -integration surface, what breaks loudly, and — more importantly — what would -degrade *silently* if you skip a step. Loud breaks announce themselves (the -runner refuses mismatched `protocol_version` requests with a structured error; -the relay's `startup` event reports the engine + protocol versions it speaks); -this file exists for everything that doesn't. +This guide covers upgrades to 0.25 from the two preceding minor releases. For +older installations, upgrade one minor version at a time using the corresponding +GitHub release notes before applying this guide. Runner and strict Trace ABI +consumers must upgrade their decoders and packaged fixtures with the engine. -Ordered newest first. "Embedder" means anything that builds on the engine -beyond the `droste` CLI: hosts calling `run_rlm` in-process, `droste_runner` -consumers, and Pyodide-substrate integrations staging the Deno relay. +## 0.25.0 from 0.24.x -## 0.25.0 +### Trace ABI v10 reports degraded execution -### Trace ABI v10: a run reports what it did without +Results and terminal `done` events now include `degradations`. An empty list is +a clean run. Each item identifies the callback site, error type, bounded detail, +and consequence of work that continued without it. -Every result now carries `degradations`, a list that is empty on a clean run -and otherwise names each thing the run continued without: `site`, -`error_type`, `detail`, and — the field that matters to a reader — -`consequence`, which records not that a callback raised but what the run then -did without it. +Hosts should surface, log, or reject non-empty degradations according to what +was lost. The affected callbacks remain observational and do not terminate the +run. -Several boundaries here deliberately refuse to let a host-supplied callback end -a run: a broken logging sink or observer must not destroy a user's work. That -resilience is correct. It was paid for with a `warnings.warn` that reaches no -consumer of the result, so a run that lost a budget event, ran its terminal -extract without host context, or discarded salvageable work returned something -byte-identical to a clean run. The loss was real and unobservable. +Update strict consumers and replace `trace_v9_*` helpers and `trace-v9-*` +fixtures with their `v10` equivalents. -Recorded today at `budget_event_sink`, `extract_context_provider`, and -`extractable_work_probe`. `BudgetLedger.dropped_events()` exposes the same -facts for a directly-held ledger. +## 0.24.0 from 0.23.x -**Hosts should treat a non-empty `degradations` as a degraded answer** — worth -surfacing, logging, or refusing, depending on what the run lost. Nothing new -can end a run; this only makes the existing fallbacks visible. +### Trace ABI v9 reports armed ready gates -The v9 -> v10 rename moves the `trace_v9_*` helpers and `trace-v9-*` fixtures -to `trace_v10_*` / `trace-v10-*`. Response builders in `droste_runner` emit -`degradations: []` on every shape, so a consumer reads one field rather than -treating a missing key as "nothing was lost". +The optional `startup.ready_gates` array names the ready-time enforcement +enabled for the run: `policy_hints`, `ready_metadata_validator`, and/or +`ready_answer_validator`. An empty array means no gate was armed. -## 0.24.0 +Hosts that require enforcement should assert the expected gates rather than +assuming configuration from a successful result. -### Trace ABI v9 reports which ready-time gates a run armed +`ready_answer_validator` receives `ReadyAnswerState`, which includes metadata, +answer content, iteration, calls made, and successful calls. The older +metadata-only validator remains supported; both run when both are configured. -The `startup` event gains an optional `ready_gates` array naming the gates the -run actually enabled: any of `policy_hints`, `ready_metadata_validator`, and -`ready_answer_validator`. An empty array means the run enforced nothing. +Update strict consumers and packaged fixtures to Trace ABI v9. -It exists because enforcement here is opt-in and defaults to off, so a host that -never passes `policy_hints` runs with `ready_violations` and -`contract_violations` permanently inert -- and until now emitted a trace -identical to a fully gated run. That is the silent degradation this file is for. -One embedder shipped for months in exactly that state while its benchmark suite -passed hints and measured the enforced configuration; no test could fail, -because nothing recorded which system was running. +## 0.23.0 from 0.22.x -**Embedders should assert on `ready_gates` in whatever checks their production -configuration**, and treat an unexpectedly empty array as a misconfiguration -rather than a default. +### Trace ABI v8 adds provider-call heartbeat -### `ready_answer_validator` gates on observed run state +The transient `heartbeat` event reports `elapsed_ms` every 15 seconds while a +provider call remains outstanding. It is liveness only: it is never retained +and does not affect budget accounting. -`run_rlm` accepts a new optional `ready_answer_validator`, called with a -`ReadyAnswerState` (metadata, answer content, iteration, calls made, successful -calls) instead of metadata alone. +No-progress watchdogs should count `heartbeat` and `reasoning_delta` as +activity. The relay cannot emit a heartbeat while generated WASM code itself is +blocked, so the event does not mask an interpreter wedge. -`ready_metadata_validator` sees only model-written metadata, so it can verify -the model's *claims* about its work but not the work. Behavioural rules -- "do -not answer without consulting the data" -- need what the engine observed. The -older hook is unchanged and still supported; both run when both are supplied, -and violations from either revoke readiness identically. - -## 0.23.0 - -### Trace ABI v8 reports liveness during provider calls - -The new transient `heartbeat` event carries `elapsed_ms` and nothing else. It -fires every 15 seconds while any provider call is outstanding, and stops when -the call completes. - -It exists because a host cannot otherwise distinguish "blocked on the network" -from "wedged in the interpreter" — both are silence — so no-progress watchdogs -were killing healthy runs whose subcall simply took a while. Batch subcalls are -the worst case: they post to `/responses/batch`, which the relay does not -stream, so they emit no `reasoning_delta` either and go completely silent -between start and completion however long they take. - -**Hosts running a no-progress watchdog should count `heartbeat` as progress.** -If yours also ignores `reasoning_delta`, count that too — it already streamed -during unary calls and is the same kind of signal. - -The heartbeat is emitted relay-side, deliberately. Pyodide runs on the relay's -thread, so the timer cannot fire while generated code spins in WASM — which is -exactly when a watchdog *should* fire. A heartbeat therefore never masks a -genuine wedge; it only reports the healthy case. - -Being transient, it is never retained and never reaches a run record, whatever -`TraceRetentionPolicy.retain` names. - -Hosts must accept Trace ABI 8 and update to the `trace_v8_*` conformance -fixtures. `droste.testing` now exports `trace_v8_execution_ndjson` and -`trace_v8_lifecycle_ndjson`; the `trace_v7_*` helpers are gone. - -## 0.22.0 - -### Trace ABI v7 publishes answer-state checkpoints - -The new configurable `checkpoint` event carries the draft the engine currently -holds — `checkpoint_seq`, `draft`, `draft_chars`, `ready`, and an opaque -`payload` — after every executed step whose draft moved. A host that loses the -process (a watchdog kill, a crashed substrate) can render the last checkpoint -instead of nothing. It is retention-gated like every other content-bearing -event, so it reaches a terminal record only when named in -`TraceRetentionPolicy.retain`. - -`RLMConfig.checkpoint_payload_provider` is the optional callable that fills -`payload`. It returns any JSON object or `None`; the engine never inspects the -value and never schema-checks it. A provider that raises is reported through a -`RuntimeWarning` and the checkpoint carries `payload: null` — a checkpoint can -never fail a run. - -Hosts must accept Trace ABI 7 and update to the `trace_v8_*` conformance -fixtures (`droste.testing.trace_v8_lifecycle_ndjson` / -`trace_v8_execution_ndjson`, backing `trace-v8-lifecycle.ndjson` / -`trace-v8-execution.ndjson`). Scaffold manifests report `abis.trace: 7`, so -every manifest id changes; pinned ids must be re-derived. Strict v6 readers -reject the new event and the new version, so this is an atomic consumer -migration. - -## 0.21.1 (from 0.21.0) - -### Streamed Responses preserve their terminal stop reason - -The Deno relay now requires `completion.stop_reason` in ModelRelay's negotiated -`responses-stream/v2` contract and carries it into the reconstructed unary -response. Missing, non-string, empty, or whitespace-padded values fail closed. -Embedders must deploy a ModelRelay version that emits this documented field -before staging the 0.21.1 relay assets. - -## 0.21.0 (from 0.20.1) - -### RLM runs have an explicit iteration ceiling - -`Budget.max_iterations` is now a required positive integer and defaults to -30. The engine refuses the first iteration beyond that ceiling before issuing -another root request. An unconfirmed run terminates with the distinct -`IterationLimitExceeded` type and may use the existing bounded extraction pass -when completed work is available. - -Runner protocol v10 requires `"max_iterations"` in the exact request budget. -Scaffold manifest v3 and Trace ABI v6 include the field in their configured -budget. Hosts must send protocol 10, accept manifest schema 3 and Trace ABI 6, -and update to the `trace_v6_*` and `runner_v10_*` conformance fixtures. -Manifest schemas 1 and 2 remain readable and resolve their historical missing -field to the current default of 30; new manifests always use schema 3. - -## 0.20.1 (from 0.20.0) - -### Hosts can validate answer metadata before the root finishes - -`run_rlm(..., ready_metadata_validator=...)` adds an optional trusted-host -gate after core ready and JSON-metadata validation. The callback receives a -detached metadata copy and returns content-free violation strings. Any -violation revokes `answer["ready"]` and enters the existing root repair path, -so embedders no longer need to discover a repairable product contract only -after the billed run has completed. - -The callback is not part of `RLMConfig` or the scaffold manifest because it is -host-side result admission, not model-visible inference configuration. It must -be synchronous, deterministic, free of model calls and external mutation, and -hosts should retain their final defense-in-depth validation. -If the callback raises or violates its return contract, Droste raises -`ReadyMetadataValidatorError` to the host without issuing a model repair call -or including the host exception in model-visible feedback. - -## 0.20.0 (from 0.19.2) - -### Trace ABI v5 reports cumulative usage at model-call boundaries - -Trace ABI v5 adds the transient, content-free `usage_progress` event. Droste -emits one after each root or subcall numeric usage observation settles. Its -`boundary` identifies the role that just settled, while its root, subcall, and -overall counters are cumulative snapshots. Consumers must replace their live -display from each event rather than sum events, and must not interpolate -between them. Partial observations retain reported counters and set the -affected scope's `complete` value to false. - -Runner protocol v9 embeds Trace ABI v5. Runner hosts must send -`"protocol_version": 9`, accept `startup` events reporting Trace ABI 5, and -forward `usage_progress` through the existing event descriptor. Strict v4/v8 -consumers will reject the new contract by design. The packaged conformance -helpers and fixtures are now named `trace_v5_execution_ndjson()`, -`trace_v5_lifecycle_ndjson()`, and `runner_v9_refusal_ndjson()`. - -## 0.19.2 (from 0.19.1) - -### Estimated categories settle complete provider token totals - -`TokenUsage.core_complete` is now the compute-budget settlement authority. It -is true for both `exact` and `estimated_categories` observations because those -bases guarantee complete provider input, output, and total counters. Root and -subcall reservations therefore reconcile to the reported provider total for -either basis instead of retaining the conservative requested-token maximum. - -`TokenUsage.exact` remains the distinct billed-category completeness signal. -Embedders should continue to use it for evidence and reporting that requires -every optional billing category to be exact; do not substitute -`core_complete` there. Existing integrations that already forward the -canonical `observation_basis` need no wire-protocol change. - -## 0.19.1 (from 0.19.0) - -### Direct ModelRelay usage preserves declared observation fidelity - -Native ModelRelay clients and Pyodide host bridges now require each -`/responses` usage mapping to include the canonical `observation_basis` and -`reasoning_tokens` fields before accepting an `exact` or -`estimated_categories` claim. Missing reasoning, a missing or unknown basis, -or counters that do not satisfy the declared basis remain incomplete evidence -and keep conservative settlement. Integrations must forward these fields from -ModelRelay rather than reconstructing completeness from structurally valid -input, output, and total counters. - -## 0.19.0 (from 0.18.0) - -### Callback usage declares observation fidelity and reasoning tokens - -`TokenUsage` now has one authoritative `observation_basis` with the values -`unavailable`, `incomplete`, `estimated_categories`, and `exact`. Its `exact` -attribute remains readable for settlement code but is derived from that basis; -the legacy `exact=` constructor input is normalized immediately and is not a -second stored field. `reasoning_tokens` is also retained as a non-negative -breakdown inside `completion_tokens`. - -Native and hosted runner callbacks must now include `observation_basis` and -`reasoning_tokens` in every usage mapping that claims `exact` or -`estimated_categories` observation. An exact callback requires signed-int64, non-negative `input_tokens`, -`output_tokens`, `total_tokens`, and `reasoning_tokens`; the total must cover -input plus output, cache classes must fit inside input, and reasoning must fit -inside output. Missing or invalid fields, an unknown basis, or a structurally -invalid exact claim remains incomplete and keeps the conservative reservation. -Use `estimated_categories` only when the overall input, output, and total are -known but one or more billing categories were estimated. Use `unavailable` -only with zero counters. - -Reasoning totals remain available on internal `ExecutionStats` while its Trace -v4 projection intentionally omits them. Provider adapters outside the runner callback boundary retain their existing -validated inference behavior when their upstream API does not expose an -observation-basis field. Runner protocol v8 and Trace ABI v4 are unchanged: -the new fields are internal callback/accounting facts and are not added to -durable trace usage. - -## 0.18.0 (from 0.17.0) - -### Runner callbacks carry exact failure usage without exposing unsafe bodies - -Native and hosted runner callbacks may now attach provider usage to a failed -root or subcall request. Exact, non-negative input/output/total counters settle -the inference reservation to the provider total before the original failure is -surfaced. Missing, malformed, incomplete, or out-of-range usage remains partial -and keeps the full conservative reservation; Droste never estimates a refund -from an error message or visible output. - -To opt in, exact root and subcall callback endpoints must return a non-2xx JSON -object with `error: "api_error"` and a mapping-valued `usage`. Use a JSON media -type (`application/json` or an `application/*+json` subtype) with no parameters -other than one UTF-8 charset. Generic URLs and untrusted media types remain -opaque transport failures. The hosted Deno relay forwards a valid typed body -only for those exact callback endpoints after independently proving the same -top-level discriminator, usage mapping, unique-key, finite-number, depth, and -integer-placement contract as native Python. It preserves raw counters without -parsing int64 values through JavaScript numbers; held credentials are redacted -from JSON strings before the body reaches Python. HTTP status is associated -with that fetch: an immediately raised callback failure keeps the status, while -a normally handled value or a later fetch consumes the association so it cannot -annotate an unrelated adapter error. - -All callback JSON now shares a strict wire grammar: UTF-8 without a BOM, unique -object keys, finite numbers, bounded nesting, and signed-int64 integers. -Successful unary and NDJSON responses use the same loader. An out-of-range -integer in a direct recognized usage counter makes that usage partial while -preserving a usable result and valid sibling counters; the same condition on -an NDJSON error is carried by `LLMUsageFailure`. An out-of-range integer -anywhere else invalidates the envelope. Embedders that previously emitted BOMs, -duplicate keys, `NaN`/`Infinity`, excessive nesting, or arbitrary-size integers -must canonicalize callback JSON before upgrading. Trace ABI v4, runner protocol -v8, and provider protocol v4 are unchanged. - -Native `llm_batch*` remains bounded client-side fan-out over the unary -`subcall_endpoint`; there is no separate native batch response schema. Each -item independently receives the strict JSON/NDJSON and typed-failure handling -above. A failed batch can settle to the sum of provider totals only when every -ordered item, including failed items, supplied exact usage. Any malformed or -unavailable item keeps the complete conservative batch reservation. - -### Prompt-pack schema v2 supplies transcript-elision text - -Live root transcripts now retain the last two completed iterations verbatim, -then use the resolved pack's `historical_stdout_elision` and -`unchanged_draft_elision` templates when older refinements enter the stable -window. Complete caller and consumer prompt packs must upgrade to -`schema_version = 2` and provide both non-empty templates. The built-in pack -IDs and profile fallback order are unchanged; their revisions are now `2.0.0`. - -Windowing affects only outbound root requests. Canonical trajectory snapshots -and repair adoption retain the complete message history. Once windowing is -active, Anthropic cache anchors mark the system prompt and stable elision -frontier rather than the mutable tail. - -Assistant code blocks and canonical repair exchanges, including adopted -missing-code repair messages, are deliberately outside this windowing scope -and remain verbatim at every distance. They preserve nonredundant evidence -about attempted programs and repair instructions, while only historical stdout -and byte-identical draft repeats are treated as disposable; code-heavy -transcripts can therefore continue to grow even after windowing activates. - -## 0.17.0 (from 0.16.0) - -### Trace ABI v4 and runner protocol v8 preserve cache token classes - -`ExecutionStats` and durable root/subcall usage now carry -`cache_read_tokens` and `cache_creation_tokens` inside the existing inclusive -`input_tokens` and `total_tokens`. Do not add the cache counters to the totals -again. ModelRelay `/responses` adapters accept absent cache fields as exact -zero, map `cache_read_input_tokens` and `cache_write_input_tokens`, and mark -usage partial when a present cache field is malformed. -Retained `replay` events copy the same complete usage projection into -`result.usage`; it reconciles exactly with the durable `usage` and `done` -values. - -The strict usage body change bumps Trace ABI to v4 and the embedding runner to -protocol v8. Request writers must send `protocol_version: 8`, event consumers -must accept `version: 4`, and cache-aware billing consumers should read both -cache classes from the root and subcall scopes. There is no compatibility -decoder for Trace ABI v3 or runner protocol v7. - -The released conformance resources are -`trace_v4_execution_ndjson()`, `trace_v4_lifecycle_ndjson()`, and -`runner_v8_refusal_ndjson()` backed by the correspondingly named NDJSON files. - -## 0.16.0 (from 0.15.5) - -### Anthropic prompt caching and inclusive usage - -Root calls mark the stable system prompt and latest message as Anthropic -ephemeral cache anchors. The markers are private loop-to-client metadata: -Anthropic converts them to at most four `cache_control` blocks, while every -other first-party adapter strips them before serialization. Terminal extraction -disables cache anchors so its one-off prompt retains the ordinary Anthropic -wire form. - -`TokenUsage` adds `cache_read_tokens` and `cache_creation_tokens` alongside the -fail-closed `exact` flag. Anthropic validates and populates the cache breakdown -for streaming and non-streaming responses, including usage-aware parsing -failures and ordered batch failures. Its `prompt_tokens` and `total_tokens` are -inclusive because Anthropic's reported `input_tokens` excludes cached input. -Use those inclusive values for billing and display; the cache fields are an -observability breakdown. `ExecutionStats`, Trace ABI v3 events, and `RLMResult` -continue to expose inclusive totals rather than separate cache counters. - -### Exact provider usage settles inference reservations - -`TokenUsage.exact` now defaults to false. First-party ModelRelay, -OpenAI-compatible, runner HTTP, and Pyodide adapters opt in only after -validating explicit non-negative input, output, and total token counts; the -provider total is preserved when it includes hidden reasoning. Anthropic usage -includes ordinary input, cache-creation input, cache-read input, and output. -Missing or malformed usage never falls back to visible-output byte estimates. - -Subcall clients implement the three `SubcallUsageProvider` companion methods -and return `SubcallQueryResult` / `SubcallBatchResult`. Batch usage is ordered -one-for-one with results. A complete batch checkpoint settles the sum of -provider totals; any unavailable item keeps the successful result but settles -the full conservative reservation. - -Fail-fast `llm_batch_with_usage` implementations preserve partial provider -facts by raising `SubcallBatchFailure` with their collected -`SubcallBatchResult`. Broker registrations record that usage once before -surfacing the original item error; plain `llm_batch` still raises the original -exception type. Root responses likewise record normalized provider usage and -success before final ledger commit, so a later token or wall overrun no longer -erases billable usage from the terminal trace. - -Exact inference usage may exceed the admitted token reservation. Such a -checkpoint now flows to final settlement instead of becoming a handler error; -`BudgetLedger.commit()` closes the reservation and raises `BudgetExhausted` -using the actual provider total while terminal usage retains the exact fact. - -Provider-completed usage now survives malformed root or subcall output through -`LLMUsageFailure`. The root loop and subcall broker settle the carried usage -before returning the original parsing error; plain client calls retain their -original exception types. Usage does not imply a usable-output success, so -malformed output increments request and token facts but not success counters. -Centralized subcall usage callbacks are serialized across concurrent broker -calls to keep `ExecutionStats` totals aligned with ledger reconciliation. - -Fan-out batches now retain `LLMUsageFailure.usage` in the failed item's ordered -slot while surfacing its original cause. Native ModelRelay batch structural -errors similarly carry the partial ordered result through -`SubcallBatchFailure`; successful earlier items remain attributed, unseen items -remain unavailable, and the broker therefore records known usage once while -settling the incomplete batch conservatively. - -Custom `RLMEnvironment.sandbox_subcalls` implementations now receive the -required keyword-only `usage_callback` and `settlement_callback` and must pass -both to `broker_subcalls`. The broker registration uses the first to record -typed per-item provider usage exactly once; subcall transports no longer add -token usage directly to `ExecutionStats`. The second marks full-reservation -fallback as partial usage, including a wall-time overrun surfaced after commit -closes the reservation. The optional capability observer is trace-only and -cannot substitute for either accounting callback. - -### Trace ABI v3 and runner protocol v7 distinguish partial usage - -Trace usage scopes now include `complete: boolean`; `kind="resolved"` requires -both root and subcall scopes to be complete, while missing provider facts emit -`kind="partial"` without inventing token counts. This changes the strict event -body and the embedded runner response, so request writers must send -`protocol_version: 7` and event consumers must accept `version: 3` atomically. -There is no compatibility decoder for Trace ABI v2 or runner protocol v6. - -The released conformance resources are now -`trace_v3_execution_ndjson()`, `trace_v3_lifecycle_ndjson()`, and -`runner_v7_refusal_ndjson()` backed by the correspondingly named NDJSON files. - -## 0.15.5 (from 0.15.4) - -### The released fixture corpus covers code/output UI projection - -`droste.testing.trace_v2_execution_ndjson()` supplies exact producer-stamped -Trace ABI v2 frames for two root iterations and a depth-one child run. It -covers `llm_response`, `code`, successful `output`, and `execution_error`; one -successful stdout deliberately begins with `ERROR:` so consumers do not infer -failure from prose. The fixture changes no event shape, Trace ABI version, -runner protocol, runtime behavior, or retention policy. - -## 0.15.4 (from 0.15.3) - -### Lifecycle conformance scenarios are available to exact-pinned embedders - -This patch publishes transport-neutral lifecycle test support under -`droste.testing`: explicit `LifecycleGate` rendezvous points, bounded -`run_while_blocked` scenarios with caller-owned timeout cleanup, immutable -thread and settlement observations, a recording attempt authority, and pure -unknown-completion and terminal-event assertions. Embedders can use the same -helpers to exercise their host adapters after pinning Droste 0.15.4 exactly. - -The helpers own test coordination only; they do not own production lifecycle, -retry, cancellation, or settlement policy. Trace ABI v2, runner protocol v6, -the dedicated Deno event descriptor contract, and the 0.15.1 conformance corpus -are unchanged. - -## 0.15.3 (from 0.15.2) - -### External Deno launchers must register the inherited event descriptor - -Version 0.15.2 documented and enforced the dedicated event descriptor, but its -relay E2E used Deno's `node:child_process` compatibility path. That path -silently registers extra numeric stdio for a Deno child. Normal shell and Go -parents do not: Deno leaves fd3 outside its JavaScript-visible descriptor table -unless its startup marker names the inherited descriptor. - -Hosts must now set both `DROSTE_RELAY_EVENT_FD=3` and -`DENO_EXTRA_STDIO_FDS=3` when they pass the event writer as child fd3. For Go, -the first `exec.Cmd.ExtraFiles` entry becomes child fd3. Deno consumes -`DENO_EXTRA_STDIO_FDS` before relay JavaScript starts; it is runtime launch -metadata, not an event or runner field. Omitting it fails closed with -`RelayEventChannelError.code = "descriptor_unavailable"` and never falls back -to fd2. Trace ABI v2, runner protocol v6, the three physical lanes, and native -`python -m droste_runner` stderr behavior are unchanged. - -## 0.15.2 (from 0.15.1) - -### The Deno relay requires a dedicated event descriptor - -Hosts that spawn `relay.ts` must open an inherited writable descriptor, set -`DROSTE_RELAY_EVENT_FD` to its decimal number (fd3 is conventional), and drain -it concurrently with fd2. stdout remains the single unary response JSON lane; -adapter-owned outcomes use the adapter HostResponse schema, while an event -channel failure before adapter ownership uses a relay-level -`RelayEventChannelError` body without runner protocol or operation fields. The -configured descriptor carries canonical Trace ABI v2 NDJSON only; -fd2 is diagnostic-only. The relay rejects fd0 through fd2 and fails closed with -`RelayEventChannelError` when the setting is missing, malformed, or unwritable. -It never falls back to stderr. Hosts that previously parsed canonical events -from the Deno relay's fd2 must stop; fd2 now contains diagnostics only. - -Preflight and pre-admission refusal still require the descriptor but write zero -event frames. Do not send or expect a probe/liveness byte. An admitted run emits -`startup` only when its first canonical event is ready. A hard cancellation, -process death, or descriptor failure may leave a valid nonterminal prefix; -never fabricate `done`, and treat a final unterminated or invalid line as a -typed transport failure rather than an event. Event bodies, Trace ABI version -2, and runner protocol version 6 are unchanged. Native -`python -m droste_runner` event sinks remain explicitly attached to the -original host stderr. - -## 0.15.1 (from 0.15.0) - -### The released Trace ABI contract includes executable cross-runtime fixtures - -Trace ABI v2 and runner protocol v6 are unchanged. This patch publishes their -previously missing conformance corpus: one deterministic NDJSON stream covering -ordinary success, unary and atomic-batch subcalls with failure attribution, -structured execution failure, repair and extract success/failure, loud output -limit failure, cancellation, and terminal reconciliation. A separate exact -runner-v6 refusal value records the pre-admission case; it is intentionally not -a Trace ABI event and relays must reject it as one. - -Python consumers read the shipped bytes with -`droste.testing.trace_v2_lifecycle_ndjson()` and -`droste.testing.runner_v6_refusal_ndjson()`. Non-Python consumers use the same -files under `conformance/` in the versioned GitHub relay artifact. Wheel and -sdist checks compare the source bytes, and the release bundle copies those same -source files rather than maintaining copied schemas. Embedders should pin -0.15.1 or newer for conformance; 0.15.0 contains the ABI implementation but not -this complete released fixture contract. - -## 0.15.0 (from 0.14.1) - -### Trace ABI v2 and runner protocol v6 add typed lifecycle events - -Trace events and terminal `RunRecord` values now carry `version: 2`. -`RUNNER_PROTOCOL_VERSION` is now 6 because a successful runner response embeds -that v2 run record and its live NDJSON stream uses the same vocabulary. Upgrade -runner request writers, event decoders, relay assets, and stored-record readers -atomically. Trace ABI v1 and runner protocol v5 fail loudly; Droste does not -reinterpret either old strict contract as v2. - -The new configurable `subcall` event is a phase-discriminated live/run-record -view over the broker's existing `call_id`, reservation, and cumulative -checkpoint. Atomic batches add `batch_count`; their existing `call_id` is the -batch identity, and they do not invent per-item identities or a second -sequence. Event-envelope `seq` remains the sole -delivery order. `repair` now carries `phase` (`start`, `completion`, or -`failure`) and `kind` (`missing_code`, `execution_error`, or `terminal`). The -new `extract` event uses the same phases, with a typed `extract_error` only on -failure. The former standalone `finalization_error` and `extract_error` event -types and the old start-only `repair.reason` shape are removed. - -All lifecycle events are configurable retention content. They never include -subcall prompts, contexts, results, or provider error messages. The existing -durable `capability`, `usage`, `budget`, `policy`, and content-free `done` -events remain the accounting and persistence authorities; stream receipt still -has no billing meaning. A successful `output` remains successful even when its -stdout begins with `ERROR:`. - -### Remote MCP sources use a trusted Streamable HTTP acquisition shell - -Hosts may call `open_mcp_http_source(ConfiguredSource(...), McpHttpHost(...))` -to freeze a remote MCP `tools/list` snapshot into the same lifecycle-owned -`BoundSource` returned by the stdio transport. Endpoint, OAuth, session, DNS/IP, -payload, and reconnect state stay in the trusted host shell. Source auth values -are secret references resolved with the exact tenant and source identity. - -Cross-language hosts can implement `McpToolTransport` and use -`bind_mcp_transport_source()` with `McpBindingPolicy`; this preserves one -manifest/schema projection without requiring the native Python HTTP transport. -Trusted runner callers may instead supply `source_opener` to `run()` or -`run_worker_request()`. Dynamic sources are acquired for both run and preflight, -and preflight therefore performs remote discovery while still forbidding -provider dispatch. Existing callers that omit the hook retain static-catalog, -connection-free preflight behavior. - -The MCP APIs do not introduce another runner or provider protocol bump. They -are additive, and no MCP authority is acquired implicitly. - -## 0.14.1 (from 0.13.1) - -The immutable `v0.14.0` tag failed in release CI before any PyPI or GitHub -release was published. Version 0.14.1 supersedes that unpublished tag and is -the first available release containing the changes below. - -### Runner protocol v5 and scaffold manifest v2 add subcall input capacity - -`RUNNER_PROTOCOL_VERSION` is now 5. Request writers may supply the closed -`subcall_input_capacity` value with `state` (`bounded`, `unbounded`, or -`unknown`) and `tokens`. Bounded requires a positive token count; the other -states require null tokens. Upgrade request writers and runners atomically. A -v4 runner refuses the v5 envelope instead of silently ignoring planning -metadata that may have selected chunking or a checkpoint. - -The value is the effective usable caller-payload bound across the complete -adapter, transport, and model path. Use `unbounded` only when arbitrary caller -payloads are guaranteed, such as by transparent chunking; a large or unknown -model context window is not unbounded. The value guides planning and does not -authorize spend or enlarge output-token ceilings. - -In-process clients may implement the optional `SubcallInputCapacityProvider`. -Its `input_token_capacity` property returns an immutable -`SubcallInputCapacity`. Protocol absence remains source-compatible and resolves -to unknown. A conflicting known `RolloutConfiguration` value fails before the -first model request. Once a client implements the property, it is a correctness -claim: getter failures and wrong return types fail before inference rather than -degrading to unknown. This is deliberately stricter than the older -best-effort output-limit metadata. - -New scaffold manifests are version 2 and record the resolved state at -`inference.input_capacity.subcall`. Stored version 1 manifests remain strictly -readable and keep their original content identity; they are not rewritten. - -### Local MCP stdio sources use one owned acquisition transaction - -Hosts may call `open_mcp_stdio_source(ConfiguredSource(...))` to initialize a -host-allowlisted local MCP process and freeze its complete `tools/list` snapshot -into a bound provider. The function returns `BoundSource`, not a reusable -`ProviderRegistration`, because MCP discovery must perform I/O before a -manifest exists. Combine it with other already-bound sources through -`ProviderRegistry`, and transfer or close that registry under the existing -runtime ownership rules. - -The configuration requires an absolute executable, an exact executable and -tool allowlist, an explicit working directory and environment, separate raw-tool/Python-binding -names, host read classification, and budget classes. MCP annotations never -supply policy; this spike rejects effectful tools. -Only protocol `2025-11-25` over local stdio is supported. Streamable HTTP and -tasks remain separate work. There is no provider or runner protocol bump; -existing sources and hosts do not acquire MCP authority implicitly. - -### Bound provider runtimes have explicit ownership - -`ProviderRuntime` accepts an optional `close_callback`, and -`ProviderRegistry.close()` releases every bound runtime exactly once in reverse -bind order. Partial catalog binds are cleaned up before the binding error is -propagated. Resource-free providers need no callback. - -Passing a registry to `create_environment()` now transfers ownership -immediately. Construction failures close it, and successful native and Pyodide -environments close it when the run or preflight exits, including exception and -Python process-control paths. Hosts that bind a registry without transferring -it to an environment must call `registry.close()` themselves. Trusted bridge -hosts may call `ProviderService.close()`; it is safe if the source's registry is -also closed. - -Do not return the same `ProviderRuntime` object from binds for multiple live -sources. A provider that shares an underlying pool must return a separate -runtime lease per source and coordinate the pool inside its close callbacks. -There is no provider or runner protocol bump because manifests and wire -envelopes are unchanged. - -The first-party filesystem provider now closes its pinned root descriptor -explicitly, and SQLite closes connections it opens from `sqlite_path`. A live -SQLite connection supplied as binder context remains host-owned and is not -closed by the provider. - -### First-party local filesystem/text provider - -Hosts may explicitly add `filesystem_text_provider()` to their -`ProviderCatalog`, then bind a named `ConfiguredSource` with an absolute `root` -and optional `include`, `exclude`, `enrichers`, and bound settings. The provider -is not added implicitly to `droste_runner`'s remote-wrapper catalog: local path -authority must remain an explicit host decision. - -The source exposes descriptor-generated `list_files`, `read`, `grep`, `search`, -and `stat` bindings. The raw manifest operation remains `list`; its Python name -is `list_files` because public bindings cannot shadow builtins. There is no -provider or runner protocol bump: existing manifests and requests are unchanged. -Bridged hosts may use the same generic unary or duplex provider transport. - -The local runtime requires an absolute non-symlink directory and secure POSIX -descriptor-relative primitives. Unsupported platforms fail at bind rather than -falling back to path resolution. Keep the provider in a trusted interpreter or -process when the configured root must be unavailable to generated code; native -in-process execution does not remove Python's ambient filesystem authority. - -### Optional provider bridge v2 duplex sessions - -`BridgeProvider` keeps its unary provider-protocol-4 behavior unless a host -explicitly passes `duplex_call`. The callable must return one per-invocation -session with `receive`, `send`, `cancellation_requested(call_id)`, and `close` -methods. Bridge v2 streams cumulative checkpoints and cooperative cancellation -through a bounded pull-based pump; do not implement it by calling back into a -suspended Pyodide interpreter. - -The bundled Deno relay now selects this duplex path for its two-interpreter -provider mode. Host adapters invoked by that relay must accept the new -`duplex_bridge_call` keyword and pass it to -`BridgeProvider(bridge_call, duplex_call=duplex_bridge_call)`. Upgrade the staged -relay, adapter, and Python package atomically. Single-interpreter mode and hosts -that construct `BridgeProvider(bridge_call)` directly remain unary. - -For the bundled relay, `SIGUSR1` requests cooperative cancellation of its one -active duplex provider call. Existing `SIGTERM`/`SIGKILL` process-control -semantics are unchanged and remain the fallback for non-cooperative handlers. - -Remote provider loss before terminal delivery now yields the stable -`bridge.transport_lost` capability error and settles once in the receiving -broker. Acknowledged checkpoints remain committed; unacknowledged remote facts -are not trusted. This post-0.13.1 work is not part of the v0.13.1 maintenance -tag. - -## 0.13.1 (from 0.13.0) - -### Runner exceptions retain their selected operation - -After runner protocol v4 accepts `run` or `preflight`, top-level worker -exceptions now carry that operation instead of `null`, so hosts can retain the -structured error without guessing which response schema applies. Missing or -mismatched protocol versions still refuse first with `operation: null`. -Preflight exceptions use the exact closed five-field preflight envelope with -`status: "error"` and `preflight: null`; they are not run envelopes relabeled -as preflight. Custom-catalog process hosts should replace outer exception -wrappers with `run_worker_request(...)`, then write `WorkerOutcome.response` -and use `WorkerOutcome.exit_code`. - -Runner requests also accept one optional non-empty `root_reasoning_effort`. -The native and Pyodide root clients send the exact value on every root callback, -and scaffold evidence records the same value under `root_sampling`. Hosts should -derive this field from their immutable run specification. A conflicting -`root_sampling.reasoning_effort` is rejected before inference. -The bundled Deno/Pyodide relay applies the same operation-specific exception -projection when an adapter raises. - -### Pyodide database access is broker-only (breaking) - -The bundled Deno relay no longer reads `RLM_DB_SERVICE` and no longer mounts a -database directory into the untrusted REPL interpreter. A request with -`db_path` always boots the trusted provider interpreter, consumes database -paths there, removes them from the sandbox request, and exposes only the -brokered `BridgeProvider` capability surface. The database-backed reference -adapter now requires a callable `bridge_call`; it has no `request["db_path"]` -fallback. - -Remove `RLM_DB_SERVICE` from host configuration and upgrade the staged relay -and any database-backed adapter atomically. Context-only adapters may still -receive `bridge_call=None` when no provider path is present, but must not treat -that as authority to open a request path inside the sandbox. The runner, -provider, and bridge protocol versions do not change: their envelopes and wire -schemas are unchanged; this removes an unversioned relay topology escape hatch. - -## 0.13.0 (from 0.12.1) - -### Runner protocol v4 adds safe, content-free scaffold preflight - -`RUNNER_PROTOCOL_VERSION` is now 4. Requests accept an explicit `operation` -discriminant: `run` (the default within v4) or `preflight`. Preflight resolves -the exact effective PromptPack, environment globals, capability manifest, -rollout configuration, and scaffold, then checks checkpoint requirements -without model or provider calls. It does not require endpoints or credentials. -Success returns the complete scaffold in a separately versioned, content-free -preflight value. Compatibility refusal preserves the public -`ScaffoldCompatibilityError` mismatch paths and has stable code -`scaffold_incompatible`. - -Upgrade request writers and runners atomically. The version bump is necessary: -an older v3 runner may ignore an unknown `operation` field and execute a request -that a newer host intended only to inspect. A v3 runner instead refuses the v4 -envelope before work. In-process hosts may call `preflight_rlm(...)`; it and -`run_rlm(...)` share one scaffold resolver. - -## 0.12.1 (from 0.12.0) - -### Scaffold concurrency now controls built-in subcall execution - -`RolloutConfiguration.concurrency` is now the effective maximum number of -in-flight items in every built-in subcall batch, rather than provenance-only -metadata. The compatibility default is 5, matching the earlier native and -runner behavior. In-process callers that choose another value must pass that -same immutable value to the built-in client's `max_parallel` constructor -argument; `run_rlm` rejects a mismatch before the first model request. Custom -third-party clients remain compatible, but are responsible for honoring the -declared value. - -Runner requests may set the positive integer `subcall_concurrency`; omitted -values resolve once to 5. Native CLI `--rollout-config` values now configure -the client as well as the scaffold manifest. The Pyodide relay preserves the -field unchanged. The effective value remains content-free provenance at -`scaffold_manifest.inference.concurrency`. The runner protocol remains v3 -because the request field is optional and additive. - -## 0.12.0 (from 0.11.0) - -### Python 3.14 is supported by the core package - -Droste continues to require Python 3.11 or newer and now tests both the 3.11 -compatibility floor and latest stable Python 3.14 in CI. The optional Verifiers -v1 harness remains limited to Python 3.11–3.13 by its upstream dependency -markers; installing the extra on 3.14 leaves it absent and its dedicated tests -skip cleanly. - -### Rollouts expose one content-addressed scaffold identity - -`RLMResult`, CLI JSON, and runner responses add `scaffold_manifest` and -`stdout_chars`, the sum of exact returned stdout lengths on retained trajectory -entries. The existing prompt-pack -`content_sha256` participates in the manifest as its prefixed `content_hash`. -Trajectory entries add `attempt_kind` and `stdout_chars`; consumers should use -the typed attempt/status fields rather than infer them from text. Droste rejects -oversized output instead of silently truncating it, so it does not publish a -truncation flag or count. Default durable terminal records retain only the -manifest ID and schema version. - -Hosts may provide resolved model/source revisions, sampling, concurrency, seed, -and runner version through `RolloutConfiguration`, plus checkpoint requirements -through `RLMConfig.checkpoint_requirements`. Incompatible requirements fail -before inference. The runner accepts the corresponding optional request fields; -the protocol remains v3 because all envelope additions are optional/additive. -The CLI adds `--prompt-profile` and `--rollout-config`. - -The `verifiers` extra adds the Verifiers v1 `droste_verifiers` harness package. -RLM skill artifacts and `rlm_skills_provider` are opt-in; no skill provider or -prompt text is registered automatically. See `docs/scaffold-manifest.md`, -`docs/verifiers-harness.md`, and `docs/rlm-skills.md`. - -## 0.11.0 (from 0.10.6) - -### Prompt-pack provenance includes canonical content identity - -Every resolved `PromptPackRecord` and its CLI, runner, report, and trace result -projection now includes the additive `content_sha256` field. It is the lowercase -SHA-256 digest of a pure canonical UTF-8 JSON serialization of the complete -validated pack value; prompt text itself is not added to result provenance. -Callers that construct `PromptPackRecord` directly remain source-compatible -because the new field is optional there, while records produced by the resolver -always populate it. - -Prompt-pack TOML may declare an optional top-level `content_sha256`. Loaders now -reject malformed declarations and declarations that do not match the validated -content. Existing artifacts without a declaration continue to load unchanged. -See `docs/prompt-packs.md` for the canonicalization rules and public helper -functions. The runner protocol version is unchanged. - -### Capability handlers receive one execution context (breaking) - -Trusted capability and `ProviderRuntime` handlers now have the single required -signature `handler(CapabilityExecutionContext, *args, **kwargs)`. Migrate every -registration atomically; there is no signature introspection or legacy adapter. -Generated sandbox bindings are unchanged and never receive the context. - -The frozen context carries call/run identity, the caller-authorized monotonic -deadline, and immutable reservation facts. Long-running trusted handlers call -`check()` to observe cancellation/deadlines and may report cumulative token and -subcall usage with `checkpoint()`. Providers do not receive the ledger or trace. -Cancellation and deadline results use stable `cancelled` and -`deadline_exceeded` codes. Admission begins the exactly-once settlement -boundary, including policy denial before handler dispatch. - -The provider protocol is now 4. `ProviderService` bridge invokes require the -versioned execution facts and return a validated cumulative checkpoint. Upgrade -both bridge interpreters and the staged relay atomically; the startup event's -`provider_protocol` detects drift. - -### Batch-item errors expose safe structured details - -Native response-batch item failures keep the existing human-readable error -string and now add an optional `details` object to `llm_batch_with_errors` and -structured JSON batch error entries. The additive fields are `request_id`, -`batch_id`, `item_id`, `layer`, `cause`, `status_code`, `code`, and -`retryable`; the frozen `BatchItemErrorDetails` value itself redacts and bounds -every accepted string, including when hosts construct it directly. Unknown -fields and payload data are not retained. The same object survives built-in -broker and environment boundaries. - -Direct `llm_batch` calls still raise a `RuntimeError`, now using the compatible -`BatchItemError` subclass when typed item details are available. Hosts can -inspect its frozen `BatchItemErrorDetails` value without parsing the unchanged -message. Custom `SubcallClient` implementations may keep returning only -`index` and `error`; no new method or required field was added. The runner -protocol version is unchanged. - -### Compute limits are one strict budget object (breaking) - -`max_iterations`, `max_calls`, `max_depth`, `max_subcalls`, and -`subcall_max_output_tokens` are removed from engine, client, CLI, and runner -configuration. There are intentionally no aliases or translation layer. Pass -one complete `Budget(tokens, subcalls, depth, wall_ms, root_output_tokens, -subcall_output_tokens)` value. Pass local REPL guardrails separately as -`SandboxLimits`. - -All root and brokered capability work reserves its maximum authorized vector -before dispatch and reconciles actual work afterward. Rejections use typed -`BudgetExhausted` values. Batch reservations are atomic; strict child ledgers -reserve from the parent and refund unused authorization on close. Runner -requests now require `protocol_version: 3` and a complete `budget` object. - -Budget mutations are durable Trace ABI v1 values from -`source="budget_ledger"`, keyed by `call_id`, with `reserve`, `commit`, -`refund`, and `exhaust` actions. See `docs/budgets.md`. - -### Subcall output limits are visible to the root model - -The built-in subcall clients now expose read-only `output_token_limit` metadata -when they know the effective value. The root authorized-compute prompt renders -positive limits as bounded, `None` as deliberately unbounded, and unavailable -metadata as unknown. The HTTP-backed runner reports an explicit positive -request override; when the field is omitted, its callback owns the default and -the limit stays unknown. This keeps custom `SubcallClient` implementations -compatible: the base protocol has no new required member. - -Custom clients may opt in by implementing the additive -`SubcallOutputTokenLimitProvider` companion protocol. Return a positive integer -for the effective per-call ceiling or `None` only when the client deliberately -leaves output unbounded. Do not report an endpoint default unless the client can -identify its effective value. The runner envelope and protocol version are -unchanged. - -The built-in generic prompt packs are revised to `1.0.2`; they distinguish -subcall input capacity from output capacity when planning structured and -map-reduce work. - -### Data providers are manifest-driven (breaking) - -The fixed `DataSource` protocol, capability booleans, universal verb table, -`extra_methods`, process-global source factories, and singular `data_source` -request sugar are removed. There are intentionally no compatibility aliases. -Hosts must construct an explicit `ProviderCatalog` from -`ProviderRegistration` values and bind declarative `ConfiguredSource` values -for each run. The bundled local provider is now `sqlite_provider()` with type -`sqlite`; its Python bindings remain `query()` and `get_schema()`, while the -stable raw operation IDs are `query` and `schema`. - -Each `ProviderManifest` is source-agnostic, immutable, revisioned, and -SHA-256-digested. Operations declare separate raw IDs and Python binding names, -schema dialect/provenance, cursor behavior, inline/handle/untyped delivery, -budget class, and descriptions. Hosts classify every operation's effect and -own policy metadata; do not trust effect annotations received over a bridge. -`ProviderService` and `BridgeProvider` replace `DataSourceService` and -`BridgeDataSource`. - -Remove imports of `DataSource`, `DataSourceCapabilities`, `SearchResult`, -`DataSourceRegistry`, `register_source_type`, `SOURCE_PROTOCOL_VERSION`, and -the old SQL factory. Use `ProviderManifest`, `ProviderOperation`, -`ProviderRegistration`, `ProviderCatalog`, `ProviderRegistry`, and -`PROVIDER_PROTOCOL_VERSION` instead. `EvidenceRef` is also replaced by -structured `EvidenceLocation`/`EvidenceRange` values. - -`RUNNER_PROTOCOL_VERSION` is now 3. The manifest migration introduced provider -protocol 3; the context-first handler migration above makes the current provider -protocol 4. Requests -must use `data_sources` as a list of `{type, name, ...config}` objects. The -relay startup event now reports `provider_protocol` instead of -`source_protocol`. Upgrade the runner request, staged relay, and provider -catalog atomically; mismatches fail before work begins. - -### Built-in sandbox capabilities use one brokered ABI - -`RunnerEnvironment` and `PyodideEnvironment` now generate their existing -`llm_query*` and provider Python APIs from one immutable capability manifest. -Built-in environments no longer put raw `SubcallClient` or provider bound -methods in the sandbox globals mapping. The loop's structured JSON -batch replacement also uses the environment's broker-backed adapter. - -Hosts that need correlation may pass additive `capability_run_id` and -`capability_parent_run_id` arguments to `create_environment`. Optional typed -`capability_guard`, `capability_annotator`, and `capability_observer` callables -are integration seams for policy/budget facts and trace projection; no default -policy, shared ledger, persistence, or transport change is implied. -Capability calls carry a frozen `CapabilityId` made of `kind`, `provider_type`, -`source_id`, and `operation`; the broker resolves the full descriptor from its -manifest so future schema, documentation, and policy metadata do not change the -wire identity. Trace integrations should persist -`CapabilityResult.to_trace_dict()`, the content-free projection. The full -`to_dict()` includes arguments, inline results, error messages, evidence -references, and result-handle locators and is suitable only for explicitly -configured replay retention. -`capability_annotator` is an exactly-once post-attempt finalizer: it runs after -each attempted handler outcome, including invalid results and propagated -cancellation, and is skipped for validation or guard exits before an attempt. -Accounting integrations can therefore reconcile a guard reservation by -`call_id` without a second broker callback. -Trusted handlers may return `CapabilityOutcome(result=..., metadata=...)` or -`CapabilityOutcome(error=CapabilityError(...), metadata=...)` to attach typed -provider failures, usage, or evidence without raising or requiring a transport -parser. Error `code` is now an extensible stable string; `CapabilityErrorCode` -remains the set of broker-defined string constants. Raw handler return values -are normalized automatically at registration. Provider sequence metadata is -preserved first and finalizer sequence metadata is appended; conflicting -singular handle/child-run facts become an annotator error rather than silently -choosing one. -The unused `BatchLLMError` compatibility type and repair branch are removed. -Plain `llm_batch` reports its brokered typed failure as `CapabilityCallError`; -callers that need ordered per-item failures use `llm_batch_json` or the trusted -`llm_batch_with_errors` adapter, both of which remain single atomic broker calls. -`DataSourceRegistry` no longer has a standalone `globals()` projection. Hosts -construct one run broker from `capability_registrations()` and pass it to -`broker_globals()`, preventing an accidental second broker without the run's -identity, guard, accounting annotator, or observer. -Custom environments must expose the resulting registry's -`accessor_manifest()` as well. There is no fixed generic-verb fallback: an -environment that omits this manifest supplies no provider accessors to the -count-policy check, so its custom bindings are not covered. - -Custom `RLMEnvironment` implementations must now implement -`sandbox_subcalls(subcalls)`. Return a broker-backed `SubcallClient`; the -`droste.capabilities.broker_subcalls()` helper supplies the standard standalone -adapter. `run_rlm` replaces all canonical subcall globals from that method and -does not retain a raw-client fallback. -### Trace ABI v1 unifies live events and terminal run records - -Every structured event now carries the required v1 envelope: `run_id`, positive -monotonic `seq`, UTC `timestamp`, `type`, `version`, `persistence_class`, and -required `depth` (`0` for roots), with optional `parent_run_id`. Partial pre-envelope dictionaries are not a -supported wire format. Pyodide relay telemetry is a child run correlated by -`parent_run_id`, so the relay and engine do not race to own one sequence. - -`RLMResult`, runner responses, and CLI JSON expose `run_record`. Durable -terminal/usage/budget/policy/capability facts are always selected; -code/output/error/repair/result/replay content requires an explicit -`TraceRetentionPolicy`; progress/deltas remain transient. The canonical -trajectory-free `result` is nevertheless always delivered live before `done`; -full `replay` is emitted only when selected. The default record retains no -configurable content. Training authorization is independent and defaults to -denied; enabling it requires an authorization reference plus the `training` -purpose. Retention policies now carry a stable `policy_id` and may record an -absolute host-managed expiry. - -Attach `on_run_record` to `RLMConfig` or `create_execution_context` to hand the -resolved immutable record to local persistence. If an existing context is -passed to `run_rlm`, it owns trace settings; explicit conflicting `RLMConfig` -settings raise `ValueError`. See [docs/trace-abi.md](docs/trace-abi.md). - -The runner protocol is now version 2. Success, refusal, and exception responses -share one field set. The unary response does not include a full trajectory; -when replay is retained, `replay.result.trajectory[].llm_input` is a structured -message list instead of a JSON-encoded string. Requests must send -`"protocol_version": 2`; mismatched v1 requests fail before work begins. - -### Hosts select environments through one substrate factory - -New in-process hosts should build an immutable `EnvironmentConfig`, then call -`create_environment_context(config, ...)` and `create_environment(config, -...)`. The CLI, HTTP runner, benchmark harness, and reference Pyodide adapter -now use this path, so execution budgets and substrate selection have one owner. - -Existing direct `RunnerEnvironment` imports remain compatible. Pyodide hosts -should migrate: `kind="pyodide"` selects the signal-free `RawExecutor` path and -requires `host_managed_timeout=True` plus `host_managed_isolation=True`. Those -flags assert that the host already supplies the external deadline and WASM -jail; they do not create either boundary. A Pyodide config with a nonzero -`exec_timeout_ms` fails loudly instead of pretending to enforce a signal timer. - -### Runner implementation modules are focused - -The former `droste_runner.runner` monolith is split into `run`, `protocol`, -`http_clients`, `sources`, and a small `environment` compatibility shim. The -generic native environment now lives at `droste.environments.RunnerEnvironment`; -the CLI and in-tree embedders use that canonical import. - -Existing imports from `droste_runner.runner` remain valid, including `run`, -`RunnerEnvironment`, HTTP clients, source helpers, and protocol constants. -This is a structure-only change: request/response fields, protocol versions, -adapter dispatch, and process entrypoint behavior are unchanged. -The supported process entrypoint remains `python -m droste_runner` from an -installed package. Direct execution of an extracted `runner.py` file is not a -supported entrypoint; the old repository-layout `sys.path` mutation was -intentionally removed. - -### Harness prompts resolve from versioned prompt packs - -The built-in system, user, refinement, repair, and extraction prompts now load -from complete TOML prompt packs and resolve once per run by `(model, profile)`. -The default generic `full` pack preserves the prior harness behavior; -`minimal` and `none` preserve the existing tips profiles. Existing complete -`system_prompt` and user/refinement template overrides remain accepted. - -`RLMConfig.prompt_profile` is the new profile spelling; when omitted, -`tips_profile` remains compatible. `RLMConfig.enforce_contract=None` delegates -to the resolved pack's policy default, while explicit booleans still win. -`run_rlm` accepts an immutable caller pack or consumer catalog. Invalid packs -fail before the first model call. - -`RLMResult.prompt_pack`, CLI JSON, and the built-in runner response now expose -the additive resolved ID, revision, profile, resolution tier, model family, and -provenance. Runner requests may send additive `prompt_profile`; protocol version -1 is unchanged. See `docs/prompt-packs.md` for authoring and fallback rules. - -### Trajectory execution status is explicit - -`IterationRecord` and each built-in runner trajectory entry now include the -additive string field `execution_status`, currently `"success"` or `"error"`. -Use it instead of interpreting the text in `execution_result`: successful -stdout may legitimately begin with `ERROR:`. The existing `execution_result` -field and runner protocol version are unchanged. -Direct positional construction that omits the new field remains accepted and -defaults conservatively to `"error"`; engine-created records always set the -status from their typed step outcome. - -### Semantic structured batches fail closed when incomplete - -`PolicyHints(semantic=True)` now keeps any `llm_batch_json` result with -unresolved item errors from confirming `answer["ready"]`. The loop returns the -violation for repair; if the iteration budget ends, the existing bounded -extraction path either produces an explicitly unconfirmed answer with -`recovered_error.type == "PolicyError"` or leaves the policy error fatal. - -A later error-free call clears an incomplete result only when it repeats the -exact prompts, contexts, schema, and validator object. A different successful -batch is not completion evidence for earlier partial work. This is a behavior -change only for callers that explicitly enable semantic policy; runs without -that hint are unchanged. - -If the remaining total subcall budget is smaller than the minimum needed to -replay every unresolved recorded exact request, the loop now stops requesting -impossible repairs and enters the existing bounded extraction/failure path -immediately. In the additive diagnostic details, `required_subcalls` is the sum -of the full batch cardinality for each unresolved recorded exact request, not -the number of failed items. Validator object identity participates in exactness, -so otherwise-identical calls made with distinct validator objects are distinct -recorded requests and each contributes its full batch cardinality. - -Successful extraction remains unconfirmed and preserves a `PolicyError` in -`recovered_error`; without extraction evidence, the same policy error remains -fatal. Its additive `details.reason` is -`"semantic_exact_retry_budget_exhausted"` and also includes the remaining -subcall count plus unresolved recorded-request and item counts. Provider errors -do not trigger this handoff while enough budget for the recorded exact retries -remains. - -When that terminal handoff has no retained `answer["content"]`, the loop now -makes one root finalization request and executes at most one returned code block -in the existing persistent REPL. No missing-code or execution repair is made. -All model-visible subcall bindings are disabled before that code executes, so -the step cannot spend further subcall budget and can synthesize already retained -work; any resulting draft still passes through the normal unconfirmed extraction -path. Incomplete exact semantic evidence continues to revoke readiness, and a -finalization that retains no draft leaves the original typed policy failure -fatal. If the root finalization request itself fails, the event stream emits an -additive `finalization_error` event with `error_type` and `message`; the original -policy failure remains authoritative and no finalization retry is made. - -To make this completeness check enforceable, `run_rlm` continues to replace any -`llm_batch_json` and `llm_query_batched_json` entries in the mapping returned by -`environment.globals()` with Droste's tracked bindings before sandbox execution. -While semantic contract enforcement is active, it now also replaces -`llm_query`, `llm_batch`, `batch_llm_query`, and `llm_query_batched` with -revocable bindings so saved aliases can be disabled during terminal -finalization. Embedders that enable semantic enforcement must treat all six -names as reserved: expose custom helpers under different names, or customize -the `SubcallClient` passed to `run_rlm` instead. Runs without semantic -enforcement retain the prior direct-binding behavior. - -The three built-in prompt packs are revised from `1.0.0` to `1.0.1` to describe -the bounded terminal mode in their error-repair templates. Caller-provided packs -remain valid without changes; the engine still enforces the no-subcall boundary -even when a custom template does not describe it. - -### ModelRelay root request accounting - -`ModelRelayClient.root_requests_issued` exposes a thread-safe cumulative count -of HTTP root requests dispatched by that client, including repair, extraction, -streaming, and failed requests. Payload validation and request-construction -failures before dispatch do not increment the count. This additive client-level -metric does not change `RLMResult` or the runner protocol. - -## 0.10.6 (from 0.10.5) - -### Confirmed answers can carry structured metadata - -Sandbox code may set a JSON object at `answer["metadata"]`. A confirmed -result exposes a validated, defensively copied version as -`RLMResult.answer_metadata`; the built-in runner and CLI JSON output include -the same additive `answer_metadata` field. Metadata is limited to 64 KiB. - -`answer["metadata"]` is now reserved and validated whenever an answer claims -readiness, even when contract enforcement is disabled. Invalid metadata blocks -confirmation and is returned to the model for repair. Emit only plain JSON -objects, arrays, strings, booleans, null, finite numbers, and integers within -JavaScript's safe range; tuples, custom scalar types, non-string object keys, -cycles, excessive nesting, and oversized structures are rejected. - -The text-only terminal extraction fallback intentionally returns empty -metadata: partial structured values are not evidence for newly synthesized -text. Embedders that construct `RLMResult` directly need no change because the -new field defaults to an empty object. - -### Successful semantic evidence and structured subcalls - -`ExecutionStats` now distinguishes attempted `calls_made` from -`successful_calls`. Subcall clients must reserve attempted calls before -dispatch as before, and increment successful calls only for items that return -usable text. Semantic policy hints now require successful evidence; an -all-failed batch no longer satisfies the ready gate. `RLMResult`, CLI JSON, and -runner protocol responses expose the successful count as -`sub_calls_succeeded` / `successful_subcalls` while preserving existing -attempted-call fields. - -Sandboxes now expose `llm_batch_json` (also `llm_query_batched_json`) for -opt-in, locally validated structured output. It supports a documented -deterministic JSON-schema subset, caller validators, ordered per-item errors, -and malformed-only bounded repair. Provider errors are attributable and are -never converted into parse retries. ModelRelay clients continue to use one -native batch request for each initial or repair batch. - -Budget rejection uses `BudgetExhausted`, carrying the exhausted resource, -requested amount, and remaining authorization. Structured batch errors expose -the stable `budget_exhausted` type without matching exception text. - -### Semantic policy is enforced when confirming an answer - -With `PolicyHints(semantic=True)`, inspection and local aggregation blocks may -now execute before a semantic subcall. The ready-time gate still refuses to -confirm an answer until at least one `llm_query` or batched equivalent succeeds. -Hosts get the same final-answer contract without forcing harmless preparation -steps through policy-repair iterations. - -## 0.10.5 (from 0.10.4) - -The Pyodide credential broker now recognizes an optional exact -`data_source_endpoint` as a short-lived runner callback. The held runner token -is injected only for that exact URL, matching the root/subcall callback rules; -data-source bearer credentials remain in the trusted host. - -## 0.10.4 (from 0.10.3) - -### ModelRelay batches use the synchronous batch endpoint - -`ModelRelaySubcallClient.llm_batch` now sends one `POST /responses/batch` -request instead of spawning per-item worker threads. Hosts that mock ModelRelay -must implement the typed batch response envelope (`results[].id/status/response/error`). -There is deliberately no runtime fallback to individual requests. - -### Pyodide relay supports hosted runners without a database - -The Deno relay no longer assumes every host request has `db_path`. Hosts may -stage a pure context/inference adapter with no DB service. A short-lived -`token` in the request is stripped before the sandbox starts and is injected -only for exact `root_endpoint`, `subcall_endpoint`, and -`subcall_batch_endpoint` callback URLs. - -## 0.10.3 (from 0.10.2) - -### Failed-only trajectories are not extraction evidence - -Failed-only trajectories with no retained draft are not extraction evidence; -their terminal error remains fatal. Because no extract call is attempted, -`extract_error` remains `None` on this path. - -## 0.10.2 (from 0.10.1) - -### Terminal failures can recover a typed best-effort answer - -When a terminal execution error, exhausted subcall budget, or unresolved -ready-policy hint leaves usable partial work, `run_rlm` now gives the existing -extract fallback one bounded chance to synthesize an answer. On success, -`RLMResult.extracted` is `True`, fatal `error` is `None`, and the superseded -step error is preserved as `recovered_error`. The runner and CLI JSON surfaces -include the same additive `recovered_error` object. Hosts should present these -answers as unconfirmed and may use `recovered_error.type` for telemetry. - -Any failed execution whose repair also fails (or returns no code) is now -retained in `trajectory`, including on mid-run iterations that later recover. -Consumers must tolerate duplicate iteration numbers and `execution_result` -values beginning with `ERROR:` even when the final run is ready. - -The exact extract response `unable to determine from the work so far` is not a -successful recovery: it produces `extract_error.type == "InsufficientEvidence"` -and retains the fatal terminal error. - -Any failed sandbox execution now revokes `answer["ready"]`, including when the -block set readiness or rebound the answer dict before raising. Such a run may -consume another root iteration instead of returning `ready=True` alongside a -fatal execution error; retained `answer["content"]` remains available to repair -or extraction. - -## 0.10.1 (from 0.10.0) - -### Event emission is now opt-in — attach sinks or loop events go silent - -`run_rlm` no longer prints NDJSON events to stderr by default (#35). If you -relied on the default stderr stream — for example to feed the relay's event -forwarder or a "watch it think" UI — attach the sinks explicitly: - -```python -from droste.execution.progress import emit_event, emit_progress - -run_rlm(..., on_progress=emit_progress, on_event=emit_event) -``` - -If you build your own `ExecutionContext` and pass it as `context=`, attach the -sinks THERE — `run_rlm`'s `on_progress`/`on_event` arguments apply only when -it creates the context for you: - -```python -context = create_execution_context(..., on_progress=emit_progress, on_event=emit_event) -run_rlm(..., context=context) -``` - -`droste_runner.run()`'s built-in HTTP path attaches them itself. An -`adapter_module` does NOT inherit that: the runner delegates to the adapter -before any sink-configured context exists, and the Deno relay likewise calls -your adapter directly — an adapter that calls `run_rlm` must attach the sinks -itself (as `examples/pyodide-host/pyodide_host_adapter.py` now does). **This -is a silent degradation if missed**: the run still succeeds; the event stream -is just empty. - -Related changes in the same release: - -- New event types `llm_response` and `execution_error`; `output` events gained - `calls_made` / `answer_ready` / `answer_content_chars`. Consumers switching - on event type should ignore unknown types (additive by design). -- `RLMConfig.verbose` is no longer read by the core. Verbose views are pure - projections of the event stream — apply - `droste.execution.progress.render_verbose(event)` in your own sink. -- Custom event emitters are validated: an event `type` outside - `droste.execution.progress.EVENT_TYPES` raises `ValueError`. - -### Relay stream + executor fixes (behavioral, no action needed) - -- A streamed LLM response that ends without the protocol's terminal event, or - carries a mid-stream `error` record, now **fails the call** instead of - returning accumulated partial text as a clean answer (#43). -- Hosted `droste_runner` subcalls now request and consume ModelRelay's - `responses-stream/v2` NDJSON contract. Callback servers should honor that - Accept header so slow generations keep bytes moving through reverse proxies; - plain JSON remains supported for local callback handlers. -- The Pyodide `RawExecutor` no longer silently truncates oversized sandbox - output; over-budget prints raise the same `SandboxError` on every substrate, - giving the model its narrow-your-query feedback (#44). - -## 0.10.0 (from 0.9.0) - -### The Deno relay ships inside the wheel — drop the tarball + SHA pin - -The relay (`relay.ts` and friends) is package data under -`droste/substrates/_relay/`. Stage it from the installed wheel: - -```sh -cp "$(droste relay-path)"/*.ts / -``` - -(or `python -c 'from droste.substrates import relay_dir; print(relay_dir())'`). -The wheel is the one pinned, hash-verified artifact in your lockfile, so relay -and engine can no longer drift; delete any release-tarball download and -SHA-256 pin from your build. **Copy every `.ts` file** — 0.10.0 added -`deps.ts` (the single Pyodide version pin), which `relay.ts` imports at -runtime; a hardcoded file list from 0.9.0 ships a relay that cannot resolve -its own imports. The release tarball still exists as a convenience for -non-Python consumers; it is no longer the embedder path. - -At startup the relay emits a `startup` event — -`{engine_version, runner_protocol, source_protocol}` — so "which contract is -this app actually running?" is a log line, not a bundle autopsy. - -### Accessor discovery reads an explicit manifest — forward it or lose it - -The `_droste_data_source` namespace marker is gone. The count contract's -`len()`-over-accessor check now reads accessor names through an optional -environment method (#31). If you compose data sources into a **custom** -`RLMEnvironment` around `DataSourceRegistry.globals()`, forward the registry's -manifest: - -```python -def accessor_manifest(self): - return self._registry.accessor_manifest() -``` - -**This is a silent degradation if missed**: enforcement falls back to the -static generic verbs and stops covering your sources' actual accessor names -(including `extra_methods`). Environments built on `droste_runner`'s -`RunnerEnvironment` need no change. - -### Smaller 0.10.0 deltas - -- `register_source_type` / `SOURCE_PROTOCOL_VERSION` moved down to - `droste.sources.registration`; `droste_runner` re-exports the old names, so - existing imports keep working. Pass the protocol version you were written - against as a literal — echoing the engine's own constant back defeats the - startup compatibility check. -- Trajectory `llm_input` is structured (the message list). On the runner wire - it is serialized as a **JSON string** — same type as before, finally - parseable (it was a Python repr). -- `llm_batch_with_errors` is now bounded (worker pool + the same 50-prompt cap - as `llm_batch`) and reports per-item errors in index order. - -## 0.9.0 (from 0.8.x) - -### `protocol_version` is required on every runner request - -Every `droste_runner` request must carry `"protocol_version": 1`. A missing or -mismatched version gets a structured refusal (`protocol_version_missing` / -`protocol_version_mismatch`) naming both sides — no partial work. See -[docs/architecture.md](docs/architecture.md), "The runner protocol". - -### The engine is domain-blind — declare your verbs - -Domain-specific verbs are no longer auto-bound by `hasattr`. A source declares -them itself: - -```python -class MySource: - extra_methods = ("get_threads", "get_labels") -``` - -Both transports honor the declaration (the registry binds them in-process; -the bridge advertises and dispatches them). `search()` also lost its -domain-specific kwargs — sources own their signatures. This is the -`SOURCE_PROTOCOL_VERSION` 1 → 2 bump: a source registering with `protocol=1` -fails loudly at startup rather than silently losing its accessors. +Update strict consumers and packaged fixtures to Trace ABI v8. diff --git a/docs/README.md b/docs/README.md index 52dcbbd..154d8b7 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,27 +1,24 @@ # Documentation -Start with the [quickstart](quickstart.md), then use a recipe that matches your -data: +Choose the page that matches the work: -- [Logs, chat archives, and SQLite](recipes.md) -- [Embed Droste in a Python application](embedding.md) -- [Architecture overview](architecture-overview.md) -- [Benchmark results](benchmarks.md) +- [Use the command line](quickstart.md) +- [Embed Droste in Python](embedding.md) +- [Understand the architecture](architecture.md) +- [Add a data provider](providers.md) +- [Review benchmark results](benchmarks.md) ## Technical reference -These documents specify contracts for hosts, provider authors, and contributors. -You do not need them to use the command line tool. +These are versioned contracts for host and protocol implementers. CLI users do +not need them. -- [Architecture and runner protocol](architecture.md) -- [Budgets](budgets.md) -- [Provider manifests](provider-manifests.md) -- [MCP over stdio](mcp-stdio.md) -- [MCP over Streamable HTTP](mcp-http.md) -- [Prompt packs](prompt-packs.md) -- [RLM skills](rlm-skills.md) -- [Scaffold manifest](scaffold-manifest.md) -- [Trace ABI](trace-abi.md) -- [Verifiers harness](verifiers-harness.md) +- [Runner protocol](reference/runner.md) +- [Trace ABI](reference/trace.md) +- [Prompt packs and RLM skills](reference/prompt-packs.md) +- [Scaffold manifest](reference/scaffold.md) -Project design notes and brand assets live under [internal](internal/). +Contributor workflow and migration guidance live in +[CONTRIBUTING.md](../CONTRIBUTING.md) and [UPGRADING.md](../UPGRADING.md). +Benchmark reproduction remains beside the tooling in +[benchmarks/README.md](../benchmarks/README.md). diff --git a/docs/architecture-overview.md b/docs/architecture-overview.md deleted file mode 100644 index da1029f..0000000 --- a/docs/architecture-overview.md +++ /dev/null @@ -1,31 +0,0 @@ -# Architecture overview - -Droste keeps the source data in an execution environment instead of placing it -in the root model prompt. On each iteration, the root model writes Python. The -environment runs that code and returns only its printed output. - -```text -question -> root model -> Python -> execution environment -> printed output - ^ | - +---------------- next iteration ---------------+ -``` - -The generated program can use ordinary Python for exact work and call a -smaller model for bounded interpretation: - -- `llm_query(prompt)` makes one model call. -- `llm_query_batched(prompts)` evaluates an ordered batch concurrently. -- `llm_batch_json(prompts, schema, ...)` validates structured batch results. - -The host decides which data and capabilities the program receives. Built-in -sources cover text files and SQLite. Provider manifests and the capability -broker allow hosts to expose additional sources without giving generated code -direct access to transport clients. - -One run-scoped ledger authorizes tokens, subcalls, depth, wall time, -iterations, and output ceilings. A separate sandbox configuration limits local -execution and captured output. Every run can emit a versioned event stream for -live progress, auditing, and replay. - -See the [technical architecture](architecture.md) for the complete broker, -runner, provider, budget, sandbox, and trace contracts. diff --git a/docs/architecture.md b/docs/architecture.md index 8020424..71d8dd7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,455 +1,67 @@ # Architecture -## The loop - -``` - -The harness strategy is resolved once at run start as one immutable -[prompt pack](prompt-packs.md). Package/file I/O ends at the loader boundary; -slot validation, `(model, profile)` fallback, and rendering are pure value -transformations. The engine never merges partial packs or changes strategy -mid-run. - -Optional [RLM skills](rlm-skills.md) are immutable, content-addressed strategy -data loaded through an explicitly registered read-only provider. They never -change the prompt-pack or loop contract. The resolved prompt, capability, -contract, inference, budget, and sandbox facts form one content-addressed -[scaffold manifest](scaffold-manifest.md), checked before inference when a -checkpoint declares requirements. -question ──▶ root LLM ──▶ python code ──▶ sandboxed REPL ──▶ output - ▲ │ - └── refinement prompt ◀───────┘ - (loop until answer["ready"], budget-bounded) -``` - -The root model's *initial prompt* contains a description of your data -(type, size, a preview for string contexts; names and sizes for files) — -not the data itself, which lives as REPL variables (`context`, or named -sources like `db`). What flows back to the root each iteration is exactly -what the executed code prints: narrow deliberately, because printed output -is root-visible by design. Each iteration it writes Python; -the sandbox executes it; stdout feeds the next iteration. Three functions -bridge back to language models: - -- `llm_query(prompt)` — one subcall. -- `llm_query_batched(prompts)` — concurrent subcalls (bounded workers), - order-preserving. Aliases: `llm_batch`, `batch_llm_query`. -- `llm_batch_json(prompts, schema, ...)` — ordered local JSON validation and - malformed-only bounded repair, with per-item errors (the error indices are - authoritative because valid JSON `null` and failed slots are both `None`). - Alias: `llm_query_batched_json`. With explicit semantic policy, an incomplete - result blocks answer confirmation until the exact prompts, contexts, schema, - and validator object later produce an error-free result. - -## Capability broker - -Generated code reaches data sources and sub-LLMs through one brokered path. -At environment construction, trusted source verbs and the subcall client become -an immutable `CapabilityManifest`. The native and Pyodide environments generate -the same model-facing Python functions from that manifest; those functions -submit immutable `CapabilityCall` values to an exact allowlist and unwrap the -single typed `CapabilityResult` envelope for compatibility with existing code. -Calls carry only a frozen `CapabilityId`—kind, reusable provider type, -configured source ID, and operation. The broker resolves the complete descriptor -from the manifest, keeping stable dispatch identity separate from documentation, -schema, pagination, budget, and policy metadata that descriptors may gain. - -The bundled Pyodide relay applies the same rule to local databases: a trusted -provider interpreter consumes host paths and serves the broker bridge, while -the untrusted REPL receives neither a database mount nor a path. There is no -direct-access topology or configuration fallback. Context-only requests may -have no provider, but generated code never gains ambient I/O as a substitute. - -Each result envelope carries the capability kind, configured source or provider -identifier, operation, call and run IDs, typed status/error, value or bounded -handle, usage and budget deltas, evidence references, and optional parent/child -run IDs. Most fields are empty until a host supplies facts through the broker's -narrow annotator seam. A guard may deny a call before dispatch, and an observer -may project the completed immutable envelope into a trace. Those are interfaces, -not policy, ledger, or trace implementations; the corresponding subsystems own -their semantics. Durable capability events use `CapabilityResult.to_trace_dict()`: -it retains IDs, status, typed error code/type, accounting facts, evidence count, -and optional result-handle media type and size, but no parameters, inline result, -error message, evidence reference, or handle locator. Full envelopes are replay -content and require an explicit content-retention policy. -At registration, raw trusted handler values are normalized into one -`CapabilityOutcome` convention. A provider can return a result or a typed -provider failure with an extensible stable error code, together with usage and -evidence metadata; unexpected exceptions remain broker `handler_error` values. -Provider sequence facts precede the exactly-once finalizer's appended facts. -Conflicting singular handle or child-run facts fail closed, so dispatch never -needs a provider-specific parser or precedence convention. -The annotator has exactly-once post-attempt semantics: it runs after success, -handler error, invalid result, or propagated cancellation, but not when run -identity, allowlist, arguments, or the guard reject a call before its handler is -attempted. The separate attempt authority starts at admission and therefore -settles even those post-admission guard exits without changing annotator scope. - -Every trusted registration has one context-first handler signature: -`handler(CapabilityExecutionContext, *args, **kwargs)`. The context is a frozen -view of call/run identity, deadline, and reservation plus two narrow mechanisms: -`check()` for cooperative cancellation/deadline observation and cumulative -`checkpoint()` for token/subcall progress. It exposes no ledger, trace, or -callback registration. A broker-owned mutable attempt controller closes the -cancellation/finalization race; admission begins the exactly-once settlement -boundary even when a later policy guard denies dispatch. - -`call_id` is an in-flight identity, not a durable idempotency key. After call -validation, the broker atomically claims a caller-supplied ID before asking the -attempt authority for admission and retains that claim through result delivery. -A concurrent duplicate is rejected without admission, checkpointing, or -settlement. The claim is released when `dispatch` returns or raises, including -after admission refusal, so a later dispatch may deliberately reuse the ID. -Hosts that require replay deduplication must enforce that separate policy before -dispatch rather than turning the budget authority into an identity registry. - -`llm_batch` is one broker operation and invokes the subcall client's batch method -once. It is not decomposed into nested `llm_query` calls, so ordering, -reservation, concurrency, and provider-native batch semantics stay intact. -`llm_batch_json` remains local deterministic composition over the broker-backed -batch adapter. `ProviderService` is a fixed bridge transport behind registered -handlers and is deliberately not a second capability ABI. - -The loop ends when the model sets `answer["ready"] = True`, or the resolved -compute budget cannot authorize more work. On a supported terminal handoff, -a trajectory containing executed work may use a single **extract pass** to -produce a best-effort answer, flagged `extracted=True` and never -presented as confirmed. Policy violations withhold gated content from the -final answer rather than silently passing it through. - -Policy is caller-supplied rather than inferred from question text. In-process -callers opt into semantic enforcement with -`RLMConfig(policy_hints=PolicyHints(semantic=True))`; without that hint, -structured batch results remain prompt-driven and the loop does not add this -completeness gate. - -## Budgets and cost - -Everything expensive is authorized by one immutable `Budget`: total tokens, -subcalls, child depth, wall time, and per-request root/subcall output ceilings. -One run-scoped `BudgetLedger` owns mutable accounting. Root calls and brokered -capabilities atomically reserve before dispatch, then commit actual work and -refund unused authorization. A failed vector reservation changes nothing; -concurrent work therefore cannot oversubscribe one dimension while another -passes. See [Budgets](budgets.md). - -The root prompt reports the effective per-call subcall output ceiling separately -from input capacity. Built-in clients implement the optional -`SubcallOutputTokenLimitProvider` companion protocol: a positive -`output_token_limit` is bounded, `None` is deliberately unbounded, and absence -is rendered as unknown rather than guessed. HTTP-backed clients report only an -explicit positive limit because a callback-owned default is not knowable from -the runner. The base `SubcallClient` protocol is unchanged, and broker-backed -and semantic-policy wrappers forward the optional metadata. - -Input planning uses the separate optional `SubcallInputCapacityProvider`. -`input_token_capacity` returns an immutable `SubcallInputCapacity`; protocol -absence alone maps legacy clients to unknown. The value is the effective -usable caller-payload bound across the adapter, transport, and model, not a raw -vendor context-window claim. `unbounded` is truthful only when arbitrary -payloads are guaranteed, such as by transparent chunking. Client and rollout -facts resolve once before inference, and conflicting known facts fail closed. -The exact bounded, unbounded, or unknown state appears in the root prompt and -scaffold manifest. This capacity guides chunking only and does not authorize -tokens or enlarge the subcall output ceiling. - -## Providers and configured sources - -Provider metadata is immutable, source-agnostic data. `ProviderManifest` -contains a revision, canonical SHA-256 digest, and ordered `ProviderOperation` -values. Each operation carries a stable raw ID, a separately validated Python -binding name, documentation, parameter/result `SchemaSpec` values with explicit -dialect and provenance, cursor semantics, inline/handle/untyped delivery, and -a budget class. Cursor operations must describe both the input cursor and -output `next_cursor`. - -The host owns the imperative edge: a `ProviderRegistration` pairs the manifest -with an exact side-effect map, optional policy metadata, and a binder returning -live handlers. Effects are never accepted from a transport. An explicit -`ProviderCatalog` binds declarative `ConfiguredSource` values into one -per-run `ProviderRegistry`; there is no process-global registry and requests -cannot name code to import. The registry snapshots descriptors once for the -run, then projects broker registrations, prompt text, Python bindings, and the -policy accessor manifest from those same values. - -Each successful bind also produces one explicit runtime ownership token. A -runtime may omit cleanup when it owns no resource; otherwise it supplies one -close callback. Registries close their runtimes once in reverse acquisition -order and clean up partial binds deterministically. Two sources cannot share a -runtime token; shared pools require separate provider-owned leases. No -finalizer, garbage collector, or process-global registry participates in this -lifecycle. - -Runtime closure starts after dispatch has quiesced; cancellation remains the -broker execution context's job. Content-free runtime counters remain readable -after close so response projection does not prolong resource lifetime. - -`ProviderService`/`BridgeProvider` carry a verified manifest plus raw operation -calls across an interpreter boundary. Invoke requests also carry exact portable -execution facts; responses carry a separately validated cumulative checkpoint -which the receiving broker applies before its sole final settlement. The -receiving host supplies its own effects and policy before binding. Unknown -operations are rejected against the bound handler map, and schema/digest -mismatches fail before dispatch. - -The bundled SQLite source is local-mode: SELECT-only policy gate (single -statement, masked-identifier keyword scanning, LIMIT injection, row caps, -statement timeout), database opened `mode=ro`, and `PRAGMA query_only=ON` -enforced on host-supplied connections. - -The bundled `filesystem_text` provider is also local-mode and read-only. One -pinned root directory descriptor and componentwise `openat`-style resolution -own all five operations (`list`, `read`, `grep`, `search`, and `stat`). Symlinks -are never followed, include/exclude policy is reapplied to explicit paths and -cursor continuations, and every scan/read/result page is bounded. Its Markdown -section support is optional enrichment over the same text contract, not another -provider or index. Cursor payloads are self-contained but authenticated to the -bound runtime, preventing callers from forging offsets or evidence coordinates. -Required POSIX security primitives are a bind-time -precondition; there is no weaker path fallback. - -In native mode, generated bindings grant no filesystem path or handle, but the -in-process REPL remains arbitrary Python and therefore may have whatever -ambient authority its host process already has. A host that needs the configured -root to be non-ambient must isolate the REPL and keep `filesystem_text` behind -the generic provider bridge. The reference two-Pyodide topology does exactly -that: only the trusted provider interpreter mounts the root. - -## The runner protocol (embedding) - -`python -m droste_runner` is a one-shot JSON worker: a host writes a request -file (question, budgets, endpoints or context, declarative data sources) and -reads a response (answer, `ready`, `extracted`, iterations, attempted and -successful subcalls, -trajectory, usage). This is how non-Python hosts embed the engine. -Each trajectory entry carries `execution_status` (`success` or `error`) beside -the unchanged `execution_result` text, an `attempt_kind` (`initial`, `repair`, -or `terminal`), and the exact character count of stdout returned by the -environment. Consumers must not infer any of these from output text. Oversized -stdout is rejected rather than silently truncated. Result-level `stdout_chars` -is the sum of these exact counts across the retained trajectory. - -The Python implementation is split by ownership: `droste.environments` -provides the generic native in-process environment; `droste_runner` keeps -HTTP clients, remote source transport, envelope helpers, and orchestration in -`http_clients`, `sources`, `protocol`, and `run` modules respectively. -`droste_runner.runner` is a convenience facade, not a second implementation. -Both successful and refused envelopes are shaped by -`protocol.build_response`, so their shared fields cannot drift. - -### Scaffold preflight - -Hosts may set `"operation": "preflight"` to resolve and validate the exact -effective scaffold before authorizing inference. Omitting `operation` preserves -the normal `"run"` behavior. Preflight requires the same budget, model, -rollout, PromptPack selector, provider configuration, sandbox configuration, -and optional `checkpoint_scaffold_requirements` as execution, but it does not -require root/subcall endpoints or a token. - -The engine and runner share one resolver for PromptPack selection, policy -defaults, environment globals, capability manifest, rollout configuration, -scaffold construction, and `ScaffoldCompatibilityError`. A successful response -is intentionally a separate, content-free envelope: - -```json -{ - "protocol_version": 10, - "operation": "preflight", - "status": "success", - "preflight": { - "schema_version": 1, - "scaffold_manifest": {"schema_version": 2, "id": "sha256:..."} - }, - "error": null -} -``` - -The abbreviated manifest above is illustrative; the actual response carries -the complete `ScaffoldManifest`. It never carries the question, context, -rendered prompt prose, answer, trajectory, trace, run identity, or usage. A -checkpoint mismatch returns `status: "refusal"`, no preflight result, and a -typed `ScaffoldCompatibilityError` with code `scaffold_incompatible` and -structured mismatch paths. Hosts can therefore distinguish a policy refusal -from worker or transport failure without parsing an error message. Preflight -installs no event sink and uses a fail-if-called capability placeholder; root, -subcall, and configured-provider callbacks cannot be dispatched. - -Configured providers are projected from their immutable registration manifest, -effect policy, source type, and source name. Preflight deliberately does not -invoke live provider binders: binders may open files, databases, or network -connections, while those runtime resources and task data do not participate in -the scaffold identity. Normal execution binds the same descriptors to their -live handlers for the run. Missing/mismatched protocol refusals and invalid -operation values carry `operation: null`. After a current-protocol request has -selected `run` or `preflight`, worker exception envelopes retain that accepted -discriminant and their structured error fields. - -`run_worker_request(...)` is the process-host boundary for embedders that -inject a trusted `ProviderCatalog`. It returns a frozen `WorkerOutcome` -containing the response and process exit code, and owns the version gate, -operation selection, and exception projection. Hosts must not wrap `run(...)` -with a second operation resolver or generic exception envelope. Preflight -exceptions use the same closed five-field shape as other preflight responses; -run exceptions use the ordinary run response shape. - -Completed responses also carry the policy-resolved -[Trace ABI v7](trace-abi.md) `run_record`. Live events and terminal records use -the same strict envelope and projection. Persistence remains a host I/O -decision; the engine never opens a trace store. - -The Deno/Pyodide relay keeps its three process output concerns physically -separate. fd1 carries exactly one unary response JSON line. A required -`DROSTE_RELAY_EVENT_FD` names one inherited writable descriptor (fd3 by -convention) that carries canonical Trace ABI v7 NDJSON only. An external -launcher also includes that number in `DENO_EXTRA_STDIO_FDS`, which Deno -consumes at startup to register inherited descriptors above fd2. Passing an -OS-level descriptor without this Deno marker leaves it unavailable to relay -code. fd2 carries only diagnostics. Missing, malformed, or unavailable event -descriptors—and fd0 through fd2 even when writable—fail closed with a typed -`RelayEventChannelError`; the relay never retries an event write on fd2. The -descriptor is mandatory for every invocation, but preflight and pre-admission -refusal produce zero frames. An admitted run lazily emits -`startup` as the prefix to its first canonical event. Hosts must drain fd2 and -the event descriptor concurrently. Cancellation or process death may leave a -valid nonterminal event prefix; a failed partial write may additionally leave -an invalid unterminated final line. Hosts discard that line and surface a typed -transport failure rather than parsing or promoting it, and must not fabricate a -terminal event. Empty preflight/refusal streams do not write a liveness marker; -peer or access-mode loss is detected on the first admitted-run frame. The -native process runner remains a separate explicit integration whose event sink -is the original host stderr. - -Completed results also expose the full scaffold manifest and aggregate stdout -facts. Default durable retention stores only the manifest ID/version; trainer -outcomes join externally by run ID and manifest ID. The optional -[Verifiers v1 harness](verifiers-harness.md) sends root and subcall traffic -through one interception endpoint without moving runtime data into the prompt. - -The optional positive `subcall_concurrency` request field resolves once before -client construction (default 5), controls every HTTP-backed subcall batch, and -is recorded at `scaffold_manifest.inference.concurrency`. Optional -`subcall_input_capacity` is a closed `{state, tokens}` request value recorded -at `scaffold_manifest.inference.input_capacity.subcall`; absent means unknown. -Native CLI rollout -configuration follows the same path. The Pyodide relay forwards the value -unchanged; it does not choose another concurrency policy. - -The optional non-empty `root_reasoning_effort` request field is also resolved -once. Native and Pyodide root clients send it unchanged on every root callback, -and the runner records the same value at -`scaffold_manifest.inference.root_sampling.reasoning_effort`. If an explicit -`root_sampling` object contains a different effort, the request fails before -inference rather than publishing evidence that differs from execution. - -**Versioned boundary**: the request/response schema and provider contract -are versioned, each by a single integer: - -- `RUNNER_PROTOCOL_VERSION` (currently 10) governs the request/response - envelope. Every request **must** carry `protocol_version` — requests are - self-describing, the same discipline as JSON-RPC's mandatory `"jsonrpc"` - field. A missing or mismatched version is answered with a structured - error (`protocol_version_missing` / `protocol_version_mismatch`) naming - both sides — a host detects incompatibility explicitly instead of - failing on a missing field. Responses carry `protocol_version`: the - engine stamps its own everywhere except an `adapter_module` response - that already claimed one (adapters own their response shape). - Protocol v4 through v9 require one exact six-field `budget` object; missing, - unknown, or invalid fields fail before endpoint dispatch. - Protocol v4 adds the `operation` discriminant and typed preflight response. - The bump is required for safety: a v3 runner could ignore an unknown - `operation` field and execute a request that a v4 host intended only to - inspect. Old runners instead reject v4 at the version gate before endpoints, - credentials, provider binding, or work. - Protocol v5 adds the strategy-relevant `subcall_input_capacity` value. The - bump prevents a v4 runner from silently ignoring capacity that a host used - to plan chunks or select a checkpoint. - Protocol v6 embeds Trace ABI v2 and emits its same live vocabulary. The bump - prevents a v5 runner or consumer from silently treating new discriminated - subcall/repair/extract lifecycle values as the former strict v1 contract. - Protocol v7 embeds Trace ABI v3, where root/subcall usage scopes declare - completeness and the top-level usage kind distinguishes resolved from - partial provider facts. The bump prevents a v6 consumer from treating - missing provider usage as a fully resolved zero-token fact. - Protocol v8 embeds Trace ABI v4, where both usage scopes add cache-read and - cache-creation token classes inside inclusive input totals. The bump prevents - a v7 consumer from silently dropping cache-rate billing evidence. - Protocol v9 embeds Trace ABI v5 and its transient cumulative - `usage_progress` event at settled root/subcall boundaries. The bump prevents - a v8 host from silently dropping the live accounting signal. - Protocol v10 embeds Trace ABI v7 and scaffold manifest v3, adding the - required seventh `max_iterations` authorization field. The exact seven-field - budget prevents a v9 host from silently submitting an unbounded loop - contract. -- `PROVIDER_PROTOCOL_VERSION` (currently 4) governs manifest parsing, - context-first provider binding, and bridge invocation facts. A mismatched - manifest fails before a source is live. - -Provider bridge v2 is a separate, explicitly selected transport contract; it -does not change provider manifest protocol 4. It adds one bounded per-call -message pump for ordered checkpoints, cancellation acknowledgement, and a -terminal outcome while preserving the same capability identity, manifest, -generated binding, and broker finalization path. Unary bridge invocation stays -the default when a host does not supply the duplex session factory. - -The rules: **adding an optional field is not a version bump** (the 0.5.x -subcall cost-control knobs are the worked example — older engines ignore -them, newer engines honor them); renaming or removing a field, or changing -a field's semantics, bumps the integer. A control plane that serves -embedded engines must tolerate a stated window of engine versions -(embedded engines in the field can't be force-upgraded). - -## Sandboxing — the honest version - -The REPL is a **guardrail, not a security boundary**. It bounds output size -and execution time and keeps well-behaved models on the rails. A hostile -workload needs real isolation, which belongs to the host: OS permissions -own file access (the SQL source's read-only policy assumes this), and -process/container/WASM isolation owns escape (hosts run the engine in a -subprocess, jail, or Pyodide/WASM runtime as their threat model requires). -The engine will not pretend otherwise, and neither should you. - -### Host environment factory - -Hosts select the execution substrate through one immutable configuration and -derive the loop context and environment from it: - -```python -from droste import ( - Budget, - EnvironmentConfig, - SandboxLimits, - create_environment, - create_environment_context, -) - -config = EnvironmentConfig( - kind="native", - budget=Budget(subcalls=50, depth=1), - sandbox=SandboxLimits(output_chars=25_000, execution_timeout_ms=5_000), -) -execution_context = create_environment_context(config) -environment = create_environment( - config, - context=data, - registry=registry, - subcalls=subcalls, - execution_context=execution_context, -) +Droste keeps source data in an execution environment instead of copying it into +the root model prompt. The root model writes Python, the environment executes +it, and only bounded stdout returns for the next iteration. + +```text +question ──▶ root model ──▶ Python ──▶ environment ──▶ stdout + ▲ │ + └─────── next iteration ◀──────────────────┘ ``` -`native` selects the signal-timed in-process environment. `pyodide` selects -the thread- and signal-free raw executor; because WASM isolation and -wall-clock termination live outside Python, that config requires explicit -`host_managed_isolation=True` and `host_managed_timeout=True` declarations and -rejects a nonzero `exec_timeout_ms`. These booleans are assertions by the host, -not security mechanisms. The host still has to provide the Deno/WASM jail and -kill deadline. This keeps substrate selection pure while the constructor is a -thin shell around live data, registry, and subcall dependencies. - -Passing `registry` to `create_environment` transfers ownership at the call -boundary. The factory closes it if validation or construction fails; otherwise -the environment closes it after success, ordinary error, or Python -process-control unwinding. Callers close registries that they bind but do not -transfer. Constructing a trusted bridge service transfers its bound source to -that service until `ProviderService.close()`. The OS remains the final -authority only when a host hard-kills the process. +The loop stops when the model sets `answer["ready"] = True`, reaches its +iteration limit, or cannot reserve more budget. A terminal extraction call may +return a best-effort answer from completed work; extracted answers remain +distinct from confirmed answers. + +## Components + +- **Prompt pack:** one immutable strategy artifact selected before inference. + It defines the initial, refinement, repair, and extraction templates. +- **Execution environment:** a persistent Python namespace containing the + supplied context and generated capability bindings. +- **Capability broker:** the only route from generated bindings to trusted + subcall or data-provider handlers. It owns allowlisting, attempt lifecycle, + cancellation, and accounting integration. +- **Budget ledger:** the single mutable authority for tokens, subcalls, depth, + wall time, iterations, and root/subcall output ceilings. +- **Scaffold manifest:** the content-addressed identity of the resolved prompt, + capabilities, inference settings, budget, sandbox, and protocol versions. +- **Trace context:** stamps and serializes the versioned live event stream and + policy-selected terminal record. + +Generated code can call `llm_query`, ordered batch helpers, and any bindings +created by the provider registry. These are flat subcalls within the current +run; Droste does not expose a built-in model-facing child-RLM helper. + +## Trust boundaries + +Provider handlers are trusted host code. They receive a frozen +`CapabilityExecutionContext` with cooperative `check()` and cumulative +`checkpoint()` methods, not the mutable ledger or trace recorder. The broker +normalizes every attempted outcome and settles it exactly once. + +The native REPL is a guardrail, not a security boundary. It limits execution +time and captured output, but arbitrary Python may retain the ambient authority +of its process. Hosts that execute untrusted workloads must add OS, container, +or WASM isolation. The Pyodide substrate keeps credentials and provider data in +trusted host/interpreter contexts and exposes only brokered calls to generated +code. + +Policy is explicit. Droste does not infer semantic or count requirements from +question wording; callers arm policy hints or ready-time validators when a +product requires them. + +## Versioned boundaries + +Current host-facing contracts are documented separately: + +- [Providers](providers.md) +- [Runner protocol](reference/runner.md) +- [Trace ABI](reference/trace.md) +- [Prompt packs and RLM skills](reference/prompt-packs.md) +- [Scaffold manifest](reference/scaffold.md) + +Those references describe current behavior only. Migration history belongs in +[UPGRADING.md](../UPGRADING.md). diff --git a/docs/budgets.md b/docs/budgets.md deleted file mode 100644 index 757214c..0000000 --- a/docs/budgets.md +++ /dev/null @@ -1,84 +0,0 @@ -# Budgets - -Droste authorizes compute with one immutable value: - -```python -Budget( - tokens=500_000, - subcalls=50, - depth=1, - wall_ms=300_000, - max_iterations=30, - root_output_tokens=4_096, - subcall_output_tokens=2_048, -) -``` - -The caller resolves presets or product policy before constructing this value. -Enforcement never interprets presets and never merges host-specific counters. -`SandboxLimits` is separate: output capture and local code timeouts describe -the REPL boundary, not provider/model spend. - -## One mutable authority - -Each run has one `BudgetLedger`. A root request or brokered capability reserves -its maximum token/subcall/wall/depth vector atomically before dispatch. If any -dimension cannot fit, the reservation is rejected without mutation as a typed -`BudgetExhausted` fact. On completion, the same `call_id` commits conservative -actual spend and refunds the unused reservation. - -Provider transports do not enforce budgets. They issue requests and report -mechanism usage. The capability broker is the admission/finalization boundary, -so handler errors, invalid results, annotator errors, and process-control exits -all settle the reservation exactly once. - -The RLM iteration ceiling is structural rather than reservable. The execution -context checks it before publishing an iteration or issuing that iteration's -root request. Reaching it produces `IterationLimitExceeded`, never -`BudgetExhausted`, so non-convergence remains distinguishable from spend -exhaustion. Completed work may still enter the one bounded extraction pass. - -Trusted handlers receive a frozen `CapabilityExecutionContext`. Its reservation -is identified only by the call's `call_id`; providers never receive the ledger. -Long-running work may report cumulative token/subcall usage with -`context.checkpoint(tokens=..., subcalls=...)`. Equal values are idempotent, -dimensions cannot move backward, and values cannot exceed the admitted vector. -Checkpoint deltas become committed ledger facts immediately; final -reconciliation cannot retract them and refunds only authorization still unused. -Wall time is always measured by the broker rather than reported by a provider. - -`context.check()` observes cooperative cancellation and the caller-authorized -monotonic deadline. Cancellation before handler dispatch refunds the admission; -after dispatch it follows the same deterministic final settlement as any other -attempt. Results distinguish the stable `cancelled` and `deadline_exceeded` -error codes under one `CapabilityStatus.CANCELLED` terminal class. - -Inference batches reserve every item together. Concurrent callers therefore -cannot pass separate check-then-increment races. The broker preserves one root -output allocation while admitting inference work so early subcalls cannot -consume the parent's ability to synthesize. - -## Child runs - -`ledger.child(call_id, child_budget)` creates a strict sub-ledger only after the -parent atomically reserves the child's token and subcall allocation plus one -depth unit. The child's own deadline must fit within the parent's shared -remaining deadline; wall time is not additive capacity. The child may spend no -more than its budget. `child.close()` reconciles its actual spend into the -parent and refunds everything unused. Closing is idempotent; closing with active -reservations fails loudly. - -## Trace facts - -Every mutation is a durable Trace ABI v7 `budget` event from -`source="budget_ledger"`: `reserve`, `commit`, `refund`, or `exhaust`. -Mutation events carry `resource`, non-negative `amount`, and `call_id`. The -terminal snapshot records the configured, consumed, and remaining vectors. -Event callbacks run outside the ledger's state lock; event observation cannot -become a second accounting authority. - -## Runner and CLI - -Runner protocol v10 requires an exact `budget` object with all seven fields. -Missing and unknown fields fail before work. The CLI resolves its seven budget -flags into the same value. There are no legacy aliases or translation rules. diff --git a/docs/embedding.md b/docs/embedding.md index 6f3593e..e97eb4b 100644 --- a/docs/embedding.md +++ b/docs/embedding.md @@ -1,87 +1,57 @@ # Embed Droste -The `droste` package has no runtime dependencies. An embedded run needs a root -model client, a subcall client, an execution environment, and one shared -execution context. +The dependency-free `droste` package can run with your model clients and data. +Install it with `uv add droste`, then start from the tested +[embedding example](../examples/embedding.py). -```bash -uv add droste -``` - -This example uses an OpenAI-compatible endpoint and a string as the dataset: +An embedded run has four shared pieces: a `Budget`, an `ExecutionContext`, model +clients, and an execution environment. Construct the context and environment +from the same `EnvironmentConfig`; conflicting budgets fail before inference. -```python -from droste import ( - Budget, - EnvironmentConfig, - OpenAICompatClient, - OpenAICompatSubcallClient, - RLMConfig, - create_environment, - create_environment_context, - run_rlm, -) +`OpenAICompatClient` reads `OPENAI_API_KEY` and `OPENAI_BASE_URL`. Anthropic and +ModelRelay clients are also included. Explicit constructor arguments override +environment values. -model = "YOUR_MODEL_ID" -environment_config = EnvironmentConfig( - kind="native", - budget=Budget(subcalls=50, depth=1), -) -context = create_environment_context(environment_config) +## Compute and local execution limits -root = OpenAICompatClient(model=model) -subcalls = OpenAICompatSubcallClient(model=model, context=context) -environment = create_environment( - environment_config, - context="your source data", - registry=None, - subcalls=subcalls, - execution_context=context, -) +`Budget` is the model-compute authorization: -result = run_rlm( - "What happened?", - environment=environment, - root_llm=root, - subcalls=subcalls, - config=RLMConfig(root_model=model), - context=context, +```python +Budget( + tokens=500_000, + subcalls=50, + depth=1, + wall_ms=300_000, + max_iterations=30, + root_output_tokens=4_096, + subcall_output_tokens=2_048, ) -print(result.answer) ``` -`OpenAICompatClient` reads `OPENAI_API_KEY` and `OPENAI_BASE_URL`. Explicit -constructor arguments take precedence. The package also provides -`AnthropicClient` and `AnthropicSubcallClient`. - -## Data providers - -Pass plain data through `context`, or bind a `ProviderRegistry` for named data -sources. Droste includes filesystem text and read-only SQLite providers. See -[provider manifests](provider-manifests.md) when exposing an application data -source or MCP server. - -## Budgets and concurrency - -`EnvironmentConfig` and its execution context must share one `Budget`. The -same ledger reserves and settles root calls, subcalls, child runs, and wall -time. +One `BudgetLedger` atomically reserves and reconciles root calls and brokered +subcalls. Failed work still settles its reservation. `depth` authorizes child +ledgers for hosts that construct child runs; Droste's built-in model-facing +helpers currently make only flat `llm_query` and batch subcalls. -Subcall clients default to five concurrent batch items. If you change -`OpenAICompatSubcallClient(max_parallel=...)`, set the same value in -`RLMConfig.rollout`. A mismatch fails before inference. +`SandboxLimits` is separate. It bounds local execution time and captured +output; it is not provider/model spend and the native REPL is not a security +boundary. -See [budgets](budgets.md) for the full authorization model. +Built-in subcall clients default to five concurrent batch items. If you change +their `max_parallel`, record the same value in `RLMConfig.rollout`; a mismatch +fails before the first model request. -## Policy and traces +## Data, policy, and events -Droste does not infer enforcement rules from the wording of a question. Pass -explicit `PolicyHints` when a product requires semantic subcalls or another -answer contract. +Pass plain data through `context`, or use a `ProviderRegistry` for named data +sources. Droste includes read-only SQLite and filesystem-text providers; see +[Providers](providers.md). -Attach `on_event` or configure trace retention through the execution context -to consume structured events. The [Trace ABI](trace-abi.md) defines the event -and retention contracts. +Droste does not infer enforcement rules from question text. Products that need +semantic-subcall or answer requirements must pass explicit `PolicyHints` or a +ready-time validator. -Hosts that run Droste through HTTP or Pyodide should use the runner protocol in -the [technical architecture](architecture.md#the-runner-protocol-embedding). +Attach `on_event` for live events and configure `TraceRetentionPolicy` for the +terminal record. See the [Trace ABI](reference/trace.md). Non-Python hosts use +the [runner protocol](reference/runner.md); the Pyodide host path is documented +beside its implementation in [pyodide/README.md](../pyodide/README.md). diff --git a/docs/internal/README.md b/docs/internal/README.md deleted file mode 100644 index d6aef48..0000000 --- a/docs/internal/README.md +++ /dev/null @@ -1,6 +0,0 @@ -# Internal project notes - -These documents guide maintainers and are not part of the user documentation. - -- [Architecture principles](architecture-principles.md) -- [Brand and assets](brand.md) diff --git a/docs/internal/architecture-principles.md b/docs/internal/architecture-principles.md deleted file mode 100644 index e56ce3f..0000000 --- a/docs/internal/architecture-principles.md +++ /dev/null @@ -1,247 +0,0 @@ -# Architecture principles - -*The north star for droste's evolution. `../architecture.md` describes what is; this -describes what everything should converge toward, and the test every new piece of -structure must pass.* - -## The test: structure must be removable - -Sutton's Bitter Lesson, applied as an engineering discipline rather than a slogan: -the only things that scale with compute are **search** and **learning**. Structure -that encodes *how humans think the problem should be solved* buys a short-term win -and becomes a ceiling. The discipline is not "no structure" — it is: - -> Every piece of human strategy in the system must be **deletable without a -> rewrite** the day a model no longer needs it. - -RLM itself passes this test: it replaces retrieval theory (embeddings, chunking -heuristics, rerankers, orchestration graphs) with a REPL — **code is the action -space**, and the model searches the problem by writing and running programs. The -structure we do ship divides into two kinds: - -- **Mechanism** — structure that scales *with* compute: the REPL, the budget, the - capability protocol. More compute → more code, deeper recursion, wider search. - This is the invariant core, and it is deliberately tiny. -- **Strategy** — structure that substitutes human judgment *for* compute: prompts, - tips, policy defaults, model pairings. All of it lives in a swappable data layer, - ships as removable defaults, and is expected to shrink toward empty as models - improve. - -The failure modes this guards against are all live in the current code: a validator -that is convention rather than boundary (since hardened by the sandbox split), -recursion capped at depth 1 by an -architectural constant (#2), and orchestration strategy welded into the engine as -Python string constants (`prompts/tips.py`). - -## The stack - -``` -┌────────────────────────────────────────────────────────────────┐ -│ PROMPT PACKS + RLM SKILLS strategy-as-data, swappable │ -├────────────────────────────────────────────────────────────────┤ -│ PROVIDERS open edge: in-proc · MCP · HTTP transports │ -├────────────────────────────────────────────────────────────────┤ -│ BROKER budget ledger · trust boundary · MCP client │ -├────────────────────────────────────────────────────────────────┤ -│ BRIDGE call / emit · JSON-RPC 2.0 │ -├────────────────────────────────────────────────────────────────┤ -│ REPL Python · persistent namespace · WASM │ -└────────────────────────────────────────────────────────────────┘ - invariant core (bottom three) · open ecosystem · deletable top -``` - -## 1. One capability protocol, location- and language-transparent - -`query()`, `search()`, `llm_query()`, a future `rlm_query()`, host tools, and -remote APIs are all the same thing: **untrusted code emitting a capability -request, and something trusted answering it.** One wire shape (JSON-RPC 2.0 with a -`register` step; descriptors shaped like tool schemas, because models are trained -on that shape). The sandbox cannot tell — and must not care — whether the provider -answering is: - -- a second interpreter in the same process (an embedded data-source service), -- host code in the embedding app (Swift, Go, anything), -- a remote HTTP API (`wrapper_v1` demotes to exactly this — a remote provider), -- **another RLM** (recursion is a provider that answers by running a child loop). - -Two primitives cross the boundary, nothing else: - -``` -bridge.call(method, params) -> result # sync from Python; request/response -bridge.emit(event) # fire-and-forget; the event stream (#1) -``` - -**Why this is not tool calling reinvented:** provider tool calling is -*decoder-mediated orchestration* — every call routes through the model's decoder, -every result lands in context, and loops/joins/batching live in the token stream, -the most expensive and least reliable computer available. Droste is -*code-mediated orchestration*: the model writes a program; the program does the -cheap deterministic control flow — loops, joins, batching, local reduction — and -the model re-enters only when judgment is needed or the reduced result matters. -Many calls happen inside one execution step, and only the reduced output (plus -traces, errors, and refinement feedback) returns to the model. The win is where -orchestration lives, not a free lunch on context. Same descriptor vocabulary, -inverted mechanics — and we converge on tool calling's *schema* precisely so we -can ride its ecosystem (next section) without adopting its economics. - -## 2. The capability ABI is ours; MCP is the default external transport - -The enduring droste idea is **model-written programs over brokered capabilities -with bounded semantic subcalls and reduced feedback** — not any particular -protocol. The stable abstraction is internal: - -``` -capability manifest → Python binding → brokered operation call → trace/budget ledger -``` - -Transports *implement* that abstraction behind the broker, and adding one must -never force a core rewrite: - -- **in-process** (direct call, no serialization) — first-class, the hot local - path: a host's embedded data source, the SQL/fs sources. -- **MCP** — the *default external* transport: the broker acts as an MCP client, - so a data source / host tool / sub-RLM can be an MCP server in any language. -- **HTTP** (`wrapper_v1` demotes to this) — remains possible; keeps its - budget/SSRF guards, relocated into the broker. - -Compatible MCP servers can be adapted without adding MCP-specific bindings to -the sandbox. The registry stops being a Python-only abstraction: manifests -generate both the sandbox bindings and the prompt's `{capabilities}` fragment, -so the prompt describes the actual API surface. MCP is how capabilities are -*populated*, not what droste *is*. - -The security posture is ours regardless of transport: **the REPL never speaks -MCP.** The sandbox calls generated Python functions; those functions go through -the broker, which owns auth, policy, budgets, validation, side-effect gating, and -trace emission — MCP assumes a trusted caller, and ours is not. - -A network boundary and a trust boundary are the same shape. Making capabilities -location-transparent *is* the sandbox split: the untrusted REPL holds no DB -handle, no credential, no raw tool — only the ability to make requests. - -## 3. Define little, but define it sharply: the five ABIs - -"Define as little as possible" is half the discipline; the other half is that what -we *do* define, we define precisely — these are boundaries embedders need to -share consistently. Five, each small, versioned, and -consumer-validated before frozen: - -1. **Kernel ABI** — the REPL contract: the `answer` dict, stdout-as-feedback, - persistent state across iterations, reserved globals. -2. **Capability ABI** — stable capability IDs, the manifest (tool-shaped - descriptors), Python binding generation, parameter/result schemas, and - **side-effect metadata** - (read-only vs effectful — the generalization of today's hard-coded - SqlValidator gate, and the hook confirmation policies hang from). -3. **Broker ABI** — `call`, `emit`, the budget ledger, policy/confirmation hooks. -4. **PromptPack ABI** — the stable slots, `(model, profile)` resolution, - provenance fields. -5. **Trace ABI** — the run record: code executed, outputs, capability calls, - usage, policy events, final answer. This unifies trajectory and event paths - into one named boundary — it is what benchmarking, citations, replay, and - billing all consume, and none of them should parse ad-hoc internals. - -## 4. Compute is a budget, not an architectural constant - -An architectural depth constant is a hard cap on how much compute the method -may spend on a hard problem — exactly the ceiling the Bitter Lesson forbids. -A fixed depth bound constrains available search regardless of task difficulty. -Depth therefore belongs in caller-authorized compute, and any claim about its -quality effect requires version-matched benchmark artifacts. - -The replacement is **one budget object**, set by the caller, metered by the broker: - -```json -{ - "tokens": 500000, - "subcalls": 100, - "depth": 3, - "wall_ms": 300000, - "root_output_tokens": 4096, - "subcall_output_tokens": 2048 -} -``` - -The broker sees only this resolved value. One ledger owns reservation and -reconciliation; transports merely report mechanism usage. Strict child ledgers -are allocations from the parent, not independent counters, and return unused -authorization when they close. - -Depth, breadth, and iteration count stop being engine constants and become -emergent consequences of how much compute the caller authorized. Child runs -(`rlm_query`) draw from the **parent's** ledger — recursion is metered, not gated. -This keeps the architecture flexible while retaining caller-set resource -ceilings with conservative defaults. - -## 5. Strategy is data: prompt packs + RLM skills - -All human strategy lives above the engine in two artifact kinds, both versioned -data, neither code: - -**Prompt packs** — deterministic harness configuration, resolved at run start by -`(model, profile)` with a fallback chain (caller pack → consumer pack → droste -default for the model family → generic). A pack carries the base prompt, the -refinement/repair/extract templates, and policy defaults. The **slot contract is -the only stable API**: the engine guarantees each template a fixed variable set — -`{capabilities}` (generated from the provider registry, never authored), -`{budget}`, `{question}`, `{history}`, `{output_contract}`. Content may change -wildly between packs; slots may not. Packs are exclusive — one governs a run — -because they carry the machinery that must be deterministic (the repair path fires -when the model is already misbehaving; that is the worst moment for model-chosen -strategy). - -**RLM skills** — additive, composable strategy in a portable -Markdown-plus-frontmatter format. The difference in genre: agent skills teach -*tool workflows*; RLM -skills teach *code-writing under a metered budget* — chunking budgets, -fat-prompts-small-batches, "string matching finds WHERE, llm_query understands -WHAT." Today's `tips.py` profiles become the first skills. Two optional -properties distinguish these artifacts: - -- **Evidence provenance**: a skill can cite immutable, version-matched benchmark - artifacts. Without that evidence, it makes no effectiveness claim. -- **Loadable mid-run through the bridge**: a `skills.*` provider lets the model - fetch strategy when it discovers it needs it, costed against the same budget. - Progressive disclosure fits RLM economics better than agent sessions: root - tokens are paid every iteration, so strategy you didn't need is strategy you - shouldn't have loaded. - -The exit ramp is built in: the pack for a strong-enough model is three lines -("REPL. Capabilities: `{capabilities}`. Budget: `{budget}`.") and the default -skill set shrinks toward empty. Deleting strategy is swapping a data file. - -Consumer extensions use the same provider, pack, and skill contracts rather -than parallel engine mechanisms. - -## 6. Trust topology - -The untrusted REPL and the trusted side are separate interpreter contexts -bridged by `call`/`emit`. One rule governs every future topology change, -including nested recursion and remote execution: - -> **Never re-enter a suspended interpreter. Every concurrently-active role gets -> its own context.** - -## Non-goals and cautions - -- **Don't over-unify.** The pull to make everything a skill (base prompt, repair - templates) trades determinism away where the harness needs it most. Two layers — - one deterministic, one discoverable — is the right amount of structure. -- **Don't strip strategy prematurely.** Compatibility profiles retain the - current guidance until versioned evidence supports changing it. The discipline - is removability, not asceticism. -- **The engine stays intent-blind.** It never infers what the caller wants from - question text (existing policy-hints rule). Callers pass policy; packs carry - defaults; the core enforces mechanics only. -- **Python is the current REPL compatibility surface.** An alternative runtime - requires comparative evidence before adoption. - -## Where the work lands - -| principle | issue | -|---|---| -| bridge (`call`/`emit`, JSON-RPC 2.0, register) | shipped; typed brokered path across native + Pyodide | -| providers are MCP; registry unification; wrapper_v1 demotion | shipped; stdio and production Streamable HTTP transports | -| one budget object; recursion as metered provider | #4 (budget) + #2 (recursion) | -| event stream over `emit` | #1 | -| prompt packs + RLM skills | #3 | diff --git a/docs/internal/brand.md b/docs/internal/brand.md deleted file mode 100644 index dc9b556..0000000 --- a/docs/internal/brand.md +++ /dev/null @@ -1,47 +0,0 @@ -# Droste brand - -Decided 2026-07-06. Brutalist: the mark and the type say what the tool is — -raw, exact, terminal-native. - -## The mark - -Self-similar square frames, each rotated **16°** and scaled **×0.70** into -the last — the *Print Gallery* transformation (an image containing itself, -turning as it shrinks) — converging on a **filled block rotated 64°**: the -next frame in the sequence, solid. Escher left the center of Print Gallery -blank; the center is what the recursion computes. **The block is the answer.** - -Geometry (viewBox −50…50): frame half-sizes 40 · 28 · 19.6 · 13.7, strokes -6.4 · 5.6 · 5.0 · 4.4 (taper ×0.88), hard corners (`rx=0`). Core: 13.2-unit -square at 64°. Small cuts drop to 2–3 frames with heavier strokes (see the -favicon cuts in the identity sheet). - -## Palette - -Escher printed black on white; color only ever flat and structural. - -| role | light | dark | -|---|---|---| -| paper | `#F3F2EE` | `#141414` | -| ink | `#131313` | `#EBEAE5` | -| print blue (the answer block) | `#1F3CC4` | `#4A63E8` | - -No other colors. No gradients, no shadows, no rounded corners. Blue-on-paper -is blueprint/cyanotype heritage and the raw web's link color — and unowned -in the AI space. Droste is cool (the print on the wall); the single accent -element is *the answer* — the recursion arrives at it. - -## Type - -Wordmark: grotesque black caps (system heavy sans), tight tracking. -Everything else: monospace, letter-spaced caps for labels. No italics. - -## Assets - -- `../assets/droste.svg` / `../assets/droste-dark.svg` is the mark in light and - dark variants. -- `../assets/og.png` is the 1200×630 social card. Upload it through GitHub - Settings → Social preview. Regenerate: the card is plain HTML + a headless - Chrome screenshot at 1200×630; composition = mark left, DROSTE / tagline / - bordered `uvx` command / metadata line right, 5px inner border (frame zero - of the recursion). diff --git a/docs/mcp-http.md b/docs/mcp-http.md deleted file mode 100644 index eb2502d..0000000 --- a/docs/mcp-http.md +++ /dev/null @@ -1,149 +0,0 @@ -# MCP Streamable HTTP provider transport - -Droste can acquire a remote MCP server through a trusted host and project its -tools through the same immutable provider and capability ABI used by in-process -and stdio sources. Generated code receives only descriptor-generated Python -bindings. It never receives the MCP endpoint, authorization configuration, -resolved secrets, session ID, JSON-RPC envelope, or raw response payload. - -The implementation targets MCP protocol `2025-11-25` and supports JSON and SSE -responses, event-ID resumption with bounded reconnect/backoff, stateful session -replacement, paginated `tools/list`, cancellation, deadlines, and explicit -session deletion. A replacement session must publish the byte-identical frozen -tool snapshot or the source fails closed. - -## Native host API - -```python -from droste import ConfiguredSource -from droste.sources import McpHttpHost, McpSecretRequest, open_mcp_http_source - -source = ConfiguredSource( - "documents", - "company_documents", - { - "endpoint": "https://mcp.example.com/mcp", - "allowed_endpoints": ["https://mcp.example.com/mcp"], - "tenant_id": "tenant-42", - "auth": {"type": "bearer", "token_ref": "secrets/mcp/documents"}, - "allowed_tools": ["documents.search", "documents.read"], - "bindings": { - "documents.search": "search", - "documents.read": "read", - }, - "effects": { - "documents.search": "read", - "documents.read": "read", - }, - "budget_classes": { - "documents.search": "data.search", - "documents.read": "data.read", - }, - "policy_metadata": { - "documents.search": {"read_only": True}, - "documents.read": {"read_only": True}, - }, - "source_description": "Company documents available to this tenant.", - }, -) - - -def resolve_secret(request: McpSecretRequest) -> str: - # Authorize the exact (tenant_id, source_id, reference) tuple here. - return secret_store.read(request.tenant_id, request.source_id, request.reference) - - -bound = open_mcp_http_source(source, McpHttpHost(resolve_secret=resolve_secret)) -``` - -The source configuration contains references, not credential values. The -resolver is live host state and is neither frozen into `ConfiguredSource` nor -included in descriptors. Every resolution carries the tenant and source IDs; -hosts must authorize that complete tuple rather than treating a reference as a -process-global secret name. - -`auth.type` supports: - -- `none`: no other auth fields; -- `bearer`: one `token_ref` resolved by the trusted host; -- `oauth_client_credentials`: `client_id_ref`, `client_secret_ref`, exact - `resource_metadata_url`, exact `authorization_server`, a non-empty exact - `token_endpoints` allowlist, and optional `scopes`. - -OAuth performs protected-resource and authorization-server discovery, verifies -the exact resource, issuer, and token endpoint, includes the RFC 8707 `resource` -parameter, caches short-lived access tokens, and reacquires or refreshes after -expiry or one `401`. Access and refresh tokens remain only in the live session. -Interactive authorization-code consent is a host responsibility; after consent, -the host can expose the resulting access token through a tenant-scoped bearer -reference. - -## Network and resource policy - -Every URL must be canonical HTTPS without userinfo, query, or fragment. The MCP -endpoint must appear in `allowed_endpoints` as an exact URL. Redirects are never -followed. OAuth resource, issuer, and token locations are independently exact. - -DNS is resolved for every exchange. Every answer is checked, then one approved -address is pinned through socket connection and TLS hostname verification. -Loopback, private, link-local, multicast, reserved, unspecified, and metadata -addresses fail closed by default. A trusted VPC host may opt into exact CIDRs -through `McpHttpHost(allowed_networks=("10.8.0.0/16",))`; DNS answers outside -those CIDRs still fail. This is authority and should be scoped as narrowly as -the deployment permits. - -Defaults and accepted configuration keys: - -| Key | Default / rule | -| --- | --- | -| `startup_timeout_ms` | 15,000; 100..120,000 | -| `request_timeout_ms` | 30,000; 100..300,000 | -| `close_timeout_ms` | 2,000; 10..30,000 | -| `max_frame_bytes` | 1 MiB; 1 KiB..16 MiB | -| `max_descriptor_bytes` | 1 MiB; 1 KiB..16 MiB | -| `max_result_bytes` | 256 KiB; 256 B..8 MiB | -| `max_tool_pages` | 32; 1..256 | -| `max_tools` | 256; 1..4,096 | -| `max_in_flight` | 64; 1..4,096 | -| `reconnect_attempts` | 2; 0..5 | -| `backoff_ms` | 100; 10..5,000 | -| `max_debug_payload_bytes` | 0 (disabled); maximum 4 KiB | - -Timeouts are total wall-clock budgets, not per-hop allowances. Startup shares -one budget across secret resolution, OAuth discovery/token acquisition, -initialization, notification, and tool discovery. Each request shares one -budget across authentication, retries, backoff, and SSE resumption. Session -close has one independent budget, uses only the last access token already held -by the session, and never reacquires credentials during teardown. - -Ordinary stats and durable capability traces contain only normalized counts, -latencies, sizes, status, and immutable capability identity. Raw MCP payload -debugging is disabled by default. Enabling a bounded `max_debug_payload_bytes` -also requires a trusted `debug_sink`; only request/response bodies are passed, -never HTTP headers, endpoint, authorization, or session values. Raw debug data -is content-bearing and must not be routed into durable default traces. -`requests_made` counts every attempted tool dispatch, including failed and -cancelled attempts; `calls` and `failures` retain the success/failure split. - -## Cross-language and runner hosts - -`bind_mcp_transport_source(source, transport, policy)` is the connector-neutral -seam. `McpToolTransport` supplies `list_tools`, `call_tool`, `stats`, and -`close`; `McpBindingPolicy` supplies the host's allowlist, bindings, effects, -budget classes, and policy. This lets a trusted Deno, Go, or service process own -HTTP/OAuth and connect through `ProviderService`/`BridgeProvider` without -copying the provider schema or exposing MCP-specific sandbox bindings. - -Trusted `droste_runner` callers can pass -`source_opener(ConfiguredSource, context) -> BoundSource` to `run()` or -`run_worker_request()`. The request cannot select this hook. Both `run` and -`preflight` acquire dynamic sources through it, so preflight records the exact -remote-discovered immutable capability shape and the runner closes the source -on every terminal path. - -The transport follows the primary MCP -[`Streamable HTTP`](https://modelcontextprotocol.io/specification/2025-11-25/basic/transports), -[`authorization`](https://modelcontextprotocol.io/specification/2025-11-25/basic/authorization), -[`lifecycle`](https://modelcontextprotocol.io/specification/2025-11-25/basic/lifecycle), -and [`tools`](https://modelcontextprotocol.io/specification/2025-11-25/server/tools) -contracts. diff --git a/docs/mcp-stdio.md b/docs/mcp-stdio.md deleted file mode 100644 index 289cadd..0000000 --- a/docs/mcp-stdio.md +++ /dev/null @@ -1,162 +0,0 @@ -# MCP stdio provider transport - -Droste can acquire a local MCP server as a data provider without exposing MCP -to generated code. The trusted host launches the server, freezes one complete -`tools/list` snapshot into a `ProviderManifest`, and returns one owned -`BoundSource`. From that point onward, prompt text, Python bindings, policy, -budgets, traces, and result envelopes use the ordinary broker path. - -This is deliberately the local stdio transport. Remote servers use the -[production Streamable HTTP transport](mcp-http.md), whose trusted host shell -owns OAuth, sessions, endpoint policy, and SSRF defenses. -MCP tasks, live tool refresh, and full cross-dialect JSON Schema validation -belong to [#91](https://github.com/tensor-systems/droste/issues/91). - -The base Droste wheel remains dependency-free. The transport is a small -spec-shaped JSON-RPC shell rather than a second capability implementation. It -owns bounds and request IDs needed for Droste cancellation; the stable MCP -Python SDK's public stdio client currently inherits ambient environment values, -does not bound newline framing, and does not expose request IDs for explicit -`notifications/cancelled` delivery. - -## Host configuration - -Launch and policy authority are explicit. Droste executes an absolute -allowlisted program directly—never through a shell—and supplies only the -declared working directory and environment object. The tool allowlist, Python binding names, -read/effectful classification, budget classes, and policy metadata are also -host facts. MCP annotations do not override them. -The current local spike accepts only tools the host classifies as `read`; it -does not expose the reference server's write/edit operations. - -```python -import os - -from droste import ConfiguredSource, ProviderRegistry -from droste.sources import open_mcp_stdio_source - -source = ConfiguredSource( - source_id="reference_docs", - provider_type="reference_filesystem", # logical type, not transport - config={ - "command": "/absolute/path/to/node", - "args": [ - "/absolute/pinned/server-filesystem/dist/index.js", - "/absolute/data/docs", - ], - "env": {}, - "cwd": "/absolute/data/docs", - "allowed_executables": ["/absolute/path/to/node"], - "allowed_tools": ["read_text_file"], - "bindings": {"read_text_file": "read_text_file"}, - "effects": {"read_text_file": "read"}, - "budget_classes": {"read_text_file": "data.read"}, - "policy_metadata": {"read_text_file": {"read_only": True}}, - "source_description": "Read-only reference documents.", - }, -) - -bound = open_mcp_stdio_source(source) -registry = ProviderRegistry((bound,)) -try: - # Pass registry to create_environment(), or project it into a broker. - ... -finally: - registry.close() -``` - -`open_mcp_stdio_source()` is an acquisition transaction rather than a static -provider factory because the manifest does not exist until after process -startup, MCP initialization, and paginated discovery. Any failure during that -transaction reaps the process. A successful runtime owns its session until the -registry or environment closes it. - -## Descriptor and result mapping - -- The supported MCP protocol is exactly `2025-11-25`. -- Every `tools/list` page is read once. Duplicate names, repeated cursors, and - independently configured page, tool, frame, and aggregate descriptor bounds - fail before binding. Initialization and discovery share one startup deadline. -- MCP tool names remain the exact, case-sensitive raw `operation_id`. The host - supplies a separate valid Python `binding_name`; collisions fail. -- Input and output schemas are retained exactly. Their explicit `$schema` is - the dialect; absence means JSON Schema 2020-12 as required by MCP. Provenance - identifies the frozen MCP snapshot and schema role. -- A declared `outputSchema` is typed inline delivery. Without one, the result - is explicitly untyped. A tool requiring MCP task execution is rejected. -- `structuredContent` is preferred exactly. Content-only results retain the - exact bounded content-block JSON in `{"content": [...], "losses": []}`; - links are never fetched. `_meta` and duplicate compatibility text do not - enter the returned structured value. A typed count records compatibility - content blocks ignored in favor of structured data. -- Discovery pagination is not operation pagination. MCP tool descriptors do - not declare cursor semantics, so the mapped operation remains - `PaginationMode.NONE`; full schema/cursor compatibility stays in #91. -- `isError`, JSON-RPC errors, protocol failures, transport loss, and oversized - normalized results have distinct stable provider error codes. - -MCP resource URIs are not promoted to Droste `EvidenceLocation` values. They -lack Droste's source revision and byte/line/section coordinates, so claiming -equivalence would manufacture evidence. Resource blocks remain available in -the untyped result; first-class evidence mapping is an explicit gap. - -## Cancellation, accounting, traces, and lifecycle - -The stdio session polls the broker's `CapabilityExecutionContext` while a tool -call is being written and while it is in flight. Stdin frames use bounded -nonblocking writes so a server that stops reading cannot defeat cancellation. -Cancellation or deadline expiry emits MCP -`notifications/cancelled`, then makes the whole session terminal and performs a -bounded stdin → process-group `TERM` → process-group `KILL` shutdown. There is -no automatic reconnect or retry: a new process would be a different session, -and an effect may already have happened. - -MCP progress is not compute usage and cannot mutate the budget ledger. Warm -call latency and response bytes are ordinary provider usage metrics. Raw -requests, results, process configuration, stderr, and error content never enter -the durable capability trace projection; the broker still owns the one -content-free `CapabilityResult.to_trace_dict()` path. - -Native hosts own the stdio process. Pyodide/WASM code does not gain ambient -process authority; a trusted host places the resulting `BoundSource` behind -the existing `ProviderService`/`BridgeProvider` boundary. The Pyodide -conformance test exercises that generic path beside SQLite without an -MCP-specific sandbox API. - -## Reference spike and measured gap - -The conformance fixture pins the official -[`@modelcontextprotocol/server-filesystem`](https://github.com/modelcontextprotocol/servers/tree/main/src/filesystem) -at `2026.7.10` in a committed npm lockfile and exposes only -`read_text_file`. CI installs it with `npm ci --ignore-scripts`; it is never a -Droste runtime dependency. - -On 2026-07-15, an Apple Silicon machine running macOS 26.5.2, Node 26.5.0, -Python 3.13, and 100 repeated reads measured: - -| Path | Median | p95 | -| --- | ---: | ---: | -| Official MCP filesystem server, warm stdio call | 0.228 ms | 0.407 ms | -| First-party in-process filesystem provider | 0.093 ms | 0.144 ms | - -Cold MCP acquisition (process launch, initialize, and discovery) was 112.720 ms; -warm median transport overhead was 0.135 ms. These are one-machine spike -measurements, not a performance guarantee. Reproduce them with: - -```bash -uv run python benchmarks/mcp_stdio_latency.py \ - --node "$(command -v node)" \ - --server tests/reference_mcp/node_modules/@modelcontextprotocol/server-filesystem/dist/index.js \ - --iterations 100 -``` - -The remaining impedance gaps are schema validation across arbitrary dialects, -task results, live in-run descriptor mutation after `listChanged`, and evidence -coordinates. They stay outside this transport so MCP does not become a second -provider or budget architecture. - -The implementation follows the current primary contracts: the -[`2025-11-25` tools specification](https://modelcontextprotocol.io/specification/2025-11-25/server/tools), -[`stdio` transport](https://modelcontextprotocol.io/specification/2025-11-25/basic/transports), -[`lifecycle`](https://modelcontextprotocol.io/specification/2025-11-25/basic/lifecycle), -and [`cancellation`](https://modelcontextprotocol.io/specification/2025-11-25/basic/utilities/cancellation). diff --git a/docs/prompt-packs.md b/docs/prompt-packs.md deleted file mode 100644 index 860e2f5..0000000 --- a/docs/prompt-packs.md +++ /dev/null @@ -1,152 +0,0 @@ -# Prompt packs - -Prompt packs are versioned, deterministic harness configuration. Exactly one -complete pack is resolved when a run starts; it supplies the system, user, -refinement, repair, and terminal-extraction templates, policy defaults, tips, -and provenance. Pack content can change by revision. The slot contract cannot. - -Prompt packs are not RLM skills. Packs govern required machinery and are never -discovered or mutated mid-run. Additive, broker-loaded skills remain tracked by -#3 and are outside this contract. - -## Stable slots - -Every template is rendered with the same immutable value containing five string -slots. A template may use its phase's required subset and any other declared -slot, but may not invent slots, use attribute/index traversal, conversions, or -format specifications. - -| Slot | Meaning | -|---|---| -| `capabilities` | Trusted environment/provider description generated by the host, plus explicit caller additions. | -| `budget` | The authorized iteration, subcall, depth, REPL output, and reported per-subcall output-token limits for this run. | -| `question` | The caller's question, unchanged. | -| `history` | Phase-specific facts: conversation context, prior answer/output, an error record, or the bounded extraction trajectory. | -| `output_contract` | The required output form for the current phase: fenced Python or terminal answer text. | - -Slot values are inserted once. Braces inside a question, capability description, -or history are ordinary data and are not interpreted as nested templates. - -## Artifact format - -Packs are TOML so loading needs no runtime dependency. `schema_version`, `id`, -`revision`, `profile`, every template, a positive provenance statement, the -unable sentinel, tips, and typed policy defaults are required. Unknown fields -fail closed to catch misspellings. A benchmark and score are optional, but must -appear together. An optional top-level `content_sha256` may declare the expected -canonical digest; loading rejects a malformed or mismatched declaration. - -```toml -schema_version = 2 -id = "example.analysis.full" -revision = "2026-07-14.1" -profile = "full" -unable_sentinel = "unable to determine from the work so far" -tips = [] - -[provenance] -source = "example" -notes = "Prompt revision selected for this analysis harness." - -[policy_defaults] -enforce_contract = true - -[templates] -system = """{output_contract} -{capabilities} -{budget}""" -user = """Question: {question} -{history}""" -refinement = """{history} -Continue. {output_contract}""" -missing_code_repair = """Return executable Python. {output_contract}""" -error_repair = """{history} -Repair the attempt. {output_contract}""" -extract_system = """Use only supported evidence. {output_contract}""" -extract_user = """Question: {question} -{history}""" -historical_stdout_elision = "\n... [historical stdout elided] ...\n" -unchanged_draft_elision = """[unchanged draft elided; see prior snapshot]""" -``` - -Literal braces authored in a template use normal Python-format escaping (`{{` -and `}}`). Braces in slot values need no escaping. - -Live root calls keep the last two completed iterations verbatim. When an older -refinement leaves that window, its stdout is reduced to a deterministic -head/tail projection around `historical_stdout_elision`. Its accumulated draft -is replaced by `unchanged_draft_elision` only when it is exactly equal to the -prior snapshot; changed drafts stay verbatim. These two constant templates must -not introduce new slots. - -## Canonical content identity - -`canonical_prompt_pack_bytes(pack)` validates the frozen pack, projects its -complete semantic value, and serializes that value as UTF-8 JSON. Object keys -are sorted lexicographically, JSON separators contain no whitespace, Unicode is -encoded directly rather than escaped, and no trailing newline is added. The -value includes schema version, authored ID and revision, profile, unable -sentinel, ordered tips, every template, policy defaults, and all provenance -fields. Optional provenance fields are represented as JSON `null` when absent. -Authored string whitespace and list order are content and therefore affect the -digest; TOML key ordering and formatting do not. - -`prompt_pack_content_sha256(pack)` returns the lowercase SHA-256 hex digest of -those canonical bytes. A declared `content_sha256` is an assertion about that -digest, not part of the semantic pack value, so it is excluded from the bytes -being hashed. Both functions are pure and perform no filesystem or package -resource I/O. - -## Loading and resolution - -Filesystem and package-resource reads are loader-boundary I/O. They return -frozen `PromptPack` values. Parsing, validation, catalog selection, and rendering -are pure functions over those values. - -```python -from droste import RLMConfig, load_prompt_pack, run_rlm - -pack = load_prompt_pack("./prompts/analysis.toml") -result = run_rlm( - question, - environment=environment, - root_llm=root, - subcalls=subcalls, - config=RLMConfig(root_model="YOUR_MODEL_ID", prompt_profile="full"), - prompt_pack=pack, -) -print(result.prompt_pack.as_dict()) -``` - -`resolve_prompt_pack` takes `(model, profile)` and selects, in order: - -1. the complete caller pack; -2. a consumer catalog match for model family, then its generic match; -3. an engine catalog match for model family; -4. the engine's generic profile; -5. generic `full` only as compatibility fallback for an unknown profile. - -Catalog selectors and packs are tuples/frozen values. Duplicate selectors fail -at construction. Selection never merges partial packs. A consumer catalog can be -passed to `run_rlm(..., consumer_prompt_catalog=catalog)`; model-family inference -is a pure selector and has no effect outside prompt resolution. - -The installed engine includes `full`, `minimal`, and `none` generic packs. The -historical `tips_profile` selects the same profiles; `prompt_profile` is the new -spelling and wins when both are present. `enforce_contract=None` delegates that -single policy default to the pack, while explicit `True` or `False` wins. - -## Compatibility and provenance - -The built-in full pack preserves the existing REPL, structured-subcall, -refinement, repair, and bounded-extraction behavior. Existing complete -`system_prompt` overrides and the older user/refinement template arguments still -work; new integrations should provide a complete caller pack instead of creating -partial combinations. - -Every `RLMResult` carries a frozen `PromptPackRecord` with pack ID, revision, -canonical `content_sha256`, profile, resolution tier, model family, and safe -benchmark provenance. The built-in runner and CLI JSON expose that record as -`prompt_pack`. The same canonical digest participates in the -[scaffold manifest](scaffold-manifest.md), with the manifest's standard -`sha256:` prefix; a revision label cannot conceal a content change. diff --git a/docs/provider-manifests.md b/docs/provider-manifests.md deleted file mode 100644 index 5e3abef..0000000 --- a/docs/provider-manifests.md +++ /dev/null @@ -1,257 +0,0 @@ -# Provider manifests - -Droste providers separate immutable description from live implementation. -There is no engine-wide data verb table and no process-global registry. - -## Values and ownership - -- `ProviderOperation` describes one source-agnostic operation: stable raw - `operation_id`, Python `binding_name`, description, parameter and result - `SchemaSpec`, pagination mode, delivery mode, and budget class. -- `ProviderManifest` groups operations for a `provider_type`, revision, and - provider protocol. Its digest is the SHA-256 of the canonical manifest - payload. Parsed manifests must reproduce the advertised digest. -- `ConfiguredSource` is only a source name, provider type, and frozen config. - It contains no handlers. -- `ProviderRegistration` is host-owned. It pairs a manifest with an exact - read/effectful classification, optional policy metadata, and a binder that - creates a `ProviderRuntime` at the edge. -- `ProviderCatalog` is an explicit set of registrations. Binding configured - sources creates one live `ProviderRegistry` whose descriptor snapshot is - immutable for the run. - -Binding is also the resource-acquisition boundary. A binder may attach an -optional `ProviderRuntime.close_callback`; resource-free providers omit it. -`ProviderRuntime.close()`, `BoundSource.close()`, and `ProviderRegistry.close()` -are idempotent and concurrency-safe. A registry closes every acquired runtime -once in reverse bind order, continues after individual close failures, and -cleans up earlier bindings if a later bind or registry validation fails. Live -resources are never assigned to a finalizer or process-global registry. - -One `ProviderRuntime` object is one ownership token and cannot be installed for -two configured sources. Providers that share an underlying pool must return a -separate runtime lease for each bind and reconcile the shared pool inside those -lease callbacks. The registry does not guess sharing from handler or callback -identity. - -The host owns a registry returned by `ProviderCatalog.bind()`. Passing it to -`create_environment()` transfers ownership immediately: construction failure -closes it, and a successful native or Pyodide environment closes it on every -run exit. A host that keeps a registry outside an environment must call -`registry.close()` explicitly. Closure begins only after broker dispatch is -quiescent; it is not a provider-cancellation mechanism. Runtime `stats` -callbacks must remain content-free and callable after close so hosts can report -final counters without reopening a resource. - -If cleanup fails after `run_rlm()` has already computed a result, the result is -preserved and Python emits a bounded `RuntimeWarning`; the relay likewise emits -a bounded stderr diagnostic without replacing its response. If execution and -cleanup fail together, Python raises a `BaseExceptionGroup` containing both -causes. Direct `close()` calls still raise their cached cleanup failure. - -Constructing `ProviderService` transfers one bound source to the trusted bridge -service for that service's lifetime. That host closes the service after the -last invocation and does not also transfer the supplying registry to an -environment. The runtime's once-only gate still makes outer failure cleanup of -the registry safe. - -The registry derives broker registrations, Python bindings, prompt text, and -the policy accessor manifest from the same operation descriptors. It snapshots -those descriptors for the run. Changing a description, schema, revision, -digest, effect, or policy value does not change the stable `CapabilityId` -`(kind, provider_type, source_id, operation_id)`. - -Every `ProviderRuntime` handler is context-first. Its first argument is a frozen -`CapabilityExecutionContext`; operation parameters follow unchanged. Generated -model bindings never expose this trusted argument. Providers use `check()` and -cumulative `checkpoint()` only—they cannot mutate the run ledger or trace. - -## Schemas, pagination, and delivery - -Every parameter schema and typed result schema names its dialect and -provenance. Droste does not guess a dialect or claim a provider-authored schema -as engine-authored. Cursor pagination requires a `cursor` parameter property -and `next_cursor` result property. - -Delivery is explicit: - -- `inline` requires a typed result schema and returns a frozen inline value; -- `handle` requires a typed result schema and a `CapabilityResultHandle`; -- `untyped` declares no result schema and permits provider-defined inline data. - -`budget_class` is stable classification data for the budget subsystem; the -provider layer does not implement a ledger. - -## Host policy and bridges - -The host must classify every operation as `read` or `effectful`. An incomplete -map or `unspecified` value fails before binding. Policy metadata is also -host-owned and frozen into the per-run descriptor. - -`ProviderService` exposes one already-bound source across the JSON bridge. It -publishes the verified manifest, source identity, and source description, then -dispatches only raw operation IDs present in the bound runtime. It never -publishes authoritative effects or policy. The receiving host constructs a -`BridgeProvider`, supplies its own exact effects and policy, and binds the -advertised source through its explicit catalog. - -Bridge `invoke` requests require a versioned `execution` object containing -call/run identity, remaining deadline, reservation facts, and the cancellation -snapshot. Every successful transport response includes a cumulative -`{tokens, subcalls}` checkpoint, including typed provider-error outcomes. The -receiving broker validates and applies that value before its final settlement. -The trusted bridge host must close the service after the last invocation. -Transport shutdown and runtime -resource ownership are separate: a duplex session closes per call, while the -provider runtime closes once for the bound source lifetime. - -Unary invocation remains the default provider-protocol-4 transport. A host that -needs live cancellation or accounting explicitly supplies a bridge-v2 duplex -session to `BridgeProvider`. Each invocation gets one bounded message pump keyed -by the existing `call_id`; it carries ordered `check`, cumulative `checkpoint`, -and exactly one `terminal` frame. The receiver applies checkpoints through the -same `CapabilityExecutionContext` and acknowledges only after ledger acceptance. -The remote handler still receives only that context. The reference pump permits -one queued frame and rejects serialized frames larger than 8 MiB. - -The pump is pull-based: the receiving interpreter returns from each transport -yield to reduce a frame, then sends one acknowledgement. Hosts must not call -back into a suspended Pyodide interpreter. A trusted host cancellation request -is sampled immediately before the receiver reduces each frame; providers cannot -set it. Once sampled, the receiver returns a cancelled acknowledgement and the -broker records that cancellation before its finalization cutoff. The terminal -sample is the transport cutoff, so a request arriving after it is late. Exact -duplicate frames are idempotent, while reordered, conflicting, wrong-call, or -over-reservation frames fail closed. - -If the remote provider interpreter or transport disappears before a valid -terminal frame, the receiving broker produces `bridge.transport_lost` and runs -its ordinary exactly-once settlement. Previously acknowledged checkpoints stay -committed. For inference attempts, the budget authority conservatively charges -the remaining reservation because unacknowledged remote spend is unknowable. -Killing the receiving broker itself is outside this in-process contract and -requires host-level durable run reconciliation. - -## Example - -```python -from droste import ConfiguredSource, ProviderCatalog -from droste.sources.sql_local import sqlite_provider - -catalog = ProviderCatalog((sqlite_provider(),)) -registry = catalog.bind( - ( - ConfiguredSource( - source_id="db", - provider_type="sqlite", - config={"sqlite_path": "local.db"}, - ), - ), - default_source_id="db", -) -# Pass registry to create_environment(), or close it explicitly in a finally block. -``` - -SQLite's raw operations are `query` and `schema`; their Python bindings are -`query` and `get_schema`. That distinction lets transports keep stable operation -identity without making Python naming the provider protocol. - -## Local filesystem/text provider - -`filesystem_text_provider()` is the dependency-free, read-only provider for a -host-selected local directory. `filesystem_text` is the reusable provider type; -each `ConfiguredSource` supplies a source name and an absolute trusted `root`: - -```python -from droste import ConfiguredSource, ProviderCatalog -from droste.sources import filesystem_text_provider - -registry = ProviderCatalog((filesystem_text_provider(),)).bind( - ( - ConfiguredSource( - source_id="docs", - provider_type="filesystem_text", - config={ - "root": "/data/docs", - "include": ["**/*.md", "**/*.txt", "**/*.log"], - "exclude": [".git/**"], - "enrichers": ["markdown"], - }, - ), - ) -) -# Pass registry to create_environment(), or close it explicitly in a finally block. -``` - -Its raw operations are `list`, `read`, `grep`, `search`, and `stat`; the Python -binding for `list` is `list_files` because provider bindings may not shadow a -Python builtin. `grep` is case-sensitive literal matching, while `search` is an -index-free case-insensitive all-terms scan. Markdown section reads are an -optional removable enrichment. The five base operations work without an index -or non-stdlib dependency. - -Paths are source-relative POSIX values. Include and exclude globs are -case-sensitive; `*`, `?`, and character classes stay within one segment, while -`**` must occupy a complete segment and spans zero or more segments. Exclusion -always wins, including for an explicit `read`, `stat`, or scan path. Directory -walks, file reads, lines, result pages, depth, and entry counts all have -configuration bounds. `max_result_bytes` limits the serialized inline result, -including JSON escaping rather than only the raw file bytes. Text is strict UTF-8 with no NUL bytes; binary files and -special files return typed unsupported outcomes rather than content. -The optional positive bounds are `max_read_bytes`, `max_scan_bytes`, -`max_result_bytes`, `max_scan_entries`, `max_results`, `max_line_bytes`, and -`max_depth`; invalid or internally inconsistent values fail at bind. - -`list_files`, `grep`, and `search` return versioned, self-contained cursors, so -the provider keeps no cursor registry. Each cursor is authenticated by and -valid only for the bound runtime that issued it. A cursor contains no path -authority: on every use the provider re-applies path policy and revalidates the -immutable file inventory. Addition, removal, replacement, or revision drift -returns retryable `filesystem.changed` instead of silently duplicating or -skipping results. `read` may also receive the opaque revision returned by -`stat`. - -The trusted runtime pins the configured root as a directory descriptor and -opens every path component relative to an already-open directory with -`O_NOFOLLOW`. It never uses string-prefix checks, `resolve()`, `os.walk`, or a -symlink-following fallback. Platforms missing the required POSIX `dir_fd`, -descriptor-listing, `O_DIRECTORY`, `O_NOFOLLOW`, `O_NONBLOCK`, or `pread` -primitives fail while binding the source. The configured absolute root is -never placed in descriptors, prompt text, results, evidence, or errors. - -Native in-process execution still runs arbitrary Python and is not an OS -security boundary. Root non-ambient access requires the host to place the -provider on the trusted side of a process or Pyodide/WASM boundary. The generic -`ProviderService`/`BridgeProvider` transport preserves the same typed result, -error, usage, cursor, and evidence values; the filesystem provider adds no -transport-specific path. - -## Evidence - -Capability metadata uses `EvidenceLocation(source_id, path, revision, ranges)`. -Each `EvidenceRange` may carry ordered byte coordinates, line coordinates, a -section identifier, or a combination. Durable trace projections retain only -the evidence count; full locations are replay content governed by the host's -retention policy. - -Filesystem byte ranges are zero-based and half-open. Line coordinates are -one-based and inclusive. Evidence paths are source-relative POSIX paths and -revisions are opaque stat-derived digests; neither exposes the configured host -root. - -## MCP transports - -`open_mcp_stdio_source()` acquires one host-configured local MCP process and -maps its complete paginated `tools/list` snapshot into the same immutable -provider values. It returns a lifecycle-owned `BoundSource`, which may be -combined directly with in-process bound sources in `ProviderRegistry`. MCP is -not a provider type, prompt vocabulary, binding namespace, policy authority, or -trace path. See the [local MCP stdio contract and spike report](mcp-stdio.md). - -`open_mcp_http_source()` performs the same acquisition transaction over MCP -Streamable HTTP. The trusted host owns exact endpoints, tenant-scoped secret -resolution, OAuth, DNS/IP policy, and sessions; none of those values enters the -manifest or generated bindings. Cross-language hosts can implement -`McpToolTransport` and call `bind_mcp_transport_source()` so the same pure -`mcp_tools_to_manifest()` projection remains authoritative. See the -[Streamable HTTP contract](mcp-http.md). diff --git a/docs/providers.md b/docs/providers.md new file mode 100644 index 0000000..80f722c --- /dev/null +++ b/docs/providers.md @@ -0,0 +1,77 @@ +# Providers (protocol v4) + +Providers expose trusted data operations to generated code without exposing +transport clients, credentials, or host paths. The same immutable description +drives the broker allowlist, prompt text, Python bindings, and policy metadata. + +## Values and ownership + +- `ProviderManifest` contains a revision, digest, and ordered + `ProviderOperation` values. +- `ConfiguredSource` is frozen configuration for one named source. +- `ProviderRegistration` pairs a manifest with host-owned side-effect policy + and a binder. +- `ProviderRuntime` owns the live handlers and optional close callback. +- `BoundSource` is one acquired runtime plus its registration. +- `ProviderRegistry` combines bound sources for one run and closes them in + reverse acquisition order. + +An operation has separate raw `operation_id` and Python `binding_name` values, +explicit parameter/result schema dialect and provenance, pagination and result +delivery modes, and a budget class. The host classifies every operation as +`read` or `effectful`; transports cannot grant effects. + +Trusted handlers use the context-first signature: + +```python +def handler(execution: CapabilityExecutionContext, *args, **kwargs): + execution.check() + return CapabilityOutcome(result={"value": "..."}) +``` + +Passing a registry to `create_environment()` transfers ownership. Otherwise, +close it explicitly. Partial binds and failed environment construction close +already-acquired runtimes deterministically. + +## Built-in providers + +The SQLite provider opens databases read-only, enables `query_only`, and admits +only bounded single-statement `SELECT` queries. The filesystem-text provider +offers bounded `list`, `read`, `grep`, `search`, and `stat` operations under one +pinned root. It does not follow symlinks and revalidates path and cursor policy +on every request. + +These policies constrain the provider operation; they do not isolate the native +Python process. Use a separate process or the Pyodide bridge when source paths +must be non-ambient. + +## MCP over local stdio + +`open_mcp_stdio_source()` launches one absolute, host-allowlisted executable, +negotiates MCP `2025-11-25`, freezes the complete paginated `tools/list` +snapshot, and returns a `BoundSource`. Generated code sees ordinary provider +bindings, not MCP vocabulary or process configuration. + +The host must provide an explicit working directory and environment, executable +and tool allowlists, Python binding names, read classifications, budget classes, +and optional policy metadata. The transport currently admits read-only tools. + +Supported result mapping prefers `structuredContent`; otherwise it returns the +bounded content-block JSON without fetching links. Cancellation terminates the +session after sending `notifications/cancelled`. There is no reconnect because +a replacement process would be a different session and an effect may already +have occurred. + +MCP tasks, resources as first-class evidence, live tool refresh, and arbitrary +cross-dialect schema validation are not supported. The integration test uses +the pinned official filesystem MCP server and routes its tool through the same +broker as SQLite. + +## Cross-interpreter bridge + +`ProviderService` and `BridgeProvider` carry a verified manifest and operation +calls across an interpreter boundary. The receiving host supplies effects and +policy, rejects unknown operations, and validates cumulative checkpoints before +the broker performs its sole final settlement. Bridge protocol v2 adds a bounded +duplex pump for mid-call checkpoints and cancellation acknowledgement; unary +invocation remains available for hosts that do not select it. diff --git a/docs/quickstart.md b/docs/quickstart.md index c09150e..d6c5f9d 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -60,3 +60,31 @@ input, and output option. The generated Python code can read the data you provide. Database access is opened read-only and policy-gated, but process isolation and operating-system permissions remain the security boundary. + +## Examples + +Interrogate a large log without putting the whole file in the model prompt: + +```bash +droste "Which failure caused the outage, and what happened immediately before it?" server.log +``` + +Ask across an export or archive. Droste recursively reads supported text files +while skipping hidden and common generated directories: + +```bash +droste "Which decisions were reversed, and why?" ./chat-export +``` + +Use SQL for exact aggregation and model calls only where interpretation is +needed: + +```bash +droste "Which plan has the highest refund rate relative to MRR?" shop.db +``` + +For multiple inputs, pass every path after the question: + +```bash +droste "What changed between these reports?" january.md february.md +``` diff --git a/docs/recipes.md b/docs/recipes.md deleted file mode 100644 index 5d201cf..0000000 --- a/docs/recipes.md +++ /dev/null @@ -1,41 +0,0 @@ -# Recipes - -These examples work with the account saved by `droste login` or any -OpenAI-compatible endpoint configured with `OPENAI_API_KEY` and, when needed, -`OPENAI_BASE_URL`. - -## 1. Interrogate a huge log file - -```bash -droste "Group errors by service and cause. Which failure started first, and -did it cascade? Include exact counts." server.log -``` - -The model inspects slices of the file through code. Python handles counting, -grouping, and time ordering. Add `--verbose` to watch the run. - -## 2. Ask questions across a chat/export archive - -Point it at Slack, WhatsApp, or another line-oriented export: - -```bash -droste export/channel-eng.txt export/channel-support.txt \ - "What did we promise customers about the migration, and did engineering's - internal discussion match what support was telling people?" -``` - -Droste searches both files, aligns relevant passages in time, and uses model -subcalls where judging consistency requires close reading. - -## 3. Analyze a SQLite database - -```bash -droste app.db \ - "Which customers churned in Q2, what did they have in common, and how much - MRR walked out the door?" -``` - -Droste detects SQLite files, reads the schema, writes bounded read-only -queries, and computes over the results. Operating-system permissions remain -the security boundary. To use a cheaper model for subcalls, add -`--subcall-model MODEL`. diff --git a/docs/reference/prompt-packs.md b/docs/reference/prompt-packs.md new file mode 100644 index 0000000..919e88f --- /dev/null +++ b/docs/reference/prompt-packs.md @@ -0,0 +1,44 @@ +# Prompt packs (schema v2) + +A prompt pack is the complete deterministic harness strategy for one run. +Droste resolves exactly one pack before inference and never merges packs or +changes strategy mid-run. + +The stable template slots are: + +| Slot | Value | +| --- | --- | +| `capabilities` | Generated model-visible bindings | +| `budget` | Resolved compute and output limits | +| `question` | Caller question | +| `history` | Bounded loop transcript | +| `output_contract` | Required model response form | + +Missing or unknown slots fail validation. A pack contains all initial, +refinement, repair, extraction, and transcript-elision templates plus policy +defaults and provenance. Parsing, validation, selection, hashing, and rendering +are pure; only loader functions read files or package resources. + +Resolution checks caller bindings, consumer model-family and generic bindings, +engine model-family bindings, then the generic fallback for the requested +profile. Built-in profiles are `full`, `minimal`, and `none`. Selection records +the tier and canonical content SHA-256 in results and the scaffold manifest. + +Use `parse_prompt_pack()`, `PromptPackCatalog`, and `resolve_prompt_pack()` for +custom packs. A declared `content_sha256` must equal the hash of the parsed +content; a revision label cannot conceal a content change. + +## RLM skills (schema v1) + +RLM skills are optional, immutable Markdown-plus-TOML strategy artifacts. They +are separate from prompt packs: loading a skill never changes the run's +deterministic prompt-pack contract. + +A skill declares `schema_version`, `id`, `revision`, `summary`, +`model_families`, provenance, and a non-empty Markdown body. Built-in skills are +loaded through the explicitly registered read-only `rlm_skills` provider. Its +`list` operation returns metadata; `load` returns one exact body and content +hash. No skill provider or prompt text is installed automatically. + +Skill bodies are configurable content. Durable capability events record only +the provider identity and outcome, never the body. diff --git a/docs/reference/runner.md b/docs/reference/runner.md new file mode 100644 index 0000000..11bc37a --- /dev/null +++ b/docs/reference/runner.md @@ -0,0 +1,81 @@ +# Runner protocol v10 + +`python -m droste_runner` is a one-shot JSON worker for non-Python hosts. It +reads a request from `RLM_RUNNER_REQUEST_PATH` or its first argument and writes +one JSON response to stdout. + +Every request must include `"protocol_version": 10`. Missing or mismatched +versions return a structured refusal before operation resolution, source +binding, credentials, or model dispatch. + +## Operations + +`operation` is `run` by default. `preflight` resolves the same prompt pack, +provider descriptors, rollout settings, budget, sandbox, and scaffold manifest +as a run without dispatching root or subcall inference. Dynamic providers may +still be acquired when their descriptors are needed, and are then closed. + +A preflight response always has five fields: + +```json +{ + "protocol_version": 10, + "operation": "preflight", + "status": "success", + "preflight": { + "scaffold_manifest": {}, + "manifest_id": "sha256:...", + "compatible": true, + "mismatches": [] + }, + "error": null +} +``` + +Checkpoint incompatibility returns the same outer shape with +`status="refusal"`, `preflight=null`, and a `scaffold_incompatible` error. + +## Required run authorization + +Run requests carry one exact seven-field budget object: + +```json +{ + "tokens": 500000, + "subcalls": 50, + "depth": 1, + "wall_ms": 300000, + "max_iterations": 30, + "root_output_tokens": 4096, + "subcall_output_tokens": 2048 +} +``` + +Unknown, missing, boolean, or out-of-range values fail before dispatch. Optional +rollout facts include model/source revisions, root and subcall sampling, +concurrency, seed, subcall input capacity, root reasoning effort, and scaffold +requirements. Conflicting duplicate evidence fails rather than being rewritten. + +Native callbacks use `root_endpoint` and `subcall_endpoint`. The root client +uses Responses-style messages. Hosted subcalls may negotiate +`responses-stream/v2` NDJSON; reconstruction requires a terminal completion and +preserves its `stop_reason`. + +## Responses and events + +Run responses have one stable field set across success, refusal, and error. The +important result fields are `answer`, `answer_metadata`, `ready`, `extracted`, +`degradations`, typed errors, iteration and subcall counts, usage, prompt-pack +and scaffold identities, `stdout_chars`, `run_id`, and `run_record`. + +Trajectory entries carry typed `execution_status` and `attempt_kind`; consumers +must not infer either from stdout text. Oversized stdout is rejected instead of +being silently truncated. + +The native process runner can emit Trace ABI events to its configured sink. The +Deno relay uses a separate inherited event descriptor; see +[pyodide/README.md](../../pyodide/README.md). Protocol refusals happen before +run admission and emit no trace events. + +Breaking request or response changes require a runner protocol bump. Current +migration guidance is in [UPGRADING.md](../../UPGRADING.md). diff --git a/docs/reference/scaffold.md b/docs/reference/scaffold.md new file mode 100644 index 0000000..c86fe24 --- /dev/null +++ b/docs/reference/scaffold.md @@ -0,0 +1,38 @@ +# Scaffold manifest v3 + +A scaffold manifest is the content-addressed identity of the facts that can +materially change a run. It lets a host reject incompatible checkpoints before +inference and join external outcome metadata without putting task content in a +durable trace. + +The closed manifest contains: + +- engine version and source revision; +- kernel, capability, trace, prompt-pack, provider, and runner versions; +- prompt-pack identity and content hash; +- capability-manifest hash and generated globals; +- terminal, subcall-identity, template, and override contracts; +- root/subcall model identities, sampling, capacities, limits, concurrency, + and seed; +- the complete budget and sandbox limits; and +- parent/child trace and call identity rules. + +`manifest_id` is `sha256:` plus the SHA-256 of canonical UTF-8 JSON from +`as_dict()`. Object keys are sorted, separators are compact, Unicode is +preserved, numbers are finite, and array order is retained. A supplied wire +`id` is verified against the content. + +New manifests use schema v3. `ScaffoldManifest.from_dict()` also reads stored +v1 and v2 manifests without rewriting their identity. Unknown fields, missing +fields, wrong scalar types, invalid digests, unsupported versions, and +duplicate or unsorted generated globals fail closed. + +`ScaffoldRequirements` may specify a complete `manifest_id`, a partial nested +`required` object, or both. `require_scaffold_compatibility()` raises an ordered +`ScaffoldCompatibilityError` on mismatch. `preflight_rlm()` and runner +`operation="preflight"` use the same resolver as `run_rlm()` and return the +complete content-free identity without dispatching model calls. + +The full manifest is returned in live results. Durable default retention stores +only its ID and schema version; task, reward, dataset, and split metadata remain +host-owned and join externally by run and manifest ID. diff --git a/docs/reference/trace.md b/docs/reference/trace.md new file mode 100644 index 0000000..0dbe8e0 --- /dev/null +++ b/docs/reference/trace.md @@ -0,0 +1,71 @@ +# Trace ABI v10 + +Droste exposes one append-only live event stream and one policy-resolved +terminal `RunRecord`. The engine creates these values; hosts choose delivery +and persistence. + +Every event has `run_id`, positive monotonic `seq`, UTC `timestamp`, `type`, +`version: 10`, `persistence_class`, `depth`, optional `parent_run_id`, and its +type-specific body. `parse_event()` rejects unsupported versions, unknown or +missing fields, invalid classifications, and invalid terminal shapes. + +## Retention classes + +| Class | Event types | Terminal record | +| --- | --- | --- | +| Durable | `usage`, `budget`, `policy`, `capability`, `done` | Always retained | +| Configurable | `iteration_start`, `llm_response`, `code`, `output`, `execution_error`, `subcall`, `repair`, `extract`, `result`, `replay`, `checkpoint` | Retained only when selected | +| Transient | `startup`, `progress`, `reasoning_delta`, `usage_progress`, `heartbeat` | Live only | + +Retention does not control ordinary live delivery. `result` is always delivered +before `done`; `replay` is emitted only when replay retention is selected. +Training authorization is separate and denied by default. + +## Current event contract + +- `startup` identifies engine, runner, provider, and scaffold versions and may + list the armed `ready_gates`. +- `heartbeat` reports content-free provider-call liveness as `elapsed_ms`. +- `checkpoint` carries the current draft and optional opaque host payload. +- `subcall`, `repair`, and `extract` use closed phase discriminators and exactly + one terminal completion or failure after their start. +- `usage_progress` is a cumulative snapshot emitted only after numeric provider + usage settles; it is never inferred from visible output. +- `capability` is the broker's content-free trace projection. Parameters, + results, error messages, and evidence locators are excluded. +- `done` includes terminal status, readiness, extraction, `degradations`, usage, + budget, policy, retention, typed error presence, scaffold identity, and total + stdout characters. It never includes answer, code, stdout, or error details. + +Root and subcall usage scopes separately report inclusive input, cache-read, +cache-creation, output, and total tokens plus request/success counts and +completeness. The top-level kind is `resolved` only when both scopes are +complete; otherwise known counters are preserved as `partial` evidence. + +## Ordering and reconciliation + +An execution context serializes stamping and callback delivery. Event `seq` is +the only event order; broker `call_id` identifies an attempt, not an event +sequence or durable idempotency key. + +Finalization emits `usage`, `budget`, and `policy`; delivers `result`; may emit +`replay`; and ends with `done`. The terminal record's final projection must +equal that `done` body. Retention can leave sequence gaps but never renumbers +events. + +## Conformance corpus + +The wheel and sdist include the authoritative strict fixture bytes. Consumers +should use these instead of copying the prose schema: + +```python +from droste.testing import ( + runner_v10_refusal_ndjson, + trace_v10_execution_ndjson, + trace_v10_lifecycle_ndjson, +) +``` + +The runner refusal fixture is intentionally not a trace event. The two Trace +ABI fixtures cover representative execution and lifecycle phases, partial and +resolved usage, retention, and terminal reconciliation. diff --git a/docs/rlm-skills.md b/docs/rlm-skills.md deleted file mode 100644 index 7c888c1..0000000 --- a/docs/rlm-skills.md +++ /dev/null @@ -1,61 +0,0 @@ -# RLM skills - -RLM skills are versioned, additive strategy documents the root model may inspect -and load during a run. They are not executable, are not subcalls, and do not -replace prompt packs. A prompt pack is the single resolved harness contract; -skills are optional data reached through the same broker as other read-only -providers. - -Each artifact is Markdown with strict `+++` TOML frontmatter: - -```toml -+++ -schema_version = 1 -id = "acme.chunking" -revision = "1.0.0" -summary = "Choose chunks that fit semantic and output limits." -model_families = ["generic"] - -[provenance] -source = "acme" -+++ -Keep bulk data in Python and print only bounded summaries. -``` - -Required fields, provenance fields, and model-family selectors fail closed. -Every parsed `RLMSkill` has a canonical `content_hash`. A `generic` skill is -eligible for every named family; a family-specific skill is never selected for -another family. Loading an ID with multiple revisions requires an exact -revision. - -Droste ships `droste.chunking@1.0.0` and a removable decomposition example. -They make no benchmark claim. Hosts may load filesystem artifacts with -`load_rlm_skill` or compose immutable values in an `RLMSkillCatalog`. - -## Broker wiring - -```python -from droste import ( - ConfiguredSource, ProviderCatalog, load_builtin_skill_catalog, - rlm_skills_provider, -) - -registration = rlm_skills_provider(load_builtin_skill_catalog()) -registry = ProviderCatalog((registration,)).bind( - (ConfiguredSource("skills", "rlm_skills"),) -) -``` - -When that registry is used to create the run environment, generated code sees: - -```python -items = skills.available(model_family="qwen") -skill = skills.load("droste.chunking", "1.0.0") -``` - -The stable provider operations are `skills.list` and `skills.load`; the Python -binding uses `available` because `list` is a Python builtin. Both operations are -read-only and their normal capability events record identity/status without the -skill body. Hosts decide whether the provider is present. Droste never -auto-registers it or silently adds skill text to a prompt. - diff --git a/docs/scaffold-manifest.md b/docs/scaffold-manifest.md deleted file mode 100644 index ed7d576..0000000 --- a/docs/scaffold-manifest.md +++ /dev/null @@ -1,128 +0,0 @@ -# Scaffold manifest v3 - -A scaffold manifest is the content-addressed identity of everything that can -materially change a Droste rollout. It lets a trainer reject a checkpoint/run -mismatch before the first model request and join trainer-owned reward metadata -without putting task content in the engine trace. - -`ScaffoldManifest.manifest_id` is `sha256:` plus the lowercase SHA-256 of the -UTF-8 JSON returned by `as_dict()`. JSON is serialized with sorted object keys, -compact separators, Unicode preserved, finite numbers only, and no `id` field. -Arrays retain order. The optional wire `id` is a claim over that content and is -verified when parsed. - -## Closed schema - -The top-level and owned nested objects are closed: unknown and missing fields, -wrong scalar types, invalid digests, duplicate/unsorted globals, and unsupported -schema versions fail. `root_sampling` and `subcall_sampling` are deliberately -open JSON objects because their provider-specific keys are themselves resolved -inference facts. - -```json -{ - "schema_version": 3, - "engine": {"version": "0.10.6", "source_revision": null}, - "abis": { - "kernel": 1, "capability": 1, "trace": 7, - "prompt_pack": 2, "provider": 4, "runner": 10 - }, - "prompt_pack": { - "id": "droste.generic.full", "revision": "2.0.0", - "profile": "full", "content_hash": "sha256:<64 lowercase hex>" - }, - "capabilities": { - "manifest_hash": "sha256:<64 lowercase hex>", - "model_visible_globals": ["answer", "context", "llm_query"] - }, - "contracts": { - "terminal": "answer-ready-v1", - "subcall_identity": "capability-call-parent-v1", - "templates": { - "refinement": "sha256:<64 lowercase hex>", - "missing_code_repair": "sha256:<64 lowercase hex>", - "error_repair": "sha256:<64 lowercase hex>", - "extract_system": "sha256:<64 lowercase hex>", - "extract_user": "sha256:<64 lowercase hex>" - }, - "overrides": { - "system_prompt": null, "system_prompt_additions": null, - "user_prompt": null, "refinement_prompt": null - } - }, - "inference": { - "root": {"id": "model", "revision": null}, - "subcall": {"id": "model", "revision": null}, - "root_sampling": {}, "subcall_sampling": {}, - "output_limits": {"root_tokens": 4096, "subcall_tokens": 2048}, - "input_capacity": { - "subcall": {"state": "unknown", "tokens": null} - }, - "concurrency": 5, "seed": null - }, - "budget": { - "tokens": 500000, "subcalls": 50, "depth": 1, "wall_ms": 300000, - "max_iterations": 30, - "root_output_tokens": 4096, "subcall_output_tokens": 2048 - }, - "sandbox": { - "output_chars": 25000, "execution_timeout_ms": 5000, - "capture_output_chars": 50000 - }, - "parent_child": { - "trace_depth": "root-zero-child-increment", - "identity": "capability-call-parent-v1" - } -} -``` - -Model IDs are either non-empty strings or explicit `null` when the host keeps -the identity opaque. A revision cannot accompany a null model ID. Source and -model revisions are never guessed by the engine; the host supplies them through -`RolloutConfiguration`. - -`inference.concurrency` is the effective maximum number of in-flight subcall -batch items. Built-in clients report their configured value, and `run_rlm` -rejects a mismatch with `RolloutConfiguration.concurrency` before inference. -The compatibility default is 5. Custom clients that do not expose the optional -read-only metadata remain source-compatible and must honor the declared value. - -`inference.input_capacity.subcall` is planning metadata, separate from both -the run token budget and `inference.output_limits`. A bounded value has -`{"state": "bounded", "tokens": }`. Deliberately unbounded -and unknown values have null tokens and the corresponding `unbounded` or -`unknown` state. Unknown is never replaced with a guessed model context -window. A rollout declaration and client report resolve to one value; two -different known values fail before the first model request. - -Version 2 adds this required closed field. `ScaffoldManifest.from_dict()` still -accepts and verifies version 1 manifests, where the field is absent, so stored -identities remain readable. New manifests are always version 2; the engine -does not silently rewrite a version 1 identity. - -Runner `root_reasoning_effort` is one host-resolved inference fact: the native -and Pyodide clients send it on root callbacks, while the scaffold records the -same value in `inference.root_sampling.reasoning_effort`. A conflicting value -inside an explicit runner `root_sampling` object is rejected before inference. - -## Compatibility and storage - -`ScaffoldRequirements` accepts an exact `manifest_id`, a partial nested -`required` object, or both. `require_scaffold_compatibility()` returns normally -or raises `ScaffoldCompatibilityError` containing typed path/expected/actual -mismatches. `run_rlm` performs this check before its first model call when -`RLMConfig.checkpoint_requirements` is set. - -Hosts that need this decision before inference authorization can call the -public `preflight_rlm(...)` API, or send `operation: "preflight"` through -`droste_runner`. Preflight and `run_rlm(...)` share the same PromptPack, -environment, capability, rollout, manifest, and compatibility resolver. The -typed preflight result contains the exact full manifest and no task or model -content; runner mismatch responses preserve the public -`ScaffoldCompatibilityError` mismatch fields. - -The full manifest is returned live as `RLMResult.scaffold_manifest` and in the -runner/CLI result. Durable default-retention terminal records store only its -content-free ID and schema version. Task, verifier, reward, split, and dataset -metadata remain trainer-owned; join them externally with -`OutcomeJoinKey(run_id, scaffold_manifest_id)`. diff --git a/docs/trace-abi.md b/docs/trace-abi.md deleted file mode 100644 index 45e0e94..0000000 --- a/docs/trace-abi.md +++ /dev/null @@ -1,266 +0,0 @@ -# Trace ABI v7 - -Droste exposes one append-only event stream and one policy-resolved terminal -`RunRecord`. The engine creates values; it does not choose a database or write -files. Hosts attach `on_event` for live delivery and optionally -`on_run_record` as their persistence I/O shell. - -The integer `version` names the complete strict contract: envelope, event -bodies, persistence classification, and ordering/terminal invariants. Changing -any of those requires a new Trace ABI version and an atomic consumer migration; -even adding an optional field is breaking because current strict readers reject -unknown fields. The runner protocol changes only when its own request/response -or negotiation contract changes, including an embedded run-record version. It -does not change merely because a released fixture is added. - -## Event envelope - -Every event is a strict v7 value with these fields: - -```json -{ - "run_id": "e0f7...", - "seq": 1, - "timestamp": "2026-07-14T05:00:00Z", - "type": "progress", - "version": 7, - "persistence_class": "transient", - "parent_run_id": "optional-parent", - "depth": 0, - "status": "Iteration 1/20: Generating code..." -} -``` - -`seq` is positive and monotonic within one `run_id`. `depth` is required: roots -use `0`; child runs use their deterministic parent depth plus one and carry -`parent_run_id`. `parse_event` validates a UTC timestamp and rejects partial, -misclassified, unsupported-version, missing-body-field, and unknown-body-field -values. - -Relay-owned Pyodide telemetry uses a child `run_id` whose `parent_run_id` is -the engine run. The relay and engine therefore each own one sequence; their -events correlate without two writers racing to stamp the same counter. - -## Retention is data selection - -The persistence class is exhaustive and fixed by event type: - -| Class | Event types | Rule | -| --- | --- | --- | -| `durable` | `usage`, `budget`, `policy`, `capability`, `done` | Always in the terminal record | -| `configurable` | `iteration_start`, `llm_response`, `code`, `output`, `execution_error`, `subcall`, `repair`, `extract`, `result`, `replay`, `checkpoint` | Included only when named by `TraceRetentionPolicy.retain` | -| `transient` | `startup`, `progress`, `reasoning_delta`, `usage_progress` | Live delivery only; never in the terminal record | - -Retention governs the terminal record, not the live channel. `result` is -always delivered once before `done`, even when it is not retained. `replay` is -different: it is emitted only when the host explicitly selects replay -retention. - -## Exhaustive v7 bodies - -Every event body has a fixed top-level schema. Optional fields are marked `?`. -Objects named below are JSON objects; all other types are primitive. - -| Type | Body | -| --- | --- | -| `startup` | `engine_version: string`, `runner_protocol?: integer|null`, `provider_protocol?: integer|null` | -| `progress` | `status: string` | -| `iteration_start` | `iteration: integer`, `remaining_tokens: integer` | -| `llm_response` | `iteration: integer`, `response: string` | -| `code` | `iteration: integer`, `code: string` | -| `output` | `iteration: integer`, `stdout: string`, `calls_made: integer`, `answer_ready: boolean`, `answer_content_chars: integer` | -| `execution_error` | `iteration: integer`, `error_type: string`, `message: string` | -| `reasoning_delta` | `text: string` | -| `subcall` | `phase`, `call_id`, `operation`, `iteration`, plus phase-specific reservation/checkpoint/error and optional batch metadata | -| `repair` | `phase: "start"|"completion"|"failure"`, `kind: "missing_code"|"execution_error"|"terminal"`, `iteration`, `error?` only on failure | -| `extract` | `phase: "start"|"completion"|"failure"`, `iteration`, `extract_error?` only on failure | -| `result`, `replay` | `result: object` | -| `checkpoint` | `iteration: integer`, `checkpoint_seq: integer`, `draft: string`, `draft_chars: integer`, `ready: boolean`, `payload: object|null` | -| `usage_progress` | `boundary: "root"|"subcall"`, `kind`, cumulative `root`, `subcall`, `unattributed`, and `total_tokens` | -| `usage` | `kind: "resolved"|"partial"`, `root: object`, `subcall: object`, `unattributed: object`, `total_tokens: integer`, `wall_time_ms: integer` | -| `budget` | `kind: "snapshot"|"mutation"`, `source: string`, plus the kind-specific fields below | -| `policy` | `contract_enforced: boolean`, `outcome: string`, `violation_type: string|null` | -| `capability` | `outcome: object` in the exact `CapabilityResult.to_trace_dict()` shape | -| `done` | `status: "success"|"error"|"cancelled"`, `ready`, `extracted`, `iterations`, and the content-free reconciliation objects/errors | - -The canonical `result.result` fields are `answer`, `answer_metadata`, `ready`, -`iterations`, `tokens_used`, `subcalls`, `successful_subcalls`, `extracted`, -`error`, `extract_error`, `recovered_error`, and `prompt_pack`. It deliberately -has no trajectory or `run_record`. A retained `replay.result` adds the full -trajectory; each `llm_input` remains a structured message list. - -The durable `done` value contains content-free terminal status, typed error -presence, and reconciled usage/budget/policy facts. It never copies answer, -code, output, trajectory, error messages, error details, or executed source. -The configurable `replay` value is the complete result snapshot for hosts that -explicitly retain replay content. Its `result.usage` is the same resolved or -partial projection emitted durably in `usage` and `done`, including both cache -token classes. - -Billing consumes the durable `usage` value. Both `root` and `subcall` carry -`input_tokens`, `cache_read_tokens`, `cache_creation_tokens`, `output_tokens`, -`total_tokens`, `requests`, `successes`, and `complete`. Cache tokens are -disjoint breakdowns inside inclusive input totals; complete scopes therefore -require their sum not to exceed `input_tokens`. They are not added again to -`input_tokens` or `total_tokens`. The top-level kind is `resolved` exactly when -both scopes are complete; otherwise it is `partial`. Partial usage preserves -known provider counts without substituting conservative budget reservations for actual usage. -Legacy/custom-client tokens that cannot be assigned safely appear under -`unattributed.total_tokens`; the three token scopes must sum to -`total_tokens`. None of these facts is inferred by counting stream events. - -`usage_progress` is the live, transient projection of those same cumulative -counters. The engine emits it only after a root or subcall provider boundary -settles a numeric usage observation. `boundary` identifies the role that just -settled; the two scope objects remain cumulative, so consumers replace their -current display rather than summing events. Exact observations therefore move -the display at model-call boundaries without estimating between them. A -partial observation preserves its reported counters and marks the affected -scope incomplete. The last progress snapshot reconciles with terminal `usage` -unless a later boundary has no numeric usage observation. - -The budget body remains a discriminated event in Trace ABI v7. The terminal snapshot uses -`kind="snapshot"`, `source="budget_ledger"`, and `configured`, `consumed`, -and `remaining` objects. The configured object includes the structural -`max_iterations` ceiling; terminal `iterations` records how many iterations -actually began. The ledger may emit any number of -`kind="mutation"` values with `action` (`reserve`, `commit`, `refund`, or -`exhaust`), `resource`, non-negative `amount`, and optional `call_id`. -Consumers must not assume exactly one mutation. - -```python -from droste import DataUseAuthorization, RLMConfig, TraceRetentionPolicy - -config = RLMConfig( - trace_retention=TraceRetentionPolicy( - retain=frozenset({"code", "output", "repair", "replay"}), - policy_id="local-training-corpus-v1", - expires_at="2026-10-14T00:00:00Z", - host_managed_expiry=True, - ), - data_use=DataUseAuthorization( - training_allowed=True, - authorization_ref="consent://training/trace-policy-1", - purposes=frozenset({"training"}), - ), - on_run_record=save_record_locally, -) -``` - -Retention never grants training permission. `DataUseAuthorization` defaults to -denied and an empty purpose set. Training requires both an auditable -`authorization_ref` and the explicit `training` purpose. `expires_at` is an -absolute, timezone-aware timestamp and is valid only with -`host_managed_expiry=True`: Droste records the policy fact while the local host -owns expiry/deletion. A local host can serialize `record.as_dict()` to its own -protected storage; cloud transport and governance integrations remain host -concerns. - -When `run_rlm` creates its context, trace settings come from `RLMConfig`. When -a caller injects an existing `ExecutionContext`, that context owns trace -identity, retention, data-use authorization, and the record sink because it -may already contain shared accounting/events. Explicit conflicting config -values fail instead of mutating a live sequence. - -## Capability outcomes - -The broker owns immutable capability call/result values and their content-free -accounting/evidence projection. Trace observation wraps only that broker-owned -projection as the `outcome` of a durable `capability` event; full parameters, -results, messages, and free-form evidence never enter the durable value. Trace -code does not copy the capability schema, dispatch calls, or participate in -authorization. This keeps tracing observational and gives native and Pyodide -paths the same identity and evidence facts. - -## Subcall, repair, and extract lifecycles - -`subcall` is the live and retainable projection of the broker-owned attempt; -it is not another dispatch or accounting protocol. `start` carries the exact -broker reservation. `progress` carries a trusted cumulative checkpoint. -`completion` and `failure` carry the last accepted checkpoint; failure adds -only the stable provider error code and type. Prompts, contexts, results, and -provider error messages never enter this event. - -Every subcall body carries the loop `iteration`; run nesting remains the -envelope's `depth`/`parent_run_id`. The broker `call_id` is the attempt identity, -while envelope `seq` is the only event order. An atomic `llm_batch` or -`llm_batch_with_errors` reports `batch_count`; its `call_id` is already the -batch identity. Droste does not synthesize item completion or IDs when the -atomic transport cannot observe them. - -`repair` and `extract` use closed phase discriminators instead of progress -strings. Missing-code, execution-error, and terminal repair paths each emit a -start and exactly one completion or failure once entered. Extract fallback does -the same; a failure carries the typed `extract_error`. The canonical `result` -still carries the unary-equivalent answer and `done` remains content-free. - -## Answer-state checkpoints - -`checkpoint` publishes the answer state the engine holds right now, so a host -that loses the process still has the last draft it saw. The engine emits one -after every executed step whose draft moved, or whose host had something of its -own to add. `checkpoint_seq` starts at one and strictly increases within a -`run_id`; `draft_chars` always equals `len(draft)`. - -`payload` is opaque. Hosts fill it through `RLMConfig.checkpoint_payload_provider`, -a callable returning any JSON object or `None`. The engine carries the value -across without inspecting it and never schema-checks its contents, exactly as -the relay carries adapter `meta`. A provider that raises is reported and the -checkpoint carries `payload: null`: a checkpoint can never fail a run. - -The event is `configurable`, not `durable`. It carries draft content, so it -reaches a terminal record only when a host names it in -`TraceRetentionPolicy.retain`. - -## Terminal reconciliation - -Finalization emits resolved `usage`, `budget`, and `policy`; always delivers the -canonical unary-equivalent `result`; optionally emits the complete `replay` -snapshot when replay retention is selected; then emits durable `done`. -`RunRecord` retains the selected subset without -renumbering it, so gaps truthfully show discarded transient or configurable -values. Its terminal projection must equal the body of its final `done` event, -and its usage totals reconcile with `RLMResult`. - -## Released conformance corpus - -Droste 0.15.1 and later ship the authoritative fixture bytes inside the wheel -and sdist. Python consumers load them through package resources: - -```python -from droste.testing import ( - runner_v10_refusal_ndjson, - trace_v10_execution_ndjson, - trace_v10_lifecycle_ndjson, -) - -execution_lines = trace_v10_execution_ndjson().splitlines() -event_lines = trace_v10_lifecycle_ndjson().splitlines() -pre_admission_refusal = runner_v10_refusal_ndjson() -``` - -The compact execution NDJSON contains a two-iteration root trace plus a -depth-one child trace. Its producer-stamped `llm_response`, `code`, successful -`output`, and `execution_error` events let consumers test iteration and depth -projection. -The successful root output deliberately starts with `ERROR:` so consumers must -use the event discriminant rather than interpreting stdout prose as status. - -The lifecycle NDJSON contains five contiguous runs whose per-run sequences -restart at one: ordinary success, successful and failed extract fallback, loud -output-limit failure, and cancellation. Together they cover unary and -atomic-batch subcall outcomes with stable call attribution, structured -execution error, repair and extract completion/failure, and exact -`result`/`usage`/`budget`/`policy`/`done` reconciliation. An output-limit -failure has no shortened `output` event: -Droste rejects oversized stdout rather than presenting clipped content as a -successful value. `stdout_chars` preserves the rejected size in the terminal -result. - -A runner protocol refusal occurs before admission, so it has no `run_id`, no -`RunRecord`, and no Trace ABI event sequence. Its fixture lives beside the -corpus so hosts test the boundary explicitly; an event relay must reject those -bytes. The GitHub release's `droste-relay-vX.Y.Z.tar.gz` exposes these same -files under `conformance/` for non-Python consumers. Consumers should pin the -release, read the fixture bytes, and use the released Python parser or Deno -filter rather than copying either schema. diff --git a/docs/verifiers-harness.md b/docs/verifiers-harness.md deleted file mode 100644 index 0996563..0000000 --- a/docs/verifiers-harness.md +++ /dev/null @@ -1,70 +0,0 @@ -# Verifiers v1 harness - -The optional `droste_verifiers` package exposes exactly one Verifiers v1 -`Harness`: `DrosteHarness`. Install it with the matching Droste release: - -```bash -uv add 'droste[verifiers]==' -``` - -The extra pins `verifiers==0.2.0`; its upstream dependency markers currently -make the harness available on Python 3.11–3.13. Core Droste supports Python -3.11–3.14. On 3.14 the optional harness is not installed and its tests skip -cleanly until upstream support lands. The harness itself creates a PEP 723 -program that pins the same exact Droste version inside the selected Verifiers -runtime. Runtime preparation remains Verifiers' content-addressed/cached -responsibility. - -A minimal Verifiers config is: - -```toml -num_tasks = 1 - -[taskset] -id = "gsm8k-v1" - -[harness] -id = "droste_verifiers" -droste_version = "" -depth = 0 -data_paths = ["."] -prompt_profile = "minimal" -token_budget = 20000 -max_subcalls = 0 -wall_ms = 60000 -root_output_tokens = 2048 -subcall_output_tokens = 1024 -``` - -`depth = 0` disables semantic subcalls and is useful for a zero-subcall control. -`depth = 1` enables Droste's current flat `llm_query`/batch calls up to -`max_subcalls`; recursive child RLM loops are not implied. `data_paths` are -runtime-local symbolic handles passed to the Droste CLI. Their contents are not -copied into the root prompt. - -Both root and subcall clients use the Verifiers interception endpoint and bearer -secret. The model ID and the canonical sampling object from `ModelContext` are -recorded in Droste's scaffold manifest. Optional `root_revision`, -`subcall_revision`, and `seed` should be resolved by the experiment controller. -Task system prompts are deterministically folded into the single task question, -matching the Verifiers base harness behavior. Verifiers MCP URLs are refused -until Droste can project them through its brokered capability ABI. - -The harness records `droste_iterations`, token/subcall counters, successful -subcalls, returned stdout characters, terminal readiness/extraction, and -configured depth as metrics. Task rewards remain taskset-owned. - -## Trace-branch limitation - -Droste emits content-free capability events with a unique call ID and root, -parent, and optional child run IDs. Verifiers v1's public Harness/interception -API records the intercepted root and subcall requests, but currently exposes no -supported field or callback through which a harness can attach Droste's call ID -as the canonical graph parent. Requests are therefore distinguishable as -intercepted message branches, while an exact Droste-call-to-Verifiers-node join -is not yet portable. - -The harness intentionally does not depend on an internal header or private -trace mutation method. Exact canonical branch correlation needs a supported -Verifiers parent/correlation seam; Droste's Trace ABI already preserves the -identity needed to use it when one exists. diff --git a/examples/embedding.py b/examples/embedding.py new file mode 100644 index 0000000..1fdc5fc --- /dev/null +++ b/examples/embedding.py @@ -0,0 +1,43 @@ +"""Minimal in-process Droste embedding using an OpenAI-compatible endpoint.""" + +from __future__ import annotations + +import os + +from droste import ( + Budget, + EnvironmentConfig, + OpenAICompatClient, + OpenAICompatSubcallClient, + RLMConfig, + create_environment, + create_environment_context, + run_rlm, +) + + +def ask(question: str, data: str, *, model: str) -> str: + config = EnvironmentConfig(kind="native", budget=Budget(subcalls=50, depth=1)) + context = create_environment_context(config) + root = OpenAICompatClient(model=model) + subcalls = OpenAICompatSubcallClient(model=model, context=context) + environment = create_environment( + config, + context=data, + registry=None, + subcalls=subcalls, + execution_context=context, + ) + result = run_rlm( + question, + environment=environment, + root_llm=root, + subcalls=subcalls, + config=RLMConfig(root_model=model), + context=context, + ) + return result.answer + + +if __name__ == "__main__": + print(ask("What happened?", "your source data", model=os.environ["DROSTE_MODEL"])) diff --git a/pyodide/README.md b/pyodide/README.md index 9d565a1..e59451c 100644 --- a/pyodide/README.md +++ b/pyodide/README.md @@ -1,106 +1,24 @@ -# Deno + Pyodide RLM substrate +# Deno and Pyodide substrate -The Deno + Pyodide substrate runs Droste in CPython on WASM with an offline -Pyodide runtime. It has two parts: +The bundled relay runs Droste in CPython on WASM. Deno owns network, process, +credential, and provider-file authority; generated Python receives only +brokered model and provider calls. -- The reusable relay and Python substrate included in the `droste` wheel. -- A host-owned adapter that supplies data sources and maps the host's request - and response types. +Relay sources ship in the wheel under `droste/substrates/_relay`. Stage the +files from the installed package so the engine and relay versions stay pinned: -The relay has no host-specific data logic. It loads the adapter module named by -the trusted process launcher and calls its fixed two-function contract. See -`examples/pyodide-host/pyodide_host_adapter.py` for a working adapter. - -The relay ships inside the wheel as package data under -`droste/substrates/_relay/`: `relay.ts`, -`event_channel.ts`, `stream.ts`, `broker.ts`, `events.ts`, `offline-probe.ts`, -and `deps.ts` (the single Pyodide version pin). Stage it in your build from the -installed package: - - relay_dir="$(droste relay-path)" - cp "$relay_dir"/*.ts / - -(Or `python -c 'from droste.substrates import relay_dir; print(relay_dir())'`.) -The relay and engine therefore come from the same pinned artifact. On an -admitted run, the relay prefixes -the first canonical event with a `startup` event -(`{engine_version, runner_protocol, provider_protocol}`). Preflight and -pre-admission refusal leave the event descriptor empty. - -In this repo the relay sources live at `src/droste/substrates/_relay/`; this -directory (`pyodide/`) keeps the substrate's tests, spikes, and this README. - -## The substrate (droste-general) - -Deno-side modules, all ModelRelay-aware but host-data-layer-agnostic: - -- `relay.ts` — the Deno process a host spawns. Reads a request JSON on stdin, - runs the RLM in Pyodide via a **host-supplied adapter module** (named on the - command line), writes a response JSON on stdout, and requires a dedicated - host-provided descriptor for Trace ABI NDJSON. Deno holds the only ambient - capabilities (narrow net to ModelRelay plus trusted provider files); the - Pyodide sandbox has none. Knows nothing about any host's data layer — see - "Writing a host adapter". -- `event_channel.ts` — the single fail-closed writer for that host event - descriptor. It does not define or transform event bodies. -- `broker.ts` — the A′-1 credential broker (see Security model). -- `events.ts` — the structured NDJSON event vocabulary + Pyodide-stderr - forwarding filter. -- `stream.ts` — reconstructs a complete `/responses` reply from a ModelRelay SSE stream. - -Python-side adapters, in **`droste.substrates.pyodide`** (part of the droste -package — installed by `pip install droste`, so a host that depends on droste -gets these for free, nothing to stage separately): - -- `RawExecutor` — a plain-interpreter executor that replaces droste's - `RestrictedExecutor`. The Deno/WASM jail *is* the sandbox here, so - RestrictedPython would be redundant; `RawExecutor` is also Pyodide-safe (no - signals, no threads). -- `EnvironmentConfig(kind="pyodide", ...)` + `create_environment(...)` — the - supported host-wiring path. It selects `RawExecutor`, shares one immutable - budget record with the loop execution context, rejects native signal - timeouts, and requires the adapter to declare that its host owns both WASM - isolation and the wall-clock deadline. The declarations are loud contract - checks, not substitutes for the actual Deno boundary. -- `BridgedLLMClient` — an `LLMClient` (droste's protocol) that calls ModelRelay - over a host-injected `host_fetch` callable instead of a real socket - (Pyodide has no sockets). Implements `responses_create`, - `get_model_context_window`, `get_model_max_output_tokens`. Under Pyodide the - host fetch is async; `_post` blocks the synchronous RLM loop on it via - `run_sync`. -- `HostFetch` — the callable type alias for that injected fetch, - `(method, url, headers_json, body) -> response_text`. -- `serialize_error` — makes `run_rlm`'s error *dataclass* JSON-serializable - for a host's stdio wire contract (otherwise `json.dumps` raises on the - dataclass and the host gets no output at all). Public API (no leading - underscore) because adapter modules in other packages call it across the - module boundary; it isn't a private helper anymore. - -These have zero third-party deps and zero knowledge of any host's product -wiring — which is exactly what lets them import under Pyodide and be reused. - -The A′-2 DB-service split's generic half — `droste.sources.bridge` -(`ProviderService`, `BridgeProvider`) — is likewise droste core; see -Security model. So is `droste.sources.sql_local` (`sqlite_provider()`, -a read-only SQLite provider), which the reference adapter uses — though note -its Pyodide timeout caveat under "Writing a host adapter" and Known gaps. - -**One honest caveat on reuse:** `BridgedLLMClient._post` and `broker.ts`'s -`isModelRelayResponsesCall` scoping are ModelRelay-specific today. A host -targeting a different LLM backend would need to extend the broker (the URL/auth -scoping) and the client itself. That's real future work adjacent to #5 -(MCP-as-transport), not solved here. +```bash +relay_dir="$(droste relay-path)" +cp "$relay_dir"/*.ts / +``` -## Writing a host adapter +The working host adapter is +[`examples/pyodide-host/pyodide_host_adapter.py`](../examples/pyodide-host/pyodide_host_adapter.py). -To embed droste under this substrate, write one Python module exposing exactly -two functions, stage it into the sources directory next to the `droste` -package, and pass its module name to `relay.ts`. That's the whole contract. -`examples/pyodide-host/pyodide_host_adapter.py` is a complete, runnable -implementation of everything below (~150 lines, thoroughly commented); read it -alongside this section. +## Host adapter contract -### The two-function contract +The trusted launcher selects an importable adapter module. Request JSON cannot +select code. The module exposes: ```python def build_db_service(db_path, contacts_db_path=None) -> tuple[ProviderService, dict]: @@ -112,70 +30,25 @@ def run_for_host_pyodide( ... ``` -Inside `run_for_host_pyodide`, build one frozen config and use it for both -pieces of loop wiring: - -```python -config = EnvironmentConfig( - kind="pyodide", - budget=Budget( - tokens=token_budget, - subcalls=subcall_budget, - depth=depth_budget, - wall_ms=wall_time_ms, - root_output_tokens=root_output_tokens, - subcall_output_tokens=subcall_output_tokens, - ), - sandbox=SandboxLimits(output_chars=max_output_chars, execution_timeout_ms=0), - host_managed_timeout=True, - host_managed_isolation=True, -) -execution_context = create_environment_context(config, on_event=emit_event) -environment = create_environment( - config, - context=data, - registry=registry, - subcalls=subcalls, - execution_context=execution_context, -) -``` - -Do not instantiate the native `RunnerEnvironment` in a Pyodide adapter. That -silently selects the wrong executor and leaves substrate assumptions scattered -through host code. +`build_db_service` runs in the trusted provider interpreter and returns one +owned service plus opaque JSON metadata. `run_for_host_pyodide` runs in the +generated-code interpreter. It must use `EnvironmentConfig(kind="pyodide")`, +declare `host_managed_timeout=True` and `host_managed_isolation=True`, and set +the Python execution timeout to zero. Those declarations verify host wiring; +the launcher must still enforce the WASM boundary and hard deadline. -- **`build_db_service(db_path, contacts_db_path=None)`** runs in the *trusted* - DB-service interpreter (the one that boots first and actually holds the DB - file). It builds a `droste.sources.bridge.ProviderService` wrapping one - already-bound source, and returns `(service, meta)`. `meta` is a plain dict of - whatever facts are only computable where the DB file is visible; it is - **opaque to `relay.ts`** (below). The reference adapter has nothing - host-specific to carry, so its `meta` is just `{}`; a production adapter - might carry things like a filesystem-probe result or a default call budget - computed from a `SELECT COUNT(*)`. -- **`run_for_host_pyodide(request, host_fetch, bridge_call, duplex_bridge_call=None, meta=None)`** - runs in the *untrusted* REPL interpreter. It wires a `BridgedLLMClient` (over - the injected `host_fetch`) and an environment into `droste.run_rlm`, and - returns a response dict. A database-backed adapter requires a callable - `bridge_call` and reaches the DB only through a - `BridgeProvider(bridge_call, duplex_call=duplex_bridge_call)` registration - with receiving-host effect policy — the DB never opens in this interpreter. - The relay supplies its bounded bridge-v2 pump as `duplex_bridge_call`; - custom hosts may omit it to retain unary provider-protocol-4 invocation. - A context-only adapter may accept `bridge_call=None`, but it must not use a - request path to open a provider inside the untrusted interpreter. `meta` is - the same opaque blob `build_db_service` returned, ferried back verbatim. +A database-backed adapter passes a verified provider through `BridgeProvider`. +The bundled relay supplies the bridge-v2 duplex pump for mid-call checkpoints +and cancellation acknowledgement. The generated-code interpreter never opens a +database or host path directly. -In two-interpreter mode, a trusted host may send `SIGUSR1` to request -cooperative cancellation of the active provider call. The relay keys that -request to the invocation's `call_id` before the first frame and the receiving broker decides -the next frame acknowledgement. `SIGTERM` and `SIGKILL` remain hard-stop paths; -a handler that never reaches `check()`/`checkpoint()` still requires the host's -ordinary hard timeout. +`host_fetch` is the only model-network seam. The bundled broker restricts it to +ModelRelay Responses endpoints and injects credentials host-side. Supporting a +different backend requires a host-specific broker/client implementation. -### How the adapter module is selected (and why it's safe) +## Process contract -`relay.ts` takes the adapter module name as its **second CLI argument**: +Launchers provide a dedicated writable event descriptor, fd3 by convention: ```bash DENO_EXTRA_STDIO_FDS=3 DROSTE_RELAY_EVENT_FD=3 \ @@ -183,235 +56,61 @@ DENO_EXTRA_STDIO_FDS=3 DROSTE_RELAY_EVENT_FD=3 \ relay.ts 3>events.ndjson ``` -The file redirect is only a shell demonstration. A host using pipes must drain -fd2 and the event descriptor concurrently while it waits for the unary response -and process exit. - -The host must open the named descriptor before launch; fd3 is the convention. -An external launcher must also include that descriptor in Deno's -`DENO_EXTRA_STDIO_FDS` startup marker. Deno consumes the marker before user -JavaScript starts and registers the inherited number for `node:fs`; merely -passing fd3 from a shell or Go `exec.Cmd.ExtraFiles` is not sufficient. Deno's -own `node:child_process` compatibility layer sets the marker automatically, -but production launchers must set it explicitly. - -The relay rejects fd0, fd1, fd2, malformed values, and descriptors it cannot -write. It never falls back to fd2. A missing marker makes the otherwise-open -descriptor unavailable to the relay and produces the same fail-closed result. -A missing or unavailable channel returns one unary response with `error.type = -"RelayEventChannelError"` when stdout remains available, writes only an -allowlisted reason code to fd2, and exits before Pyodide work begins. +The three output lanes are independent: -That transport failure body is relay-level and intentionally does not claim a -runner protocol or operation: descriptor validation occurs before the -host-selected adapter owns its response schema. Consumers branch on the closed -`RelayEventChannelError` type/code, not on adapter or runner fields. +| Lane | Content | +| --- | --- | +| fd1 | Exactly one adapter-owned JSON response line | +| configured descriptor | Trace ABI v10 NDJSON only | +| fd2 | Diagnostics only | -The process has three independent output lanes: +External launchers must include the event descriptor in +`DENO_EXTRA_STDIO_FDS`; inheriting fd3 at the OS level alone is insufficient. +Descriptors 0–2, missing markers, malformed values, and unwritable channels +fail before Pyodide work with `RelayEventChannelError`. Preflight and +pre-admission refusal intentionally emit no event frames. -| Descriptor | Contract | -|------------|----------| -| fd1 | Exactly one unary response JSON line. Adapter-owned responses use its HostResponse schema; pre-adapter event-channel failures use the relay-level error above. | -| configured event descriptor (fd3 by convention) | Canonical Trace ABI v7 NDJSON only. | -| fd2 | Diagnostics only; never parse or promote these bytes as events. | +Drain fd2 and the event descriptor concurrently. A hard stop may leave a valid +nonterminal prefix. Treat an incomplete final frame as a transport error and do +not fabricate `done`. -Drain fd2 and the event descriptor concurrently. A hard cancellation or -process failure may end fd3 after a valid nonterminal prefix; the transport -does not invent a `done` event. Reconcile terminal state from the unary response -when one exists, otherwise report a transport failure. A descriptor can also -fail after part of a frame was written; treat a final unterminated or invalid -line as a typed transport failure and never promote it to a Trace event. An -empty preflight/refusal stream intentionally performs no peer-liveness write; -peer or access-mode loss is detected fail-closed on the first admitted-run -frame. +## Security boundary -The name is validated against `^[A-Za-z_][A-Za-z0-9_.]*$`, then set as a Pyodide -Python global and resolved with `importlib.import_module` — never -string-interpolated into a Python source template. Crucially, the name comes -from `Deno.args` (the same trust class as ``: set by whatever spawns -the process — the control plane), **never from the request JSON on stdin** (the -data plane). This mirrors droste's native path, where `droste_runner.run()`'s -request-file entrypoint explicitly rejects an `adapter_module` in the request -body: code-selection is a trusted-caller decision, not something an LLM-facing -request may steer. +The provider interpreter owns corpus files and live provider runtimes. The Deno +broker owns model credentials. The generated-code interpreter has neither. Its +only external effects are the model and provider calls that those trusted +components admit. -The module just needs to be importable inside Pyodide, so `relay.ts` stages it -into the same sources directory as the `droste` package (mounted at `/app`, -which is on `sys.path`) — `importlib.import_module("your_adapter_module")` then -resolves it like any other top-level module. +`RawExecutor` intentionally runs ordinary Python because WASM is the isolation +boundary. `SandboxLimits` still bounds captured output, while the host owns the +wall-clock kill. Under Pyodide, SQLite's thread-based per-query timer is not +available; host termination is the final timeout. -### The opaque `meta` blob +## Tests -`meta` is the adapter's private channel between its two halves, which run in two -*different interpreters*. `relay.ts` never looks inside it: whatever -`build_db_service` returns is `json.dumps`'d in the trusted interpreter, carried -across the boundary, `json.loads`'d, and handed back to -`run_for_host_pyodide(..., meta=...)` completely unexamined. Only your adapter -(present in both interpreters) knows or cares what's in it. Put anything -JSON-serializable there that the trusted side computes and the untrusted side -needs. (An earlier version of `relay.ts` hardcoded one host's field names into -its own Python template; making `meta` opaque is what removed the last -host-specific knowledge from the relay.) - -### `sql_local` timeouts - -Pyodide has no OS threads, so `LocalSqlRuntime.query()` cannot enforce its -`threading.Timer` timeout. It emits one warning and continues. The host must -enforce the wall-clock deadline and terminate the process when needed. - -## Configuration flags - -`relay.ts` reads three independent Deno environment variables at process start. -(These are `relay.ts`'s flags; a different host would define its own.) - -| Flag | Default | Set it to... | ...to get | -|------|---------|---------------|-----------| -| `DENO_EXTRA_STDIO_FDS` | none (required for external launchers) | A comma-separated list that includes the event descriptor (`3` conventionally). | Registers extra inherited numeric stdio with Deno before relay code starts; Deno consumes this marker. | -| `DROSTE_RELAY_EVENT_FD` | none (required) | A decimal inherited descriptor of 3 or greater; fd3 is conventional. | The sole canonical Trace ABI NDJSON output lane. Missing, malformed, or unwritable descriptors fail closed. | -| `RLM_BRIDGE` | (unset) | `legacy` | Pre-A′-1 behavior: the ModelRelay credential is a visible global inside the untrusted REPL interpreter, which assembles its own auth header. Kill switch only — the split is one `host_fetch` call site catching a real design mistake, not a feature. | -| `RLM_STREAM` | on | `0` | Legacy unary ModelRelay call (no SSE), for when NDJSON streaming from `/responses` is suspected of causing an issue. | - -## Security model - -Two orthogonal splits between untrusted (LLM-generated code) and trusted -(host-controlled) territory. The mechanisms live in the substrate; `relay.ts` -wires them and the host adapter fills in the host-specific pieces. - -**A′-1 — credential broker (`broker.ts`).** The untrusted REPL interpreter -never holds the ModelRelay API key or customer token. `splitCredentials` -strips them from the request before it becomes a sandbox global; the broker -holds them host-side and `stripAndInjectAuth` overwrites whatever auth header -the sandbox's `host_fetch` call tried to set — scoped to the exact -`POST https://api.modelrelay.ai/api/v1/responses(/batch)` calls -(`isModelRelayResponsesCall`), never a general-purpose credential. Proven by -`broker_test.ts` (pure unit tests) and `broker_integration_test.ts` (real -Pyodide interpreter, a scripted *sync* `host_fetch`). The async branch — -`relay.ts`'s real `host_fetch` is `async`, so `BridgedLLMClient._post` must -`run_sync` the awaitable — isn't exercised by a dedicated in-repo broker test -anymore (that was `broker_batch_integration_test.ts`, which moved out to the -host repo it depended on); -it's still covered for real by `examples/pyodide-host/e2e_test.ts`, which -spawns the actual `relay.ts` subprocess with its actual async `host_fetch`. - -**A′-2 — provider-service split (`droste.sources.bridge`, wired by `relay.ts` + the -host adapter's `build_db_service`).** The untrusted REPL interpreter never holds -the corpus DB either. A second, trusted Pyodide interpreter boots first, holds -a bound provider behind a `ProviderService` (a fixed operation allowlist from -the verified manifest — never a generic `getattr` on caller-controlled strings), -and the REPL interpreter reaches it only through a `BridgeProvider` registration. -`ProviderService` and `BridgeProvider` are droste-general -(`droste.sources.bridge`); the adapter's `build_db_service` is the host-specific -step that puts a real data source behind the service. -`bridge_source_integration_test.ts` proves the generic wire contract. -`filesystem_provider_integration_test.ts` mounts real SQLite and -`filesystem_text` sources in the trusted interpreter and proves the separate -REPL interpreter can read the configured text root only through generated -broker bindings; the root path and provider runtime are non-ambient there. And -`examples/pyodide-host/e2e_test.ts` proves the full path end to end against a -real SQLite file (see Testing). - -The sandbox that runs LLM-generated code has no filesystem access to the -corpus, no network of its own (Pyodide has no sockets), and no live credential. -Its only channels to the outside world are the brokered provider call and the -brokered ModelRelay call, both host-mediated and host-scoped. Database paths -are consumed by the trusted interpreter and removed before the sandbox request -is constructed. - -## Testing - -Deno suite (the substrate's own tests, all in this directory): +Run the substrate suite: ```bash -cd pyodide -deno test --allow-read --allow-env --allow-ffi . +deno test --allow-read --allow-env --allow-ffi pyodide/ ``` -First run on a fresh machine also needs `--allow-net=cdn.jsdelivr.net` plus -`--allow-write` pointed at your Deno cache dir (`deno info` prints it): the -npm pyodide package unvendors stdlib `sqlite3`, so `loadPackage("sqlite3")` -fetches the wheel from the CDN once and writes it alongside the pyodide npm -install (neither flag needed thereafter — CI warms its cache the same way). - -droste's end-to-end proof of the whole relay + adapter path (in `examples/`, -needs `--allow-run` to spawn the relay subprocess, `--allow-write` for its temp -sources/DB dirs, and loopback net for its mock ModelRelay server): +On a cold cache, allow network access to `cdn.jsdelivr.net` and writes to the +Deno cache so Pyodide can fetch its SQLite wheel. Run the complete relay and +adapter path with: ```bash -deno test --allow-run --allow-read --allow-write --allow-net=127.0.0.1 --allow-env \ - examples/pyodide-host/e2e_test.ts +deno test --allow-run --allow-read --allow-write \ + --allow-net=127.0.0.1 --allow-env examples/pyodide-host/e2e_test.ts ``` -Python tests (from the repo root, auto-discovered — no special config): - -```bash -uv run pytest -``` - -Roughly three tiers of coverage: - -- **Pure / unit** (no Pyodide, no network): `broker_test.ts`, `events_test.ts`, - `event_channel_test.ts`, `stream_test.ts` (Deno); - `tests/test_pyodide_error_serialization.py` and - `tests/test_sql_local.py` (Python) — the latter includes the #8 - regressions proving an explicit `timeout_ms: 0` survives (isn't defaulted back - to 5000) and that a query then runs under it. `examples/pyodide-host/` - `test_pyodide_host_adapter.py` is a fast *native* (no-Pyodide) sanity check of - the reference adapter's brokered provider path, rejection of direct database - access, and error serialization on a root LLM failure. -- **Real-Pyodide, substrate-only** (no host adapter, no host data layer): - `broker_integration_test.ts`, `bridge_source_integration_test.ts`, and - `filesystem_provider_integration_test.ts` load one or two - real Pyodide interpreters with a scripted `host_fetch` and prove the broker / - bridge wire contracts directly. The filesystem test uses the real first-party - provider and proves its configured root is absent from the generated-code - interpreter. A few seconds each. -- **Real-Pyodide against the real `relay.ts`** (droste's own E2E): - `examples/pyodide-host/e2e_test.ts` spawns the actual `relay.ts` as a Deno - subprocess — real process boundary, real Pyodide interpreter, real dynamic - `importlib.import_module` of `pyodide_host_adapter`, real stdin/stdout - request/response contract — against a local mock HTTP server standing in for - ModelRelay. It builds a real temp SQLite fixture, asserts a real brokered - `query()` round-trip (`SELECT COUNT(*)` → answer `"3"`), and proves a sibling - file in the database directory is not visible to generated code. **Zero real - network and zero sibling checkout**, so it runs unconditionally in CI — no - skip. This replaces - two former host-coupled tests (`db_service_integration_test.ts`, - `broker_batch_integration_test.ts`), which required a sibling host-repo - checkout and moved to that host's repo alongside its own adapter. - -## Known gaps +The end-to-end test spawns the real relay, imports the real adapter, executes in +real Pyodide, queries a temporary SQLite database through the broker, and uses a +loopback model server. Python-side substrate tests run with `uv run pytest`. -- **Custom hosts that select only the unary provider bridge cannot deliver a - new soft-cancellation request during a synchronous remote call.** Provider protocol 4 carries the - cancellation snapshot and remaining deadline at dispatch, and returns a - validated cumulative checkpoint at completion. A remote handler can poll - `CapabilityExecutionContext.check()` and enforce that dispatched deadline, - but the current request/response bridge has no reverse channel for a - cancellation requested afterward and cannot stream mid-call checkpoints. - The bundled two-interpreter Deno relay explicitly selects bridge v2 to add - that channel. Custom unary-only hosts still require a wall-clock timeout and - process termination as their hard Pyodide boundary. +## Limitations -- **`sql_local.py`'s per-query `timeout_ms` is unenforced under Pyodide (#8, - fixed to degrade).** The timeout is a `threading.Timer`; where thread - creation is unavailable the source now skips the timer with a - `RuntimeWarning` instead of crashing on the first query, so the default - policy works. But no in-interpreter timeout exists in that substrate — the - host's wall-clock kill is the only enforcement (see "Writing a host - adapter", Gotcha 2). -- **Typed server-side batch is a host extension today (#21).** droste's - `LLMClient` protocol deliberately has no batch method (#6 removed the stale, - never-called `batch_responses`): the loop parallelizes via - `subcalls.llm_batch`, and how that is transported is each client's own - implementation. The typed `batch_responses_typed(...) -> BatchResponse` - contract against a real server-side batch endpoint lives on host adapters' - own client subclasses; #21 designs its first-class home (one wire call on - batch-capable gateway clients — no fallbacks, no capability sniffing in the - loop). -- **Extract-fallback failure rate is unknown.** When `max_iterations` is - exhausted without `answer["ready"]`, one more LLM call tries to synthesize a - best-effort answer; a failure there now surfaces as a structured - `extract_error` (result field + the Trace ABI v7 `extract` failure event) instead of - silently falling back to raw loop output, but there's no data yet on how - often that call actually fails or why. No retry has been added — that's a - decision for once real failure data exists, not before. +- Unary-only custom provider bridges cannot receive a new soft-cancellation + request during a synchronous remote call; the host still needs a hard + timeout. +- SQLite per-query timers are unavailable under Pyodide. +- The bundled model broker is ModelRelay-specific. diff --git a/src/droste/execution/trace.py b/src/droste/execution/trace.py index 53f42ae..1869d40 100644 --- a/src/droste/execution/trace.py +++ b/src/droste/execution/trace.py @@ -60,8 +60,9 @@ class PersistenceClass(str, Enum): EventFieldType = type | tuple[type, ...] EventBodySchema = tuple[Mapping[str, EventFieldType], Mapping[str, EventFieldType]] -# One exhaustive v7 table. The first mapping is required fields; the second is -# optional fields. Nested broker/result values keep their own schema authority. +# One exhaustive table for the current ABI. The first mapping is required +# fields; the second is optional fields. Nested broker/result values keep their +# own schema authority. EVENT_BODY_SCHEMAS: Mapping[str, EventBodySchema] = MappingProxyType( { "startup": ( @@ -207,11 +208,13 @@ def _matches_field_type(value: Any, expected: EventFieldType) -> bool: def validate_event_body(event_type: str, body: Mapping[str, Any]) -> None: - """Validate one body against the exhaustive Trace ABI v7 table.""" + """Validate one body against the exhaustive current Trace ABI table.""" try: required, optional = EVENT_BODY_SCHEMAS[event_type] except KeyError as exc: - raise ValueError(f"event type {event_type!r} has no v7 body schema") from exc + raise ValueError( + f"event type {event_type!r} has no Trace ABI v{TRACE_ABI_VERSION} body schema" + ) from exc missing = required.keys() - body.keys() if missing: raise ValueError(f"event {event_type!r} missing body fields: " + ", ".join(sorted(missing))) @@ -254,9 +257,11 @@ def _validate_structured_body(event_type: str, body: Mapping[str, Any]) -> None: if event_type == "subcall": phase = body["phase"] if phase not in {"start", "progress", "completion", "failure"}: - raise ValueError("subcall phase is not recognized by Trace ABI v7") + raise ValueError(f"subcall phase is not recognized by Trace ABI v{TRACE_ABI_VERSION}") if body["operation"] not in {"llm_query", "llm_batch", "llm_batch_with_errors"}: - raise ValueError("subcall operation is not recognized by Trace ABI v7") + raise ValueError( + f"subcall operation is not recognized by Trace ABI v{TRACE_ABI_VERSION}" + ) if not body["call_id"]: raise ValueError("subcall call_id must not be empty") if body["iteration"] < 1: @@ -315,9 +320,9 @@ def _validate_structured_body(event_type: str, body: Mapping[str, Any]) -> None: raise ValueError("subcall batch_count must be non-negative") elif event_type == "repair": if body["phase"] not in {"start", "completion", "failure"}: - raise ValueError("repair phase is not recognized by Trace ABI v7") + raise ValueError(f"repair phase is not recognized by Trace ABI v{TRACE_ABI_VERSION}") if body["kind"] not in {"missing_code", "execution_error", "terminal"}: - raise ValueError("repair kind is not recognized by Trace ABI v7") + raise ValueError(f"repair kind is not recognized by Trace ABI v{TRACE_ABI_VERSION}") if body["iteration"] < 1: raise ValueError("repair iteration must be positive") error = body.get("error") @@ -333,7 +338,7 @@ def _validate_structured_body(event_type: str, body: Mapping[str, Any]) -> None: raise ValueError("repair start/completion cannot carry error") elif event_type == "extract": if body["phase"] not in {"start", "completion", "failure"}: - raise ValueError("extract phase is not recognized by Trace ABI v7") + raise ValueError(f"extract phase is not recognized by Trace ABI v{TRACE_ABI_VERSION}") if body["iteration"] < 1: raise ValueError("extract iteration must be positive") extract_error = body.get("extract_error") @@ -438,7 +443,9 @@ def _validate_structured_body(event_type: str, body: Mapping[str, Any]) -> None: if not required <= body.keys(): raise ValueError("budget mutation requires action, resource, and amount") if body["action"] not in {"reserve", "commit", "refund", "exhaust"}: - raise ValueError("budget mutation action is not recognized by Trace ABI v7") + raise ValueError( + f"budget mutation action is not recognized by Trace ABI v{TRACE_ABI_VERSION}" + ) if body["amount"] < 0: raise ValueError("budget mutation amount must be non-negative") else: @@ -498,10 +505,10 @@ def _validate_structured_body(event_type: str, body: Mapping[str, Any]) -> None: ScaffoldManifest.from_dict(scaffold_manifest) elif event_type == "policy": if body["outcome"] not in {"passed", "violated", "not_evaluated", "not_enforced"}: - raise ValueError("policy outcome is not recognized by Trace ABI v7") + raise ValueError(f"policy outcome is not recognized by Trace ABI v{TRACE_ABI_VERSION}") elif event_type == "done": if body["status"] not in {"success", "error", "cancelled"}: - raise ValueError("done status is not recognized by Trace ABI v7") + raise ValueError(f"done status is not recognized by Trace ABI v{TRACE_ABI_VERSION}") if body["iterations"] < 0: raise ValueError("done iterations must be non-negative") stdout_chars = body.get("stdout_chars") @@ -652,7 +659,7 @@ def as_dict(self) -> dict[str, Any]: def parse_event(value: RunEvent | Mapping[str, Any]) -> RunEvent: - """The one strict parser for Trace ABI v7 event values.""" + """The one strict parser for current Trace ABI event values.""" if isinstance(value, RunEvent): return value required = { diff --git a/src/droste/testing/__init__.py b/src/droste/testing/__init__.py index b845644..3cd2367 100644 --- a/src/droste/testing/__init__.py +++ b/src/droste/testing/__init__.py @@ -39,13 +39,13 @@ def conformance_fixture_names() -> tuple[str, ...]: def trace_v10_lifecycle_ndjson() -> bytes: - """Return the shared Trace ABI v7 lifecycle conformance corpus.""" + """Return the shared Trace ABI v10 lifecycle conformance corpus.""" return files(__package__).joinpath("fixtures/trace-v10-lifecycle.ndjson").read_bytes() def trace_v10_execution_ndjson() -> bytes: - """Return the shared Trace ABI v7 response/code/output/error conformance corpus.""" + """Return the shared Trace ABI v10 response/code/output/error conformance corpus.""" return files(__package__).joinpath("fixtures/trace-v10-execution.ndjson").read_bytes() diff --git a/tests/test_docs.py b/tests/test_docs.py new file mode 100644 index 0000000..86a68b4 --- /dev/null +++ b/tests/test_docs.py @@ -0,0 +1,124 @@ +from __future__ import annotations + +import re +import runpy +import unicodedata +from pathlib import Path +from urllib.parse import unquote + +from droste.execution.manifest import SCAFFOLD_MANIFEST_VERSION +from droste.execution.trace import TRACE_ABI_VERSION +from droste.prompts.pack import PROMPT_PACK_SCHEMA_VERSION +from droste.providers import PROVIDER_PROTOCOL_VERSION +from droste_runner.protocol import RUNNER_PROTOCOL_VERSION + +REPO = Path(__file__).resolve().parents[1] +DOCS = REPO / "docs" +LINK = re.compile(r"!?\[[^]]*\]\(([^)]+)\)") +HEADING = re.compile(r"^#{1,6}\s+(.+?)\s*$") + + +def _maintained_markdown() -> tuple[Path, ...]: + roots = ( + REPO / "README.md", + REPO / "CONTRIBUTING.md", + REPO / "UPGRADING.md", + REPO / "benchmarks" / "README.md", + REPO / "integrations" / "README.md", + REPO / "pyodide" / "README.md", + ) + docs = tuple(path for path in DOCS.rglob("*.md") if "assets" not in path.parts) + return tuple(sorted((*roots, *docs))) + + +def _destination(raw: str) -> str: + value = raw.strip() + if value.startswith("<") and ">" in value: + return value[1 : value.index(">")] + return value.split(maxsplit=1)[0] + + +def _slug(value: str) -> str: + value = re.sub(r"<[^>]+>", "", value) + value = re.sub(r"[`*_~]", "", value) + value = unicodedata.normalize("NFKD", value).casefold() + value = "".join(character for character in value if character.isalnum() or character in " -_") + return re.sub(r"[ _]+", "-", value).strip("-") + + +def _anchors(path: Path) -> set[str]: + anchors: set[str] = set() + counts: dict[str, int] = {} + fenced = False + for line in path.read_text(encoding="utf-8").splitlines(): + if line.startswith("```"): + fenced = not fenced + continue + match = None if fenced else HEADING.match(line) + if match is None: + continue + base = _slug(match.group(1)) + count = counts.get(base, 0) + counts[base] = count + 1 + anchors.add(base if count == 0 else f"{base}-{count}") + return anchors + + +def _local_links(path: Path) -> tuple[tuple[Path, str | None], ...]: + links: list[tuple[Path, str | None]] = [] + for raw in LINK.findall(path.read_text(encoding="utf-8")): + destination = unquote(_destination(raw)) + if destination.startswith(("http://", "https://", "mailto:")): + continue + target_name, separator, anchor = destination.partition("#") + target = path if not target_name else (path.parent / target_name).resolve() + links.append((target, anchor if separator else None)) + return tuple(links) + + +def test_documented_contract_versions_match_runtime_constants() -> None: + headings = { + DOCS / "providers.md": f"# Providers (protocol v{PROVIDER_PROTOCOL_VERSION})", + DOCS / "reference" / "runner.md": f"# Runner protocol v{RUNNER_PROTOCOL_VERSION}", + DOCS / "reference" / "trace.md": f"# Trace ABI v{TRACE_ABI_VERSION}", + DOCS + / "reference" + / "prompt-packs.md": f"# Prompt packs (schema v{PROMPT_PACK_SCHEMA_VERSION})", + DOCS / "reference" / "scaffold.md": f"# Scaffold manifest v{SCAFFOLD_MANIFEST_VERSION}", + } + for path, expected in headings.items(): + assert path.read_text(encoding="utf-8").splitlines()[0] == expected + + +def test_maintained_markdown_links_and_anchors_resolve() -> None: + failures: list[str] = [] + for source in _maintained_markdown(): + for target, anchor in _local_links(source): + if not target.exists(): + failures.append(f"{source.relative_to(REPO)} -> missing {target}") + continue + if anchor and target.suffix.casefold() == ".md" and anchor not in _anchors(target): + failures.append( + f"{source.relative_to(REPO)} -> missing #{anchor} in {target.relative_to(REPO)}" + ) + assert not failures, "\n".join(failures) + + +def test_every_public_doc_is_reachable_from_the_index() -> None: + public = set(DOCS.glob("*.md")) | set((DOCS / "reference").glob("*.md")) + reachable: set[Path] = set() + pending = [DOCS / "README.md"] + while pending: + current = pending.pop() + if current in reachable: + continue + reachable.add(current) + for target, _ in _local_links(current): + if target in public and target not in reachable: + pending.append(target) + assert public <= reachable, sorted(str(path.relative_to(REPO)) for path in public - reachable) + + +def test_embedding_example_imports_without_dispatch() -> None: + namespace = runpy.run_path(str(REPO / "examples" / "embedding.py"), run_name="docs_test") + assert callable(namespace["ask"])