diff --git a/capabilities/computation/src/capability/computation-function.ts b/capabilities/computation/src/capability/computation-function.ts index 38c302100..2df4f995c 100644 --- a/capabilities/computation/src/capability/computation-function.ts +++ b/capabilities/computation/src/capability/computation-function.ts @@ -2,14 +2,14 @@ import { defineCapability, functionCategoryLabels, functionResourceLabels, - inWords, + outputInWords, type DefinitionSummary, type Capability, } from '@beonauto/definitions'; import { issueText } from '@beonauto/definitions/document'; -import { asSentence, InvalidInput } from '@beonauto/operations'; +import { InvalidInput } from '@beonauto/operations'; import type { ProgramPool } from '@beonauto/workflow-engine/dsl'; -import { Effect, Result, type Schema } from 'effect'; +import { Effect, Result } from 'effect'; import type { ComputationFunctionDefinitionDocument } from '../document/computation-document.ts'; import { parseComputationDocument } from '../document/document-parsing.ts'; @@ -42,13 +42,6 @@ function summarize({ description, input, output }: ComputationFunctionDefinition }; } -function describeResult(output: Schema.Json): string { - const words = inWords(output); - return words === undefined - ? 'Its result is too long to repeat here; the whole of it is in the details below.' - : asSentence(`Its result: ${words}`); -} - export function makeComputationFunctionAdapter({ pool, deadlineMs = computationBounds.deadlineMs, @@ -59,7 +52,7 @@ export function makeComputationFunctionAdapter({ title: functionCategoryLabels.computation, guide: { name: 'computation-function' }, noun: { one: functionResourceLabels.computation.singular, other: functionResourceLabels.computation.plural }, - describeOutput: describeResult, + describeOutput: outputInWords('result'), mediaType: 'text/markdown', parse, summarize, diff --git a/capabilities/coordination/src/calls/function-run.ts b/capabilities/coordination/src/calls/function-run.ts index 73d3a146e..367bd41d8 100644 --- a/capabilities/coordination/src/calls/function-run.ts +++ b/capabilities/coordination/src/calls/function-run.ts @@ -5,7 +5,7 @@ import type { Lineage, RejectionKind, RejectionReason, - UnavailableBecause, + RejectionBecause, } from '@beonauto/operations'; import type { Effect, Schema } from 'effect'; @@ -31,7 +31,7 @@ export type EndedRunResult = readonly detail: string; readonly issues?: readonly Issue[]; readonly kind?: RejectionKind; - readonly because?: UnavailableBecause; + readonly because?: RejectionBecause; } | { readonly status: 'failed'; readonly detail: string }; diff --git a/capabilities/coordination/src/capability/workflow.ts b/capabilities/coordination/src/capability/workflow.ts index 7e48361b9..565a3ccc8 100644 --- a/capabilities/coordination/src/capability/workflow.ts +++ b/capabilities/coordination/src/capability/workflow.ts @@ -1,5 +1,11 @@ -import { defineCapability, inWords, type RunContext, type FinishesLater, type Capability } from '@beonauto/definitions'; -import { asSentence, Conflict, type InvalidInput, type Unavailable } from '@beonauto/operations'; +import { + defineCapability, + outputInWords, + type RunContext, + type FinishesLater, + type Capability, +} from '@beonauto/definitions'; +import { Conflict, type InvalidInput, type Unavailable } from '@beonauto/operations'; import type { StartAnswer, WorkflowHost } from '@beonauto/workflow-host'; import { Effect, Random, type Schema } from 'effect'; @@ -22,13 +28,6 @@ const notNow = 'The workflow cannot start now; try again shortly'; const mostSeed = 2_147_483_647; -function describeResult(output: Schema.Json): string { - const words = inWords(output); - return words === undefined - ? 'Its result is too long to repeat here; the whole of it is in the details below.' - : asSentence(`Its result: ${words}`); -} - function finishedLaterOr(answer: StartAnswer): Effect.Effect { return answer === 'settled' ? Effect.fail(new Conflict({ detail: ranBefore, kind: 'unworkable' })) @@ -68,7 +67,7 @@ export function makeWorkflowAdapter(dependencies: WorkflowAdapterDependencies): title: 'Workflow', guide: { name: 'workflow' }, noun: { one: 'workflow', other: 'workflows' }, - describeOutput: describeResult, + describeOutput: outputInWords('result'), mediaType: 'application/yaml', parse: (source: string): Effect.Effect => parseWorkflowDocument(source, dependencies.mostDurationMs), diff --git a/capabilities/coordination/src/presenting/run-presenter.test.ts b/capabilities/coordination/src/presenting/run-presenter.test.ts index 8c0fe59d5..618f7b2c6 100644 --- a/capabilities/coordination/src/presenting/run-presenter.test.ts +++ b/capabilities/coordination/src/presenting/run-presenter.test.ts @@ -135,8 +135,9 @@ describe('the words of an input a workflow took', () => { }); }); -function rejectedWith(kind: string, because: string): InputReceipt { - return { kind: 'call_answered', key: callKeyText(callKey), at, status: 'rejected', rejection: { kind, because } }; +function rejectedWith(kind: string, because?: string): InputReceipt { + const rejection = because === undefined ? { kind } : { kind, because }; + return { kind: 'call_answered', key: callKeyText(callKey), at, status: 'rejected', rejection }; } describe('the words of a rejection of a function a workflow called', () => { @@ -144,7 +145,7 @@ describe('the words of a rejection of a function a workflow called', () => { expect( present(recordOf(eventOf(rejectedWith('tools_unfinished', 'server_failed'), stepsOf(1), [settled])))?.summary, ).toBe( - 'A function the workflow called did not succeed, and 1 step moved; the workflow ended. It called tools but could not finish, because a tool server kept failing.', + 'A function the workflow called did not succeed, and 1 step moved; the workflow ended. It called tools but could not finish, because a tool server failed.', ); }); @@ -158,9 +159,13 @@ describe('the words of a rejection of a function a workflow called', () => { 'A function the workflow called did not succeed. A tool server it needs could not be used, because the tool server asked it to slow down for longer than a run waits.', ], [ - rejectedWith('tools_called', 'no_answer'), + rejectedWith('tools_called'), 'A function the workflow called did not succeed. This run calls tools, and an attempt of it under the same id may still be in progress or did not succeed, so its tools may have changed something.', ], + [ + rejectedWith('effect_unknown', 'server_failed'), + 'A function the workflow called did not succeed. It could not finish after calling a tool that may change something, so whether that happened is not known, because a tool server failed.', + ], [rejectedWith('something_new', 'some_reason'), 'A function the workflow called did not succeed.'], ])('says each kind and because it knows: %#', (receipt, summary) => { expect(present(recordOf(eventOf(receipt)))?.summary).toBe(summary); diff --git a/capabilities/coordination/src/presenting/run-words.ts b/capabilities/coordination/src/presenting/run-words.ts index 4c1973722..aaeb40dd5 100644 --- a/capabilities/coordination/src/presenting/run-words.ts +++ b/capabilities/coordination/src/presenting/run-words.ts @@ -2,7 +2,7 @@ import { CancelledKindSchema, ConflictKindSchema, UnansweredKindSchema, - UnavailableBecauseSchema, + RejectionBecauseSchema, UnavailableKindSchema, asSentence, capitalized, @@ -43,7 +43,7 @@ const decodeCancelledKind = Schema.decodeUnknownOption(CancelledKindSchema); const decodeUnansweredKind = Schema.decodeUnknownOption(UnansweredKindSchema); -const decodeBecause = Schema.decodeUnknownOption(UnavailableBecauseSchema); +const decodeBecause = Schema.decodeUnknownOption(RejectionBecauseSchema); type Rejection = NonNullable['rejection']>; @@ -56,7 +56,7 @@ function whyNot({ kind, because }: Rejection): string { } const conflict = Option.getOrUndefined(decodeConflictKind(kind)); if (conflict !== undefined) { - return explanationOf({ reason: 'conflict', kind: conflict }).why; + return explanationOf({ reason: 'conflict', kind: conflict, ...withBecause }).why; } const cancelled = Option.getOrUndefined(decodeCancelledKind(kind)); if (cancelled !== undefined) { diff --git a/capabilities/interaction/README.md b/capabilities/interaction/README.md index 27f20dfa0..8ef57f1a6 100644 --- a/capabilities/interaction/README.md +++ b/capabilities/interaction/README.md @@ -1,6 +1,6 @@ # @beonauto/interaction -Interaction functions: a brain asks a person or a system and takes the answer later. An interaction function is a definition, saved and versioned as every function is; its run renders a request from its input, sends it through the tool of a tool server its document names or leaves it in the brain's inbox, and waits, with nothing in memory, until the request is answered, expires or is cancelled. A function that sends through a tool may also read the replies to its message and take its party's reply as the answer. The answer is the run's output, checked against the answer schema the definition gives. [Decision 0010](../../docs/decisions/0010-interaction-functions.md) records why it is built this way, with its amendment on a function that owns its delivery, [decision 0019](../../docs/decisions/0019-the-answer-shape-of-an-open-request.md) why the listing of open requests shows the answer schema each request recorded, and [decision 0020](../../docs/decisions/0020-replies-through-a-channel.md) how replies are read and taken. +Interaction functions: a brain asks a system and answers at once, or asks a person and takes the answer later. An interaction function is a definition, saved and versioned as every function is. One that asks a system names one tool of a tool server in `call`; its run makes one recorded call and answers with the part of the tool's answer that `read` points at. One that asks a person renders a request from its input, sends it through the tool of a tool server its document names or leaves it in the brain's inbox, and waits, with nothing in memory, until the request is answered, expires or is cancelled. A function that sends through a tool may also read the replies to its message and take its party's reply as the answer. The answer is the run's output, checked against the answer schema the definition gives. [Decision 0010](../../docs/decisions/0010-interaction-functions.md) records why it is built this way, with its amendment on a function that owns its delivery, [decision 0019](../../docs/decisions/0019-the-answer-shape-of-an-open-request.md) why the listing of open requests shows the answer schema each request recorded, [decision 0020](../../docs/decisions/0020-replies-through-a-channel.md) how replies are read and taken, and [decision 0021](../../docs/decisions/0021-asking-a-system.md) why a function may ask a system and answer at once, and the one rule for a run that called tools and could not finish. ## The document @@ -45,15 +45,46 @@ output: Here is the draft: {{ input.draft }} ``` -`parseInteractionDocument(source)` reads it with the shared document reader of `@beonauto/definitions/document` (`src/document`). The front matter takes `description`, `to`, a Liquid template that names the party, `from`, one that names the answerer, `expires`, an ISO 8601 duration from one minute to thirty days read by `readDuration` of `@beonauto/workflow-engine/dsl`, `deliver` and `replies`, `input` and `output`, each with a `schema`, and `reply`, the reply rule. Every other key, `model` and `tools` among them, is refused with its line, within the nested blocks too. Without `output.schema` the function is a notification: it takes no answer, and its run succeeds, with the output `{}`, once its delivery lands, at once without `deliver`. The body is the message, a Liquid template. `to`, `from` and the message read `input`, `today` and `now` alone, through an instance of the engine of `@beonauto/definitions/template` with no registrations of its own, and a template that does not compile, reads another name, or a property a closed input schema does not have, is refused with its line. +A function that asks a system names the tool in `call` and has no body: -`src/route` holds the two blocks, written once as effect schemas, `ToolDeliverySchema` and `RepliesSchema`, and `compiledRoute(written, lines, inputSchema)`, which checks them at save: the server and tool names by their shapes, `isServerName` and `isToolName` of `@beonauto/mcp`; every template of `deliver.with`, `replies.conversation`, `replies.with` and `tell.with` against the names its place reads, with no `${`; each pointer of `deliver.sent` and `replies.read` a JSON Pointer that starts with a slash, refused in one sentence; `read.order`; `wait` from `PT5S` to `PT1H`. `replies` needs `deliver`, its `sent` and `output.schema`. Arguments are typed as written (`renderedArguments`): a number, boolean, null, list or object is sent as written with the templates in its strings rendered, a string that is one `{{ expression }}` alone is sent as the value it reads, and any other string is rendered as text, so a structure inside a longer text is refused at save. `routeOf(blocks)` answers what an attempt, a read or a telling works with, `{ kind: 'inbox' }` or `{ kind: 'tool', delivery, replies? }`, and `throughWords` says it, the same sentence the definitions package's `deliveryStarted` says. +```markdown +--- +description: Read the replies of a thread in the team chat, oldest first +call: + server: chat + tool: thread_replies + with: + channel: '{{ input.channel }}' + ts: '{{ input.thread }}' + read: /messages +input: + schema: + { type: object, required: [channel, thread], properties: { channel: { type: string }, thread: { type: string } } } +output: + schema: + type: array + items: + { + type: object, + required: [user, ts], + properties: { user: { type: string }, text: { type: string }, ts: { type: string } }, + } +--- +``` + +`parseInteractionDocument(source)` reads it with the shared document reader of `@beonauto/definitions/document` (`src/document`). The front matter takes `description`, `to`, a Liquid template that names the party, `from`, one that names the answerer, `expires`, an ISO 8601 duration from one minute to thirty days read by `readDuration` of `@beonauto/workflow-engine/dsl`, `deliver` and `replies`, `input` and `output`, each with a `schema`, and `reply`, the reply rule. Every other key, `model` and `tools` among them, is refused with its line, within the nested blocks too. Without `output.schema` the function is a notification: it takes no answer, and its run succeeds, with the output `{}`, once its delivery lands, at once without `deliver`. The body is the message, a Liquid template. A document is one of two shapes, `{ shape: 'call' }` or `{ shape: 'request' }` (`interaction-document.ts`): with `call`, `output.schema` is required and `to`, `expires`, `deliver`, `replies`, `from`, `reply` and a body are refused with their line (`call-parts.ts`); without it, `to` and `expires` are required (`request-parts.ts`). `to`, `from` and the message read `input`, `today` and `now` alone, through an instance of the engine of `@beonauto/definitions/template` with no registrations of its own, and a template that does not compile, reads another name, or a property a closed input schema does not have, is refused with its line. + +`src/tool-blocks` holds the blocks, written once as effect schemas on one `ToolUseSchema`, `{ server, tool, with }`: `DeliverBlockSchema`, which adds `sent`, `CallBlockSchema`, which adds `read`, and `RepliesSchema`; `compiledToolBlock(written, lines, inputSchema)` checks a delivery and its reading at save, and `compiledCallBlock` a call: the server and tool names by their shapes, `isServerName` and `isToolName` of `@beonauto/mcp`; every template of `call.with`, `deliver.with`, `replies.conversation`, `replies.with` and `tell.with` against the names its place reads, with no `${`; arguments of a call that read no `input` within 16 KiB as JSON; each pointer of `call.read`, `deliver.sent` and `replies.read` a JSON Pointer that starts with a slash, refused in one sentence; `read.order`; `wait` from `PT5S` to `PT1H`. `replies` needs `deliver`, its `sent` and `output.schema`. Arguments are typed as written (`renderedArguments`): a number, boolean, null, list or object is sent as written with the templates in its strings rendered, a string that is one `{{ expression }}` alone is sent as the value it reads, and any other string is rendered as text, so a structure inside a longer text is refused at save; `momentVariables(input, at)` gives a call and a request the same `input`, `today` and `now`. `routeOf(blocks)` answers what an attempt, a read or a telling works with, `{ kind: 'inbox' }` or `{ kind: 'tool', delivery, replies? }`, and `throughWords` says it, the same sentence the definitions package's `deliveryStarted` says. The reply rule (`src/replies/reply-rule.ts`, checked by `rule-checks.ts`) maps a top-level string property of the answer to `word`, the first word with the `words` that mean each value, `rest` or `text`; left out, it is derived for an answer whose one required property is a string, and is none otherwise. At most 16 values, 16 words a value, 64 bytes a word. ## A run -`makeInteractionFunctionAdapter({ tools, openRequests, mostOpenRequests })` is the capability (`src/capability`, `src/run`). Its prepared definition finishes later, but for a notification without `deliver`, and its longest run is its `expires`, so a workflow step that calls it waits that long and a minute more. It reaches outside while any tool server is configured, `tools.configured`. A run validates its input, checks by name alone that every tool its route names is offered to the brain, `tools.named`, the words of `list_tool_servers` and `unavailable`, kind `tool_not_offered`, otherwise; counts the brain's open requests through `openRequests` and refuses a run past `mostOpenRequests` as `requests_full`. It renders the party, the answerer and the message as text, and the arguments of `deliver` once to check them: an empty party or one with a control character or past 256 bytes, a message past 8 KiB, arguments past 16 KiB as JSON or one that renders no text, end the run `conflict`, kind `unworkable`, with the place in its record. The run then defers, and the deferral's record is the request: `{ to, message, answer_schema, answerer, reply, expires_at, requested_at, deliver, replies }`, the answering fields left out of a notification and the blocks out of a request in the inbox. A run that is cancelled ends `rejected` as `cancelled`, unless an answer a reply brought came first, which settles it. +`makeInteractionFunctionAdapter({ tools, openRequests, mostOpenRequests })` is the capability (`src/capability`, `src/run`, `src/call`), which runs a call or a request by the document's shape. A request's prepared definition finishes later, but for a notification without `deliver`, and its longest run is its `expires`, so a workflow step that calls it waits that long and a minute more. It reaches outside while any tool server is configured, `tools.configured`. A run validates its input, checks by name alone that every tool its route names is offered to the brain, `tools.named`, the words of `list_tool_servers` and `unavailable`, kind `tool_not_offered`, otherwise; counts the brain's open requests through `openRequests` and refuses a run past `mostOpenRequests` as `requests_full`. It renders the party, the answerer and the message as text, and the arguments of `deliver` once to check them: an empty party or one with a control character or past 256 bytes, a message past 8 KiB, arguments past 16 KiB as JSON or one that renders no text, end the run `conflict`, kind `unworkable`, with the place in its record. The run then defers, and the deferral's record is the request: `{ to, message, answer_schema, answerer, reply, expires_at, requested_at, deliver, replies }`, the answering fields left out of a notification and the blocks out of a request in the inbox. A run that is cancelled ends `rejected` as `cancelled`, unless an answer a reply brought came first, which settles it. + +## A call + +A run of a function with `call` (`src/call`) validates its input, renders the arguments over `momentVariables`, and calls once through `tools.callOnce` of `@beonauto/mcp`, with the run's id as the call's id and as `com.beonauto/run_id` in its metadata, the run's journal, which records `tool_call_started` once the tool is found listed and before it is sent and `tool_call_answered` after, and `toolBounds.longestRetryWaitMs` as the longest wait for a 429. `endingOf` (`call-endings.ts`) reads what the call answered. A result's document, `answerDocument`, at `read`, checked against `output.schema`, 64 KiB as JSON and 512 levels, is the output, and `{ server, tool, read }` the record. Nothing to read, nothing at `read`, a value the schema refuses, and arguments the tool refused end `conflict`, kind `unworkable`; when the tool that answered is not marked read-only, the detail of an answer the run cannot use ends "The tool ran and may have changed something; a run again calls it again". A tool not offered, or a server that could not be opened or did not list the tool, ends `unavailable`, kind `tool_not_offered` or `mcp_server_failed`, before anything is sent. A tool error, a server failure or no answer after sending ends `unavailable`, kind `tools_unfinished`, when the server marks the tool `readOnlyHint`, and `conflict`, kind `effect_unknown`, otherwise, with `because` `tool_error` or `server_failed` and the wait a 429 asked for as `retry_after_ms` in the record. Its prepared definition calls tools and finishes within its call, and its longest run is 70 seconds, `longestCallRunMs`: 10 to open, 10 to list, 10 to restart a process that exited, 30 to call and 10 to reopen a session the server forgot. A call makes no request, so the open requests never hold it and `answer_interaction` and `cancel_run` refuse its run. ## The open requests @@ -79,25 +110,28 @@ Attempts follow `attemptSchedule` (`src/schedule/attempt-schedule.ts`): five, th ## Words -The capability's run words show the deferral as `interaction_requested`, a type the brain reserves, "A request is waiting for an answer, through the tool post_message of chat, until …", or "… in the inbox …", with the delivery, the party, the answerer where it differs, the size of the message, whether it takes an answer and the expiry, and never the message. An answer is described as the output of any run. +`describeAnswer(output, record)` says what a call answered from the record, and that a notification was delivered where the record is a request that takes no answer or a delivery's `{ delivered_at }`, "It asked the thread replies tool of chat; its answer: …", or how many items there are when the words would be too long, and what a request was answered with otherwise. The capability's run words show the deferral as `interaction_requested`, a type the brain reserves, "A request is waiting for an answer, through the post message tool of chat, until …", or "… in the inbox …", with the delivery, the party, the answerer where it differs, the size of the message, whether it takes an answer and the expiry, and never the message. ## Bounds -| Bound | Value | When it is reached | -| ------------------------------------------- | ------------------------------------------ | ---------------------------------------------------------------------- | -| The rendered party, and the answerer | 256 bytes, no control character | `conflict`, `unworkable` | -| The rendered message | 8 KiB | `conflict`, `unworkable` | -| The arguments of a call | 16 KiB as JSON | `conflict`, `unworkable` at the start; a refused attempt at a delivery | -| What `sent` reads, and the conversation key | 256 bytes each, no control character | delivered without `delivered_as`; the request takes no reply | -| A read's answer | 64 KiB | a failed read | -| A reply's words | 8 KiB | `reply_refused`, `too_long` | -| Refused replies of a request | 10 recorded, 3 told | no trace, or nobody told | -| Open requests read in a conversation | 100, oldest first | later ones are read for once earlier ones end | -| `expires` | 1 minute to 30 days | refused at save | -| An answer | 64 KiB as JSON, 512 levels | `invalid_input` | -| `claimed_for` | 256 bytes, event text | `invalid_input` | -| Attempts | 5, at once and after 1, 2, 4 and 8 minutes | a question stays open; a notification ends `undelivered` | -| Open requests of a brain | 10,000 unless the server sets another | `unavailable`, `requests_full` | +| Bound | Value | When it is reached | +| ------------------------------------------- | ----------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------- | +| The rendered party, and the answerer | 256 bytes, no control character | `conflict`, `unworkable` | +| The rendered message | 8 KiB | `conflict`, `unworkable` | +| The arguments of a call | 16 KiB as JSON | refused at save for a call that reads no `input`; otherwise `conflict`, `unworkable` at the start; a refused attempt at a delivery | +| A function's call of its tool | 10 seconds to open and 10 to list, 30 seconds to answer, a 429 of at most 10 seconds waited out | `mcp_server_failed` before sending; `tools_unfinished` or `effect_unknown` after | +| What a call's `read` takes | 64 KiB as JSON, 512 levels | `conflict`, `unworkable` | +| The longest run of a call | 70 seconds | a workflow step waits 70 seconds and a minute | +| What `sent` reads, and the conversation key | 256 bytes each, no control character | delivered without `delivered_as`; the request takes no reply | +| A read's answer | 64 KiB | a failed read | +| A reply's words | 8 KiB | `reply_refused`, `too_long` | +| Refused replies of a request | 10 recorded, 3 told | no trace, or nobody told | +| Open requests read in a conversation | 100, oldest first | later ones are read for once earlier ones end | +| `expires` | 1 minute to 30 days | refused at save | +| An answer | 64 KiB as JSON, 512 levels | `invalid_input` | +| `claimed_for` | 256 bytes, event text | `invalid_input` | +| Attempts | 5, at once and after 1, 2, 4 and 8 minutes | a question stays open; a notification ends `undelivered` | +| Open requests of a brain | 10,000 unless the server sets another | `unavailable`, `requests_full` | `interactionBounds` holds those of the run. @@ -107,8 +141,8 @@ The capability's run words show the deferral as `interaction_requested`, a type ## Testing -`@beonauto/interaction/testing` holds a harness over the in-memory ledger with the projections, the operations of the brain and the due work (`interactionHarness`, and `chatHarness` with the reading), fake tool access over a chat board that posts, lists a thread and takes replies (`fakeTools`), documents of a question, a notification and one read in a thread, and a request asked through the chat. +`@beonauto/interaction/testing` holds a harness over the in-memory ledger with the projections, the operations of the brain and the due work (`interactionHarness`, and `chatHarness` with the reading), fake tool access over a chat board that posts, lists a thread and takes replies (`fakeTools`), documents of a question, a notification and one read in a thread, a request asked through the chat, and documents of a call with runs of it against the fake MCP server of `@beonauto/mcp/testing` (`callDocument`, `call-runs.ts`). ## Source -`src/document` reads the definition, `src/route` the delivery and the reading, `src/run` renders and defers a request, `src/requests` holds the projection of open requests and the two operations, `src/delivery` the attempt as a recorded call, `src/schedule` the due work and its schedule, `src/conversations` the projection of conversations and the reading, `src/replies` the reply rule, the taking and the tellings, `src/capability` the capability, its guide, the public reference page served to agents as `interaction-function`, and its words, and `src/testing` what the tests share. +`src/document` reads the definition, `src/tool-blocks` the call, the delivery and the reading, `src/call` runs a call and reads its endings, `src/run` renders and defers a request, `src/requests` holds the projection of open requests and the two operations, `src/delivery` the attempt as a recorded call, `src/schedule` the due work and its schedule, `src/conversations` the projection of conversations and the reading, `src/replies` the reply rule, the taking and the tellings, `src/capability` the capability, its guide, the public reference page served to agents as `interaction-function`, and its words, and `src/testing` what the tests share. diff --git a/capabilities/interaction/src/call/call-endings.test.ts b/capabilities/interaction/src/call/call-endings.test.ts new file mode 100644 index 000000000..9523f4cfe --- /dev/null +++ b/capabilities/interaction/src/call/call-endings.test.ts @@ -0,0 +1,167 @@ +import { recordingJournal } from '@beonauto/definitions/testing'; +import { deniedText, patientTiming } from '@beonauto/mcp/testing'; +import { Cause, Exit } from 'effect'; +import { describe, expect, it } from 'vitest'; + +import { callRuns } from '../testing/call-runs.ts'; +import { callDocument } from '../testing/index.ts'; + +const anyOutput = ['output:', ' schema: {}']; + +function asking(tool: string, written: readonly string[] = []): string { + return callDocument({ tool, read: null, with: written, output: anyOutput }); +} + +function defectOf(ended: unknown): unknown { + return Exit.isExit(ended) && Exit.isFailure(ended) ? Cause.squash(ended.cause) : undefined; +} + +function rejectedWith(fields: Readonly>) { + return Exit.fail(expect.objectContaining(fields)); +} + +describe('a run that asks a system and sends nothing', () => { + it('is unavailable when its server does not serve the brain, or its operator does not allow the tool, whether its server lists it or not', async () => { + const elsewhere = await callRuns({ entry: { org: 'globex' } }); + const narrow = await callRuns({ entry: { allowed: ['search'] } }); + + expect([ + await elsewhere.run(callDocument()), + await narrow.run(callDocument()), + await narrow.run(callDocument({ tool: 'gone', read: null })), + ]).toEqual([ + rejectedWith({ kind: 'tool_not_offered', because: 'mcp_server_not_configured' }), + rejectedWith({ kind: 'tool_not_offered', because: 'tool_not_allowed' }), + rejectedWith({ kind: 'tool_not_offered', because: 'tool_not_allowed' }), + ]); + expect([elsewhere.fake.seen(), narrow.fake.received(), elsewhere.journal.recorded()]).toEqual([[], [], []]); + }); + + it('is unavailable when its server does not list the tool or cannot be reached, and records no call', async () => { + const { fake, journal, run } = await callRuns(); + const unlisted = await run(callDocument({ tool: 'gone', read: null })); + await fake.close(); + + expect([unlisted, await run(callDocument())]).toEqual([ + rejectedWith({ + kind: 'tool_not_offered', + because: 'tool_not_listed', + detail: 'The MCP server chat does not list the tool gone', + }), + rejectedWith({ kind: 'mcp_server_failed', because: 'unreachable' }), + ]); + expect(journal.recorded()).toEqual([]); + }); + + it('dies, sending nothing, when its run records no start', async () => { + const { fake, run } = await callRuns({ journal: recordingJournal(() => true) }); + + const ended = await run(callDocument()); + + expect(defectOf(ended)).toEqual( + new Error('The start of the call could not be recorded on its run, so the call was not sent'), + ); + expect(fake.received()).toEqual([]); + }); +}); + +describe('a run that asks a system with arguments it cannot render', () => { + it('rejects an input that does not match, or lacks what an argument reads, or that a filter refuses', async () => { + const { fake, run } = await callRuns(); + const reading = callDocument({ input: ['input:', ' schema: { type: object }'] }); + const filtered = callDocument({ with: [" channel: '{{ input.channel | divided_by: 0 }}'"] }); + + expect([ + await run(callDocument(), { channel: 7 }), + await run(reading, { channel: 'C0123' }), + await run(filtered), + ]).toEqual([ + rejectedWith({ _tag: 'invalid_input' }), + rejectedWith({ _tag: 'invalid_input', detail: 'The definition reads a field the input does not have' }), + rejectedWith({ + _tag: 'invalid_input', + detail: 'The argument channel of the call cannot be rendered with this input', + }), + ]); + expect(fake.received()).toEqual([]); + }); + + it('is unworkable when an argument renders a structure among text, or the arguments take more than 16 KiB', async () => { + const { run } = await callRuns(); + const open = ['input:', ' schema: { type: object }']; + const among = callDocument({ with: [" channel: 'In {{ input.channel }}'"], input: open }); + const whole = callDocument({ with: [" a: '{{ input.a }}'", " b: '{{ input.b }}'"], input: open }); + const text = callDocument({ with: [" channel: 'In {{ input.channel }}'"], input: open }); + + expect([ + await run(among, { channel: ['C0123'] }), + await run(whole, { a: 'x'.repeat(8200), b: 'x'.repeat(8200) }), + await run(text, { channel: 'x'.repeat(16_400) }), + ]).toEqual([ + rejectedWith({ + kind: 'unworkable', + detail: expect.stringContaining('renders a value that is not text among text'), + }), + rejectedWith({ kind: 'unworkable', detail: expect.stringContaining('take 16415 bytes, more than the 16384') }), + rejectedWith({ kind: 'unworkable', detail: expect.stringContaining('renders more than the 16384 bytes') }), + ]); + }); +}); + +describe('a run whose call was sent and could not finish', () => { + it('is unavailable, safe to try again, when its server marks the tool read-only, and a conflict whose effect is unknown when not', async () => { + const readOnly = await callRuns(); + const writing = await callRuns({ server: { hints: { denied: { readOnlyHint: false }, broken: {} } } }); + + expect([ + await readOnly.run(asking('denied')), + await writing.run(asking('denied')), + await readOnly.run(asking('broken')), + await writing.run(asking('broken')), + ]).toEqual([ + rejectedWith({ + kind: 'tools_unfinished', + because: 'tool_error', + detail: `The denied tool of chat answered an error: ${deniedText}`, + }), + rejectedWith({ _tag: 'conflict', kind: 'effect_unknown', because: 'tool_error' }), + rejectedWith({ + kind: 'tools_unfinished', + because: 'server_failed', + detail: expect.stringMatching( + /^The MCP server chat failed: .*, after the run called the broken tool of chat$/u, + ), + }), + rejectedWith({ _tag: 'conflict', kind: 'effect_unknown', because: 'server_failed' }), + ]); + }); + + it('keeps the wait a 429 asked for past the longest a run waits, and ends a call that took too long', async () => { + const { fake, run } = await callRuns({ timing: { ...patientTiming, callMs: 300 } }); + fake.answerNextOf('tools/call', 429, 1, { 'retry-after': '20' }); + + expect([await run(asking('search', [' query: acme'])), await run(asking('sleep', [' ms: 5000']))]).toEqual([ + rejectedWith({ kind: 'tools_unfinished', because: 'server_failed', record: { retry_after_ms: 20_000 } }), + rejectedWith({ + kind: 'tools_unfinished', + because: 'server_failed', + detail: expect.stringContaining('did not answer within 300 ms'), + }), + ]); + }); +}); + +describe('a run whose tool refused its arguments', () => { + it('is unworkable, which trying again would meet again', async () => { + const { run } = await callRuns(); + + expect(await run(asking('strict', [" limit: '15'"]))).toEqual( + rejectedWith({ + kind: 'unworkable', + detail: expect.stringMatching( + /^The strict tool of chat refused the arguments: .*limit must be a whole number/u, + ), + }), + ); + }); +}); diff --git a/capabilities/interaction/src/call/call-endings.ts b/capabilities/interaction/src/call/call-endings.ts new file mode 100644 index 000000000..4962d9f04 --- /dev/null +++ b/capabilities/interaction/src/call/call-endings.ts @@ -0,0 +1,120 @@ +import { Buffer } from 'node:buffer'; + +import { issuesDetail } from '@beonauto/definitions/json-schema'; +import { answerDocument, isReadOnly, toolInWords, type AnsweredOnce, type CalledOnce } from '@beonauto/mcp'; +import { capitalized, Conflict, Unavailable } from '@beonauto/operations'; +import { measureOf, mostValueDepth } from '@beonauto/workflow-engine/dsl'; +import { Effect, Result, Schema } from 'effect'; + +import type { CallDocument } from '../document/interaction-document.ts'; +import { checkedAnswer } from '../requests/answer-check.ts'; +import { interactionBounds } from '../run/run-bounds.ts'; +import { valueAt } from '../tool-blocks/json-pointers.ts'; + +const isJson = Schema.is(Schema.Json); + +interface CallFinished { + readonly output: Schema.Json; + readonly record: Schema.JsonObject; +} + +type Ending = Effect.Effect; + +function unworkable(detail: string): Ending { + return Effect.fail(new Conflict({ detail, kind: 'unworkable' })); +} + +function recordOf({ call }: CallDocument): Schema.JsonObject { + const { server, tool, read } = call; + return read === undefined ? { server, tool } : { server, tool, read }; +} + +const mayHaveActed = 'The tool ran and may have changed something; a run again calls it again'; + +type Answered = Extract; + +interface Reading { + readonly named: string; + readonly at: string; + readonly unreadable: (detail: string) => Ending; +} + +function unreadableAfter({ annotations }: Answered): (detail: string) => Ending { + return (detail) => unworkable(isReadOnly(annotations) ? detail : `${detail}. ${mayHaveActed}`); +} + +function checkedValue(document: CallDocument, value: Schema.Json, { named, at, unreadable }: Reading): Ending { + const what = `What ${named} answered${at}`; + const bytes = Buffer.byteLength(JSON.stringify(value), 'utf8'); + if (bytes > interactionBounds.answerBytes) { + return unreadable( + `${what} takes ${bytes} bytes as JSON, more than the ${interactionBounds.answerBytes} an answer may`, + ); + } + if (measureOf(value) === undefined) { + return unreadable(`${what} nests deeper than the ${mostValueDepth} levels a value may`); + } + return Result.match(checkedAnswer(value, document.output.schema.document), { + onSuccess: (output) => Effect.succeed({ output, record: recordOf(document) }), + onFailure: (issues) => unreadable(`${what} does not match the output schema: ${issuesDetail(issues, 'answer')}`), + }); +} + +function answeredWith(document: CallDocument, answer: Answered): Ending { + const named = toolInWords(document.call); + const read = document.call.read ?? ''; + const unreadable = unreadableAfter(answer); + const found = answerDocument(answer.answer); + if (found === undefined) { + return unreadable( + `${capitalized(named)} answered neither structured content nor text, so there is nothing to read`, + ); + } + const value = valueAt(found, read); + if (!isJson(value)) { + return unreadable( + typeof found === 'string' + ? `${capitalized(named)} answered text that is not JSON, which holds nothing at ${read}` + : `The answer of ${named} holds nothing at ${read}`, + ); + } + return checkedValue(document, value, { named, at: read === '' ? '' : ` at ${read}`, unreadable }); +} + +function afterSending(called: AnsweredOnce, because: 'tool_error' | 'server_failed', detail: string): Ending { + const ending = { + detail, + because, + ...(called.retryAfterMs === null ? {} : { record: { retry_after_ms: called.retryAfterMs } }), + }; + return Effect.fail( + isReadOnly(called.annotations) + ? new Unavailable({ ...ending, kind: 'tools_unfinished' }) + : new Conflict({ ...ending, kind: 'effect_unknown' }), + ); +} + +function endedWith(document: CallDocument, called: AnsweredOnce): Ending { + const named = toolInWords(document.call); + if (called.outcome === 'result') { + return answeredWith(document, called); + } + if (called.outcome === 'tool_error' && called.fields.result_bytes === null) { + return unworkable(`${capitalized(named)} refused the arguments: ${called.detail}`); + } + return called.outcome === 'tool_error' + ? afterSending(called, 'tool_error', `${capitalized(named)} answered an error: ${called.detail}`) + : afterSending( + called, + 'server_failed', + `The MCP server ${document.call.server} failed: ${called.detail}, after the run called ${named}`, + ); +} + +export function endingOf(document: CallDocument, called: CalledOnce): Ending { + if (called.kind === 'unopened') { + const { refused, because, detail } = called; + return Effect.fail(new Unavailable({ detail, kind: refused, because })); + } + return endedWith(document, called); +} diff --git a/capabilities/interaction/src/call/call-reading.test.ts b/capabilities/interaction/src/call/call-reading.test.ts new file mode 100644 index 000000000..49a2988b6 --- /dev/null +++ b/capabilities/interaction/src/call/call-reading.test.ts @@ -0,0 +1,87 @@ +import type { Schema } from 'effect'; +import { Exit } from 'effect'; +import { describe, expect, it } from 'vitest'; + +import { callRuns } from '../testing/call-runs.ts'; +import { callDocument } from '../testing/index.ts'; + +const anyOutput = ['output:', ' schema: {}']; + +const open = ['input:', ' schema: { type: object }']; + +function unworkable(detail: unknown) { + return Exit.fail(expect.objectContaining({ _tag: 'conflict', kind: 'unworkable', detail })); +} + +function nestedIn(levels: number): Schema.Json { + return Array.from({ length: levels }).reduce((inner) => ({ n: inner }), 'leaf'); +} + +describe('a run that asks a system and cannot read what the tool answered', () => { + it('is unworkable when the answer has no document, or nothing at read, in a structure or in a text', async () => { + const { run } = await callRuns(); + + expect([ + await run(callDocument({ tool: 'photo', read: null, with: [], output: anyOutput })), + await run(callDocument({ read: '/nothing' })), + await run(callDocument({ tool: 'search', read: '/rows', with: [' query: acme'], output: anyOutput })), + ]).toEqual([ + unworkable('The photo tool of chat answered neither structured content nor text, so there is nothing to read'), + unworkable('The answer of the thread tool of chat holds nothing at /nothing'), + unworkable('The search tool of chat answered text that is not JSON, which holds nothing at /rows'), + ]); + }); + + it('is unworkable when what it reads takes more than 64 KiB, nests deeper than 512 levels, or does not match the output schema', async () => { + const { run } = await callRuns(); + const deep = callDocument({ + tool: 'echo', + read: null, + with: [" deep: { inner: '{{ input.deep }}' }"], + input: open, + output: anyOutput, + }); + + expect([ + await run(callDocument({ tool: 'large', read: null, with: [' kib: 70'], output: anyOutput })), + await run(deep, { deep: nestedIn(511) }), + await run(callDocument({ output: ['output:', ' schema: { type: string }'] })), + ]).toEqual([ + unworkable('What the large tool of chat answered takes 71682 bytes as JSON, more than the 65536 an answer may'), + unworkable( + 'What the echo tool of chat answered nests deeper than the 512 levels a value may. The tool ran and may have changed something; a run again calls it again', + ), + unworkable( + expect.stringMatching( + /^What the thread tool of chat answered at \/messages does not match the output schema: the answer: /u, + ), + ), + ]); + }); +}); + +const countOutput = [ + 'output:', + ' schema: { type: object, required: [count], properties: { count: { type: number } } }', +]; + +describe('a run that asks a system whose answer the brain cannot use', () => { + it('checks what a tool answered against its own output schema, whatever schema the tool declares', async () => { + const { fake, run } = await callRuns(); + + expect(await run(callDocument({ tool: 'promised', read: null, with: [], output: countOutput }))).toEqual( + unworkable('What the promised tool of chat answered does not match the output schema: /count: Expected number'), + ); + expect(fake.received()).toHaveLength(1); + }); + + it('says that running it again calls the tool again when the tool may change something', async () => { + const { run } = await callRuns({ server: { hints: { thread: { readOnlyHint: false } } } }); + + expect(await run(callDocument({ read: '/nothing' }))).toEqual( + unworkable( + 'The answer of the thread tool of chat holds nothing at /nothing. The tool ran and may have changed something; a run again calls it again', + ), + ); + }); +}); diff --git a/capabilities/interaction/src/call/call-run.test.ts b/capabilities/interaction/src/call/call-run.test.ts new file mode 100644 index 000000000..f7a423bd0 --- /dev/null +++ b/capabilities/interaction/src/call/call-run.test.ts @@ -0,0 +1,106 @@ +import { threadReplies } from '@beonauto/mcp/testing'; +import { Effect, Exit } from 'effect'; +import { describe, expect, it } from 'vitest'; + +import { makeInteractionFunctionAdapter } from '../capability/interaction-function.ts'; +import { callRunId, callRuns } from '../testing/call-runs.ts'; +import { callDocument, noTools } from '../testing/index.ts'; + +const aNumber: unknown = expect.any(Number); + +const aDigest: unknown = expect.stringMatching(/^[0-9a-f]{64}$/u); + +const asSystem = makeInteractionFunctionAdapter({ + tools: noTools, + openRequests: () => Effect.succeed(0), + mostOpenRequests: 10, +}); + +describe('a run of an interaction function that asks a system', () => { + it('calls the tool once and ends at once with the value at read as its output, recording the call on the run', async () => { + const { fake, journal, run } = await callRuns(); + + expect(await run(callDocument())).toEqual( + Exit.succeed({ output: threadReplies, record: { server: 'chat', tool: 'thread', read: '/messages' } }), + ); + expect(fake.received()).toEqual([ + { + tool: 'thread', + arguments: { channel: 'C0123', ts: '1728379900.000050', limit: 100, inclusive: false }, + meta: { 'com.beonauto/run_id': callRunId }, + }, + ]); + expect(journal.recorded()).toEqual([ + { + type: 'tool_call_started', + call_id: callRunId, + server: 'chat', + tool: 'thread', + arguments_bytes: aNumber, + arguments_sha256: aDigest, + read_only: true, + number: 1, + }, + expect.objectContaining({ type: 'tool_call_answered', number: 1, outcome: 'result', duration_ms: aNumber }), + ]); + }); + + it('sends a value written alone in one {{ }} as the value it reads, and answers with the whole answer without read', async () => { + const { fake, run } = await callRuns(); + const echoing = callDocument({ + tool: 'echo', + read: null, + with: [ + " limit: '{{ input.limit }}'", + " filters: '{{ input.filters }}'", + " note: 'For {{ input.channel }}'", + ], + input: ['input:', ' schema: { type: object }'], + output: ['output:', ' schema: { type: object }'], + }); + const input = { limit: 15, filters: { open: true }, channel: 'C0123' }; + const sent = { limit: 15, filters: { open: true }, note: 'For C0123' }; + + expect(await run(echoing, input)).toEqual(Exit.succeed({ output: sent, record: { server: 'chat', tool: 'echo' } })); + expect(fake.received()).toMatchObject([{ tool: 'echo', arguments: sent }]); + }); +}); + +describe('what a run that asks a system answers with', () => { + it('answers with a text that is not JSON as that text', async () => { + const { run } = await callRuns(); + const searching = callDocument({ + tool: 'search', + read: null, + with: [" query: '{{ input.channel }}'"], + output: ['output:', ' schema: { type: string }'], + }); + + expect(await run(searching)).toMatchObject({ value: { output: 'Found 2 rows for C0123.' } }); + }); +}); + +describe('an interaction function that asks a system, as the brain runs it', () => { + it('calls tools, finishes within its call, and runs at most 70 seconds', () => { + const prepared = Effect.runSync(asSystem.prepare(callDocument())); + + expect(prepared).toMatchObject({ callsTools: true, finishesLater: false, longestRunMs: 70_000 }); + }); + + it('says what the tool answered, the records of a list in parentheses, or how many items there are', () => { + const record = { server: 'chat', tool: 'thread_replies', read: '/messages' }; + const many = Array.from({ length: 14 }, (_, index) => ({ user: `member-${index}`, text: 'x'.repeat(40) })); + + expect([ + asSystem.describeOutput(threadReplies, record), + asSystem.describeOutput(many, record), + asSystem.describeOutput({ text: 'x'.repeat(400) }, record), + asSystem.describeOutput({}, record), + ]).toEqual([ + 'It asked the thread replies tool of chat; its answer: (user: “member-17”, text: “Shipped the fix to staging.”, and ts: “1728380000.000100”) and (user: “member-42”, text: “Thanks, closing the ticket.”, and ts: “1728380100.000200”).', + 'It asked the thread replies tool of chat, whose answer of 14 items is too long to repeat here; the whole of it is in the details below.', + 'It asked the thread replies tool of chat, whose answer is too long to repeat here; the whole of it is in the details below.', + 'It asked the thread replies tool of chat; its answer: nothing.', + ]); + }); +}); diff --git a/capabilities/interaction/src/call/call-run.ts b/capabilities/interaction/src/call/call-run.ts new file mode 100644 index 000000000..310c8c890 --- /dev/null +++ b/capabilities/interaction/src/call/call-run.ts @@ -0,0 +1,68 @@ +import type { CapabilityAnswer, CapabilityRejection, RunContext } from '@beonauto/definitions'; +import { toolBounds, type ToolAccess } from '@beonauto/mcp'; +import { runIdKey } from '@beonauto/mcp/policy'; +import type { Conflict, InvalidInput } from '@beonauto/operations'; +import { Clock, Effect, Result, type Schema } from 'effect'; + +import type { CallDocument } from '../document/interaction-document.ts'; +import { unrenderableFor, unworkable } from '../run/request-rendering.ts'; +import { preparedInput } from '../run/run-input.ts'; +import { + argumentsFailureWords, + momentVariables, + renderedArguments, + type ArgumentsFailure, +} from '../tool-blocks/rendered-arguments.ts'; +import { endingOf } from './call-endings.ts'; + +export const longestCallRunMs = 4 * toolBounds.openMs + toolBounds.callMs; + +export interface CallPorts { + readonly tools: Pick; +} + +function argumentsRejection(failure: ArgumentsFailure): Conflict | InvalidInput { + if (failure.reason === 'too_large') { + return unworkable('/call/with', argumentsFailureWords(failure, 'the call')); + } + const { argument, failure: rendering } = failure; + return rendering.reason === 'not_text' || rendering.reason === 'too_long' + ? unworkable(`/call/with/${argument}`, argumentsFailureWords(failure, 'the call')) + : unrenderableFor(`The argument ${argument} of the call`, rendering); +} + +function argumentsOf( + { call }: CallDocument, + variables: Readonly>, +): Effect.Effect>, Conflict | InvalidInput> { + return Result.match(renderedArguments(call.with ?? {}, variables), { + onSuccess: ({ input }) => Effect.succeed(input), + onFailure: (failure) => Effect.fail(argumentsRejection(failure)), + }); +} + +export function callRun(ports: CallPorts) { + return ( + document: CallDocument, + input: Schema.Json, + context: RunContext, + ): Effect.Effect => + Effect.gen(function* () { + const admitted = yield* preparedInput(input, document.input); + const at = new Date(yield* Clock.currentTimeMillis).toISOString(); + const rendered = yield* argumentsOf(document, momentVariables(admitted, at)); + const { server, tool } = document.call; + const called = yield* ports.tools.callOnce( + { + org: context.org, + brain: context.brain, + reference: { server, tool }, + input: rendered, + meta: { [runIdKey]: context.id }, + longestRetryWaitMs: toolBounds.longestRetryWaitMs, + }, + { callId: context.id, journal: context.journal }, + ); + return yield* endingOf(document, called); + }); +} diff --git a/capabilities/interaction/src/call/call-words.ts b/capabilities/interaction/src/call/call-words.ts new file mode 100644 index 000000000..ad3af7ed8 --- /dev/null +++ b/capabilities/interaction/src/call/call-words.ts @@ -0,0 +1,35 @@ +import { inWords } from '@beonauto/definitions'; +import { toolInWords } from '@beonauto/mcp'; +import { asSentence, counted } from '@beonauto/operations'; +import { Option, Schema } from 'effect'; + +const CallRecordSchema = Schema.Struct({ + server: Schema.String, + tool: Schema.String, + read: Schema.optionalKey(Schema.String), +}); + +type CallRecord = typeof CallRecordSchema.Type; + +const decodeCallRecord = Schema.decodeUnknownOption(CallRecordSchema); + +const isList = Schema.is(Schema.Array(Schema.Json)); + +const item = { one: 'item', other: 'items' }; + +const inTheDetails = 'is too long to repeat here; the whole of it is in the details below.'; + +function askedInWords(call: CallRecord, output: Schema.Json): string { + const asked = `It asked ${toolInWords(call)}`; + const words = inWords(output); + if (words !== undefined) { + return asSentence(`${asked}; its answer: ${words}`); + } + return isList(output) + ? `${asked}, whose answer of ${counted(output.length, item)} ${inTheDetails}` + : `${asked}, whose answer ${inTheDetails}`; +} + +export function callOutputInWords(output: Schema.Json, record: unknown): string | undefined { + return Option.getOrUndefined(Option.map(decodeCallRecord(record), (call) => askedInWords(call, output))); +} diff --git a/capabilities/interaction/src/capability/definition-reading.ts b/capabilities/interaction/src/capability/definition-reading.ts index 62f6a56f1..87975c949 100644 --- a/capabilities/interaction/src/capability/definition-reading.ts +++ b/capabilities/interaction/src/capability/definition-reading.ts @@ -1,7 +1,7 @@ import type { DefinitionSummary } from '@beonauto/definitions'; import { issueText } from '@beonauto/definitions/document'; import { InvalidInput } from '@beonauto/operations'; -import { Effect, Result } from 'effect'; +import { Effect, Result, type Schema } from 'effect'; import { parseInteractionDocument } from '../document/document-parsing.ts'; import type { InteractionFunctionDefinitionDocument } from '../document/interaction-document.ts'; @@ -19,16 +19,22 @@ export function parse(source: string): Effect.Effect { }, { summary: - 'A notification is waiting to be delivered, through the tool post_message of chat, until 2026-10-09T09:00:00.000Z.', + 'A notification is waiting to be delivered, through the post message tool of chat, until 2026-10-09T09:00:00.000Z.', data: { delivery: { server: 'chat', tool: 'post_message' }, to: 'a'.repeat(256), @@ -135,14 +135,29 @@ describe('the words of a request answered by reply', () => { }); describe('the words of an answer', () => { - it('say what the answer was, or that a notification was delivered', () => { + it('say that a notification was delivered, read from the record of its request or of its delivery', () => { + const notified = { + to: 'ada', + message: 'Shipped', + expires_at: '2026-10-09T09:00:00.000Z', + requested_at: '2026-10-07T09:00:00.000Z', + }; + expect([ - describeAnswer({}), - describeAnswer({ choice: 'approve' }), - describeAnswer('x'.repeat(5000)), - describeAnswer([]), + describeAnswer({}, notified), + describeAnswer({}, { delivered_at: '2026-10-07T09:00:01.000Z' }), + describeAnswer({}, { ...notified, answer_schema: {} }), + describeAnswer({}, { answered_by: 'acme-admin', answered_at: '2026-10-07T09:00:01.000Z' }), ]).toEqual([ 'It delivered its notification.', + 'It delivered its notification.', + 'Its answer: nothing.', + 'Its answer: nothing.', + ]); + }); + + it('say what the answer was', () => { + expect([describeAnswer({ choice: 'approve' }), describeAnswer('x'.repeat(5000)), describeAnswer([])]).toEqual([ expect.stringContaining('Its answer:'), 'Its answer is too long to repeat here; the whole of it is in the details below.', expect.stringContaining('Its answer'), diff --git a/capabilities/interaction/src/capability/interaction-function.ts b/capabilities/interaction/src/capability/interaction-function.ts index c8fa3c040..10932acc0 100644 --- a/capabilities/interaction/src/capability/interaction-function.ts +++ b/capabilities/interaction/src/capability/interaction-function.ts @@ -5,15 +5,30 @@ import { type Capability, } from '@beonauto/definitions'; -import { finishesLater, interactionRun } from '../run/interaction-run.ts'; +import { callRun, longestCallRunMs } from '../call/call-run.ts'; +import type { InteractionFunctionDefinitionDocument } from '../document/interaction-document.ts'; +import { asksLater, requestRun } from '../run/interaction-run.ts'; import type { InteractionPorts } from '../run/request-reach.ts'; import { parse, summarize } from './definition-reading.ts'; import { interactionType } from './interaction-type.ts'; import { describeAnswer, interactionRunWords } from './interaction-words.ts'; import { cancelledRequest } from './request-cancels.ts'; +function finishesLater(document: InteractionFunctionDefinitionDocument): boolean { + return document.shape === 'request' && asksLater(document); +} + +function callsTools({ shape }: InteractionFunctionDefinitionDocument): boolean { + return shape === 'call'; +} + +function longestRunOf(document: InteractionFunctionDefinitionDocument): number { + return document.shape === 'call' ? longestCallRunMs : document.expiresMs; +} + export function makeInteractionFunctionAdapter(ports: InteractionPorts): Capability { - const run = interactionRun(ports); + const asked = requestRun(ports); + const called = callRun(ports); const reachesOutside = ports.tools.configured; return defineCapability({ type: interactionType, @@ -24,12 +39,14 @@ export function makeInteractionFunctionAdapter(ports: InteractionPorts): Capabil mediaType: 'text/markdown', parse, summarize, - run: (document, input, context) => run(document, input, context), + run: (document, input, context) => + document.shape === 'call' ? called(document, input, context) : asked(document, input, context), whenCancelled: 'finish', reachesOutside, mayChangeOutside: reachesOutside, + callsTools, finishesLater, - longestRunOf: ({ expiresMs }) => expiresMs, + longestRunOf, runWords: interactionRunWords, cancel: cancelledRequest, }); diff --git a/capabilities/interaction/src/capability/interaction-words.ts b/capabilities/interaction/src/capability/interaction-words.ts index 80ecbeede..2cb1cf6f0 100644 --- a/capabilities/interaction/src/capability/interaction-words.ts +++ b/capabilities/interaction/src/capability/interaction-words.ts @@ -1,12 +1,12 @@ import { Buffer } from 'node:buffer'; -import { defaultRunWords, inWords, type RunAccount, type RunWords } from '@beonauto/definitions'; -import { asSentence } from '@beonauto/operations'; +import { defaultRunWords, outputInWords, type RunAccount, type RunWords } from '@beonauto/definitions'; import type { Schema } from 'effect'; -import { routeOf, throughWords } from '../route/routes.ts'; -import { requestRecordOf, takesAnswer } from '../run/request-record.ts'; +import { callOutputInWords } from '../call/call-words.ts'; +import { isNotification, requestRecordOf, takesAnswer } from '../run/request-record.ts'; import { interactionBounds } from '../run/run-bounds.ts'; +import { routeOf, throughWords } from '../tool-blocks/routes.ts'; function cutParty(party: string): string { const bytes = Buffer.from(party, 'utf8'); @@ -44,12 +44,12 @@ export const interactionRunWords: RunWords = { deferral: requestAccount, }; -export function describeAnswer(output: Schema.Json): string { - if (output !== null && typeof output === 'object' && !Array.isArray(output) && Object.keys(output).length === 0) { - return 'It delivered its notification.'; +const answerInWords = outputInWords('answer'); + +export function describeAnswer(output: Schema.Json, record?: Schema.JsonObject): string { + const asked = callOutputInWords(output, record); + if (asked !== undefined) { + return asked; } - const words = inWords(output); - return words === undefined - ? 'Its answer is too long to repeat here; the whole of it is in the details below.' - : asSentence(`Its answer: ${words}`); + return isNotification(record) ? 'It delivered its notification.' : answerInWords(output); } diff --git a/capabilities/interaction/src/conversations/cadence.ts b/capabilities/interaction/src/conversations/cadence.ts index 8df3b6417..fecad09f6 100644 --- a/capabilities/interaction/src/conversations/cadence.ts +++ b/capabilities/interaction/src/conversations/cadence.ts @@ -1,6 +1,6 @@ import { readDuration } from '@beonauto/workflow-engine/dsl'; -import type { Replies } from '../route/route-schemas.ts'; +import type { Replies } from '../tool-blocks/tool-block-schemas.ts'; export const firstReadWaitMs = 5000; diff --git a/capabilities/interaction/src/conversations/conversation-parts.ts b/capabilities/interaction/src/conversations/conversation-parts.ts index 9c86f5187..d3c1b5f10 100644 --- a/capabilities/interaction/src/conversations/conversation-parts.ts +++ b/capabilities/interaction/src/conversations/conversation-parts.ts @@ -2,8 +2,8 @@ import type { ToolAccess } from '@beonauto/mcp'; import type { BrainAddress, ProjectionAdvancer, ProjectionReader } from '@beonauto/operations'; import type { Effect } from 'effect'; -import type { Replies } from '../route/route-schemas.ts'; import type { RequestLedger } from '../schedule/request-ledger.ts'; +import type { Replies } from '../tool-blocks/tool-block-schemas.ts'; import { advancedOf, conversationsName, type Cadence, type ConversationRow } from './conversation-rows.ts'; export interface ConversationLedger diff --git a/capabilities/interaction/src/conversations/conversation-reads.ts b/capabilities/interaction/src/conversations/conversation-reads.ts index 6316d56e9..32c8b5616 100644 --- a/capabilities/interaction/src/conversations/conversation-reads.ts +++ b/capabilities/interaction/src/conversations/conversation-reads.ts @@ -5,7 +5,7 @@ import { Effect, Result } from 'effect'; import { handledReply, type Reading } from '../replies/reply-handling.ts'; import { nothingRead, readArguments, type ReadingState } from '../replies/reply-recording.ts'; import type { KeptRequest } from '../replies/reply-taking.ts'; -import { argumentsFailureWords } from '../route/rendered-arguments.ts'; +import { argumentsFailureWords } from '../tool-blocks/rendered-arguments.ts'; import { failedRead, readAnswerOf, type ReadAnswer } from './read-answers.ts'; import { cursorOf } from './read-cadence.ts'; import { recordedRead } from './read-records.ts'; diff --git a/capabilities/interaction/src/conversations/open-in-conversation.ts b/capabilities/interaction/src/conversations/open-in-conversation.ts index c6cc2139d..57251c4ea 100644 --- a/capabilities/interaction/src/conversations/open-in-conversation.ts +++ b/capabilities/interaction/src/conversations/open-in-conversation.ts @@ -4,7 +4,7 @@ import { ReplyRuleSchema } from '../replies/reply-rule.ts'; import type { KeptRequest } from '../replies/reply-taking.ts'; import { openRequestsName } from '../requests/open-requests.ts'; import { requestRowFrom, type OpenRequestRow } from '../requests/request-rows.ts'; -import { RepliesSchema } from '../route/route-schemas.ts'; +import { RepliesSchema } from '../tool-blocks/tool-block-schemas.ts'; import type { ConversationParts, ConversationPlace, ReadingRoute } from './conversation-parts.ts'; const conversationBounds = { requestsRead: 100 } as const; diff --git a/capabilities/interaction/src/conversations/read-answers.ts b/capabilities/interaction/src/conversations/read-answers.ts index f3e85fb3f..aa68aab4c 100644 --- a/capabilities/interaction/src/conversations/read-answers.ts +++ b/capabilities/interaction/src/conversations/read-answers.ts @@ -1,12 +1,12 @@ import { Buffer } from 'node:buffer'; -import { toolBounds, type CalledOnce, type ReadOutcome } from '@beonauto/mcp'; +import { answerDocument, toolBounds, type CalledOnce, type ReadOutcome } from '@beonauto/mcp'; import { endOfCall } from '../delivery/call-ends.ts'; -import { documentOf, isBoundedPart } from '../delivery/sent-messages.ts'; +import { isBoundedPart } from '../delivery/sent-messages.ts'; import type { Reply } from '../replies/reply-taking.ts'; -import { textAt, valueAt } from '../route/json-pointers.ts'; -import type { Replies } from '../route/route-schemas.ts'; +import { textAt, valueAt } from '../tool-blocks/json-pointers.ts'; +import type { Replies } from '../tool-blocks/tool-block-schemas.ts'; type ReplyPointers = Replies['read']['each']; @@ -50,17 +50,19 @@ function listed(replies: Replies, document: unknown): ReadAnswer { } export function readAnswerOf(replies: Replies, called: CalledOnce): ReadAnswer { - if (called.kind !== 'answered') { - return failedRead(called.kind === 'not_offered' ? 'tool_not_offered' : 'server_failure'); + if (called.kind === 'unopened') { + return failedRead(called.refused === 'tool_not_offered' ? 'tool_not_offered' : 'server_failure'); } const end = endOfCall(called); if ('failed' in end) { const { because, retryAfterMs } = end.failed; return failedRead(because === 'timed_out' || because === 'tool_error' ? because : 'server_failure', retryAfterMs); } - if (Buffer.byteLength(JSON.stringify(end.answered), 'utf8') > toolBounds.resultBytes) { - return failedRead('too_large'); + const document = answerDocument(end.answered); + if (document === undefined) { + return failedRead('unreadable'); } - const document = documentOf(end.answered); - return document === undefined ? failedRead('unreadable') : listed(replies, document); + return Buffer.byteLength(JSON.stringify(document), 'utf8') > toolBounds.resultBytes + ? failedRead('too_large') + : listed(replies, document); } diff --git a/capabilities/interaction/src/conversations/read-cadence.ts b/capabilities/interaction/src/conversations/read-cadence.ts index e17c71813..d8d2acc5f 100644 --- a/capabilities/interaction/src/conversations/read-cadence.ts +++ b/capabilities/interaction/src/conversations/read-cadence.ts @@ -1,4 +1,4 @@ -import type { Replies } from '../route/route-schemas.ts'; +import type { Replies } from '../tool-blocks/tool-block-schemas.ts'; import { waitBefore } from './cadence.ts'; import type { Cadence, ConversationRow } from './conversation-rows.ts'; diff --git a/capabilities/interaction/src/conversations/read-failures.test.ts b/capabilities/interaction/src/conversations/read-failures.test.ts index 230fd3c08..10c845c6f 100644 --- a/capabilities/interaction/src/conversations/read-failures.test.ts +++ b/capabilities/interaction/src/conversations/read-failures.test.ts @@ -45,6 +45,7 @@ describe('a read that fails', () => { answered({ messages: [], padding: 'x'.repeat(70_000) }), textOf('Nothing new.'), answered({ messages: { ts: '1' } }), + { outcome: 'result', answer: { content: [{ type: 'image' }] }, detail: '', retryAfterMs: null }, { outcome: 'tool_error', detail: 'channel_not_found', retryAfterMs: null }, { outcome: 'timed_out', detail: 'The MCP server did not answer within 30000 ms', retryAfterMs: null }, { outcome: 'cancelled', detail: '', retryAfterMs: null }, @@ -56,6 +57,7 @@ describe('a read that fails', () => { [{ outcome: 'too_large', replies: 0 }, true], [{ outcome: 'unreadable' }, true], [{ outcome: 'unreadable' }, true], + [{ outcome: 'unreadable' }, true], [{ outcome: 'tool_error' }, true], [{ outcome: 'timed_out' }, true], [{ outcome: 'server_failure' }, true], diff --git a/capabilities/interaction/src/delivery/attempt-delivery.ts b/capabilities/interaction/src/delivery/attempt-delivery.ts index a03dd4b82..22ef9396c 100644 --- a/capabilities/interaction/src/delivery/attempt-delivery.ts +++ b/capabilities/interaction/src/delivery/attempt-delivery.ts @@ -1,11 +1,11 @@ -import type { DeliveryCall, StartedFields } from '@beonauto/mcp'; +import type { OneCall, StartedFields } from '@beonauto/mcp'; import { deliveryIdKey, runIdKey } from '@beonauto/mcp/policy'; import { Effect, Result, type Schema } from 'effect'; -import { argumentsFailureWords, deliveryVariablesOf, renderedArguments } from '../route/rendered-arguments.ts'; import type { DeliveringRecord } from '../run/request-record.ts'; import type { DeliveryParts, DueRequest } from '../schedule/delivery-parts.ts'; import { recordedCall } from '../schedule/request-ledger.ts'; +import { argumentsFailureWords, deliveryVariablesOf, renderedArguments } from '../tool-blocks/rendered-arguments.ts'; import { endOf, type AttemptEnd } from './attempt-end.ts'; import { startedFact, type Attempting } from './attempt-facts.ts'; @@ -32,7 +32,7 @@ function recordedStart( ); } -function callOf({ request, record }: DeliveryPlan, input: DeliveryCall['input']): DeliveryCall { +function callOf({ request, record }: DeliveryPlan, input: OneCall['input']): OneCall { const { address, row } = request; return { org: address.org, diff --git a/capabilities/interaction/src/delivery/attempt-end.ts b/capabilities/interaction/src/delivery/attempt-end.ts index 8313ec431..a073d428a 100644 --- a/capabilities/interaction/src/delivery/attempt-end.ts +++ b/capabilities/interaction/src/delivery/attempt-end.ts @@ -33,11 +33,13 @@ function endOfAnswer(called: AnsweredOnce, record: DeliveringRecord): AttemptEnd } export function endOf(called: CalledOnce, record: DeliveringRecord): AttemptEnd { - if (called.kind === 'not_offered') { - return { outcome: 'failed', because: 'tool_not_offered', ...detailOf(called.detail) }; + if (called.kind === 'answered') { + return endOfAnswer(called, record); } - if (called.kind === 'unopened') { - return { outcome: 'failed', because: 'server_failure', ...detailOf(called.detail) }; - } - return endOfAnswer(called, record); + const { refused, detail } = called; + return { + outcome: 'failed', + because: refused === 'tool_not_offered' ? 'tool_not_offered' : 'server_failure', + ...detailOf(detail), + }; } diff --git a/capabilities/interaction/src/delivery/call-ends.test.ts b/capabilities/interaction/src/delivery/call-ends.test.ts index 069855edf..a9f13a90c 100644 --- a/capabilities/interaction/src/delivery/call-ends.test.ts +++ b/capabilities/interaction/src/delivery/call-ends.test.ts @@ -6,6 +6,7 @@ const answered = { kind: 'answered', fields: { result_bytes: 52, result_sha256: null, jsonrpc_id: 1 }, durationMs: 3, + annotations: undefined, } as const; describe('the end of one call of a tool the brain makes', () => { diff --git a/capabilities/interaction/src/delivery/kept-messages.test.ts b/capabilities/interaction/src/delivery/kept-messages.test.ts index 1a24b4031..385f2389e 100644 --- a/capabilities/interaction/src/delivery/kept-messages.test.ts +++ b/capabilities/interaction/src/delivery/kept-messages.test.ts @@ -89,15 +89,11 @@ describe('a delivery whose answer names no message the brain can keep', () => { const ends = await Promise.all(answers.map(async (answer) => (await deliveredWith(answer)).ended)); expect(ends.map((ended) => keptOf(ended))).toEqual([ + [undefined, undefined, `The answer of the tool holds neither structured content nor text${takesNoReply}`], [ undefined, undefined, - `The answer of the tool holds no JSON document, in its structured content or its first text${takesNoReply}`, - ], - [ - undefined, - undefined, - `The answer of the tool holds no JSON document, in its structured content or its first text${takesNoReply}`, + `The answer of the tool has no text or whole number at /channel, its conversation${takesNoReply}`, ], [undefined, undefined, `The answer of the tool has no text or whole number at /ts, its message${takesNoReply}`], [ diff --git a/capabilities/interaction/src/delivery/sent-messages.ts b/capabilities/interaction/src/delivery/sent-messages.ts index aebc026fd..068a48c45 100644 --- a/capabilities/interaction/src/delivery/sent-messages.ts +++ b/capabilities/interaction/src/delivery/sent-messages.ts @@ -2,27 +2,17 @@ import { Buffer } from 'node:buffer'; import { refusingForbiddenCharacters, type DeliveredAs, type RepliesIn } from '@beonauto/definitions'; import { parsedTemplate } from '@beonauto/definitions/template'; -import type { CallAnswer } from '@beonauto/mcp'; -import { Option, Result, Schema } from 'effect'; +import { answerDocument, type CallAnswer } from '@beonauto/mcp'; +import { Result, Schema } from 'effect'; import { interactionEngine } from '../document/request-templates.ts'; -import { textAt } from '../route/json-pointers.ts'; -import type { Replies } from '../route/route-schemas.ts'; import type { DeliveringRecord } from '../run/request-record.ts'; import { renderedText } from '../run/text-rendering.ts'; +import { textAt } from '../tool-blocks/json-pointers.ts'; +import type { Replies } from '../tool-blocks/tool-block-schemas.ts'; const sentBounds = { partBytes: 256 } as const; -const decodeJson = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Json)); - -export function documentOf({ content, structuredContent }: CallAnswer): Schema.Json | undefined { - if (structuredContent !== undefined) { - return structuredContent; - } - const text = content.find(({ type }) => type === 'text')?.text; - return text === undefined ? undefined : Option.getOrUndefined(decodeJson(text)); -} - const isPartText = Schema.is(Schema.String.check(refusingForbiddenCharacters)); export function isBoundedPart(part: string): boolean { @@ -85,10 +75,10 @@ export function keptOf(record: DeliveringRecord, answer: CallAnswer): Kept { if (sent === undefined) { return {}; } - const document = documentOf(answer); + const document = answerDocument(answer); const kept = document === undefined - ? Result.fail('The answer of the tool holds no JSON document, in its structured content or its first text') + ? Result.fail('The answer of the tool holds neither structured content nor text') : Result.flatMap( Result.all({ conversation: partAt(document, sent.conversation, 'conversation'), diff --git a/capabilities/interaction/src/document/call-parsing.test.ts b/capabilities/interaction/src/document/call-parsing.test.ts new file mode 100644 index 000000000..0b9c44aba --- /dev/null +++ b/capabilities/interaction/src/document/call-parsing.test.ts @@ -0,0 +1,136 @@ +import { Result } from 'effect'; +import { describe, expect, it } from 'vitest'; + +import { summarize } from '../capability/definition-reading.ts'; +import { callDocument, replyRuleLines } from '../testing/index.ts'; +import { problemsOf, reading } from '../testing/route-documents.ts'; +import { parseInteractionDocument } from './document-parsing.ts'; + +const pointerWords = + 'Expected a JSON Pointer such as /messages: it starts with a slash, writes ~ as ~0 and a slash within a name as ~1'; + +const asksNoParty = 'A function with call answers at once and asks no party, so it takes no'; + +describe('an interaction function that asks a system', () => { + it('reads its call as written, and is summarised with the tool it calls', () => { + const parsed = Result.getOrThrow(parseInteractionDocument(callDocument())); + + expect(parsed).toMatchObject({ + shape: 'call', + call: { + server: 'chat', + tool: 'thread', + with: { channel: '{{ input.channel }}', ts: '{{ input.thread }}', limit: 100, inclusive: false }, + read: '/messages', + }, + output: { schema: { document: { type: 'array' } } }, + }); + expect(summarize(parsed)).toMatchObject({ + description: 'Read the replies of a thread in the team chat, oldest first', + details: { call: { server: 'chat', tool: 'thread' } }, + }); + }); + + it('takes a call without arguments and without read, which answers with the whole answer', () => { + expect(problemsOf(callDocument({ with: [], read: null }))).toEqual([]); + }); +}); + +describe('the keys a function that asks a system takes', () => { + it('takes a call without a description or an input schema, and refuses an input schema it cannot compile', () => { + const plain = ['---', 'call:', ' server: chat', ' tool: thread', 'output:', ' schema: {}', '---', ''].join('\n'); + + expect(problemsOf(plain)).toEqual([]); + expect(problemsOf(callDocument({ input: ['input:', ' schema: { type: thing }'] }))).toEqual([ + expect.stringMatching(/^Line 13, \/input\/schema/u), + ]); + }); + + it('refuses a key of call it does not hold, with its line', () => { + expect(problemsOf(callDocument({ with: [' channel: C0123'], front: [' limit: 3'] }))).toEqual([ + 'Line 9, /call/limit: limit is not a key of call; it takes server, tool, with, read', + ]); + }); + + it.each([ + ["to: '{{ input.channel }}'", 'to'], + ['expires: P2D', 'expires'], + ['from: ada', 'from'], + ['deliver: { server: chat, tool: post_message, with: {} }', 'deliver'], + ])('refuses %s beside call, at its line', (line, key) => { + expect(problemsOf(callDocument({ front: [line] }))).toContainEqual(`Line 12, /${key}: ${asksNoParty} ${key}`); + }); + + it('refuses replies and a reply rule beside call, at their lines', () => { + const problems = problemsOf(callDocument({ front: [...reading, ...replyRuleLines] })); + + expect(problems).toContainEqual(`Line 13, /replies: ${asksNoParty} replies`); + expect(problems.filter((problem) => problem.endsWith(`${asksNoParty} reply`))).toHaveLength(1); + }); + + it('refuses a body beside call at its first written line, and never reads it as a message', () => { + expect(problemsOf(callDocument({ body: '\n\nHello {{ oops' }))).toEqual([ + 'Line 33: A function with call sends no message: leave the body after the front matter empty, and say what the function does in description', + ]); + }); + + it('refuses a call without an output schema, at its line', () => { + expect(problemsOf(callDocument({ output: [] }))).toEqual([ + 'Line 4, /call: A function with call answers with what the tool answered, so output.schema says what that is', + ]); + }); +}); + +describe('the names and the pointer of a call', () => { + it('refuse a server or a tool of the wrong shape', () => { + expect(problemsOf(callDocument({ server: 'Chat', tool: 'thread replies' }))).toEqual([ + 'Line 4, /call/server: Expected a server name of 1 to 32 lowercase letters, digits and hyphens, starting with a letter', + 'Line 5, /call/tool: Expected a tool name of 1 to 128 letters, digits, underscores, hyphens and dots', + ]); + }); + + it.each([["''"], ['messages'], ['/messages/~']])('refuse %s as read, as every pointer of the format', (read) => { + expect(problemsOf(callDocument({ read }))).toEqual([`Line 11, /call/read: ${pointerWords}`]); + }); +}); + +describe('the arguments of a call', () => { + it('refuse a template that reads a name the call does not have, or a property the input schema does not allow', () => { + const closed = [ + 'input:', + ' schema: { type: object, additionalProperties: false, properties: { channel: { type: string } } }', + ]; + + expect([ + problemsOf(callDocument({ with: [" channel: '{{ to }}'"] })), + problemsOf(callDocument({ with: [" channel: '{{ input.thread }}'"], input: closed })), + ]).toEqual([ + ['Line 7, /call/with/channel: Reads to, which a template of a call does not have; it reads input, today and now'], + ['Line 7, /call/with/channel: input.thread is not a property of the input schema, which allows no others'], + ]); + }); + + it('refuse a secret, and a text that renders a structure among text without reading the input', () => { + expect([ + problemsOf(callDocument({ with: [" channel: '${CHAT_KEY}'"] })), + problemsOf(callDocument({ with: [' channel: \'In {{ "a,b" | split: "," }}\''] })), + ]).toEqual([ + ['Line 7, /call/with/channel: Expected no ${ in a template, which never holds a secret'], + [ + 'Line 7, /call/with/channel: Renders a value that is not text among text; write | json after it, or write it alone as one {{ }} to send it as it is', + ], + ]); + }); + + it('refuse arguments that read no input and take more than 16 KiB as JSON, or one that does alone', () => { + const half = 'x'.repeat(8200); + + expect([ + problemsOf(callDocument({ with: [` a: ${half}`, ` b: ${half}`] })), + problemsOf(callDocument({ with: [` note: ${half}${half}`] })), + ]).toEqual([ + ['Line 7, /call/with: The arguments of the call take 16415 bytes, more than the 16384 a call may send'], + ['Line 7, /call/with/note: The argument note of the call renders more than the 16384 bytes a call may send'], + ]); + }); +}); diff --git a/capabilities/interaction/src/document/call-parts.ts b/capabilities/interaction/src/document/call-parts.ts new file mode 100644 index 000000000..53c283b0a --- /dev/null +++ b/capabilities/interaction/src/document/call-parts.ts @@ -0,0 +1,67 @@ +import { issueAt, type DocumentIssue, type DocumentParts, type SourceLines } from '@beonauto/definitions/document'; +import { Predicate, Result } from 'effect'; + +import { compiledCallBlock } from '../tool-blocks/compiled-tool-block.ts'; +import type { CallBlock } from '../tool-blocks/tool-block-schemas.ts'; +import type { InteractionFrontMatter } from './front-matter.ts'; +import type { CallDocument } from './interaction-document.ts'; +import { contractOf, issuesOf, type Checked } from './written-parts.ts'; + +const writesACall = /^call:/mu; + +const keysOfARequest = ['to', 'expires', 'deliver', 'replies', 'from', 'reply'] as const; + +const noMessage = + 'A function with call sends no message: leave the body after the front matter empty, and say what the function does in description'; + +const outputNeeded = 'A function with call answers with what the tool answered, so output.schema says what that is'; + +export function asksASystem(root: unknown): boolean { + return Predicate.hasProperty(root, 'call'); +} + +export function seemsToAskASystem({ frontMatter }: Pick): boolean { + return writesACall.test(frontMatter); +} + +function requestKeyIssues(written: InteractionFrontMatter, lines: SourceLines): readonly DocumentIssue[] { + return keysOfARequest + .filter((key) => written[key] !== undefined) + .map((key) => + issueAt(lines, `/${key}`, `A function with call answers at once and asks no party, so it takes no ${key}`), + ); +} + +function bodyIssues({ body, bodyLine }: DocumentParts): readonly DocumentIssue[] { + const firstWritten = body.split('\n').findIndex((line) => line.trim() !== ''); + return firstWritten === -1 ? [] : [{ line: bodyLine + firstWritten, pointer: '', detail: noMessage }]; +} + +export function callFrom( + written: InteractionFrontMatter, + call: CallBlock, + lines: SourceLines, + parts: DocumentParts, +): Checked { + const input = contractOf(written.input, 'input', lines); + const output = contractOf(written.output, 'output', lines); + const compiled = compiledCallBlock(call, lines, Result.isSuccess(input) ? input.success.schema?.document : undefined); + const answered = Result.flatMap(output, ({ schema }) => + schema === undefined ? Result.fail([issueAt(lines, '/call', outputNeeded)]) : Result.succeed({ schema }), + ); + const issues = [ + ...requestKeyIssues(written, lines), + ...bodyIssues(parts), + ...issuesOf(() => compiled), + ...issuesOf(() => input), + ...issuesOf(() => answered), + ]; + return issues.length > 0 + ? Result.fail(issues) + : Result.map(Result.all({ input, output: answered }), (contracts: Pick) => ({ + shape: 'call', + ...(written.description === undefined ? {} : { description: written.description }), + call, + ...contracts, + })); +} diff --git a/capabilities/interaction/src/document/document-parsing.test.ts b/capabilities/interaction/src/document/document-parsing.test.ts index 7e8fb7fc5..f068255f8 100644 --- a/capabilities/interaction/src/document/document-parsing.test.ts +++ b/capabilities/interaction/src/document/document-parsing.test.ts @@ -3,6 +3,7 @@ import { Result } from 'effect'; import { describe, expect, it } from 'vitest'; import { approvalDocument, notificationDocument } from '../testing/index.ts'; +import { requestDocumentOf } from '../testing/route-documents.ts'; import { parseInteractionDocument } from './document-parsing.ts'; function problemsOf(source: string): readonly string[] { @@ -16,11 +17,11 @@ function documentWith(frontMatter: readonly string[], body = 'Approve {{ input.c const essentials = ["to: '{{ input.owner }}'", 'expires: P2D']; -const keys = 'description, to, from, expires, deliver, replies, input, output, reply'; +const keys = 'description, call, to, from, expires, deliver, replies, input, output, reply'; describe('an interaction function definition', () => { it('reads its party, expiry, schemas and message, and waits in the inbox without deliver', () => { - const parsed = Result.getOrThrow(parseInteractionDocument(approvalDocument())); + const parsed = requestDocumentOf(approvalDocument()); expect(parsed).toMatchObject({ description: 'Ask the campaign owner to approve a brief', @@ -48,7 +49,7 @@ describe('an interaction function definition', () => { }); function expiresOf(expires: string): number { - return Result.getOrThrow(parseInteractionDocument(documentWith(["to: 'x'", `expires: ${expires}`]))).expiresMs; + return requestDocumentOf(documentWith(["to: 'x'", `expires: ${expires}`])).expiresMs; } describe('the expiry of a definition', () => { @@ -71,6 +72,27 @@ describe('the expiry of a definition', () => { }); }); +describe('a definition that names neither the tool it calls nor the party it asks', () => { + it('refuses a request without its party or its expiry, and an empty front matter, in words that name call', () => { + expect([ + problemsOf(documentWith(['description: Ask someone'])), + problemsOf(documentWith(["to: '{{ input.owner }}'"])), + problemsOf('---\n---\nHello'), + ]).toEqual([ + [ + 'Line 2, /to: to is required, the party the request goes to, unless the function asks a system with call', + 'Line 2, /expires: expires is required, how long the request waits, unless the function asks a system with call', + ], + [ + 'Line 2, /expires: expires is required, how long the request waits, unless the function asks a system with call', + ], + [ + 'Line 2: The front matter is empty; it names at least the tool it calls, or the party it goes to and when it expires', + ], + ]); + }); +}); + describe('the refusals of a definition', () => { it('refuse a party or a message that does not compile, or reads what a request does not have', () => { expect([ diff --git a/capabilities/interaction/src/document/document-parsing.ts b/capabilities/interaction/src/document/document-parsing.ts index 3d2b74055..23324d71e 100644 --- a/capabilities/interaction/src/document/document-parsing.ts +++ b/capabilities/interaction/src/document/document-parsing.ts @@ -6,102 +6,37 @@ import { type DocumentParts, type ReadFrontMatter, } from '@beonauto/definitions/document'; -import type { ParsedTemplate } from '@beonauto/definitions/template'; import { Result } from 'effect'; -import type { ReplyRule } from '../replies/reply-rule.ts'; -import type { WrittenRoute } from '../route/compiled-route.ts'; -import { decodeFrontMatter, interactionFrontMatter, type InteractionFrontMatter } from './front-matter.ts'; -import type { InteractionFunctionDefinitionDocument, ValueContract } from './interaction-document.ts'; -import { fromOf, replyOf } from './reply-parts.ts'; -import { routeIn } from './route-parts.ts'; -import { contractOf, expiresOf, messageOf, toOf, type Checked } from './written-parts.ts'; +import { asksASystem, callFrom, seemsToAskASystem } from './call-parts.ts'; +import { decodeFrontMatter, interactionFrontMatter } from './front-matter.ts'; +import type { InteractionFunctionDefinitionDocument } from './interaction-document.ts'; +import { requestFrom } from './request-parts.ts'; +import { messageOf, type Checked } from './written-parts.ts'; -function issuesOf(check: () => Checked): readonly DocumentIssue[] { - const checked = check(); - return Result.isFailure(checked) ? checked.failure : []; -} - -type CheckedParts = { - readonly route: Checked; - readonly expires: Checked; - readonly input: Checked; - readonly output: Checked; - readonly to: Checked; - readonly from: Checked; - readonly message: Checked; - readonly reply: Checked; -}; - -function partsOf(written: InteractionFrontMatter, { lines }: ReadFrontMatter, parts: DocumentParts): CheckedParts { - const input = contractOf(written.input, 'input', lines); - const output = contractOf(written.output, 'output', lines); - const inputSchema = Result.isSuccess(input) ? input.success.schema?.document : undefined; - return { - route: routeIn(written, lines, inputSchema), - expires: expiresOf(written.expires, lines), - input, - output, - to: toOf(written.to, lines, inputSchema), - from: fromOf(written.from, lines, inputSchema), - message: messageOf(parts, inputSchema), - reply: Result.isFailure(output) - ? Result.succeed(null) - : replyOf( - { - reply: written.reply, - answerSchema: output.success.schema?.document, - readsReplies: written.replies !== undefined, - }, - lines, - ), - }; -} - -type DoneParts = { readonly [Part in keyof CheckedParts]: Result.Result.Success }; - -function documentOfParts(written: InteractionFrontMatter, done: DoneParts): InteractionFunctionDefinitionDocument { - const { route, expires, input, output, to, from, message, reply } = done; - return { - ...(written.description === undefined ? {} : { description: written.description }), - ...(route === null ? {} : { route }), - to, - ...(from === null ? {} : { from }), - expires: written.expires, - expiresMs: expires, - input, - output, - message, - ...(reply === null ? {} : { reply }), - }; +function bodyIssuesOf(parts: DocumentParts, callsATool: boolean): readonly DocumentIssue[] { + if (callsATool) { + return []; + } + const message = messageOf(parts); + return Result.isFailure(message) ? message.failure : []; } function documentFrom(reading: ReadFrontMatter, parts: DocumentParts): Checked { const decoded = decodeFrontMatter(reading.root); if (reading.issues.length > 0 || Result.isFailure(decoded)) { - return Result.fail([...reading.issues, ...issuesOf(() => messageOf(parts))]); + return Result.fail([...reading.issues, ...bodyIssuesOf(parts, asksASystem(reading.root))]); } const written = decoded.success; - const checked = partsOf(written, reading, parts); - const found = [ - ...issuesOf(() => checked.route), - ...issuesOf(() => checked.expires), - ...issuesOf(() => checked.input), - ...issuesOf(() => checked.output), - ...issuesOf(() => checked.to), - ...issuesOf(() => checked.from), - ...issuesOf(() => checked.message), - ...issuesOf(() => checked.reply), - ]; - return found.length > 0 - ? Result.fail(found) - : Result.map(Result.all(checked), (done) => documentOfParts(written, done)); + return written.call === undefined + ? requestFrom(written, reading, parts) + : callFrom(written, written.call, reading.lines, parts); } function documentOf(parts: DocumentParts): Checked { const reading = frontMatterIn(parts.frontMatter, parts.frontMatterLine, interactionFrontMatter); return Result.isFailure(reading) - ? Result.fail([...reading.failure, ...issuesOf(() => messageOf(parts))]) + ? Result.fail([...reading.failure, ...bodyIssuesOf(parts, seemsToAskASystem(parts))]) : documentFrom(reading.success, parts); } diff --git a/capabilities/interaction/src/document/front-matter.ts b/capabilities/interaction/src/document/front-matter.ts index 88c7af4e0..6b792eb8a 100644 --- a/capabilities/interaction/src/document/front-matter.ts +++ b/capabilities/interaction/src/document/front-matter.ts @@ -3,13 +3,14 @@ import { Schema } from 'effect'; import { WrittenRuleSchema } from '../replies/reply-rule.ts'; import { + CallBlockSchema, + DeliverBlockSchema, EachSchema, ReadSchema, RepliesSchema, SentSchema, TellSchema, - ToolDeliverySchema, -} from '../route/route-schemas.ts'; +} from '../tool-blocks/tool-block-schemas.ts'; const descriptionLength = 1000; @@ -17,10 +18,11 @@ const SectionSchema = Schema.Struct({ schema: Schema.optionalKey(Schema.JsonObje const FrontMatterSchema = Schema.Struct({ description: Schema.optionalKey(Schema.String.check(Schema.isMinLength(1), Schema.isMaxLength(descriptionLength))), - to: Schema.String, + call: Schema.optionalKey(CallBlockSchema), + to: Schema.optionalKey(Schema.String), from: Schema.optionalKey(Schema.String), - expires: Schema.String, - deliver: Schema.optionalKey(ToolDeliverySchema), + expires: Schema.optionalKey(Schema.String), + deliver: Schema.optionalKey(DeliverBlockSchema), replies: Schema.optionalKey(RepliesSchema), input: Schema.optionalKey(SectionSchema), output: Schema.optionalKey(SectionSchema), @@ -33,7 +35,8 @@ const sections: readonly FrontMatterSection[] = [ { name: undefined, keys: Object.keys(FrontMatterSchema.fields) }, { name: 'input', keys: Object.keys(SectionSchema.fields) }, { name: 'output', keys: Object.keys(SectionSchema.fields) }, - { name: 'deliver', keys: Object.keys(ToolDeliverySchema.fields) }, + { name: 'call', keys: Object.keys(CallBlockSchema.fields) }, + { name: 'deliver', keys: Object.keys(DeliverBlockSchema.fields) }, { name: 'deliver.sent', keys: Object.keys(SentSchema.fields) }, { name: 'replies', keys: Object.keys(RepliesSchema.fields) }, { name: 'replies.read', keys: Object.keys(ReadSchema.fields) }, @@ -46,5 +49,5 @@ export const decodeFrontMatter = Schema.decodeUnknownResult(FrontMatterSchema, { export const interactionFrontMatter: FrontMatterShape = { sections, decode: decodeFrontMatter, - required: 'the party it goes to and when it expires', + required: 'the tool it calls, or the party it goes to and when it expires', }; diff --git a/capabilities/interaction/src/document/interaction-document.ts b/capabilities/interaction/src/document/interaction-document.ts index 4d333d10e..a21663e4e 100644 --- a/capabilities/interaction/src/document/interaction-document.ts +++ b/capabilities/interaction/src/document/interaction-document.ts @@ -2,21 +2,34 @@ import type { CompiledSchema } from '@beonauto/definitions/document'; import type { ParsedTemplate } from '@beonauto/definitions/template'; import type { ReplyRule } from '../replies/reply-rule.ts'; -import type { WrittenRoute } from '../route/compiled-route.ts'; +import type { WrittenRoute } from '../tool-blocks/compiled-tool-block.ts'; +import type { CallBlock } from '../tool-blocks/tool-block-schemas.ts'; export interface ValueContract { readonly schema?: CompiledSchema; } -export interface InteractionFunctionDefinitionDocument { +interface WrittenFunction { readonly description?: string; + readonly input: ValueContract; +} + +export interface CallDocument extends WrittenFunction { + readonly shape: 'call'; + readonly call: CallBlock; + readonly output: { readonly schema: CompiledSchema }; +} + +export interface RequestDocument extends WrittenFunction { + readonly shape: 'request'; + readonly output: ValueContract; readonly to: ParsedTemplate; readonly from?: ParsedTemplate; readonly expires: string; readonly expiresMs: number; - readonly input: ValueContract; - readonly output: ValueContract; readonly message: ParsedTemplate; readonly route?: WrittenRoute; readonly reply?: ReplyRule; } + +export type InteractionFunctionDefinitionDocument = CallDocument | RequestDocument; diff --git a/capabilities/interaction/src/document/request-parts.ts b/capabilities/interaction/src/document/request-parts.ts new file mode 100644 index 000000000..286d2eccc --- /dev/null +++ b/capabilities/interaction/src/document/request-parts.ts @@ -0,0 +1,110 @@ +import { issueAt, type DocumentParts, type ReadFrontMatter, type SourceLines } from '@beonauto/definitions/document'; +import type { ParsedTemplate } from '@beonauto/definitions/template'; +import { Result } from 'effect'; + +import type { ReplyRule } from '../replies/reply-rule.ts'; +import type { WrittenRoute } from '../tool-blocks/compiled-tool-block.ts'; +import type { InteractionFrontMatter } from './front-matter.ts'; +import type { RequestDocument, ValueContract } from './interaction-document.ts'; +import { fromOf, replyOf } from './reply-parts.ts'; +import { routeIn } from './route-parts.ts'; +import { contractOf, expiresOf, issuesOf, messageOf, toOf, type Checked } from './written-parts.ts'; + +interface Asked { + readonly to: string; + readonly expires: string; +} + +type CheckedParts = { + readonly route: Checked; + readonly expires: Checked; + readonly input: Checked; + readonly output: Checked; + readonly to: Checked; + readonly from: Checked; + readonly message: Checked; + readonly reply: Checked; +}; + +function partsOf( + written: InteractionFrontMatter, + asked: Asked, + lines: SourceLines, + parts: DocumentParts, +): CheckedParts { + const input = contractOf(written.input, 'input', lines); + const output = contractOf(written.output, 'output', lines); + const inputSchema = Result.isSuccess(input) ? input.success.schema?.document : undefined; + return { + route: routeIn(written, lines, inputSchema), + expires: expiresOf(asked.expires, lines), + input, + output, + to: toOf(asked.to, lines, inputSchema), + from: fromOf(written.from, lines, inputSchema), + message: messageOf(parts, inputSchema), + reply: Result.isFailure(output) + ? Result.succeed(null) + : replyOf( + { + reply: written.reply, + answerSchema: output.success.schema?.document, + readsReplies: written.replies !== undefined, + }, + lines, + ), + }; +} + +type DoneParts = { readonly [Part in keyof CheckedParts]: Result.Result.Success }; + +function documentOfParts(written: InteractionFrontMatter, expires: string, done: DoneParts): RequestDocument { + const { route, input, output, to, from, message, reply } = done; + return { + shape: 'request', + ...(written.description === undefined ? {} : { description: written.description }), + ...(route === null ? {} : { route }), + to, + ...(from === null ? {} : { from }), + expires, + expiresMs: done.expires, + input, + output, + message, + ...(reply === null ? {} : { reply }), + }; +} + +const unaskedWords = { + to: 'to is required, the party the request goes to, unless the function asks a system with call', + expires: 'expires is required, how long the request waits, unless the function asks a system with call', +} as const; + +export function requestFrom( + written: InteractionFrontMatter, + { lines }: Pick, + parts: DocumentParts, +): Checked { + const { to, expires } = written; + if (to === undefined || expires === undefined) { + const unasked = (['to', 'expires'] as const).filter((key) => written[key] === undefined); + return Result.fail([ + ...unasked.map((key) => issueAt(lines, `/${key}`, unaskedWords[key])), + ...issuesOf(() => messageOf(parts)), + ]); + } + const checked = partsOf(written, { to, expires }, lines, parts); + const found = [ + ...issuesOf(() => checked.route), + ...issuesOf(() => checked.expires), + ...issuesOf(() => checked.input), + ...issuesOf(() => checked.output), + ...issuesOf(() => checked.to), + ...issuesOf(() => checked.from), + ...issuesOf(() => checked.message), + ...issuesOf(() => checked.reply), + ]; + return found.length > 0 + ? Result.fail(found) + : Result.map(Result.all(checked), (done) => documentOfParts(written, expires, done)); +} diff --git a/capabilities/interaction/src/document/route-parts.ts b/capabilities/interaction/src/document/route-parts.ts index 21a4aa2eb..c903b71c8 100644 --- a/capabilities/interaction/src/document/route-parts.ts +++ b/capabilities/interaction/src/document/route-parts.ts @@ -1,7 +1,7 @@ import { issueAt, type DocumentIssue, type SourceLines } from '@beonauto/definitions/document'; import { Result, type Schema } from 'effect'; -import { compiledRoute, type WrittenRoute } from '../route/compiled-route.ts'; +import { compiledToolBlock, type WrittenRoute } from '../tool-blocks/compiled-tool-block.ts'; import type { InteractionFrontMatter } from './front-matter.ts'; type Checked = Result.Result; @@ -31,7 +31,7 @@ export function routeIn( return replies === undefined ? Result.succeed(null) : Result.fail([issueAt(lines, '/replies', inboxReadsNothing)]); } const shape = replies === undefined ? [] : readingIssues(written, lines); - const compiled = compiledRoute({ deliver, replies }, lines, inputSchema); + const compiled = compiledToolBlock({ deliver, replies }, lines, inputSchema); if (shape.length > 0) { return Result.fail([...shape, ...(Result.isFailure(compiled) ? compiled.failure : [])]); } diff --git a/capabilities/interaction/src/document/written-parts.ts b/capabilities/interaction/src/document/written-parts.ts index 669ea67e7..9f9e65682 100644 --- a/capabilities/interaction/src/document/written-parts.ts +++ b/capabilities/interaction/src/document/written-parts.ts @@ -15,6 +15,11 @@ import { compiledTemplate } from './request-templates.ts'; export type Checked = Result.Result; +export function issuesOf(check: () => Checked): readonly DocumentIssue[] { + const checked = check(); + return Result.isFailure(checked) ? checked.failure : []; +} + type ValueSection = { readonly schema?: Schema.JsonObject } | undefined; export function messageOf({ body, bodyLine }: DocumentParts, inputSchema?: Schema.JsonObject): Checked { diff --git a/capabilities/interaction/src/replies/reply-recording.ts b/capabilities/interaction/src/replies/reply-recording.ts index e276268cd..a06a8b17d 100644 --- a/capabilities/interaction/src/replies/reply-recording.ts +++ b/capabilities/interaction/src/replies/reply-recording.ts @@ -2,7 +2,7 @@ import { Result } from 'effect'; import type { ReadingRoute } from '../conversations/conversation-parts.ts'; import type { ConversationRow } from '../conversations/conversation-rows.ts'; -import { renderedArguments } from '../route/rendered-arguments.ts'; +import { renderedArguments } from '../tool-blocks/rendered-arguments.ts'; import type { KeptRequest } from './reply-taking.ts'; export interface ReadingState { diff --git a/capabilities/interaction/src/replies/rule-checks.test.ts b/capabilities/interaction/src/replies/rule-checks.test.ts index c2b8169ae..19165c454 100644 --- a/capabilities/interaction/src/replies/rule-checks.test.ts +++ b/capabilities/interaction/src/replies/rule-checks.test.ts @@ -12,7 +12,7 @@ function problemsOf(source: string): readonly string[] { function ruleOf(source: string) { const parsed = parseInteractionDocument(source); - return Result.isSuccess(parsed) ? parsed.success.reply : 'refused'; + return Result.isSuccess(parsed) && parsed.success.shape === 'request' ? parsed.success.reply : 'refused'; } function withAnswer(schema: readonly string[], rule: readonly string[] = []): string { diff --git a/capabilities/interaction/src/requests/request-rows.ts b/capabilities/interaction/src/requests/request-rows.ts index 8a282a53f..ec76423ec 100644 --- a/capabilities/interaction/src/requests/request-rows.ts +++ b/capabilities/interaction/src/requests/request-rows.ts @@ -1,8 +1,8 @@ import type { ProjectedRow } from '@beonauto/operations'; import { Option, Schema } from 'effect'; -import { RepliesSchema } from '../route/route-schemas.ts'; -import { routeOf, type Route } from '../route/routes.ts'; +import { routeOf, type Route } from '../tool-blocks/routes.ts'; +import { RepliesSchema } from '../tool-blocks/tool-block-schemas.ts'; export const StandingSchema = Schema.Literals([ 'in_inbox', diff --git a/capabilities/interaction/src/run/interaction-run.ts b/capabilities/interaction/src/run/interaction-run.ts index a597c9b66..589071bae 100644 --- a/capabilities/interaction/src/run/interaction-run.ts +++ b/capabilities/interaction/src/run/interaction-run.ts @@ -2,8 +2,9 @@ import type { CapabilityAnswer, CapabilityRejection, RunContext } from '@beonaut import type { Conflict, InvalidInput } from '@beonauto/operations'; import { Clock, Effect, type Schema } from 'effect'; -import type { InteractionFunctionDefinitionDocument } from '../document/interaction-document.ts'; -import { routeOf, toolsOf } from '../route/routes.ts'; +import type { RequestDocument } from '../document/interaction-document.ts'; +import { momentVariables } from '../tool-blocks/rendered-arguments.ts'; +import { routeOf, toolsOf } from '../tool-blocks/routes.ts'; import { checkedArguments, offered, roomFor, type InteractionPorts } from './request-reach.ts'; import type { RequestRecord } from './request-record.ts'; import { checkedAnswerer, checkedParty, fromPart, messagePart, renderedPart, toPart } from './request-rendering.ts'; @@ -12,7 +13,7 @@ import { preparedInput } from './run-input.ts'; type Variables = Readonly>; function answeringOf( - document: InteractionFunctionDefinitionDocument, + document: RequestDocument, variables: Variables, to: string, ): Effect.Effect, Conflict | InvalidInput> { @@ -27,13 +28,13 @@ function answeringOf( } function requestOf( - document: InteractionFunctionDefinitionDocument, + document: RequestDocument, input: Schema.Json, ): Effect.Effect { return Effect.gen(function* () { const at = yield* Clock.currentTimeMillis; const now = new Date(at).toISOString(); - const variables = { input, today: now.slice(0, 10), now }; + const variables = momentVariables(input, now); const to = yield* Effect.flatMap(renderedPart({ ...toPart, template: document.to }, variables), checkedParty); const message = yield* renderedPart({ ...messagePart, template: document.message }, variables); const answerSchema = document.output.schema?.document; @@ -54,13 +55,13 @@ function requestOf( }); } -export function finishesLater(document: InteractionFunctionDefinitionDocument): boolean { +export function asksLater(document: RequestDocument): boolean { return document.route !== undefined || document.output.schema !== undefined; } -export function interactionRun(ports: InteractionPorts) { +export function requestRun(ports: InteractionPorts) { return ( - document: InteractionFunctionDefinitionDocument, + document: RequestDocument, input: Schema.Json, context: RunContext, ): Effect.Effect => @@ -68,7 +69,7 @@ export function interactionRun(ports: InteractionPorts) { const admitted = yield* preparedInput(input, document.input); const brain = { org: context.org, brain: context.brain }; yield* offered(toolsOf(routeOf(document.route ?? {})), ports, brain); - if (!finishesLater(document)) { + if (!asksLater(document)) { const notified = yield* requestOf(document, admitted); return { output: {}, record: notified }; } diff --git a/capabilities/interaction/src/run/request-reach.ts b/capabilities/interaction/src/run/request-reach.ts index 2eeb584cf..5a0de29b9 100644 --- a/capabilities/interaction/src/run/request-reach.ts +++ b/capabilities/interaction/src/run/request-reach.ts @@ -3,12 +3,12 @@ import type { ToolReference } from '@beonauto/mcp/policy'; import { Unavailable, type BrainAddress, type Conflict } from '@beonauto/operations'; import { Effect, Result, type Schema } from 'effect'; -import { deliveryVariablesOf, renderedArguments, type ArgumentsFailure } from '../route/rendered-arguments.ts'; +import { deliveryVariablesOf, renderedArguments, type ArgumentsFailure } from '../tool-blocks/rendered-arguments.ts'; import type { RequestRecord } from './request-record.ts'; import { unworkable } from './request-rendering.ts'; export interface InteractionPorts { - readonly tools: Pick; + readonly tools: Pick; readonly openRequests: (brain: BrainAddress) => Effect.Effect; readonly mostOpenRequests: number; } diff --git a/capabilities/interaction/src/run/request-record.ts b/capabilities/interaction/src/run/request-record.ts index 729e9a5aa..831203dd0 100644 --- a/capabilities/interaction/src/run/request-record.ts +++ b/capabilities/interaction/src/run/request-record.ts @@ -1,7 +1,7 @@ import { Option, Schema } from 'effect'; import { ReplyRuleSchema } from '../replies/reply-rule.ts'; -import { RepliesSchema, ToolDeliverySchema } from '../route/route-schemas.ts'; +import { RepliesSchema, DeliverBlockSchema } from '../tool-blocks/tool-block-schemas.ts'; const RequestRecordSchema = Schema.Struct({ to: Schema.String, @@ -11,13 +11,13 @@ const RequestRecordSchema = Schema.Struct({ reply: Schema.optionalKey(ReplyRuleSchema), expires_at: Schema.String, requested_at: Schema.String, - deliver: Schema.optionalKey(ToolDeliverySchema), + deliver: Schema.optionalKey(DeliverBlockSchema), replies: Schema.optionalKey(RepliesSchema), }); export type RequestRecord = typeof RequestRecordSchema.Type; -const DeliveringRecordSchema = Schema.Struct({ ...RequestRecordSchema.fields, deliver: ToolDeliverySchema }); +const DeliveringRecordSchema = Schema.Struct({ ...RequestRecordSchema.fields, deliver: DeliverBlockSchema }); export type DeliveringRecord = typeof DeliveringRecordSchema.Type; @@ -33,3 +33,14 @@ export function requestRecordOf(record: unknown): RequestRecord | undefined { export function takesAnswer({ answer_schema: schema }: Pick): boolean { return schema !== undefined; } + +const DeliveredRecordSchema = Schema.Struct({ delivered_at: Schema.String }); + +export type DeliveredRecord = typeof DeliveredRecordSchema.Type; + +const isDeliveredRecord = Schema.is(DeliveredRecordSchema); + +export function isNotification(record: unknown): boolean { + const request = requestRecordOf(record); + return request === undefined ? isDeliveredRecord(record) : !takesAnswer(request); +} diff --git a/capabilities/interaction/src/run/request-rendering.ts b/capabilities/interaction/src/run/request-rendering.ts index b639bf89f..d19c1ff83 100644 --- a/capabilities/interaction/src/run/request-rendering.ts +++ b/capabilities/interaction/src/run/request-rendering.ts @@ -35,16 +35,10 @@ function invalidWith(detail: string, issue: string, pointer = ''): InvalidInput return new InvalidInput({ detail, issues: [{ pointer, detail: issue }] }); } -function rejectionOf({ pointer, what, mostBytes }: RenderedPart, failure: TextFailure): Conflict | InvalidInput { - if (failure.reason === 'not_text') { - return unworkable( - pointer, - `${what} renders a value that is not text on line ${failure.line}; render a field that is text, or write | json after it`, - ); - } - if (failure.reason === 'too_long') { - return unworkable(pointer, `${what} renders to more than the ${mostBytes} bytes a request may hold`); - } +export function unrenderableFor( + what: string, + failure: Exclude, +): InvalidInput { if (failure.reason === 'missing_variable') { return invalidWith( 'The definition reads a field the input does not have', @@ -58,6 +52,19 @@ function rejectionOf({ pointer, what, mostBytes }: RenderedPart, failure: TextFa ); } +function rejectionOf({ pointer, what, mostBytes }: RenderedPart, failure: TextFailure): Conflict | InvalidInput { + if (failure.reason === 'not_text') { + return unworkable( + pointer, + `${what} renders a value that is not text on line ${failure.line}; render a field that is text, or write | json after it`, + ); + } + if (failure.reason === 'too_long') { + return unworkable(pointer, `${what} renders to more than the ${mostBytes} bytes a request may hold`); + } + return unrenderableFor(what, failure); +} + export function renderedPart( part: RenderedPart, variables: Readonly>, diff --git a/capabilities/interaction/src/schedule/request-edges.test.ts b/capabilities/interaction/src/schedule/request-edges.test.ts index 728de3367..b52583e41 100644 --- a/capabilities/interaction/src/schedule/request-edges.test.ts +++ b/capabilities/interaction/src/schedule/request-edges.test.ts @@ -37,7 +37,7 @@ describe('a notification whose attempts fail', () => { reason: 'unanswered', kind: 'undelivered', detail: - 'The notification could not be delivered through the tool post_message of chat, though every attempt was made', + 'The notification could not be delivered through the post message tool of chat, though every attempt was made', }, }, }); diff --git a/capabilities/interaction/src/schedule/request-endings.ts b/capabilities/interaction/src/schedule/request-endings.ts index a16fed441..cc0791bc2 100644 --- a/capabilities/interaction/src/schedule/request-endings.ts +++ b/capabilities/interaction/src/schedule/request-endings.ts @@ -2,7 +2,8 @@ import type { ReplyIdentity, Settlement } from '@beonauto/definitions'; import { brainCallerOf, type BrainAddress } from '@beonauto/operations'; import type { Schema } from 'effect'; -import { throughWords, type Route } from '../route/routes.ts'; +import type { DeliveredRecord } from '../run/request-record.ts'; +import { throughWords, type Route } from '../tool-blocks/routes.ts'; export function expiredSettlement({ expires_at: expiresAt }: { readonly expires_at: number }): Settlement { return { @@ -25,7 +26,8 @@ export function answeredSettlement(brain: BrainAddress, { answer, at, reply }: B } export function deliveredSettlement(at: string): Settlement { - return { status: 'succeeded', output: {}, record: { delivered_at: at } }; + const record: DeliveredRecord = { delivered_at: at }; + return { status: 'succeeded', output: {}, record }; } export function undeliveredSettlement(route: Route): Settlement { diff --git a/capabilities/interaction/src/testing/call-documents.ts b/capabilities/interaction/src/testing/call-documents.ts new file mode 100644 index 000000000..c38c35311 --- /dev/null +++ b/capabilities/interaction/src/testing/call-documents.ts @@ -0,0 +1,75 @@ +export interface CallOptions { + readonly server?: string; + readonly tool?: string; + readonly read?: string | null; + readonly with?: readonly string[]; + readonly front?: readonly string[]; + readonly input?: readonly string[]; + readonly output?: readonly string[]; + readonly body?: string; +} + +const threadArguments: readonly string[] = [ + " channel: '{{ input.channel }}'", + " ts: '{{ input.thread }}'", + ' limit: 100', + ' inclusive: false', +]; + +const threadInput: readonly string[] = [ + 'input:', + ' schema:', + ' type: object', + ' required: [channel, thread]', + ' properties:', + ' channel: { type: string }', + ' thread: { type: string }', +]; + +export const repliesOutput: readonly string[] = [ + 'output:', + ' schema:', + ' type: array', + ' maxItems: 100', + ' items:', + ' type: object', + ' required: [user, text, ts]', + ' properties:', + ' user: { type: string }', + ' text: { type: string }', + ' ts: { type: string }', +]; + +function argumentsOf(written: readonly string[]): readonly string[] { + return written.length === 0 ? [] : [' with:', ...written]; +} + +function readOf(read: string | null): readonly string[] { + return read === null ? [] : [` read: ${read}`]; +} + +export function callDocument({ + server = 'chat', + tool = 'thread', + read = '/messages', + with: written = threadArguments, + front = [], + input = threadInput, + output = repliesOutput, + body = '', +}: CallOptions = {}): string { + return [ + '---', + 'description: Read the replies of a thread in the team chat, oldest first', + 'call:', + ` server: ${server}`, + ` tool: ${tool}`, + ...argumentsOf(written), + ...readOf(read), + ...front, + ...input, + ...output, + '---', + body, + ].join('\n'); +} diff --git a/capabilities/interaction/src/testing/call-runs.ts b/capabilities/interaction/src/testing/call-runs.ts new file mode 100644 index 000000000..e7125479a --- /dev/null +++ b/capabilities/interaction/src/testing/call-runs.ts @@ -0,0 +1,67 @@ +import type { CapabilityAnswer, CapabilityRejection, RunContext } from '@beonauto/definitions'; +import { noLongestRuns, recordingJournal, type RecordingJournal } from '@beonauto/definitions/testing'; +import type { Timing } from '@beonauto/mcp'; +import { reportingAccess, serveFakeMcp, type FakeMcpOptions, type FakeMcpServer } from '@beonauto/mcp/testing'; +import { allPermissions } from '@beonauto/operations'; +import { Effect, Function, type Exit, type Schema } from 'effect'; +import { onTestFinished } from 'vitest'; + +import { makeInteractionFunctionAdapter } from '../capability/interaction-function.ts'; + +export const callRunId = '0199a3c4-7d2e-7c1a-9b3f-2f1e0d9c8b7a'; + +const chatCallKey = 'chat-api-key-5a1c9e27'; + +const threadInput = { channel: 'C0123', thread: '1728379900.000050' }; + +export interface CallRunOptions { + readonly server?: FakeMcpOptions; + readonly entry?: Readonly>; + readonly timing?: Timing; + readonly journal?: RecordingJournal; +} + +export interface CallRuns { + readonly fake: FakeMcpServer; + readonly journal: RecordingJournal; + readonly run: (source: string, input?: Schema.Json) => Promise>; +} + +function contextOf(journal: RecordingJournal): RunContext { + return { + id: callRunId, + org: 'acme', + brain: 'alpha', + caller: { id: 'acme-admin', org: 'acme', permissions: allPermissions, brains: '*' }, + definition: { name: 'thread-replies', version: 1 }, + journal, + lineage: { startId: '5d0e9f6a-1b2c-5d3e-8f4a-6b7c8d9e0f1a', correlationId: callRunId }, + depth: 0, + callDepth: 0, + longestRunOf: noLongestRuns, + }; +} + +export async function callRuns(options: CallRunOptions = {}): Promise { + const fake = await serveFakeMcp({ bearer: chatCallKey, data: true, ...options.server }); + onTestFinished(fake.close); + const chat = { url: fake.url, headers: { Authorization: 'Bearer ${CHAT_KEY}' }, org: 'acme', ...options.entry }; + const { access } = reportingAccess( + { chat }, + { environment: { CHAT_KEY: chatCallKey }, ...(options.timing === undefined ? {} : { timing: options.timing }) }, + ); + onTestFinished(access.close); + const capability = makeInteractionFunctionAdapter({ + tools: access, + openRequests: Function.constant(Effect.succeed(0)), + mostOpenRequests: 10, + }); + const journal = options.journal ?? recordingJournal(); + const context = contextOf(journal); + return { + fake, + journal, + run: (source, input = threadInput) => + Effect.runPromiseExit(Effect.flatMap(capability.prepare(source), (prepared) => prepared.run(input, context))), + }; +} diff --git a/capabilities/interaction/src/testing/chat-board.ts b/capabilities/interaction/src/testing/chat-board.ts index 5bf8de905..246005eff 100644 --- a/capabilities/interaction/src/testing/chat-board.ts +++ b/capabilities/interaction/src/testing/chat-board.ts @@ -1,8 +1,8 @@ -import type { AnsweredOnce, DeliveryCall } from '@beonauto/mcp'; +import type { AnsweredOnce, OneCall } from '@beonauto/mcp'; export type BoardAnswer = AnsweredOnce extends infer Answer ? Answer extends AnsweredOnce - ? Omit + ? Omit : never : never; @@ -22,14 +22,14 @@ export interface Replying { } export interface ChatBoard { - readonly answerOf: (call: DeliveryCall) => BoardAnswer; + readonly answerOf: (call: OneCall) => BoardAnswer; readonly posted: () => readonly ChatMessage[]; readonly reply: (replying: Replying) => string; } export const brainUser = 'brain'; -function argument(call: DeliveryCall, name: string): string { +function argument(call: OneCall, name: string): string { const value = call.input[name]; return typeof value === 'string' ? value : ''; } @@ -50,7 +50,7 @@ function threadOf(thread: string) { export function chatBoard(): ChatBoard { const messages: ChatMessage[] = []; const nextTs = () => `1699.${String(messages.length + 1).padStart(6, '0')}`; - const post = (call: DeliveryCall): BoardAnswer => { + const post = (call: OneCall): BoardAnswer => { const ts = nextTs(); const channel = argument(call, 'channel'); messages.push({ @@ -62,7 +62,7 @@ export function chatBoard(): ChatBoard { }); return answered({ ok: true, channel, ts }); }; - const read = (call: DeliveryCall): BoardAnswer => { + const read = (call: OneCall): BoardAnswer => { const ts = argument(call, 'ts'); const oldest = Number(argument(call, 'oldest')); const inThread = (message: ChatMessage) => ts === '' || message.ts === ts || message.thread_ts === ts; diff --git a/capabilities/interaction/src/testing/fake-tools.ts b/capabilities/interaction/src/testing/fake-tools.ts index 8cc2f60da..01f12c6ee 100644 --- a/capabilities/interaction/src/testing/fake-tools.ts +++ b/capabilities/interaction/src/testing/fake-tools.ts @@ -1,6 +1,6 @@ import { Buffer } from 'node:buffer'; -import type { CalledOnce, DeliveryCall, NotOfferedBecause, StartedFields, ToolAccess } from '@beonauto/mcp'; +import type { CalledOnce, NotOfferedBecause, OneCall, StartedFields, ToolAccess } from '@beonauto/mcp'; import { ToolNotOffered } from '@beonauto/mcp'; import { Effect, Result } from 'effect'; @@ -14,7 +14,7 @@ export type FakeAnswer = export type ToolPorts = Pick; export interface FakeTools extends ToolPorts, Pick { - readonly calls: () => readonly DeliveryCall[]; + readonly calls: () => readonly OneCall[]; readonly answerNext: (...answers: readonly FakeAnswer[]) => void; readonly disallow: (tool: string) => void; } @@ -38,32 +38,32 @@ interface Offering { readonly disallowed: ReadonlySet; } -function refusalOf({ server, tool }: DeliveryCall['reference'], { disallowed }: Offering): Refusal | undefined { +function refusalOf({ server, tool }: OneCall['reference'], { disallowed }: Offering): Refusal | undefined { if (server !== 'chat') { return notConfigured(server); } return disallowed.has(tool) ? notAllowed(tool) : undefined; } -function notOfferedBy({ because, detail }: Refusal): CalledOnce { - return { kind: 'not_offered', because, detail }; +function notOfferedBy(refusal: Refusal): CalledOnce { + return { kind: 'unopened', refused: 'tool_not_offered', ...refusal }; } -function startOf({ reference, input }: Pick): StartedFields { +function startOf({ reference, input }: Pick): StartedFields { const argumentsJson = JSON.stringify(input); return { ...reference, arguments_bytes: Buffer.byteLength(argumentsJson), arguments_sha256: digest }; } -function unsentBy(answer: Exclude, { reference }: DeliveryCall): CalledOnce { +function unsentBy(answer: Exclude, { reference }: OneCall): CalledOnce { return answer.kind === 'unopened' - ? { kind: 'unopened', because: 'mcp_server_failed', detail: answer.detail } + ? { kind: 'unopened', refused: 'mcp_server_failed', because: 'failing', detail: answer.detail } : notOfferedBy(notAllowed(reference.tool)); } function answeredWith(answer: BoardAnswer, number: number): CalledOnce { const bytes = answer.outcome === 'result' ? Buffer.byteLength(JSON.stringify(answer.answer)) : null; const fields = { result_bytes: bytes, result_sha256: bytes === null ? null : digest, jsonrpc_id: number }; - return { ...answer, kind: 'answered', fields, durationMs: 3 }; + return { ...answer, kind: 'answered', fields, durationMs: 3, annotations: undefined }; } function namedBy(refusals: readonly (Refusal | undefined)[]): Result.Result { @@ -71,12 +71,12 @@ function namedBy(refusals: readonly (Refusal | undefined)[]): Result.Result Effect.Effect = () => Effect.void): FakeTools { +export function fakeTools(duringCall: (call: OneCall) => Effect.Effect = () => Effect.void): FakeTools { const board = chatBoard(); - const calls: DeliveryCall[] = []; + const calls: OneCall[] = []; const queued: FakeAnswer[] = []; const offering = { disallowed: new Set() }; - const sent = (call: DeliveryCall): Effect.Effect => { + const sent = (call: OneCall): Effect.Effect => { const refusal = refusalOf(call.reference, offering); if (refusal !== undefined) { return Effect.succeed(notOfferedBy(refusal)); diff --git a/capabilities/interaction/src/testing/index.ts b/capabilities/interaction/src/testing/index.ts index 59f57b9e1..fae372c8d 100644 --- a/capabilities/interaction/src/testing/index.ts +++ b/capabilities/interaction/src/testing/index.ts @@ -16,6 +16,7 @@ export { threadReplies, type ThreadOptions, } from './documents.ts'; +export { callDocument, repliesOutput, type CallOptions } from './call-documents.ts'; export { askedRunId, askedThroughChat, type AskedRequest, type AskingOptions } from './asked-requests.ts'; export { attemptedThenStopped } from './stopped-requests.ts'; export { broughtBeforeSettling, recordedReply, takenReply, type RacingDelivery } from './racing-deliveries.ts'; diff --git a/capabilities/interaction/src/testing/route-documents.ts b/capabilities/interaction/src/testing/route-documents.ts index 4cc046350..e7fec7e84 100644 --- a/capabilities/interaction/src/testing/route-documents.ts +++ b/capabilities/interaction/src/testing/route-documents.ts @@ -1,7 +1,8 @@ import { issueText } from '@beonauto/definitions/document'; -import { Result } from 'effect'; +import { Option, Result } from 'effect'; import { parseInteractionDocument } from '../document/document-parsing.ts'; +import type { InteractionFunctionDefinitionDocument, RequestDocument } from '../document/interaction-document.ts'; export const delivering: readonly string[] = [ "to: '{{ input.owner }}'", @@ -64,3 +65,11 @@ export function problemsOf(source: string): readonly string[] { const parsed = parseInteractionDocument(source); return Result.isFailure(parsed) ? parsed.failure.map((issue) => issueText(issue)) : []; } + +function isRequest(document: InteractionFunctionDefinitionDocument): document is RequestDocument { + return document.shape === 'request'; +} + +export function requestDocumentOf(source: string): RequestDocument { + return Option.getOrThrow(Option.liftPredicate(Result.getOrThrow(parseInteractionDocument(source)), isRequest)); +} diff --git a/capabilities/interaction/src/route/route-templates.test.ts b/capabilities/interaction/src/tool-blocks/block-templates.test.ts similarity index 94% rename from capabilities/interaction/src/route/route-templates.test.ts rename to capabilities/interaction/src/tool-blocks/block-templates.test.ts index bef15da76..7788de8ac 100644 --- a/capabilities/interaction/src/route/route-templates.test.ts +++ b/capabilities/interaction/src/tool-blocks/block-templates.test.ts @@ -54,7 +54,7 @@ describe('a delivery argument and the values it holds', () => { problemsOf(documentOf(deliveringText('Input: {{ input }}'))), ]).toEqual([ [ - 'Line 9, /deliver/with/text: Renders a value that is not text inside a longer text; write | json after a structured value such as answer_schema, or write the expression alone to send the value as it is', + 'Line 9, /deliver/with/text: Renders a value that is not text among text; write | json after it, or write it alone as one {{ }} to send it as it is', ], [], [], @@ -119,7 +119,7 @@ describe('the values a reading and a telling send', () => { problemsOf(documentOf(delivering, keyed, answering)), ]).toEqual([ [ - 'Line 18, /replies/with/ts: Renders a value that is not text inside a longer text; write | json after a structured value such as sent, or write the expression alone to send the value as it is', + 'Line 18, /replies/with/ts: Renders a value that is not text among text; write | json after it, or write it alone as one {{ }} to send it as it is', ], [], [ diff --git a/capabilities/interaction/src/route/route-templates.ts b/capabilities/interaction/src/tool-blocks/block-templates.ts similarity index 93% rename from capabilities/interaction/src/route/route-templates.ts rename to capabilities/interaction/src/tool-blocks/block-templates.ts index c164d39cc..338d4061b 100644 --- a/capabilities/interaction/src/route/route-templates.ts +++ b/capabilities/interaction/src/tool-blocks/block-templates.ts @@ -6,7 +6,7 @@ import { interactionEngine } from '../document/request-templates.ts'; import { renderedText } from '../run/text-rendering.ts'; import { isLoneExpression } from '../run/value-rendering.ts'; import { childrenOf, isJsonParent, type JsonParent } from './json-children.ts'; -import type { Templates } from './rendered-arguments.ts'; +import { notTextRemedy, type Templates } from './rendered-arguments.ts'; import { nameOf, reads, type TemplateSet } from './template-sets.ts'; export interface TemplatePlace { @@ -46,8 +46,7 @@ type Rendering = 'text' | 'argument'; const notTextWords: Readonly string>> = { text: (structured) => `Renders a value that is not text; write | json after a structured value such as ${structured}`, - argument: (structured) => - `Renders a value that is not text inside a longer text; write | json after a structured value such as ${structured}, or write the expression alone to send the value as it is`, + argument: () => `Renders a value that is not text among text; ${notTextRemedy}`, }; function templateDetails(set: TemplateSet, text: string, place: TemplatePlace, rendering: Rendering) { diff --git a/capabilities/interaction/src/route/compiled-route.ts b/capabilities/interaction/src/tool-blocks/compiled-tool-block.ts similarity index 72% rename from capabilities/interaction/src/route/compiled-route.ts rename to capabilities/interaction/src/tool-blocks/compiled-tool-block.ts index 0ee87acd7..2a5bc9ffb 100644 --- a/capabilities/interaction/src/route/compiled-route.ts +++ b/capabilities/interaction/src/tool-blocks/compiled-tool-block.ts @@ -3,14 +3,25 @@ import { isServerName, isToolName, serverNameShape, toolNameShape } from '@beona import { readDuration } from '@beonauto/workflow-engine/dsl'; import { JsonPointer, Result, type Schema } from 'effect'; +import { templateIssues, templatesIssues, type TemplatePlace } from './block-templates.ts'; import { isJsonPointer } from './json-pointers.ts'; -import { argumentsFailureWords, renderedArguments } from './rendered-arguments.ts'; -import type { Replies, ToolDelivery } from './route-schemas.ts'; -import { templateIssues, templatesIssues, type TemplatePlace } from './route-templates.ts'; -import { conversationTemplates, deliveryTemplates, readingTemplates, tellingTemplates } from './template-sets.ts'; +import { + argumentsFailureWords, + renderedArguments, + type ArgumentsFailure, + type Templates, +} from './rendered-arguments.ts'; +import { + callTemplates, + conversationTemplates, + deliveryTemplates, + readingTemplates, + tellingTemplates, +} from './template-sets.ts'; +import type { CallBlock, DeliverBlock, Replies } from './tool-block-schemas.ts'; export interface WrittenRoute { - readonly deliver: ToolDelivery; + readonly deliver: DeliverBlock; readonly replies?: Replies | undefined; } @@ -119,7 +130,7 @@ function readingIssues(replies: Replies, place: TemplatePlace): readonly Documen ]; } -export function compiledRoute( +export function compiledToolBlock( written: WrittenRoute, lines: SourceLines, inputSchema: Schema.JsonObject | undefined, @@ -131,3 +142,36 @@ export function compiledRoute( ]; return issues.length > 0 ? Result.fail(issues) : Result.succeed(written); } + +function isOversized(failure: ArgumentsFailure): boolean { + return failure.reason === 'too_large' || failure.failure.reason === 'too_long'; +} + +function callArgumentIssues(written: Templates, place: TemplatePlace): readonly DocumentIssue[] { + const pointer = placeOf('call', 'with'); + const issues = templatesIssues(callTemplates, written, { ...place, pointer }); + if (issues.length > 0) { + return issues; + } + const rendered = renderedArguments(written, callTemplates.sample); + if (Result.isSuccess(rendered) || !isOversized(rendered.failure)) { + return []; + } + const { failure } = rendered; + const at = failure.reason === 'too_large' ? pointer : `${pointer}/${failure.argument}`; + return [issueAt(place.lines, at, argumentsFailureWords(failure, 'the call'))]; +} + +export function compiledCallBlock( + call: CallBlock, + lines: SourceLines, + inputSchema: Schema.JsonObject | undefined, +): Result.Result { + const issues = [ + ...nameIssues(lines, [[placeOf('call', 'server'), call.server]], isServerName, serverNameShape), + ...nameIssues(lines, [[placeOf('call', 'tool'), call.tool]], isToolName, toolNameShape), + ...callArgumentIssues(call.with ?? {}, { lines, pointer: '', inputSchema }), + ...pointerIssues(lines, [[placeOf('call', 'read'), call.read]]), + ]; + return issues.length > 0 ? Result.fail(issues) : Result.succeed(call); +} diff --git a/capabilities/interaction/src/route/json-children.ts b/capabilities/interaction/src/tool-blocks/json-children.ts similarity index 100% rename from capabilities/interaction/src/route/json-children.ts rename to capabilities/interaction/src/tool-blocks/json-children.ts diff --git a/capabilities/interaction/src/route/json-pointers.test.ts b/capabilities/interaction/src/tool-blocks/json-pointers.test.ts similarity index 100% rename from capabilities/interaction/src/route/json-pointers.test.ts rename to capabilities/interaction/src/tool-blocks/json-pointers.test.ts diff --git a/capabilities/interaction/src/route/json-pointers.ts b/capabilities/interaction/src/tool-blocks/json-pointers.ts similarity index 100% rename from capabilities/interaction/src/route/json-pointers.ts rename to capabilities/interaction/src/tool-blocks/json-pointers.ts diff --git a/capabilities/interaction/src/route/rendered-arguments.ts b/capabilities/interaction/src/tool-blocks/rendered-arguments.ts similarity index 91% rename from capabilities/interaction/src/route/rendered-arguments.ts rename to capabilities/interaction/src/tool-blocks/rendered-arguments.ts index c6158058b..01d901541 100644 --- a/capabilities/interaction/src/route/rendered-arguments.ts +++ b/capabilities/interaction/src/tool-blocks/rendered-arguments.ts @@ -88,8 +88,10 @@ export function renderedArguments( }); } +export const notTextRemedy = 'write | json after it, or write it alone as one {{ }} to send it as it is'; + const failureWords: Readonly> = { - not_text: 'renders a value that is not text', + not_text: `renders a value that is not text among text; ${notTextRemedy}`, too_long: `renders more than the ${toolBounds.argumentBytes} bytes a call may send`, missing_variable: 'reads a value it does not have', limit_exceeded: 'cannot be rendered within the limits of a template', @@ -108,12 +110,13 @@ export interface DeliveryContext { readonly functionName: string; } +export function momentVariables(input: Schema.Json, at: string): TemplateVariables { + return { input, today: at.slice(0, 10), now: at }; +} + export function deliveryVariablesOf(record: RequestRecord, { input, runId, functionName }: DeliveryContext) { - const now = record.requested_at; return { - input, - today: now.slice(0, 10), - now, + ...momentVariables(input, record.requested_at), to: record.to, message: record.message, run_id: runId, diff --git a/capabilities/interaction/src/route/routes.test.ts b/capabilities/interaction/src/tool-blocks/routes.test.ts similarity index 99% rename from capabilities/interaction/src/route/routes.test.ts rename to capabilities/interaction/src/tool-blocks/routes.test.ts index 185023601..ce7489506 100644 --- a/capabilities/interaction/src/route/routes.test.ts +++ b/capabilities/interaction/src/tool-blocks/routes.test.ts @@ -40,7 +40,7 @@ describe('the route of a request', () => { it('is told in the same words by the capability and by the history of a delivery', () => { const through = throughWords(routeOf({ deliver: chat })); - expect([through, throughWords(routeOf({}))]).toEqual(['through the tool post_message of chat', 'in the inbox']); + expect([through, throughWords(routeOf({}))]).toEqual(['through the post message tool of chat', 'in the inbox']); expect(throughTheTool(chat)).toBe(through); expect(deliveryStarted({ number: 1, ...chat })).toBe(`Delivery attempt 1 of the request started, ${through}.`); }); diff --git a/capabilities/interaction/src/route/routes.ts b/capabilities/interaction/src/tool-blocks/routes.ts similarity index 84% rename from capabilities/interaction/src/route/routes.ts rename to capabilities/interaction/src/tool-blocks/routes.ts index 1fead5401..b95926083 100644 --- a/capabilities/interaction/src/route/routes.ts +++ b/capabilities/interaction/src/tool-blocks/routes.ts @@ -1,6 +1,7 @@ +import { toolInWords } from '@beonauto/mcp'; import type { ToolReference } from '@beonauto/mcp/policy'; -import type { Replies } from './route-schemas.ts'; +import type { Replies } from './tool-block-schemas.ts'; export interface RecordedBlocks { readonly deliver?: ToolReference | undefined; @@ -22,9 +23,7 @@ export function routeOf({ deliver, replies }: RecordedBlocks): Route { } export function throughWords(route: Route): string { - return route.kind === 'inbox' - ? 'in the inbox' - : `through the tool ${route.delivery.tool} of ${route.delivery.server}`; + return route.kind === 'inbox' ? 'in the inbox' : `through ${toolInWords(route.delivery)}`; } export function toolsOf(route: Route): readonly ToolReference[] { diff --git a/capabilities/interaction/src/route/template-sets.ts b/capabilities/interaction/src/tool-blocks/template-sets.ts similarity index 91% rename from capabilities/interaction/src/route/template-sets.ts rename to capabilities/interaction/src/tool-blocks/template-sets.ts index a20640b93..b590f2dca 100644 --- a/capabilities/interaction/src/route/template-sets.ts +++ b/capabilities/interaction/src/tool-blocks/template-sets.ts @@ -13,6 +13,13 @@ const momentOfTheSample = '2026-10-07T09:00:00.000Z'; const sent = { conversation: 'C0123', id: '1699.1' }; +export const callTemplates: TemplateSet = { + names: ['input', 'today', 'now'], + what: 'a template of a call', + sample: { today: momentOfTheSample.slice(0, 10), now: momentOfTheSample }, + structured: 'input', +}; + export const deliveryTemplates: TemplateSet = { names: ['input', 'today', 'now', 'to', 'message', 'run_id', 'function', 'expires_at', 'answer_schema'], what: 'a template of a delivery', diff --git a/capabilities/interaction/src/route/route-parsing.test.ts b/capabilities/interaction/src/tool-blocks/tool-block-parsing.test.ts similarity index 97% rename from capabilities/interaction/src/route/route-parsing.test.ts rename to capabilities/interaction/src/tool-blocks/tool-block-parsing.test.ts index 7b9e18373..3a02a2091 100644 --- a/capabilities/interaction/src/route/route-parsing.test.ts +++ b/capabilities/interaction/src/tool-blocks/tool-block-parsing.test.ts @@ -1,7 +1,5 @@ -import { Result } from 'effect'; import { describe, expect, it } from 'vitest'; -import { parseInteractionDocument } from '../document/document-parsing.ts'; import { answering, delivering, @@ -9,6 +7,7 @@ import { problemsOf, reading, replacing, + requestDocumentOf, without, } from '../testing/route-documents.ts'; @@ -18,7 +17,7 @@ const toolShape = 'Expected a tool name of 1 to 128 letters, digits, underscores describe('a definition that delivers through a tool and reads its replies', () => { it('reads deliver and replies as written', () => { - const parsed = Result.getOrThrow(parseInteractionDocument(documentOf(delivering, reading, answering))); + const parsed = requestDocumentOf(documentOf(delivering, reading, answering)); expect(parsed.route).toMatchObject({ deliver: { diff --git a/capabilities/interaction/src/route/route-schemas.ts b/capabilities/interaction/src/tool-blocks/tool-block-schemas.ts similarity index 73% rename from capabilities/interaction/src/route/route-schemas.ts rename to capabilities/interaction/src/tool-blocks/tool-block-schemas.ts index 524699ec3..497dcdd35 100644 --- a/capabilities/interaction/src/route/route-schemas.ts +++ b/capabilities/interaction/src/tool-blocks/tool-block-schemas.ts @@ -11,18 +11,35 @@ export const SentSchema = Schema.Struct({ }), }); -export const ToolDeliverySchema = Schema.Struct({ +const ArgumentsSchema = Templates.annotate({ + description: + "The arguments of the call, named as the tool's input_schema names them; a number, boolean, null, list or object is sent as written, with the Liquid templates in its strings rendered; a string that is one {{ expression }} alone is sent as the value it reads; any other string is rendered as text", +}); + +const ToolUseSchema = Schema.Struct({ server: Schema.String.annotate({ description: 'The tool server, as list_tool_servers names it' }), - tool: Schema.String.annotate({ description: 'The tool of that server that sends the request' }), - with: Templates.annotate({ - description: - 'The arguments of the call over input, today, now, to, message, run_id, function, expires_at and answer_schema; a number, boolean, null, list or object is sent as written, with the Liquid templates in its strings rendered; a string that is one {{ expression }} alone is sent as the value it reads; any other string is rendered as text', - }), + tool: Schema.String.annotate({ description: 'The tool of that server, as list_tool_servers lists it' }), + with: ArgumentsSchema, +}); + +export const DeliverBlockSchema = Schema.Struct({ + ...ToolUseSchema.fields, sent: Schema.optionalKey( SentSchema.annotate({ description: 'Where, in what the tool answered, the message that was sent is named' }), ), }); +export const CallBlockSchema = Schema.Struct({ + ...ToolUseSchema.fields, + with: Schema.optionalKey(ArgumentsSchema), + read: Schema.optionalKey( + Schema.String.annotate({ + description: + "A JSON Pointer into the document of the tool's answer, to the value the function answers with; the whole document when left out", + }), + ), +}); + const ReadOrderSchema = Schema.Literals(['oldest_first', 'newest_first']); export const EachSchema = Schema.Struct({ @@ -72,6 +89,8 @@ export const RepliesSchema = Schema.Struct({ tell: Schema.optionalKey(TellSchema), }); -export type ToolDelivery = typeof ToolDeliverySchema.Type; +export type DeliverBlock = typeof DeliverBlockSchema.Type; + +export type CallBlock = typeof CallBlockSchema.Type; export type Replies = typeof RepliesSchema.Type; diff --git a/capabilities/reasoning/README.md b/capabilities/reasoning/README.md index d698f975a..8f38a9e40 100644 --- a/capabilities/reasoning/README.md +++ b/capabilities/reasoning/README.md @@ -40,7 +40,7 @@ A request has `model`, optional `instructions`, `messages` (roles `user` and `as ## Tools -A reasoning function whose front matter names `tools` (each `server/tool`, or `server/*`) runs with the tools of the MCP servers configured for its brain, as [decision 0003](../../docs/decisions/0003-mcp-servers.md) sets out. `makeReasoningFunctionAdapter` takes the server's `ToolAccess` from [`@beonauto/mcp`](../../packages/mcp) as `tools`, as a type only: it imports the helpers it needs to parse `tools` and bound a run from `@beonauto/mcp/policy`, which loads no transport code; without one, or with no server configured, a function that names tools is rejected as `unavailable`, kind `tool_not_offered`. The adapter then says it may change something outside (`mayChangeOutside`), so `run_definition` is destructive. +A reasoning function whose front matter names `tools` (each `server/tool`, or `server/*`) runs with the tools of the MCP servers configured for its brain, as [decision 0003](../../docs/decisions/0003-mcp-servers.md) sets out. `makeReasoningFunctionAdapter` takes the server's `ToolAccess` from [`@beonauto/mcp`](../../packages/mcp) as `tools`, as a type only: it imports the helpers it needs to parse `tools` and bound a run from `@beonauto/mcp/policy`, which loads no transport code; `tools` is required, and a test with no tool server gives `noToolServers` of `@beonauto/mcp/testing`; with no server configured, a function that names tools is rejected as `unavailable`, kind `tool_not_offered`, in the words of `namedLinks`. The adapter then says it may change something outside (`mayChangeOutside`), so `run_definition` is destructive. A function that names tools calls tools (`callsTools`), so the start of its run records that, and a run of it that is still `started` is not run again under its id. A run first admits its request with the model's `admit`, so a run whose model is not offered reaches no server and starts no process; it then opens its tools with its id, org, brain and journal before the model is called, and rejects as `unavailable` with kind `tool_not_offered` (because `mcp_server_not_configured`, `tool_not_allowed` or `tool_not_listed`) or `mcp_server_failed` (because `unreachable`, `failing` or `rate_limited`). Otherwise the request carries `tools`: the offered tools under their model-facing names with their input schemas unchanged, whether the calls have ended, a signal that ends them, and the bound of the whole run. In the adapter, the AI SDK's loop: @@ -50,7 +50,7 @@ A function that names tools calls tools (`callsTools`), so the start of its run - fails as `tools_stopped` when that last step still calls tools (`no_answer`), when the run reaches its bound (`run_bound`), or when its tool servers failed too often (`server_failed`); - adds to the result's `warnings`, after the provider's, one `compatibility` warning for each place in a tool's input schema where the provider of the run's model may refuse the schema or not hold the tool's arguments to it, named as the tool's name, `inputSchema#` and the JSON pointer, such as `mcp__graph__search inputSchema#/properties/query/maxLength`. The rules are those of a JSON output schema (`src/schema/schema-portability.ts`) that hold for a tool's arguments: a root that is not an object, and for Anthropic models the bounds and `oneOf` they do not enforce and the formats they do not accept, each worded for a tool's arguments. The rules for open objects and optional properties bind strict structured outputs only, and the AI SDK sends tools as not strict (`@ai-sdk/openai` 4.0.83 sets `strict: false` unless a tool asks), so they are left out. A schema of more than 64 KiB or 64 levels is not checked. -Once a run has recorded a tool call, every `unavailable` ending it meets has the kind `tools_unfinished`, with because `server_failed`, `model_unavailable`, `run_bound` or `no_answer`, and words that name the tools it called in words, never as `server/tool`, and never ask to try again. The run's sessions end however it ends. +Once a run has recorded a tool call, every `unavailable` ending it meets has the kind `tools_unfinished` when every tool it called is one its server marks `readOnlyHint` (`calledOnlyReadOnly` of its tools), and becomes a `Conflict` of the kind `effect_unknown` when any is not (`src/tools/tool-endings.ts`), with because `server_failed`, `model_unavailable`, `run_bound` or `no_answer`, and words that name the tools it called in words, never as `server/tool`. The words of `tools_unfinished` say that a new run, or a workflow's retry, may make it; those of `effect_unknown` that whether the tool changed something is not known, and never ask to try again. The run's sessions end however it ends. A run that calls tools costs more: every step resends the conversation so far, so a run that uses its whole budget of 256 KiB of results sends on the order of a million tokens. diff --git a/capabilities/reasoning/src/capability/definition-request.ts b/capabilities/reasoning/src/capability/definition-request.ts index e48b01c62..6a69bbba3 100644 --- a/capabilities/reasoning/src/capability/definition-request.ts +++ b/capabilities/reasoning/src/capability/definition-request.ts @@ -22,8 +22,8 @@ export function longestRunMsOf({ settings, tools }: ReasoningFunctionDefinitionD return tools.length === 0 ? timeoutMs : runBoundMs(timeoutMs); } -function modelToolsOf({ offered, callsEnded, ended }: RunTools, timeoutMs: number): ModelTools { - return { offered, callsEnded, ended, runBoundMs: runBoundMs(timeoutMs) }; +function modelToolsOf({ offered, callsEnded, calledAny, ended }: RunTools, timeoutMs: number): ModelTools { + return { offered, callsEnded, calledAny, ended, runBoundMs: runBoundMs(timeoutMs) }; } export function requestFor( diff --git a/capabilities/reasoning/src/capability/definition-run.ts b/capabilities/reasoning/src/capability/definition-run.ts index e49861b32..c1cbaaa72 100644 --- a/capabilities/reasoning/src/capability/definition-run.ts +++ b/capabilities/reasoning/src/capability/definition-run.ts @@ -13,7 +13,7 @@ import { finishedWith } from './spending-record.ts'; export interface RunServices { readonly languageModel: LanguageModel['Service']; readonly clock?: Clock.Clock; - readonly tools?: ToolAccess; + readonly tools: ToolAccess; } function outputOf({ text, json }: { readonly text: string; readonly json?: Schema.Json }): Schema.Json { diff --git a/capabilities/reasoning/src/capability/gateway-options.test.ts b/capabilities/reasoning/src/capability/gateway-options.test.ts index df1f46ceb..1ebc14c4c 100644 --- a/capabilities/reasoning/src/capability/gateway-options.test.ts +++ b/capabilities/reasoning/src/capability/gateway-options.test.ts @@ -5,16 +5,15 @@ import { describe, expect, it } from 'vitest'; import { accessFor } from '../testing/adapter-harness.ts'; import { documentOf } from '../testing/definition-documents.ts'; import { chatCompletion } from '../testing/provider-replies.ts'; -import { runContext } from '../testing/reasoning-runs.ts'; +import { adapterWithoutTools, runContext } from '../testing/reasoning-runs.ts'; import { jsonResponse, recordingFetch } from '../testing/recording-fetch.ts'; -import { makeReasoningFunctionAdapter } from './reasoning-function.ts'; const gateway = { name: 'internal', base_url: 'https://llm.internal.example/v1', allowed_provider_options: ['user'] }; async function executedWith(providerOptions: string) { const recording = recordingFetch(() => jsonResponse(chatCompletion('Hello'))); const access = await accessFor({ MODEL_GATEWAYS: JSON.stringify([gateway]) }, { fetch: recording.fetch }); - const capability = makeReasoningFunctionAdapter({ languageModel: access.languageModel, offered: access.offered }); + const capability = adapterWithoutTools(access); const document = documentOf(`model: internal/llama-3.3-70b\nprovider_options:\n internal: ${providerOptions}`, 'Hi'); const prepared = Effect.runSync(capability.prepare(document)); const exit = await Effect.runPromiseExit(prepared.run({}, runContext)); diff --git a/capabilities/reasoning/src/capability/provider-messages.test.ts b/capabilities/reasoning/src/capability/provider-messages.test.ts index 13c4b5eb7..2dd7d580f 100644 --- a/capabilities/reasoning/src/capability/provider-messages.test.ts +++ b/capabilities/reasoning/src/capability/provider-messages.test.ts @@ -6,9 +6,8 @@ import { accessFor } from '../testing/adapter-harness.ts'; import { documentOf } from '../testing/definition-documents.ts'; import { exposedText } from '../testing/exposure.ts'; import { gatewayError, gatewayErrorText, gatewayInternals, recordingReporter } from '../testing/provider-errors.ts'; -import { runContext } from '../testing/reasoning-runs.ts'; +import { adapterWithoutTools, runContext } from '../testing/reasoning-runs.ts'; import { jsonResponse, recordingFetch } from '../testing/recording-fetch.ts'; -import { makeReasoningFunctionAdapter } from './reasoning-function.ts'; async function executedThroughGateway(gateway: object) { const reporter = recordingReporter(); @@ -16,7 +15,7 @@ async function executedThroughGateway(gateway: object) { { MODEL_GATEWAYS: JSON.stringify([gateway]) }, { fetch: recordingFetch(() => jsonResponse(gatewayError, 404)).fetch, reportProviderMessage: reporter.report }, ); - const capability = makeReasoningFunctionAdapter({ languageModel: access.languageModel, offered: access.offered }); + const capability = adapterWithoutTools(access); const prepared = Effect.runSync(capability.prepare(documentOf('model: gateway/no-such-model-xyz', 'Say hello.'))); const exit = await Effect.runPromiseExit(prepared.run({}, runContext)); return { exit, reports: reporter.reports() }; diff --git a/capabilities/reasoning/src/capability/reasoning-function.test.ts b/capabilities/reasoning/src/capability/reasoning-function.test.ts index b147e3716..3ff365ea2 100644 --- a/capabilities/reasoning/src/capability/reasoning-function.test.ts +++ b/capabilities/reasoning/src/capability/reasoning-function.test.ts @@ -1,3 +1,4 @@ +import { noToolServers } from '@beonauto/mcp/testing'; import { InvalidInput } from '@beonauto/operations'; import { Effect, Exit } from 'effect'; import { TestClock } from 'effect/testing'; @@ -138,6 +139,7 @@ describe('the moment a run renders', () => { languageModel: scripted.languageModel, clock, offered: { providers: ['anthropic'], aliases: [] }, + tools: noToolServers, }); const definition = Effect.runSync( timed.prepare(documentOf('model: openai/gpt-5', 'Today is {{ today }}, now {{ now }}')), diff --git a/capabilities/reasoning/src/capability/reasoning-function.ts b/capabilities/reasoning/src/capability/reasoning-function.ts index 284a606f2..74ccc6dea 100644 --- a/capabilities/reasoning/src/capability/reasoning-function.ts +++ b/capabilities/reasoning/src/capability/reasoning-function.ts @@ -2,13 +2,13 @@ import { defineCapability, functionCategoryLabels, functionResourceLabels, - inWords, + outputInWords, type DefinitionSummary, type Capability, } from '@beonauto/definitions'; import { issueText } from '@beonauto/definitions/document'; -import { asSentence, InvalidInput } from '@beonauto/operations'; -import { Effect, Result, type Schema } from 'effect'; +import { InvalidInput } from '@beonauto/operations'; +import { Effect, Result } from 'effect'; import { parseDefinitionDocument } from '../definition/definition-parsing.ts'; import type { ReasoningFunctionDefinitionDocument } from '../definition/reasoning-function-definition.ts'; @@ -43,13 +43,6 @@ function summarize({ description, input, output, warnings }: ReasoningFunctionDe }; } -function describeAnswer(output: Schema.Json): string { - const words = inWords(output); - return words === undefined - ? 'Its answer is too long to repeat here; the whole of it is in the details below.' - : asSentence(`Its answer: ${words}`); -} - export function makeReasoningFunctionAdapter(options: ReasoningFunctionAdapterOptions): Capability { const run = definitionRun(options); return defineCapability({ @@ -57,10 +50,10 @@ export function makeReasoningFunctionAdapter(options: ReasoningFunctionAdapterOp title: functionCategoryLabels.reasoning, guide: { name: 'reasoning-function', - onThisServer: onThisServer(options.offered, options.tools?.configured === true), + onThisServer: onThisServer(options.offered, options.tools.configured), }, noun: { one: functionResourceLabels.reasoning.singular, other: functionResourceLabels.reasoning.plural }, - describeOutput: describeAnswer, + describeOutput: outputInWords('answer'), mediaType: 'text/markdown', parse, summarize, @@ -68,7 +61,7 @@ export function makeReasoningFunctionAdapter(options: ReasoningFunctionAdapterOp longestAnyRunMs: longestRequestMs, longestRunOf: longestRunMsOf, reachesOutside: true, - mayChangeOutside: options.tools?.configured === true, + mayChangeOutside: options.tools.configured, callsTools: ({ tools }) => tools.length > 0, }); } diff --git a/capabilities/reasoning/src/capability/run-answer.ts b/capabilities/reasoning/src/capability/run-answer.ts index c5d42fc61..1b01cefdd 100644 --- a/capabilities/reasoning/src/capability/run-answer.ts +++ b/capabilities/reasoning/src/capability/run-answer.ts @@ -12,7 +12,7 @@ import { rejections, spendingSince, type DefinitionRejection } from './model-rej export interface Answering { readonly languageModel: LanguageModel['Service']; - readonly access: ToolAccess | undefined; + readonly access: ToolAccess; readonly definition: ReasoningFunctionDefinitionDocument; readonly prompt: RenderedPrompt; readonly run: RunContext; diff --git a/capabilities/reasoning/src/model/model-request.ts b/capabilities/reasoning/src/model/model-request.ts index f451bb482..262fbaa3c 100644 --- a/capabilities/reasoning/src/model/model-request.ts +++ b/capabilities/reasoning/src/model/model-request.ts @@ -69,6 +69,7 @@ export interface ModelTool { export interface ModelTools { readonly offered: readonly ModelTool[]; readonly callsEnded: () => boolean; + readonly calledAny: () => boolean; readonly ended: Readonly; readonly runBoundMs: number; } diff --git a/capabilities/reasoning/src/testing/reasoning-runs.ts b/capabilities/reasoning/src/testing/reasoning-runs.ts index d94a25e2a..1228220fb 100644 --- a/capabilities/reasoning/src/testing/reasoning-runs.ts +++ b/capabilities/reasoning/src/testing/reasoning-runs.ts @@ -1,11 +1,15 @@ import type { CapabilityAnswer, RunContext, PreparedDefinition, Capability } from '@beonauto/definitions'; import { noLongestRuns, recordingJournal, type RecordingJournal } from '@beonauto/definitions/testing'; import type { ToolAccess } from '@beonauto/mcp'; +import { noToolServers } from '@beonauto/mcp/testing'; import { allPermissions, type Conflict, type InvalidInput, type Unavailable } from '@beonauto/operations'; import { DateTime, Effect, type Exit, type Schema } from 'effect'; import { TestClock } from 'effect/testing'; -import { makeReasoningFunctionAdapter } from '../capability/reasoning-function.ts'; +import { + makeReasoningFunctionAdapter, + type ReasoningFunctionAdapterOptions, +} from '../capability/reasoning-function.ts'; import type { ModelRequest } from '../model/model-request.ts'; import { scriptedLanguageModel, type ScriptedReply } from './scripted-language-model.ts'; @@ -39,16 +43,12 @@ export interface ToolRun extends ReasoningRun { readonly journal: RecordingJournal; } -function reasoningOf( - tools: ToolAccess | undefined, - replies: readonly ScriptedReply[], - context: RunContext, -): ReasoningRun { +function reasoningOf(tools: ToolAccess, replies: readonly ScriptedReply[], context: RunContext): ReasoningRun { const scripted = scriptedLanguageModel(...replies); const capability = makeReasoningFunctionAdapter({ languageModel: scripted.languageModel, offered: anthropicOnly, - ...(tools === undefined ? {} : { tools }), + tools, }); const prepared = (source: string): PreparedDefinition => Effect.runSync(capability.prepare(source)); return { @@ -65,8 +65,15 @@ function reasoningOf( }; } +export function adapterWithoutTools({ + languageModel, + offered, +}: Pick): Capability { + return makeReasoningFunctionAdapter({ languageModel, offered, tools: noToolServers }); +} + export function reasoningWith(...replies: readonly ScriptedReply[]): ReasoningRun { - return reasoningOf(undefined, replies, runContext); + return reasoningOf(noToolServers, replies, runContext); } export function reasoningWithTools(tools: ToolAccess, ...replies: readonly ScriptedReply[]): ToolRun { diff --git a/capabilities/reasoning/src/testing/scripted-tools.test.ts b/capabilities/reasoning/src/testing/scripted-tools.test.ts new file mode 100644 index 000000000..17747db03 --- /dev/null +++ b/capabilities/reasoning/src/testing/scripted-tools.test.ts @@ -0,0 +1,15 @@ +import { describe, expect, it } from 'vitest'; + +import { scriptedTools } from './scripted-tools.ts'; + +describe('scripted tools', () => { + it('say whether the run called any of them', async () => { + const { tools } = scriptedTools(); + const before = tools.calledAny(); + const signal = new AbortController().signal; + + await tools.offered[0]?.call({ callId: 'call-1', input: { query: 'acme' } }, { signal, cancelled: signal }); + + expect([before, tools.calledAny()]).toEqual([false, true]); + }); +}); diff --git a/capabilities/reasoning/src/testing/scripted-tools.ts b/capabilities/reasoning/src/testing/scripted-tools.ts index 1c284c4e8..d8192eb45 100644 --- a/capabilities/reasoning/src/testing/scripted-tools.ts +++ b/capabilities/reasoning/src/testing/scripted-tools.ts @@ -46,6 +46,7 @@ export function scriptedTools({ }, ], callsEnded: () => calls.length >= endAfter, + calledAny: () => calls.length > 0, ended: ending.signal, runBoundMs, }, diff --git a/capabilities/reasoning/src/tools/tool-endings.test.ts b/capabilities/reasoning/src/tools/tool-endings.test.ts index 97f25aa22..e43d61d3c 100644 --- a/capabilities/reasoning/src/tools/tool-endings.test.ts +++ b/capabilities/reasoning/src/tools/tool-endings.test.ts @@ -120,6 +120,39 @@ describe('a run whose tool servers kept failing', () => { }); }); +function unknown(because: string, detail: string) { + return Exit.fail(expect.objectContaining({ _tag: 'conflict', kind: 'effect_unknown', because, detail })); +} + +const echoed: ScriptedCall = ['mcp__graph__echo', { said: 'acme' }]; + +const usedBoth = 'after the run called the search and echo tools of graph'; + +describe('a run that could not finish after calling a tool its server does not mark read-only', () => { + it('is a conflict whose effect is unknown, with the because and detail it would have had, whether that tool came first or last', async () => { + const afterReading = callingTools([...searched, echoed], stoppedBy('no_answer', 'anthropic kept calling tools')); + const beforeReading = callingTools([echoed, ...searched], timedOut); + + expect(await ending(await graphServer(), afterReading)).toEqual( + unknown('no_answer', `anthropic kept calling tools, ${usedBoth}`), + ); + expect(await ending(await graphServer(), beforeReading)).toEqual( + unknown( + 'model_unavailable', + 'anthropic did not answer within 60000 ms, after the run called the echo and search tools of graph', + ), + ); + }); + + it('is a conflict whose effect is unknown when its run reached its bound', async () => { + const reply = callingTools([echoed], stoppedBy('run_bound', 'The run went on for 600000 ms')); + + expect(await ending(await graphServer(), reply)).toEqual( + unknown('run_bound', 'The run went on for 600000 ms, after the run called the echo tool of graph'), + ); + }); +}); + describe('a run that could not finish before it called a tool', () => { it('is unavailable as before, without saying that tools were called', async () => { const fake = await graphServer(); diff --git a/capabilities/reasoning/src/tools/tool-endings.ts b/capabilities/reasoning/src/tools/tool-endings.ts index c80d99f8e..7ca9de3a4 100644 --- a/capabilities/reasoning/src/tools/tool-endings.ts +++ b/capabilities/reasoning/src/tools/tool-endings.ts @@ -1,11 +1,11 @@ import type { RunTools } from '@beonauto/mcp'; -import { Unavailable, type UnavailableBecause } from '@beonauto/operations'; +import { Conflict, Unavailable, type RejectionBecause } from '@beonauto/operations'; import { Effect, type Schema } from 'effect'; import type { ToolsStoppedBecause } from '../failure/tools-stopped.ts'; import type { TokenUsage } from '../model/model-result.ts'; -type UnfinishedBecause = Extract; +type UnfinishedBecause = Extract; const serverEndings = { failing: 'A tool server kept failing', @@ -16,13 +16,11 @@ export interface Spent { readonly record?: Schema.JsonObject; } -function unfinished(tools: RunTools, because: UnfinishedBecause, detail: string, spent: Spent): Unavailable { - return new Unavailable({ - detail: `${detail}, after the run called ${tools.usedInWords()}`, - kind: 'tools_unfinished', - because, - ...spent, - }); +function unfinished(tools: RunTools, because: UnfinishedBecause, detail: string, spent: Spent): Unavailable | Conflict { + const ending = { detail: `${detail}, after the run called ${tools.usedInWords()}`, because, ...spent }; + return tools.calledOnlyReadOnly() + ? new Unavailable({ ...ending, kind: 'tools_unfinished' }) + : new Conflict({ ...ending, kind: 'effect_unknown' }); } export interface Stopped { @@ -40,7 +38,7 @@ export function stoppedEnding( tools: RunTools | undefined, stopped: Stopped, spent: Spent = {}, -): Effect.Effect { +): Effect.Effect { return Effect.fail( tools?.calledAny() === true ? unfinished(tools, stopped.because, stoppedDetail(tools, stopped), spent) @@ -53,7 +51,7 @@ export function unavailableAfter( detail: string, advice: string, spent: Spent = {}, -): Effect.Effect { +): Effect.Effect { return Effect.fail( tools?.calledAny() === true ? unfinished(tools, 'model_unavailable', detail, spent) diff --git a/capabilities/reasoning/src/tools/tool-opening.test.ts b/capabilities/reasoning/src/tools/tool-opening.test.ts index 5572c33a1..d221d1b52 100644 --- a/capabilities/reasoning/src/tools/tool-opening.test.ts +++ b/capabilities/reasoning/src/tools/tool-opening.test.ts @@ -97,8 +97,7 @@ describe('a tool a reasoning function names that is not offered', () => { expect.objectContaining({ kind: 'tool_not_offered', because: 'mcp_server_not_configured', - detail: - 'The reasoning function names graph/search and crm/find, but no MCP server is configured on this server', + detail: 'No MCP server named graph or crm is configured for this brain', }), ), ); diff --git a/capabilities/reasoning/src/tools/tool-opening.ts b/capabilities/reasoning/src/tools/tool-opening.ts index 0a044e034..004ae62e2 100644 --- a/capabilities/reasoning/src/tools/tool-opening.ts +++ b/capabilities/reasoning/src/tools/tool-opening.ts @@ -1,6 +1,6 @@ import type { RunContext } from '@beonauto/definitions'; import type { NotOfferedBecause, RunTools, ServerFailedBecause, ToolAccess } from '@beonauto/mcp'; -import { runIdKey, writtenOf, type ToolReference } from '@beonauto/mcp/policy'; +import { runIdKey, type ToolReference } from '@beonauto/mcp/policy'; import { Unavailable } from '@beonauto/operations'; import { Effect } from 'effect'; @@ -9,17 +9,6 @@ interface Refused { readonly detail: string; } -const conjunction = new Intl.ListFormat('en', { style: 'long', type: 'conjunction' }); - -function noServerFor(references: readonly ToolReference[]): Unavailable { - const named = conjunction.format(references.map((reference) => writtenOf(reference))); - return new Unavailable({ - detail: `The reasoning function names ${named}, but no MCP server is configured on this server`, - kind: 'tool_not_offered', - because: 'mcp_server_not_configured', - }); -} - function opened( access: ToolAccess, references: readonly ToolReference[], @@ -36,7 +25,7 @@ function opened( } export function withTools( - access: ToolAccess | undefined, + access: ToolAccess, references: readonly ToolReference[], run: RunContext, use: (tools?: RunTools) => Effect.Effect, @@ -44,9 +33,6 @@ export function withTools( if (references.length === 0) { return use(); } - if (access === undefined) { - return Effect.fail(noServerFor(references)); - } return opened(access, references, run).pipe( Effect.flatMap((tools) => use(tools).pipe(Effect.ensuring(Effect.promise(() => tools.close())))), ); diff --git a/capabilities/recall/src/capability/recall-function.test.ts b/capabilities/recall/src/capability/recall-function.test.ts index ece14a0cb..d0e330778 100644 --- a/capabilities/recall/src/capability/recall-function.test.ts +++ b/capabilities/recall/src/capability/recall-function.test.ts @@ -18,7 +18,7 @@ describe('a recall function', () => { it('describes its result in words', () => { const { capability } = recallWith(poolOf({ workers: 1 })); - expect(capability.describeOutput([{ verdict: 'approve' }])).toBe('Its result: verdict: “approve”.'); + expect(capability.describeOutput([{ verdict: 'approve' }])).toBe('Its result: (verdict: “approve”).'); expect(capability.describeOutput('x'.repeat(5000))).toBe( 'Its result is too long to repeat here; the whole of it is in the details below.', ); diff --git a/capabilities/recall/src/capability/recall-function.ts b/capabilities/recall/src/capability/recall-function.ts index 38846424d..f355c5f85 100644 --- a/capabilities/recall/src/capability/recall-function.ts +++ b/capabilities/recall/src/capability/recall-function.ts @@ -2,13 +2,11 @@ import { defineCapability, functionCategoryLabels, functionResourceLabels, - inWords, + outputInWords, type Capability, } from '@beonauto/definitions'; -import { asSentence } from '@beonauto/operations'; import type { ProgramPool } from '@beonauto/workflow-engine/dsl'; import type { ViewsPort } from '@beonauto/workflow-host'; -import type { Schema } from 'effect'; import { recallBounds, recallDefinitionType } from '../run/recall-bounds.ts'; import { recallRun } from '../run/recall-run.ts'; @@ -22,13 +20,6 @@ export interface RecallFunctionAdapterOptions { readonly deadlineMs?: number; } -function describeResult(output: Schema.Json): string { - const words = inWords(output); - return words === undefined - ? 'Its result is too long to repeat here; the whole of it is in the details below.' - : asSentence(`Its result: ${words}`); -} - export function makeRecallFunctionAdapter({ pool, views, @@ -41,7 +32,7 @@ export function makeRecallFunctionAdapter({ title: functionCategoryLabels.recall, guide: { name: 'recall-function' }, noun: { one: functionResourceLabels.recall.singular, other: functionResourceLabels.recall.plural }, - describeOutput: describeResult, + describeOutput: outputInWords('result'), mediaType: 'text/markdown', parse, summarize, diff --git a/docs/concepts/functions.md b/docs/concepts/functions.md index 128b5257c..80aa21140 100644 --- a/docs/concepts/functions.md +++ b/docs/concepts/functions.md @@ -38,7 +38,7 @@ An interaction function defines input or output between the brain and people or Human approval is one interaction pattern; machine-to-machine input and output belong here too. -In the current runtime, an interaction function asks one party and takes the answer later. Its run renders a request from its input, a message and the party it goes to, and sends it through a tool of a tool server, which the function names, or leaves it in the brain's inbox. The run then waits, holding nothing of the runtime, until the request is answered, expires or is cancelled. An answer must match the answer schema the function declares, and it becomes the run's output, so a workflow that asked continues with the decision. A function without an answer schema is a notification, which succeeds once it is delivered. +In the current runtime, an interaction function asks a system and answers at once, or asks one party and takes the answer later. One that asks a system names a tool of a tool server, calls it once and answers with the part of the tool's answer it names, checked against the shape it declares, with no model in between. One that asks a party renders a request from its input, a message and the party it goes to, and sends it through a tool of a tool server, which the function names, or leaves it in the brain's inbox. The run then waits, holding nothing of the runtime, until the request is answered, expires or is cancelled. An answer must match the answer schema the function declares, and it becomes the run's output, so a workflow that asked continues with the decision. A function without an answer schema is a notification, which succeeds once it is delivered. The inbox is the brain's own: `list_interactions` lists the requests waiting in it, and a caller who may write to the brain answers one with `answer_interaction`. A function that sends its request through a tool, such as one that posts in a chat, may also read the replies to its message and take the party's reply as the answer. The runtime's operator decides which tool servers a brain may use and which of their tools. A request nobody answers before it expires ends the run as `unanswered`, which a workflow can catch and handle. @@ -64,7 +64,7 @@ See [Recall function format](../reference/recall-format.md). A computation function executes specified code or expressions with defined inputs and outputs, such as a calculation or data transformation. It need not be mathematically pure. AI-assisted authoring can help write the logic; once defined, the operation executes that specified logic. -In the current runtime, a computation function is a program in jq with schemas for its input and output. A run applies the program to its input and answers with exactly one output, the same output for the same input every time, within bounds on its work, memory and time. It reaches nothing outside the brain. Use it for the arithmetic a language model should not do: a workflow can read figures through a reasoning function's tools, compute totals, paces and projections exactly with a computation function, and have another reasoning function write about them. +In the current runtime, a computation function is a program in jq with schemas for its input and output. A run applies the program to its input and answers with exactly one output, the same output for the same input every time, within bounds on its work, memory and time. It reaches nothing outside the brain. Use it for the arithmetic a language model should not do: a workflow can read figures through an interaction function that calls a tool, compute totals, paces and projections exactly with a computation function, and have a reasoning function write about them. See [Computation function format](../reference/computation-format.md). diff --git a/docs/concepts/terminology.md b/docs/concepts/terminology.md index def91ee1b..4d0fd348a 100644 --- a/docs/concepts/terminology.md +++ b/docs/concepts/terminology.md @@ -63,13 +63,13 @@ See [Workflows and runs](workflows.md) for the implemented version, retry, waiti | Term | Meaning | | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------- | -| Request | What a run of an interaction function asks: a message, the party it goes to and when it expires. | +| Request | What a run of an interaction function that asks a person asks: a message, the party it goes to and when it expires. | | Inbox | The brain's own place for requests, where a caller who may write to the brain reads and answers them. | | Delivery | How a request reaches a party outside the brain: one call of a tool of a tool server the brain may use, which the function names in `deliver`. | -| Answer | What the party gives back; it must match the function's answer schema and becomes the run's output. | +| Answer | What the party or the tool gives back; it must match the function's answer schema and becomes the run's output. | | Notification | A request that takes no answer; its run succeeds once it is delivered. | -An interaction function asks a person or a system and takes the answer later. One run makes one request; the answer settles that run, and a request nobody answers before it expires ends it as unanswered. Answering a request is not sending an event: an event goes to a waiting workflow run, and an answer goes to the run of the interaction function that asked. +An interaction function asks a system and answers at once, or asks a person and takes the answer later. A run that asks a person makes one request; the answer settles that run, and a request nobody answers before it expires ends it as unanswered. Answering a request is not sending an event: an event goes to a waiting workflow run, and an answer goes to the run of the interaction function that asked. ## Supporting assets diff --git a/docs/concepts/workflows.md b/docs/concepts/workflows.md index 3db290b61..b9579ae21 100644 --- a/docs/concepts/workflows.md +++ b/docs/concepts/workflows.md @@ -28,7 +28,7 @@ Keep the saved work separate from what happens when it executes: Saving a workflow creates its definition at version 1, and each change to its document adds a version. A run uses the active latest version when it starts and keeps that version until it ends; the run records it as `definition_version`. A step that calls a function runs the function's active latest version at the time of the call. -Each call to a function starts a run of that function, recorded under its own run id. A retry is a new attempt at the step: it starts another run of the function, not a new function. When the runtime resumes a step after an interruption, the step keeps its run id, so a function run that already has a final result is not run again. A reasoning function's run is not run again under its id either when it had recorded a tool call and did not succeed, or when its function names tools and the run had started without ending, since its tools may have changed something. The step then fails with an error of the type `https://on.auto/problems/tools_called`, which a retry of runtime or communication errors does not catch; a retry that catches every error, or names that type, calls the function again under a new run id, and so calls its tools again. The run's history shows what the function called. +Each call to a function starts a run of that function, recorded under its own run id. A retry is a new attempt at the step: it starts another run of the function, not a new function. When the runtime resumes a step after an interruption, the step keeps its run id, so a function run that already has a final result is not run again. The run of a function that calls tools, a reasoning function that names them or an interaction function that calls one, is not run again under its id either when it had recorded a tool call and did not succeed, or when the run had started without ending, since its tools may have changed something. The step then fails with an error of the type `https://on.auto/problems/tools_called`, which a retry of runtime or communication errors does not catch; a retry that catches every error, or names that type, calls the function again under a new run id, and so calls its tools again. The run's history shows what the function called. ## Starting a run @@ -75,17 +75,17 @@ A run ends in one of three states: A run that was cancelled, or that ran as long as a run may last, is `rejected` with the reason `cancelled` and the kind of its cancellation, a run that did not catch a request nobody answered is `rejected` as `unanswered` with the kind `expired` or `undelivered`, and a run whose output is too large to record is `rejected` as a `conflict` of the kind `oversized`. -A rejection's reason is `invalid_input` when the error says the input or the document led to it, and `unavailable` when it was a timeout or a function that could not be reached, failed or was unavailable; a new run may then succeed. Two endings are different, because a reasoning function's tools may have changed something: `unavailable` of the kind `tools_unfinished`, when it called tools and could not finish, and `conflict` of the kind `tools_called`, when a step met a function run whose tools may already have been called. After either, check what the run's history shows the function called before starting a new run. The [workflow format](../reference/workflow-format.md#how-a-run-ends) lists the exact rules. +A rejection's reason is `invalid_input` when the error says the input or the document led to it, and `unavailable` when it was a timeout or a function that could not be reached, failed or was unavailable; a new run may then succeed. Three endings are different, because a function's tools may have changed something: `unavailable` of the kind `tools_unfinished`, when it called tools that only read and could not finish, which a new run, or a retry on status 503, may make again; `conflict` of the kind `effect_unknown`, when it called a tool that may change something and could not finish, so whether it did is not known; and `conflict` of the kind `tools_called`, when a step met a function run whose tools may already have been called. After the last two, check what the run's history shows the function called before starting a new run. The [workflow format](../reference/workflow-format.md#how-a-run-ends) lists the exact rules. ## Calling another workflow -A workflow calls another workflow with `run_definition`, as it calls any function of its brain; the workflow it calls is a subworkflow. A run of a reasoning, computation or recall function finishes within its call. A run of a workflow, like a run of an interaction function, finishes later, so the calling workflow waits for that run, holding nothing of the server while it waits, and continues with its output, or catches its rejection like any error. A server that restarts meanwhile keeps the wait, and the subworkflow's ending answers it on whichever server that ending is recorded. +A workflow calls another workflow with `run_definition`, as it calls any function of its brain; the workflow it calls is a subworkflow. A run of a reasoning, computation or recall function, or of an interaction function that calls a tool, finishes within its call. A run of a workflow, like a run of an interaction function that asks a person, finishes later, so the calling workflow waits for that run, holding nothing of the server while it waits, and continues with its output, or catches its rejection like any error. A server that restarts meanwhile keeps the wait, and the subworkflow's ending answers it on whichever server that ending is recorded. A call waits as long as the function it names may take, plus a minute; past that, it raises a `timeout` error and the run it waited for is cancelled. Workflows that call workflows reach 8 calls deep, and the runs under one workflow wait for at most 1,000 calls at once. ## Cancelling a run -`cancel_run` cancels a workflow run that has not ended, with a reason the run keeps. The request is recorded at once, so it can reach any server and outlasts a restart; the run then stops, cancels each run it waits for, and ends `rejected` as `cancelled` with the kind `requested`. `cancel_run` also cancels the run of an interaction function whose request waits, which ends `rejected` as `cancelled`. A run of a reasoning, computation or recall function ends within its call, and cannot be cancelled from outside it. +`cancel_run` cancels a workflow run that has not ended, with a reason the run keeps. The request is recorded at once, so it can reach any server and outlasts a restart; the run then stops, cancels each run it waits for, and ends `rejected` as `cancelled` with the kind `requested`. `cancel_run` also cancels the run of an interaction function whose request waits, which ends `rejected` as `cancelled`. A run of a reasoning, computation or recall function, or of an interaction function that calls a tool, ends within its call, and cannot be cancelled from outside it. ## Inspecting a run diff --git a/docs/decisions/0003-mcp-servers.md b/docs/decisions/0003-mcp-servers.md index 377fae781..d6bff6d4c 100644 --- a/docs/decisions/0003-mcp-servers.md +++ b/docs/decisions/0003-mcp-servers.md @@ -1,6 +1,6 @@ # 0003 — MCP servers: a brain reaches the outside world through configured MCP servers -**Status:** accepted (2026-10-05), revised four times after audit and the gateway's answers the same day; amended (2026-10-06) with what the build of step 1 and its review learned; amended (2026-10-07) with `list_tool_servers`; amended (2026-10-08) with a server's tools allowed and marked testable on its own entry +**Status:** accepted (2026-10-05), revised four times after audit and the gateway's answers the same day; amended (2026-10-06) with what the build of step 1 and its review learned; amended (2026-10-07) with `list_tool_servers`; amended (2026-10-08) with a server's tools allowed and marked testable on its own entry; amended (2026-10-09) with the two endings of a run that called tools and could not finish ## Context @@ -184,3 +184,7 @@ mcp_servers: - **Verification the build must include.** Settings: both fields read per entry from the file and from `MCP_SERVERS`, `null` and none when left out; each refusal above at its pointer and line, never a value; the file's two global keys refused at their line with their sentence; a server with neither field. Names: `server/*` offers exactly the allowed tools; `server/tool` outside `allowed` is `tool_not_allowed`; a server with no `allowed` offers every tool it lists. The listing: every server serving the brain is asked; `testable` true for a read-only tool and a marked one, false otherwise. The test: `not_testable` with the words; `mayChangeOutside` true only while an entry has `testable`. The note: once per server per process, not when an allowed tool is read-only while a read-only tool outside `allowed` still gives it, not when the entry has `testable`, never a secret, pinned on the fake server. The texts: the recipe at 2,052 bytes, the instructions at 1,990, 1,495 and 1,950, the remedy at 214 and the detail at 119. The schema as generated, equal to the committed file. Self-check (2026-10-08), against `origin/main` at 498219da: every file and line cited was read there; the three clients' pages and OpenAI's were fetched and quoted from; the counts were measured with `wc` over the sentence as written and over the recipe with the one substitution, and the instruction counts are those `tool-testing-instructions.test.ts` pins, which the amendment leaves untouched; the note and the example line had landed with `fix/tool-servers-without-a-brain`, reading the global keys, and their words for the entry are given here; the amendment names no vendor, platform or business. + +## Amendment (2026-10-09): a run that called tools and could not finish + +Which of two endings it meets is read over every call the run started: when every tool it called is one its server marks `readOnlyHint`, it ends `unavailable`, kind `tools_unfinished`, status 503, which says running it again is safe; when any is not, it ends `conflict`, kind `effect_unknown`, status 409, since whether the tool's effect happened is not known and a person or a workflow rule that names the kind decides. Either carries its `because` as section 4 gave it, and a `catch` on status 503 never repeats a call that may have acted, as section 6 meant ([decision 0021](0021-asking-a-system.md)). diff --git a/docs/decisions/0018-testing-a-tool-call.md b/docs/decisions/0018-testing-a-tool-call.md index 2401957af..b5acf637e 100644 --- a/docs/decisions/0018-testing-a-tool-call.md +++ b/docs/decisions/0018-testing-a-tool-call.md @@ -1,6 +1,6 @@ # 0018 — Testing a tool call: an agent tries one tool of a tool server through the brain, as a run would call it -**Status:** accepted (2026-10-08), designed against `feat/speaking-to-agents` at 09b727a2 and verified again against `origin/main` at f4a7d7f6, where that branch has merged; built on 2026-10-08, with the amendments from the build after the appendix and the verifications of the record against the code at the end +**Status:** accepted (2026-10-08), designed against `feat/speaking-to-agents` at 09b727a2 and verified again against `origin/main` at f4a7d7f6, where that branch has merged; built on 2026-10-08, with the amendments from the build after the appendix and the verifications of the record against the code at the end; amended (2026-10-09) by [decision 0021](0021-asking-a-system.md), a test also answering the document a call reads ## Context @@ -272,6 +272,7 @@ Built on 2026-10-08 on `origin/main` at f4a7d7f6. The texts and counts the recor - **The fake server learns to answer without sessions** (`issuesSessionIds: false`), to count the five requests of a test of a server that issues no session id, and a tool that lists the channels of a workspace, `list_channels`, read-only, for episode 8. - **The answer schema of an open request**, which section 12 leaves to a record of its own, is that record, [decision 0019](0019-the-answer-shape-of-an-open-request.md), built beside this one; nothing here touches it. - **The image's smoke test** checks that `test_tool_call` is served, that a key that may only read is refused it over HTTP and is not offered it over MCP, and that a server without tool servers answers a test with `mcp_server_not_configured`. +- **The document a call reads (2026-10-09).** `test_tool_call` also answers `answer`, the document a call's `read` points into: the structured content, else the first text block parsed as JSON, else that text, built by the `answerDocument` an interaction function's call reads with, scrubbed as `text` is and left out past 64 KiB ([decision 0021](0021-asking-a-system.md)). Section 1 answered only what a model sees, since the next thing written from a test was a prompt; a call's `read` and `output.schema` are written against the document, which `text` hides where a tool gives structured content beside a summary, so the test answers both. ## Verification of this record diff --git a/docs/decisions/0020-replies-through-a-channel.md b/docs/decisions/0020-replies-through-a-channel.md index 5f344a006..9b74ec7e5 100644 --- a/docs/decisions/0020-replies-through-a-channel.md +++ b/docs/decisions/0020-replies-through-a-channel.md @@ -1,6 +1,6 @@ # 0020 — Replies to what the brain sent: a person answers where the question reached them, and the brain takes the reply -**Status:** proposed (2026-10-08), designed against `origin/main` at 6c9fd7e8 with `feat/testing-a-tool-call` (0018, head 0e07646f) and `fix/interaction` (0010's amendments, head 826ef99e) read as merged, since both land before this is built; verified again the same day against `origin/main` at a06e9692, where several triggers (0015, #130) landed and touched nothing this record cites; amended the same day by [decision 0010's amendment, a function owns its delivery](0010-interaction-functions.md#amendment-2026-10-08-a-function-owns-its-delivery), read against `origin/main` at c4d01f90, which redirects this record's feature half: a function's own delivery reads its replies as a channel's does, the reading is keyed by the tool and never by a channel's name, and the delivery's `sent` pointers live on the delivery; its data-store half, the keyed projection over named stream kinds and the reader's advance, stands as designed and is being built; built the same day with that amendment, whose section 8 edits are applied to this text +**Status:** proposed (2026-10-08), designed against `origin/main` at 6c9fd7e8 with `feat/testing-a-tool-call` (0018, head 0e07646f) and `fix/interaction` (0010's amendments, head 826ef99e) read as merged, since both land before this is built; verified again the same day against `origin/main` at a06e9692, where several triggers (0015, #130) landed and touched nothing this record cites; amended the same day by [decision 0010's amendment, a function owns its delivery](0010-interaction-functions.md#amendment-2026-10-08-a-function-owns-its-delivery), read against `origin/main` at c4d01f90, which redirects this record's feature half: a function's own delivery reads its replies as a channel's does, the reading is keyed by the tool and never by a channel's name, and the delivery's `sent` pointers live on the delivery; its data-store half, the keyed projection over named stream kinds and the reader's advance, stands as designed and is being built; built the same day with that amendment, whose section 8 edits are applied to this text; [decision 0021](0021-asking-a-system.md) (2026-10-09) applies its edits to sections 2, 4, 6 and 11: the document a read takes, the start a delivery records, and the bounds of a read ## Context @@ -118,9 +118,9 @@ replies: | `tell.tool` | The tool that tells the party how to answer when a reply was refused; the delivery's `tool` when left out; the whole `tell` block optional: without it a refusal is recorded and nobody is told | | `tell.with` | Its arguments, Liquid templates over `to`, `sent.conversation`, `sent.id`, those of the request whose reply was refused, and `message`, the words of §6 | -Liquid renders what the brain sends, over named fields, as a delivery's `with` does; pointers read what a tool answered. A template over a tool's answer would be the brain reading data as a program, and an answer is data and never instructions (0003 §3); a pointer names a place and nothing else, is checked for its shape at start, and is the form the API already uses to name a place in a value. The **document** a pointer reads is the tool's `structuredContent` when the tool gives one, otherwise the text of its first text block read as JSON; a tool that gives neither, or text that is not JSON, has no document, and a pointer over none fails as `unreadable`. The pointers' values must be text, or whole numbers, which are written as text: one chat's `ts` is a string, another chat's id a string. +Liquid renders what the brain sends, over named fields, as a delivery's `with` does; pointers read what a tool answered. A template over a tool's answer would be the brain reading data as a program, and an answer is data and never instructions (0003 §3); a pointer names a place and nothing else, is checked for its shape at start, and is the form the API already uses to name a place in a value. The **document** a pointer reads is the tool's `structuredContent` when the tool gives one, otherwise its first text block parsed as JSON, else that text; a tool that gives neither has no document, and a pointer that finds nothing, in a text or in no document, fails as `unreadable`, the rule of `answerDocument` ([decision 0021](0021-asking-a-system.md), §1). The pointers' values must be text, or whole numbers, which are written as text: one chat's `ts` is a string, another chat's id a string. -The templates are compiled when the document is saved, each with the names it may read, and refused with the document's line, as a delivery's are, through the one `compiledRoute` of 0010's amendment §4 (`default` is among the filters the engine keeps, `packages/specs/src/template/template-engine.ts:34`); a pointer that is not a JSON Pointer, a `read.order` that is neither value, a `wait` outside its bounds, and a `replies` block without `deliver.sent` or without `output.schema`, are refused with their line and never a value. A function's `tool` and `tell.tool` outside the server's `allowed` end its run at the start, `unavailable` `tool_not_offered`, as its `deliver.tool` does (0010, amendment, §2). A server's secrets never reach a template, a pointer or a record, as today: a template may not hold `${`. +The templates are compiled when the document is saved, each with the names it may read, and refused with the document's line, as a delivery's are, through the one `compiledRoute` of 0010's amendment §4 (`default` is among the filters the engine keeps, `packages/specs/src/template/template-engine.ts:34`); a pointer that is not a JSON Pointer, the empty pointer among them, in the one sentence every pointer of the format shares, "Expected a JSON Pointer such as /messages: it starts with a slash, writes ~ as ~0 and a slash within a name as ~1", a `read.order` that is neither value, a `wait` outside its bounds, and a `replies` block without `deliver.sent` or without `output.schema`, are refused with their line and never a value. A function's `tool` and `tell.tool` outside the server's `allowed` end its run at the start, `unavailable` `tool_not_offered`, as its `deliver.tool` does (0010, amendment, §2). A server's secrets never reach a template, a pointer or a record, as today: a template may not hold `${`. ### 3. The function document: `from` and `reply` @@ -193,7 +193,7 @@ Each named property must be a top-level string property of `output.schema`; each `delivery_ended` gains two fields. **`delivered_as`**, `{ conversation, id }`, the two parts the delivery's `sent` pointers read from the document the send's tool answered, each at most 256 bytes of text without a control character, present whenever the delivery writes `sent` and both could be read, whether or not it reads replies, so the history says what the message was delivered as. **`replies_in`**, `{ server, tool, key }`, the server, the reading tool and the conversation key the `conversation` template rendered, present when the request takes an answer and its delivery reads replies: the projection of §5 keys its row from this one fact, by the tool; the server and the tool of the attempt are on `delivery_started` (0010, amendment, §3). When a pointer cannot be read, the attempt is still `delivered`, since the message went out, with `detail` saying which pointer could not be read, and neither field: such a request takes no reply and waits for `answer_interaction`, and its listing says so (§10). A delivery without `sent` reads no pointer and delivers as today. -For that, the mcp package's `callOnce` answers the call whole, as 0010's amendment §3 shapes it: the outcome, the fields `callAnswered` builds under the server's `Recording`, the result as `content` and `structuredContent` scrubbed of the server's secrets, the failure's words and `Retry-After`, in place of its JSON cut at 4 KiB; `deliveryBounds.resultBytes` goes, since no caller cut by it meant anything; and the caller records the start before the call is sent, with the fields `callStarted` builds, which `startOf` gives, so a delivery is one recorded call as a run's are. `callOnce` takes its metadata from the caller, `meta` on the call as `CallerContext` carries it since 0018 (`caller-context.ts:5-11`): a delivery gives `{ execution_id, delivery_id }` as today, a read and a telling give `{ "com.beonauto/conversation_call_id": }`, and `DeliveryCall` loses `executionId` and `deliveryId`. +For that, the mcp package's `callOnce` answers the call whole, as 0010's amendment §3 shapes it: the outcome, the fields `callAnswered` builds under the server's `Recording`, the result as `content` and `structuredContent` scrubbed of the server's secrets, the failure's words and `Retry-After`, in place of its JSON cut at 4 KiB; `deliveryBounds.resultBytes` goes, since no caller cut by it meant anything; and the delivery records its own `delivery_started` before the call is sent, with the fields `callStarted` builds less `call_id`, which `startOf` gives (0010, amendment, §3). `callOnce` takes its metadata from the caller, `meta` on the call as `CallerContext` carries it since 0018 (`caller-context.ts:5-11`): a delivery gives `{ execution_id, delivery_id }` as today, a read and a telling give `{ "com.beonauto/conversation_call_id": }`, and `DeliveryCall` loses `executionId` and `deliveryId`. ### 5. Conversations are the schedule of reading, kept as a projection its reader advances @@ -245,8 +245,8 @@ The perform of a due conversation: 1. Advances the row, `next_read_at` a minute ahead, so a read lost to a stop is read again a minute later and never twice at once. 2. Reads the open requests of the conversation from the open-requests projection, `open` and `conversation` equal, oldest first, at most 100 (`by_conversation`), and keeps those that stand `delivered` or `retrying`: a row that stands `answered` or `cancelling` takes no reply. When none is kept, the row rests, `open` false and `next_read_at` none, with no call and no fact; a delivery wakes it. -3. Takes the reading of the oldest kept request through `routeOf`, the `replies` block its row records; renders `replies.with` over that request's `to` and `sent`, the key, and `since`, the row's cursor or an empty text, and calls `replies.tool` through `callOnce` with the delivery's server, under `namedLinks` as the delivery was, within its bounds, 10 s to open and 30 s to answer, scrubbed, as a delivery is; a tool the server's `allowed` no longer names is a failed read, `tool_not_offered`, recorded in step 7. -4. Reads the document, the list at `read.list`, and considers every reply in it, in the tool's order, which the result's 64 KiB bounds (`toolBounds.resultBytes`); a document that cannot be read, a list that is not an array, or a result past the bound ends the read `unreadable` or `too_large`, a failed read recorded in step 7, which whoever reads the history fixes in the function's `replies` or the tool's `limit`. +3. Takes the reading of the oldest kept request through `routeOf`, the `replies` block its row records; renders `replies.with` over that request's `to` and `sent`, the key, and `since`, the row's cursor or an empty text, and calls `replies.tool` through `callOnce` with the delivery's server, under `namedLinks` as the delivery was, within its bounds, 10 s to open and to list, and 30 s to answer, scrubbed, as a delivery is; a tool the server's `allowed` no longer names is a failed read, `tool_not_offered`, recorded in step 7. +4. Reads the document, the list at `read.list`, and considers every reply in it, in the tool's order, which bounds the document the read takes at 64 KiB as JSON (`toolBounds.resultBytes`); a document that cannot be read, a list that is not an array, or a result past the bound ends the read `unreadable` or `too_large`, a failed read recorded in step 7, which whoever reads the history fixes in the function's `replies` or the tool's `limit`. 5. For each reply, in order: its `id`, `sender`, `text` and `to` by the pointers; a reply without an `id` or a `sender` is passed over. It belongs to the kept request whose `delivered_as.id` equals its `to`; a reply with no `to`, or on a reading without the pointer, belongs to the conversation's one kept request, and when several are kept it is **ambiguous**: recorded as refused on the newest of them, since that is the request a guess would have answered and its `tell` is the one that speaks, and never taken, because a guess that silently answers the wrong question is worse than a refusal with words. A reply that belongs to none, or whose `sender` is not the request's answerer, is nothing and leaves no trace: a stranger's words are never recorded in a reply fact, though where the server's entry records content the read's `result_json` keeps the whole answer the tool gave, as §12 allows. A reply from the answerer whose `id` the request's stream already holds is nothing too: the decider refuses it (§7), as it refuses a second reply to a request this read already answered. Otherwise its `text`, bounded at 8 KiB (`interactionBounds.messageBytes`, a longer one refused as `too_long`), is mapped by the request's recorded reply rule to an answer and checked by `checkedAnswer` against the recorded `answer_schema`, the one answering check: - an answer that fits: `reply_taken` on the request's stream, `{ server, tool, reply: { id, sender }, answer, at }`, with the server and the tool that read it, caused by the row's `last_fact`, and then the settlement of §7, which the due work also makes from the fact should the server stop between the two; - a first word the rule has no value for, or no words at all, as a reply that is an attachment alone: `reply_refused` with `because` `not_an_answer`; an answer that fails the schema: `reply_refused` with `because` `invalid` and the issues under `/answer`; a reply past its bound: `too_long`; a reply that names no request where several are open: `ambiguous`; each `{ server, tool, reply: { id, sender }, because, issues?, told }`, caused by the row's `last_fact`, and at most ten per request, past which such a reply leaves no trace. @@ -285,7 +285,7 @@ A reply posted to the brain, rather than read by it, is not decided: it waits fo | ---------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- | | `sent.id`, `sent.conversation`, the conversation key | 256 bytes of text each, no control character | `delivered` without `delivered_as`, with the detail; the request takes no reply | | `from`, the answerer | 256 bytes, as `to` (`interactionBounds.toBytes`) | `conflict`, `unworkable`, as a `to` past its bound | -| A read's result | 64 KiB (`toolBounds.resultBytes`), the bound a run's call has | The read ends `too_large`, recorded; the cadence goes on | +| A read's result | The document, 64 KiB as JSON (`toolBounds.resultBytes`) | The read ends `too_large`, recorded; the cadence goes on | | Replies considered in one read | Every reply of the document, which its 64 KiB bounds; the tool's `limit` in `with` bounds each read further | In `oldest_first` order the rest come next time, from the cursor; in `newest_first` order, as §2 says | | Open requests read for in one conversation | 100, oldest first | Later ones are read for once earlier ones end | | A reply's words | 8 KiB (`interactionBounds.messageBytes`) | `reply_refused`, `too_long` | @@ -293,7 +293,7 @@ A reply posted to the brain, rather than read by it, is not decided: it waits fo | Refusals recorded per request | 10 | A further reply the rule cannot read leaves no trace; `repliesSeen` holds at most 11 identities | | Refusals told per request | 3 | Further refusals recorded, nobody told | | Reads of one conversation | The cadence, floored by `wait`, raised by `Retry-After` to at most an hour | — | -| A read in flight | A minute (`attemptInFlightMs`); the call itself 10 s to open and 30 s to answer (`deliveryBounds`) | Read again a minute after it began | +| A read in flight | A minute (`attemptInFlightMs`); the call itself 10 s to open and to list, and 30 s to answer (`toolBounds.openMs`, `toolBounds.callMs`) | Read again a minute after it began | | Reads at once | The conversations' outbound lane, 256 in flight, 16 at once | The rest wait for the next tick | | Conversations open per brain | At most the open requests, 10,000 unless set (`INTERACTION_OPEN_REQUESTS`) | — | | Facts a conversation writes | None while quiet; 1 per read that found a reply or failed, 2 per telling, 1 per reply taken or refused; a server that fails every read, 302 a day at the cadence | Stated as the price of reading where nothing can push; `list_brain_events` filters by type | diff --git a/docs/decisions/0021-asking-a-system.md b/docs/decisions/0021-asking-a-system.md new file mode 100644 index 000000000..bbadcce7f --- /dev/null +++ b/docs/decisions/0021-asking-a-system.md @@ -0,0 +1,412 @@ +# 0021 — Asking a system: an interaction function calls a tool and answers with its result + +**Status:** accepted (2026-10-09); designed on 2026-10-08 against `origin/main` at c4377676 with [decision 0010's amendment, a function owns its delivery](0010-interaction-functions.md#amendment-2026-10-08-a-function-owns-its-delivery-and-channels-go), and the per-server tool lists of decision 0003's amendment it builds on, read as merged, and [decision 0020](0020-replies-through-a-channel.md), replies, read as that amendment's section 8 amends it; written in the words of [decision 0007](0007-one-vocabulary.md), each file cited under the name that rename gave it, at the line read at c4377676; built on 2026-10-09 on `origin/main` at b753ded3, with the amendments from the build after the appendix + +## Context + +A workflow often needs data from a system, or an effect on one, with nothing to reason about between the asking and the answer. The only way today is a reasoning function whose prompt says "call this tool and answer with the rows it gives": the computation reference teaches exactly that, "A workflow reaches an MCP server only through a reasoning function's tools, so the first function reads the rows and passes them on unchanged" (`docs/reference/computation-format.md:181`), with a function whose prompt reads "Call ads/campaign-costs for the month {{ input.month }} and answer with the rows it gives, exactly as it gives them. Do not add, round or calculate anything." (`:213`). It costs a model call for every read, puts a tool's answer through a model that may drop, reorder or invent rows, exposes the run to whatever text the tool returns, and still hands the workflow a shape only the prompt promised. The owner decided: this is an interaction function in the shape "ask a system, answer now". Its document names the tool of a tool server it calls, the arguments as templates over its input, where in the tool's answer the data is, and what the data must look like; a run makes one recorded call through the brain's one call path, reads, checks, bounds and records the answer, and ends at once with the data as its output or with a rejection a workflow can catch. No new capability, no bare tool step in the workflow language, no model; MCP stays the transport and its raw result never reaches a workflow. An interaction function then has three shapes, which share one document format and one run path wherever they do the same thing: **ask a system now** (this record), **ask a party and take the answer later** through a tool, `deliver` and `replies` (0010's amendment, 0020), and the **inbox**, which names neither. + +What the code does, read on `origin/main` at c4377676: + +- The document takes `description`, `channel`, `to`, `expires`, `input` and `output`, all but three optional, and refuses every other key with "x is not a key of the front matter; it takes …" (`capabilities/interaction/src/document/front-matter.ts:8-15,19-23`; `packages/definitions/src/document/front-matter-keys.ts:32-50`); the words of an empty front matter name "the channel, the party it goes to and when it expires" (`front-matter.ts:30`; `packages/definitions/src/document/front-matter-reading.ts:147`). The body is the message, refused when empty with "The definition has no message: write it after the front matter" (`document/document-parsing.ts:31-38`). The amendment replaces `channel` by `deliver` and `replies` and keeps `to` and `expires` required (amendment §1). +- A run validates its input, renders `to` and the message over `{ input, today, now }` built from the run's moment (`run/interaction-run.ts:12-34`, the variables at `:18-20`), and either succeeds at once, a notification to the inbox with the output `{}`, or defers with the request as its record (`:36-38,50-57`). The capability declares that its runs finish later per document, run on to their end when their caller goes away, and last as long as `expires` (`capability/interaction-function.ts:23,26,27`; `packages/definitions/src/operations/run-requests.ts:115`). +- A delivery's arguments are rendered as text and only as text: `renderedArguments` answers `Record` (`channels/channel-arguments.ts:31-34,40-56`), a template that renders a structure is refused (`channels/argument-templates.ts:52-71`), and `| json` gives a structure as JSON text, so a tool whose `input_schema` takes `limit` as an integer receives `"15"`, as 0020's own example sends `limit: '15'` (0020 §2). The amendment's one renderer, `renderedArguments(templates, variables)` (§4), is being built with typed values for every `with` of the document format (§1 of this record says the rule). +- `callOnce` is the single-call path (`packages/mcp/src/access/tool-access.ts:27`, `access/linked-access.ts:33`, `delivery/delivery-call.ts:8-17`): `namedLinks`, then a connection taken within 10 s, then `forwarded` with the call bound clamped to 30 s and no wait for a 429 (`delivery/call-once.ts:29-61`, the clamps at `:35,53-54`, `delivery/connection-bound.ts:15-17`), a failure to take the connection answered as a call's `server_failure` or `timed_out` (`call-once.ts:37-45`) where a run's opening answers the same failure as `mcp_server_failed` with its because (`access/server-listing.ts:18-34`), and the answer cut to 4 KiB of JSON (`call-once.ts:16-27`); it lists nothing, where a run's opening takes a connection, lists the server's tools within the open bound and refuses a tool the server does not list (`server-listing.ts:36-58`; `access/tool-naming.ts:53-60,99-103`; `access/run-opening.ts:30-49`), each listed tool with the hints its server gives it (`bounds/result-text.ts:25-30`). The amendment has `callOnce(call)` answer the call whole, `CalledOnce`, with the metadata from the caller, and learn at the attempt whether the server lists the tool (§2, §3); a delivery records its own `delivery_started` before it calls, numbered by the decider, with the server, the tool and the arguments' fields it knows before any connection. +- A run's tool call records `tool_call_started` before it is sent and `tool_call_answered` after, built by `callStarted` and `callAnswered` under the server's `Recording` and appended through the run's journal (`packages/mcp/src/calls/tool-caller.ts:41-53`, `calls/recorded-calls.ts:13-16,48-80`, `packages/definitions/src/tool-calls/tool-call-journal.ts:54-58`, `packages/definitions/src/runs/run-events.ts:83-87`). A start makes the run one that may have changed something (`runs/run-state.ts:119-121`), so a run that did not succeed is not made again under its id (`runs/run-decisions.ts:37-47`), and a start that declares `calls_tools` refuses a second start while it runs (`:107-109`; `operations/run-requests.ts:69`). A run that runs within its call cannot be cancelled (`run-decisions.ts:247-253`). +- The model reads a tool's answer through `resultText`: every text block joined, structured content as JSON only when there is no text block, other content as a placeholder (`packages/mcp/src/bounds/result-text.ts:62-67`). `test_tool_call` answers exactly that as `text` (`calls/call-replies.ts:61-69`; `tool-tests/tool-test-schemas.ts:41-44`). 0020 defines the document its pointers read the other way round: `structuredContent` when the tool gives one, else the first text block read as JSON (0020 §2). A tool that follows the protocol's advice repeats its structured content as JSON text, and the two agree; a tool whose text is a summary shows the summary to a test and hides the data a pointer would read. +- An `unavailable` run carries a kind and a because (`packages/operations/src/outcome/unavailable.ts:3-30`), worded by `explanation.ts:81-90,139-163`; once a reasoning run has recorded a call, every `unavailable` ending is `tools_unfinished` (`capabilities/reasoning/src/tools/tool-endings.ts:19-26`; 0003 §4). A workflow raises `invalid_input` as `validation` 400, `conflict` as `runtime` 409 and `unavailable` as `communication` 503, `tools_called` and `tools_unfinished` under types of their own, a `conflict` at 409 and an `unavailable` at 503 (`packages/workflow-engine/src/dsl/raised-error.ts:116-126,193-201,218-241`; `packages/api/src/problem/problem.ts:69-73`; `docs/reference/workflow-format.md:217-232`), and a `catch` matches by comparing each field it names (`dsl/task-outcomes.ts:51-60`), so the reference's own retry on `{ status: 503 }` (`workflow-format.md:43-46`) matches `tools_unfinished` as well, where its prose says a `catch` that retries communication errors does not run tools again (`:234`). The API sends `Retry-After: 5` for an `unavailable` a retry may resolve and none for `tools_unfinished` or `tool_not_offered` (`packages/api/src/problem/problem.ts:86-97`). +- The outcome sentence of `run_definition` is "Ran ‹the function›." and the capability's `describeOutput(output)` (`packages/definitions/src/plain-language/run-words.ts:24-26,56-57`; `capability/capability.ts:100`), within 400 characters (`packages/operations/src/plain-language/word-bounds.ts:1`; `packages/api/src/tools/tool-result.ts:56`). The interaction function's says "It delivered its notification." for the output `{}` and "Its answer: …" otherwise (`capability/interaction-words.ts:46-54`). +- The instructions take 1,990 characters on `/mcp`, 1,495 on the org endpoint and 1,950 on the brain endpoint, pinned at `packages/api/src/mcp/tool-testing-instructions.test.ts:14-17` and strictly under 2,000 at `instructions.test.ts:133`; the interaction sentence is "An interaction function asks a person or a system and takes the answer later." (`mcp/instructions.ts:36`) and the run sentence "A run of an interaction function or a workflow answers started; get_run shows whether it ended or still waits." (`:126-135`). The amendment leaves both, with nine characters of room on `/mcp` (§6). A recipe may take 4,096 bytes and there are four (`packages/api/src/bounds/served-bounds.ts:17,23`); `give-tools` takes 2,052 bytes after the per-server amendment and 2,312 with the amendment's sentence, which is appended where interaction functions are served (§6). + +### How products call a system without a model + +Read on 2026-10-08 from each product's public reference; this record names no vendor or platform, so each product is named for what it is, and its pages are cited by what they say. The open workflow standard whose DSL 1.0 the workflow format follows names the task `call`: it "Enables the execution of a specified function within a workflow, allowing seamless integration with custom business logic or external services", with HTTP, OpenAPI, gRPC, AsyncAPI, A2A and MCP calls, the last written `call: mcp` with `with.method: tools/call` and `parameters` holding the tool's `name` and `arguments`, a reusable function declared once under `use.functions` and called by its name, an HTTP call's `output` of `raw`, `content` ("outputs the content of HTTP response, possibly deserialized", the default) or `response`, a task's output reshaped by `output.as` in jq, and errors caught by `catch.errors.with`, "a static filter, used to catch only the errors whose properties match the specified values", over standard types such as `communication` and `runtime`. A cloud provider's workflow service has an HTTP task, "a type of Task workflow state state that you can use to call HTTPS APIs in your workflows", its credentials in a connection that "helps you avoid hard-coding secrets"; its "HTTP task requests will timeout after 60 seconds", it sends `Range: bytes=0-262144`, retries and catches by names declared on the state for a status such as 429 or 503, and fails as a runtime error when "The response can't be read as a valid string"; a state's parameters "can either be static values that you include in your state machine definition, or selected from either the input or the Context object with a path", and a result selector "lets you create a collection of key value pairs, where the values are static or selected from the state's result" before a result path places it. A durable-execution engine's activity "is a normal function or method (an Activity Definition) that executes a single, well-defined action", recommended idempotent "so retries can be processed without duplicate side effects", and retried by default, an activity that fails being retried automatically with "the maximum attempt of retries ... set to zero which is evaluated as unlimited", and bounded by a start-to-close timeout, "the maximum time allowed for a single Activity Task Execution", whose expiry writes "an ActivityTaskTimedOut Event ... to the Workflow Execution's mutable state". A scripting platform's scripts "have input parameters defined by a JSON Schema", validated "During script execution", and a flow step's inputs "can be configured in three ways: Templatable string, Dynamic, or Static". A GraphQL gateway's connector directive "defines the API data source of a GraphQL schema field", its `selection` "Used to map an API's JSON response to GraphQL fields", in which "A leading `$.` is required when unwrapping a single property". An automation tool's HTTP request node lets a workflow "make HTTP requests to query data from any app or service with a REST API"; "By default, the node returns success only when the response returns with a 2xx code", it has Pagination "for handling query results that are too big for the API to return in a single call", and Retry On Fail is a setting under which "the node reruns until it succeeds". An event-driven function platform's `step.fetch()` is "a shorthand for making HTTP requests from within" one of its functions and offloads the request to the platform, and `step.run()` runs code "as a retriable step in your function", "All data returned from `step.run` is serialized as JSON". The Model Context Protocol returns structured content "as a JSON object in the `structuredContent` field of a result", says "a tool that returns structured content SHOULD also return the serialized JSON in a TextContent block", and reports tool execution errors, "API failures", "Invalid input data" and "Business logic errors", "in tool results with `isError: true`", apart from protocol errors such as an unknown tool or invalid arguments (https://modelcontextprotocol.io/specification/2025-06-18/server/tools). + +What the comparison settles, point by point: + +1. **The word is `call`.** The open workflow standard names the task `call` and its MCP form calls `tools/call`, the protocol's own method; the HTTP task exists "to call HTTPS APIs". The other words name something else: `fetch` an HTTP read, where a call may write; `connect` a standing binding of a field to its source, not one act; `invoke` is the HTTP task's resource name and would be a second word for the act. This repository already says it: a workflow step is `call: run_definition` with `with` (`workflow-format.md:209`), the facts are `tool_call_started` and `tool_call_answered`, the bound is `toolBounds.callMs` (`packages/mcp/src/bounds/call-bounds.ts:11`). So the block is `call`, a verb beside `with`, as `deliver` is for the shape that asks a party (amendment §1, "The word."), and the facts, the bound and the words keep the one word they have. +2. **Selecting is a place, and reshaping lives where it already lives.** Every product that selects lets the selection reshape: key-value pairs over the result, a mapping language, jq over the output. Here reshaping has two homes, a workflow's `output.as` in jq and a computation function with its own bounds (0005), and the document format names places in a tool's answer with JSON Pointers, `deliver.sent` and `replies.read`, because a tool's answer is data and never instructions (0003 §3; 0020 §2). So `read` is one JSON Pointer (RFC 6901), checked for its shape at save, and a call answers with the value it points at, as it is. +3. **Arguments keep their types.** A static value goes as written and a selected one keeps its type: the HTTP task's static or path-selected values, the three kinds of step input, "Templatable string, Dynamic, or Static". So `with` takes values as written, text templates that render text, and a template that is one `{{ }}` and nothing else, which sends the value it reads as it is. +4. **A failure is the workflow's to retry, by what it is.** The HTTP task declares its retries and catches on the state, the open standard on the `try`, the node opts into Retry On Fail; only the activity is retried by default, and it is told to be idempotent for that reason. Whether a tool may be called again is the server's own word, its `readOnlyHint`, trusted as far as the operator's `allowed` admitted the tool (0018 §3). So a run makes one call and never retries it; its ending is one of the outcomes that exist; a failure after sending, a tool's `isError` among them, which the protocol uses for "API failures" that may pass, is an `unavailable` a workflow may retry when the tool only reads, and a `conflict` that no retry on 503 repeats when it may write. +5. **A call is bounded in time and in what it keeps.** 60 seconds and 256 KiB for the HTTP task; here 10 s to open, 30 s to answer and 64 KiB kept as the output, the bounds a run's call and an interaction function's answer already have (§4). + +## Decision + +### 1. The document: `call`, `read` and `output.schema` + +A function that asks a system names in **`call`** a tool of a tool server the brain may use, the arguments the call takes and, in `read`, where in the tool's answer its data is; `output.schema` says what that data must look like. It sends no message and makes no request, so it writes no body, no `to`, `expires`, `deliver`, `replies`, `from` or `reply`. + +```markdown +--- +description: Read the replies of a thread in the team chat, oldest first +call: + server: chat + tool: thread_replies + with: + channel: '{{ input.channel }}' + ts: '{{ input.thread }}' + limit: 100 + inclusive: false + read: /messages +input: + schema: + type: object + required: [channel, thread] + properties: + channel: { type: string } + thread: { type: string } +output: + schema: + type: array + maxItems: 100 + items: + type: object + required: [user, text, ts] + properties: + user: { type: string } + text: { type: string } + ts: { type: string } +--- +``` + +The names `channel`, `ts`, `limit` and `inclusive` are the chat tool's own, as its `input_schema` gives them, and the brain knows none of them; `limit` and `inclusive` go as the number and the boolean they are written as. `list_tool_servers` lists the tool with `readOnlyHint: true`. Tested with `test_tool_call` and `{ "channel": "C0123", "ts": "1728379900.000050", "limit": 100 }`, the tool answered the `text` "2 replies in the thread." and the `answer` `{ "ok": true, "messages": [ { "user": "member-17", "text": "Shipped the fix to staging.", "ts": "1728380000.000100" }, { "user": "member-42", "text": "Thanks, closing the ticket.", "ts": "1728380100.000200" } ], "has_more": false }`, its structured content, so `read` is `/messages` and the output schema is a list of those objects; a message's other fields pass through, since the schema allows them. A workflow calls it as any function: + +```yaml +- replies: + try: + - read: + call: run_definition + with: + type: interaction + name: thread-replies + input: { channel: '${ .channel }', thread: '${ .thread }' } + catch: + errors: + with: { status: 503 } + retry: + delay: { seconds: 5 } + limit: + attempt: { count: 2 } +``` + +Its server marks the tool read-only, so a failure after the call was sent ends the run `unavailable` as `tools_unfinished`, and the retry on status 503 matches it, as it matches `mcp_server_failed` and `tool_not_offered`, which sent nothing; it never matches `unworkable`, nor the `effect_unknown` of a tool not so marked, both of which raise 409 (§2). + +| Field | What it holds | +| --------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `call.server` | The tool server, as `list_tool_servers` names it: 1 to 32 lowercase letters, digits and hyphens, starting with a letter (`packages/mcp/src/names/tool-reference.ts:8`); required | +| `call.tool` | The tool of that server, as `list_tool_servers` lists it: 1 to 128 letters, digits, underscores, hyphens and dots (`tool-reference.ts:10`); required. Whether the server serves the brain, its `allowed` names the tool and the server lists it, with the hints it gives the tool, is learned at the run, as for a reasoning function's `tools` and a `deliver.tool` | +| `call.with` | The tool's arguments, an object named as its `input_schema` names them, `{}` when left out. A number, `true`, `false` or `null`, and a list or an object of such values at any depth, is sent as written; a text is a Liquid template over `input`, `today` and `now` and is sent as the text it renders; a text that is one `{{ }}` and nothing else is sent as the value it reads, of whatever JSON type it is | +| `call.read` | A JSON Pointer into the document of the tool's answer, to the value the function answers with; left out, the whole document | +| `output.schema` | Required with `call`: the JSON Schema the value must match, which becomes the run's output | +| `input.schema` | Optional, as today | +| `description` | Optional, as today | + +**The document a call reads.** The **document** of a tool's answer is its `structuredContent` when the answer has one; otherwise the text of its first text block, read as JSON when it is JSON and as that text when it is not; an answer with neither has no document. One function says it, `answerDocument(result)`, in the module that says what a model sees, `resultText` (`result-text.ts:62-67`), so the two readings of one answer are written side by side and compared in one test: they agree whenever the tool repeats its structured content as JSON text in one block, as the protocol asks; they part where the tool gives both and its text is not that JSON, where `resultText` shows the text and the document is the structure; where it gives several text blocks, which the model reads joined and the document reads the first of; and where it gives only content that is not text, which the model sees as placeholders and the document does not have. 0020's pointers read the same document: its rule said text that is not JSON has none, and a pointer that is not empty finds nothing in a text either, so every reading it describes ends as it said (§9). A test shows what a model sees, which is not what `read` points into where the two part, so `test_tool_call` answers the document as well, in a field **`answer`**, "the document a call's `read` points into: the structured content, else the first text block parsed as JSON, else that text", built by the same `answerDocument`, scrubbed as `text` is, and left out when the answer has no document or its JSON takes more than 64 KiB, the bound `text` has. It amends 0018 §1, in the paragraph of the appendix, which the build appends to that record. + +**Arguments.** A tool takes typed arguments, an integer `limit`, a boolean `inclusive`, an object of filters, and the rule that every argument is rendered text (`channel-arguments.ts:44,50`) cannot send them. The rule of every `with` of the document format, a delivery's, a reading's, a telling's and a call's, is therefore one, and it is being built with the delivery now, in the amendment's one renderer, `renderedArguments(templates, variables)` (§4), which answers JSON values: a value written as a number, a boolean, `null`, a list or an object goes as written, its texts rendered in place; a text renders as text, refusing a structure among its text as today (`run/text-rendering.ts:29-44`); and a text that is one output and nothing else, `'{{ input.filters }}'`, takes the value the output yields, which the emitter already sees before it becomes text (`text-rendering.ts:33`). `{{ x | json }}` still renders text, so a template that renders text sends what it sends today. This record adds no second renderer: a call's `with` is rendered by that one, over the call's variables. + +**What is checked at save**, in the parser, pure, each refusal with its line: + +| Written | Refused with | +| --------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| A key the front matter or `call` does not hold | "x is not a key of call; it takes server, tool, with, read", the shared refusal of an unknown key (`front-matter-keys.ts:46`), `call` being a section of its own | +| `to`, `expires`, `deliver`, `replies`, `from` or `reply` beside `call`, at that key's line | "A function with call answers at once and asks no party, so it takes no to" (and the key's name in its place) | +| A body beside `call`, at the body's first line | "A function with call sends no message: leave the body after the front matter empty, and say what the function does in description"; the body of a document with `call` is never compiled as a message, where the parser compiles it today on every path, its failures' too (`document/document-parsing.ts:93,132`) | +| `call` without `output.schema`, at `call`'s line | "A function with call answers with what the tool answered, so output.schema says what that is" | +| Neither `call` nor `to`, or `to` without `expires` | "to is required, the party the request goes to, unless the function asks a system with call", and the same for expires, "how long the request waits"; an empty front matter names "the tool it calls, or the party it goes to and when it expires" (`front-matter.ts:30`) | +| `call.server`, `call.tool` | "Expected a server name of 1 to 32 lowercase letters, digits and hyphens, starting with a letter", "Expected a tool name of 1 to 128 letters, digits, underscores, hyphens and dots" (§9) | +| A template of `call.with` that does not compile, or reads a name other than `input`, `today` and `now` | The engine's words, and "x is not a variable of the arguments of a call, which reads input, today and now; assign it first" (`packages/definitions/src/template/template-variables.ts:37`) | +| A template reading a property the input schema does not allow | "input.x is not a property of the input schema, which allows no others" (`template-variables.ts:28`) | +| `${` in a template | "Expected no ${ in a template, which never holds a secret" (amendment §1) | +| A text template that reads no `input` and renders a structure among its text, found by rendering every such template once over a sample `today` and `now` | "Renders a value that is not text among text; write \| json after it, or write it alone as one {{ }} to send it as it is" | +| Arguments that read no `input` and take more than 16 KiB as JSON so rendered | "The arguments take N bytes as JSON, more than the 16384 a call may send" (the words of `tool-tests/tool-test-schemas.ts:11`) | +| `call.read` that is not a JSON Pointer, the empty text among them | "Expected a JSON Pointer such as /messages: it starts with a slash, writes ~ as ~0 and a slash within a name as ~1", 113 characters, the one sentence and the one check of every pointer of the format, `call.read`, `deliver.sent.conversation`, `deliver.sent.id` and each of `replies.read`, so the empty pointer, the whole document, is refused in each; a call that answers with the whole answer leaves `read` out | + +The server and the tool are not checked against any setting at save, as a `deliver` is not (amendment §1): `create_definition` holds no settings and a function saved on one server runs on another. `output.schema` and `input.schema` are compiled as every definition's are (`document-parsing.ts:40-49`). A definition's summary keeps `details: { call: { server, tool } }`, where the other two shapes keep `{ expires }` and `{ expires, deliver: { server, tool } }` (`capability/definition-reading.ts:22-35`, amendment §3). + +### 2. The run: one recorded call, and the endings it may meet + +A run of a function with `call`: + +1. validates the input against `input.schema` (`run/run-input.ts:12-25`); +2. renders the arguments with `renderedArguments` over `momentVariables(input, at)`, `{ input, today, now }` at the run's moment, the one function the request's `to` and message and the amendment's `deliveryVariablesOf` read (§5); an argument reading what the input lacks is `invalid_input` with the words of `request-rendering.ts:48-53`, a text template rendering a structure or arguments over 16 KiB `conflict` `unworkable`, before anything is sent; +3. calls `callOnce(call, recordStart)` with `{ org, brain, reference, input, meta, callId }`, `meta` `{ "com.beonauto/run_id": }` as a reasoning run's calls carry (`capabilities/reasoning/src/tools/tool-opening.ts:28`), the run's longest wait for a 429 (§4), and the run's journal's `started` as the start hook. `callId` is the `call_id` the start records (`packages/mcp/src/calls/call-facts.ts:7`), which every caller gives through `StartedCall.callId` (`calls/recorded-calls.ts:19,51`): a reasoning run's calls give the model's tool-call id (`capabilities/reasoning/src/tools/offered-tools.ts:33`), a test its `test_id` (0018 §4), and a call the run's id, since the run makes one call. `callOnce` checks the names with `namedLinks`, takes a connection and lists the server's tools within the open bound, finds the tool listed with its hints, calls the hook, which appends `tool_call_started` built by `callStarted` and answers its number, sends the call through `forwarded`, and answers `CalledOnce` (§5); +4. on `answered`, appends `tool_call_answered` through the journal's `answered`, built from the fields `answered` carries, the number the hook answered, its `outcome` and its `duration_ms`, as a run's caller appends it whatever the append answers (`tool-caller.ts:51-53`); on `not_sent`, the hook having recorded no start, sends nothing and ends as the table says; +5. on a result, takes the document, the value at `read`, and checks it with `checkedAnswer` against `output.schema`, the one answering check an answer given to `answer_interaction` and a reply taken by 0020 pass (`requests/answer-check.ts:9-26`), with the depth bound a run's input has (`run-input.ts:13`); +6. ends `succeeded` with the value as its output and the record `{ server, tool, read? }`, or with the rejection below. + +There is no step 0: the names check of the amendment's §2, `ToolAccess.named` at a request's start, exists because a request is attempted later; a call is made in the same run, and `callOnce`'s own `namedLinks` refuses before anything is appended, so the check is made once. + +The run declares `calls_tools` on its start and no `finishes_later` (`run-requests.ts:69-70`): `callsTools` is true for a document with `call`, `finishesLater` false, and `longestRunOf` 70 s, the bounds of §4 laid end to end, so a workflow waits for it as for a run that ends within its call, 70 s and the minute every task adds (`workflow-format.md:213`). The capability's runs run on to their end when their caller goes away (`interaction-function.ts:23`; `run-requests.ts:115`), so a call once sent is answered and recorded whatever the caller does, and `forwarded`'s `cancelled` never reaches it. A run in flight cannot be cancelled: `cancel_run` answers `conflict`, "The run runs within the call that started it…" (`run-decisions.ts:60-63,251-252`), as for a reasoning run. + +| What happened | On the run's stream | Ending | In a workflow | HTTP `Retry-After` | +| ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------ | ------------------ | +| The input does not match, or an argument reads what it lacks | the start | `invalid_input`, with the pointer | `validation`, 400 | — | +| A render stopped by the engine's limit or by a filter that refused the value | the start | `invalid_input`, "… cannot be rendered with this input" (`run/request-rendering.ts:55-58`) | `validation`, 400 | — | +| An argument renders a structure among text, or the arguments take more than 16 KiB | the start | `conflict`, `unworkable` | `runtime`, 409 | — | +| The server does not serve the brain, or its `allowed` does not name the tool | the start | `unavailable`, `tool_not_offered`, because `mcp_server_not_configured` or `tool_not_allowed`, in `namedLinks`' words (`access/tool-naming.ts:32-46`) | `communication`, 503 | none | +| No connection or no listing within 10 s, or the server refused either with a 401, a 429 or an error | the start | `unavailable`, `mcp_server_failed`, because `unreachable`, `key_refused`, `rate_limited` or `failing`, "The MCP server chat could not be used: …" (`server-listing.ts:29-34`) | `communication`, 503 | 5 s | +| The server does not list the tool | the start | `unavailable`, `tool_not_offered`, because `tool_not_listed`, "The MCP server chat does not list the tool thread_replies" (`tool-naming.ts:53-60`), as a reasoning run's opening ends | `communication`, 503 | none | +| `CalledOnce` is `not_sent`: the journal recorded no start | the start | `failed`, with an incident: the run dies with "The start of the call could not be recorded on its run, so the call was not sent", and the run records a failed attempt where the ledger takes it (`packages/definitions/src/operations/run-requests.ts:115-119`) | `runtime`, 500 | — | +| The tool refused the arguments, JSON-RPC -32602 (`connections/server-failures.ts:61-64`; `calls/tool-calls.ts:102-104`) | both call facts, `tool_error`, no result | `conflict`, `unworkable`, "The thread replies tool of chat refused the arguments: ‹its words›": the tool never ran and the same arguments meet the same refusal; the run tells it from a tool's error because the answer has no result | `runtime`, 409 | — | +| A tool its server marks `readOnlyHint`: the tool answered `isError`; the server failed on the call, a 429 asked for more than the wait, or no answer came in 30 s | both call facts, `tool_error`, `server_failure` or `timed_out` | `unavailable`, `tools_unfinished`, the new because **`tool_error`** or `server_failed`, in the details of the next two rows; `retry_after_ms` in the rejection's record where the server gave one | `https://on.auto/problems/tools_unfinished`, 503 | none | +| Any other tool: the tool answered `isError` | both call facts, `tool_error` | `conflict`, the new kind **`effect_unknown`**, because `tool_error`, "The thread replies tool of chat answered an error: ‹its words›" | `https://on.auto/problems/effect_unknown`, 409 | — | +| Any other tool: the server failed on the call, a 429 asked for more than the wait, or no answer came in 30 s | both call facts, `server_failure` or `timed_out` | `conflict`, `effect_unknown`, because `server_failed`, "The MCP server chat failed: ‹words›, after the run called the thread replies tool of chat"; `retry_after_ms` in the rejection's record where the server gave one | `https://on.auto/problems/effect_unknown`, 409 | — | +| The answer has no document | both call facts, `result` | `conflict`, `unworkable`, "The thread replies tool of chat answered neither structured content nor text, so there is nothing to read" | `runtime`, 409 | — | +| Nothing at `read`, a text that is not JSON among them | both call facts | `conflict`, `unworkable`, "The answer of the thread replies tool of chat holds nothing at /messages", or "… answered text that is not JSON, which holds nothing at /messages" | `runtime`, 409 | — | +| The value takes more than 64 KiB as JSON or nests deeper than 512 levels | both call facts | `conflict`, `unworkable`, "What the thread replies tool of chat answered at /messages takes N bytes as JSON, more than the 65536 an answer may", or "… nests deeper than the 512 levels a value may" | `runtime`, 409 | — | +| The value does not match `output.schema` | both call facts | `conflict`, `unworkable`, "What the thread replies tool of chat answered at /messages does not match the output schema: ‹the issues›" (`issuesDetail`, as `capabilities/computation/src/run/run-outcome.ts:98-101`) | `runtime`, 409 | — | +| It fits | both call facts | `succeeded`, the value as the output | the task's output | — | + +Each ending is an outcome that exists. `invalid_input` and `conflict` `unworkable` end a run whose definition or input is wrong, as a computation function's do (0005 §2), and a retry on status 503 never matches them. `tool_not_offered` and `mcp_server_failed` end a run before anything was sent, in the words a reasoning run's opening uses, so the run may be made again under its id (`run-state.ts:160-165,183-185`) and `mcp_server_failed`'s remedy, "Nothing was called through it", is true; a tool the server does not list is one of them, refused as a reasoning run's opening refuses it, and never a tool's error. A start the journal refuses is a call not sent: a reasoning run tells its model so and goes on (`packages/mcp/src/calls/tool-caller.ts:7,42-45`), and when the ledger that refused the start refuses the run's ending as well, the run records a failed attempt where it can and its operation answers `failed` with an incident (`run-requests.ts:115-119`); a call has nothing to go on with, so it meets that ending at once. A tool that refused the arguments never ran, and the same arguments meet the same refusal, so that run is `unworkable`; the answer carries no result where the server refused (`calls/tool-calls.ts:102-104`), and a result where the tool answered `isError`, which is how the run tells the two apart. Once the call is sent and the tool may have run, what decides the ending is whether the tool may be called again, and the server's `readOnlyHint` says so: it is trusted here because the operator's `allowed` admitted the tool, which is as far as a brain trusts a server's hints (0018 §3), and the hint decides nothing but whether a retry is safe, which is what the hint is for. + +**One rule for every run that called tools and could not finish.** Today such a run ends `unavailable`, kind `tools_unfinished`, at 503 (`packages/operations/src/outcome/unavailable.ts:7`; `problem.ts:70`), so a workflow's retry on status 503, the workflow reference's own example among them (`workflow-format.md:43-46`), runs again a function whose tool may already have posted, paid or written, which 0003 §6 meant no retry to do. The rule, read over every call the run started, answered or not, since running it again makes every one of them again: when every tool it called is one its server marks `readOnlyHint`, the run ends `unavailable`, kind `tools_unfinished`, 503, whose words now say that the tools only read and running it again is safe, with `retry_after_ms` where the server gave one, and the API sends no `Retry-After` for it, as today: the same `run_id` answers `tools_called` once a call is recorded (`run-decisions.ts:37-41,79`), so a retry of the same request resolves nothing, and `tools_unfinished` stays among the kinds resolved only by change (`problem.ts:88-93`; `docs/reference/http.md:392`; `packages/api/README.md:34`); a run again is a new run, a workflow's retry or a new `run_definition`; when any is not so marked, it ends `conflict`, kind **`effect_unknown`**, 409, under a problem type of its own, since the brain cannot say whether the tool's effect happened, so a person decides, or a workflow rule that names the kind, and the history shows the call. A `conflict` is the outcome for it because the effect does not pass by waiting; `tools_called` already means "this id cannot run again" and `unworkable` "the definition or the input is wrong" (`explanation.ts:60,70-75`), so neither fits. The history still holds the call in either case, so the run is not made again under its own id, and a workflow's retry is a new run. `Conflict` gains the optional `because` `Unavailable` has (`unavailable.ts:34-39`), so `effect_unknown` carries the becauses `tools_unfinished` carries, everywhere a rejection travels (Build, step 1, lists each place); the workflow's mapping settles `effect_unknown` as a `conflict` with its because, reading the reason of the kind from `reasonOfKind` (`packages/operations/src/outcome/problem-types.ts:21`), where its `tools_called` branch keeps none (`packages/workflow-engine/src/dsl/raised-error.ts:104,116-126`); and 0003 §4 and §6 are read as amended in the paragraph of the appendix, which the build appends to that record. + +**The word.** The owner's decision says a call brings data from a system or "an effect on" one, the capability declares whether it may have one, `mayChangeOutside` (`packages/definitions/src/capability/capability.ts:112`), and 0018 §3 says of a tool without the hint that it "may change something"; 0003 §5 calls a call without its answer "an outcome unknown", and `outcome` already names a recorded call's `result`, `tool_error` and the rest (`packages/mcp/src/calls/call-facts.ts:3`). So the kind says the effect, the thing a retry would repeat, and what the brain knows of it, `effect_unknown`; `may_have_changed` would be a second name for the `mayHaveChanged` every cancellation's words carry (`explanation.ts:10`), and `unsafe_to_retry` names one consequence where the person needs the fact. Its words: the why "it could not finish after calling a tool that may change something, so whether that happened is not known", 105 characters, and the remedy "It is not run again by itself: a person decides, or a workflow rule that names this kind; its history shows the call.", 117. `tools_unfinished` keeps its why, "it called tools but could not finish", and its remedy reads "Every tool it called only reads, by its server's own account, so running it again is safe: a new run, or a workflow's retry, may make it; its history shows what it called.", 171, and loses `mayHaveChanged`, so `run_definition`'s refusal adds "Nothing was changed." (`explanation.ts:139-144,197`). Each is measured in the two forms a run's ending is said in: `get_run`'s outcome sentence, "The run of ‹the function› did not go through: ‹why›, ‹because›. ‹remedy›" (`packages/definitions/src/plain-language/run-words.ts:46-49`), cut at 400 (`packages/api/src/tools/tool-result.ts:56`), and `run_definition`'s refusal, "Could not run ‹the function›: ‹why›, ‹because›.‹ Nothing was changed.› ‹remedy›", cut at 600 (`tool-result.ts:41`). `effect_unknown` takes 363 characters in the first with `tool_error` and the name "thread-replies", and 397 with a name of 48 characters, the longest a name may be (`packages/definitions/src/operations/definition-fields.ts:13`), so no because is cut; 381 in the second. `tools_unfinished` takes at most 382 in the first, with `tool_error` and a 48-character name, and 387 in the second. + +**For a reasoning run** the same rule reads over every tool its run called. The adapter knows the hints: its opening lists each named server's tools to offer them, and each offered tool carries the listed tool's `annotations` (`packages/mcp/src/access/run-opening.ts:30-49`; `calls/run-tools.ts:53-59`); the run records each tool it calls once its start is appended (`calls/tool-caller.ts:46`), so `RunTools` answers, beside `calledAny` and `usedInWords` (`run-tools.ts:65-66`), whether every tool it called is marked read-only, and `unfinished` in `capabilities/reasoning/src/tools/tool-endings.ts:19-26` builds `Unavailable` `tools_unfinished` when it is and `Conflict` `effect_unknown` when it is not, with the becauses and the detail it builds today; `stoppedEnding` and `unavailableAfter` (`:39-62`) and their callers (`capability/model-rejection.ts:80,111`) answer either. + +The one new because, `tool_error`, serves both kinds: their becauses say a server failed, a model stopped, a bound or a model that would not answer (`explanation.ts:159-162`), and a tool that answered an error is none of those; its words, "because the tool answered an error, which the details below give", 64 characters, put the operator in front of the tool's answer and not of the server, and a workflow tells it from a server failure with `when: '${ $error.because == "tool_error" }'`. The words of `server_failed`, "because a tool server kept failing" (`explanation.ts:159`), are false for one call, so they read "because a tool server failed", true of a reasoning run's five failures and of a call's one. A server's `Retry-After` stays in the rejection's record as `retry_after_ms`, as a delivery keeps it (`run-events.ts:122`); the HTTP problem's `Retry-After` is the API's own, a constant five seconds that never reads `retry_after_ms` (`problem.ts:86,114`), sent for an `unavailable` a retry of the same request may resolve and never for `tools_unfinished` or a `conflict` (`:88-97`), and a workflow's retry waits by its own `delay`, since an error carries no wait (`raised-error.ts:65-75`). + +What the run records is a reasoning run's: `tool_call_started` with `server`, `tool`, the arguments' size and digest, `read_only: true` where the server marks the tool `readOnlyHint` (an amendment from the build), and, where the entry says `record_content`, their text; `tool_call_answered` with the outcome, the result's size and digest, `duration_ms`, `jsonrpc_id`, the server's request id where the entry names one and, with `record_content`, the result's text, each scrubbed and cut to 4 KiB as stored (`recorded-calls.ts:44-80`); the history shows them at 2 KiB, "A run made tool call 1, to the thread replies tool of chat." and "Tool call 1 answered." (`packages/definitions/src/plain-language/event-words.ts:63-69`), since `plainNumber` spells out 100 to 599 alone (`packages/operations/src/plain-language/phrasing.ts:34-36,64-67`). There is no deferral, no request, no `list_interactions` row and no delivery fact. The run's record `{ server, tool, read? }` lets `get_run` say what gave the output without the history. + +### 3. Guards + +- **Who may write it** is who may write the brain, `brain:write`, the permission a reasoning function naming the same tool needs; anyone who may write a brain can already read and change whatever the tool reaches by writing that function and running it, so nothing new is reachable (0018 §3; amendment §2). The call reaches less than that function: its arguments are written by its author and rendered from its input, never chosen by a model reading the tool's answers. +- **What it may call** is what the server's `allowed` names on the server's entry (0003's per-server amendment), checked live by `namedLinks` at every run, before anything is appended. +- **What is kept.** `record_content` decides, per server, whether the call's arguments and answer are kept in its facts, as for every call (0003 §5). The run's output is not a fact of the call: it is the value the function's author chose with `read` and `output.schema`, kept and shown to every reader of the brain as every run's output is, as a reasoning function's answer may repeat what its tool answered under the same rule. A server whose content must not be kept is one whose tools a brain's writers should not be allowed to call, which `allowed` says. +- **Secrets.** A template may not hold `${` and a server is named by name; `callOnce` scrubs the result, content and structured content alike, with every server's secrets, in the JSON and JSON-escaped forms the scrubber knows (`bounds/secrets.ts:14-33`), before the run reads it, so the output and the words are scrubbed too. +- **What a client is told.** `run_definition` is destructive, and the capability reaches outside, while any tool server is configured, `ToolAccess.configured`, as the amendment has it (§3) and as a reasoning function's `mayChangeOutside` is (`capabilities/reasoning/src/capability/reasoning-function.ts:71`). Its description's sentence "A reasoning function that names tools may change something outside the brain…" (`packages/definitions/src/operations/run-definition.ts:17`) reads "A function that calls a tool may change something outside the brain…", 789 characters of 800. + +### 4. Bounds + +| Bound | Value | When reached | +| -------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------- | +| The arguments, rendered | 16 KiB as JSON, `toolBounds.argumentBytes` (`call-bounds.ts:7`) | Refused at save when none reads `input`; otherwise `conflict`, `unworkable`, nothing sent | +| A template of `call.with` | The engine's limits, 65,536 characters, 1,000 names, 200 ms and 5 MB to render (`packages/definitions/src/template/template-engine.ts:13-19`) | Refused at save; `invalid_input` at the run, as a request's template | +| Opening a connection; listing the tools | 10 s each, `toolBounds.openMs` (`call-bounds.ts:12`): the taking raced against the bound by `takenWithin` (`delivery/connection-bound.ts:28-35`), an HTTP opening waiting out a 429 within it (`connections/server-links.ts:48-55`), the listing within the same bound, as a run's opening and a test have it (0018 §6) | `unavailable`, `mcp_server_failed`, `unreachable`, nothing sent | +| The call | 30 s, `toolBounds.callMs` (`call-bounds.ts:11`), from the first sending (`calls/tool-calls.ts:141-144`); a 429 waited out within it | A failure after sending, `server_failed`: `unavailable` `tools_unfinished` for a read-only tool, `conflict` `effect_unknown` for any other (§2) | +| A 429 on the call | waited out when it asks for at most 10 s, `toolBounds.longestRetryWaitMs` (`call-bounds.ts:10`; `tool-calls.ts:94-98`) | Past that, a failure after sending, with `retry_after_ms` | +| A process that exited; a session the server forgot | restarted once before the call's 30 s begin, and reopened once outside them, each within the open bound by `boundedSlot` (`connection-bound.ts:43-55`; `tool-calls.ts:99,142-143`) | Then a failure after sending | +| The value the function answers with | 64 KiB as JSON and 512 levels, `interactionBounds.answerBytes` and `answerDepth`, the bound of every answer of an interaction function (`run/run-bounds.ts:5-6`) | `conflict`, `unworkable` | +| The tool's whole answer | Received whole, as a run's call receives it; kept as its size and digest, and its content at 4 KiB with `record_content` | — | +| Calls in one run | 1 | There is no loop | +| The longest run | 70 s: 10 to open, 10 to list, 10 to restart a process that exited, 30 to call and 10 to reopen a forgotten session | A workflow waits 70 s and a minute | +| `test_tool_call`'s `answer` | Shown when its JSON takes at most 64 KiB, `toolBounds.resultBytes`, the bound of `text` | Left out; `result_bytes` says how large the answer was | + +No bound is added. The call waits out a short 429 as a reasoning run's call and a test do (0018 §6): `forwarded` sends again a request the server refused to take, or one a session forgot, which is not a second call of a tool that may have acted; the delivery keeps its own zero, since its schedule honours a 429 (0010 §3), so the longest wait becomes the caller's, where `callOnce` fixes it at zero today (`call-once.ts:54`). `takenWithin` and `boundedSlot` stay as they are (`connection-bound.ts:28-55`): they bound the taking, a restart and a reopening by the open bound, which the call's longest run adds up. + +### 5. One path: what the three shapes share and what this one adds + +| Piece | Shared with | What a call adds or renames | +| ----------------------------------------------------------------------- | ------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `callOnce(call, recordStart?)`, answering `CalledOnce` (amendment §3) | The delivery, which passes no hook, 0020's read and telling | The optional hook, which the call alone passes, run once the connection is taken and the tool found listed, right before the call is sent; `answered` carries, with a hook and without, the fields the amendment lists, those `callAnswered` builds less `type`, `number`, `outcome` and `duration_ms`, with the outcome, the duration, the scrubbed result, the failure's words and `Retry-After`, and also the listed tool's hints; `not_sent` when the hook recorded no start; the caller gives the longest 429 wait | +| `unopened`, a connection not opened or a tool not listed | Every caller of `callOnce`, as a run's opening | Nothing: it is the `ToolsNotOpened` a run's opening answers (`access/caller-context.ts:20`), from `serverFailed` and `notListed` (`server-listing.ts:18-34`; `tool-naming.ts:53-60`), the slot taken within `takenWithin` and the tools listed within the open bound as `connectedTo` lists them (`server-listing.ts:36-58`) | +| `journalledCall(slot, call, recording, recordStart?)` | A reasoning run's `caller` (`tool-caller.ts:41-53`) | The one signature `caller` and `callOnce` read for the one sequence, start built and recorded where a hook is given, call forwarded, answer fields built; `caller` keeps its admission and passes its journal's `started`, `callOnce` its names check and opening | +| `callStarted`, `callAnswered` under the server's `Recording` | Every recorded call | Nothing | +| The run's journal (`tool-call-journal.ts:54-58`) | A reasoning run | Nothing: its `started` is the hook | +| `answerDocument(result)` beside `resultText` | 0020's `deliver.sent` and `replies.read` | `call.read`, and `test_tool_call`'s `answer` | +| The pointer reader and the pointer's save check | `deliver.sent`, `replies.read` | `call.read` | +| `ToolUseSchema`, `{ server, tool, with }` | `DeliverBlockSchema`, the amendment's `ToolDeliverySchema`, adds `sent` | `CallBlockSchema` adds `read` (§9) | +| `compiledToolBlock(written, line, names)` (amendment's `compiledRoute`) | `deliver`, `replies`, `tell` | `call.with` with the names `input`, `today`, `now` | +| `renderedArguments(templates, variables)`, typed (§1) | The delivery, the read, the telling, built with the delivery | Nothing: a call's `with` is rendered by it | +| `momentVariables(input, at)` | `to` and the message (`interaction-run.ts:18-20`), `deliveryVariablesOf` | The call's variables | +| `checkedAnswer` | `answer_interaction`, 0020's taking | The call's value | +| `namedLinks` and its words | Every caller | Nothing | +| `toolInWords(reference)` | The history, the refusals, the delivery's words, the test's words | The outcome sentence (§7) | +| `routeOf` (amendment §4) | A request's attempt, read and telling | Nothing: a call makes no request, so it has no route | + +**The start hook is this record's.** A delivery records its own `delivery_started` before it calls, as today (`capabilities/interaction/src/schedule/request-attempts.ts:46-48,55`), numbered by the decider and holding the server, the tool and the arguments' fields it knows before any connection, and calls `callOnce(call)` without a hook; `callOnce` answers it whole, `unopened` among its answers when the connection could not be opened or the server does not list the tool, which the delivery records as a failed attempt (amendment §3), and `answered` with the same fields as for a call. The one call path is `callOnce(call, recordStart?)`, and the call shape alone passes the hook: `callOnce(call, recordStart)` runs it after the connection is taken and the tool is found listed, and before the call is sent, so `tool_call_started` means that the call was sent, as a reasoning run's does, whose calls start only on slots its opening took and tools it found listed (`access/run-opening.ts:30-49`; `calls/tool-caller.ts:41-46`). A run whose server could not be reached, or does not list the tool, records no call and can be made again under its id. Listing at every call costs an `http` server that no run holds six requests, as a test does (0018 §2), and gives the call the tool's hints, which §2 reads. + +**Where the code lives.** The amendment's names were made for a delivery and serve three shapes; the build renames each (§9, Build): the `route` folder (amendment §4) becomes `tool-blocks`, holding `ToolUseSchema`, `{ server, tool, with }`, with `DeliverBlockSchema`, the amendment's `ToolDeliverySchema`, adding `sent`, `CallBlockSchema` adding `read`, and `RepliesSchema` as it is, `compiledToolBlock`, the amendment's `compiledRoute`, `renderedArguments`, `momentVariables` and the amendment's `deliveryVariablesOf`, which builds on it, the pointer reader with the one pointer check, and `routeOf`, which keeps its name since only a request has a route; a folder `call` holds the run of a call and its endings, beside `run`, which keeps the request's. The document becomes a union over its shape, `{ shape: 'call', call }` or `{ shape: 'request', to, expires, message, deliver?, replies?, from?, reply? }`, with `description`, `input` and `output` common, and `finishesLater`, `callsTools` and `longestRunOf` read the shape. The adapter's ports take the tool access, `callOnce` with `named` and `configured` as the amendment gives them (`run/request-reach.ts:12-16`; `packages/server/src/composition/served-interaction.ts:26,37`, where only the due work has it today). + +### 6. What is not built + +- **No retry inside the run.** One call, answered or failed; the workflow's `catch` and `retry` are the one retry, as the open standard and the HTTP task place it (Context, point 4). `forwarded`'s resend of a request a server refused to take or forgot is the call path's, shared by every caller, and is not a retry of a call. +- **No model**, no prompt, no body. +- **No request**: no deferral, no `list_interactions` row, no expiry, no open-requests count, so `requests_full` is never checked, and `answer_interaction` on such a run answers "The run is not a request of an interaction function that waits for an answer" (`requests/answer-interaction.ts:52-54`). +- **No pagination.** A tool that pages answers one page for each run. Its function answers with the part of the answer that holds both the items and the cursor, leaving `read` out or pointing at a parent of both, with an output schema of that object, and a workflow calls it again with the cursor while there is one, keeping what it read in `$context` with `export.as` and jumping back with `then`, or with `for` and `while` (`workflow-format.md:168-191`); each page is a recorded run of its own, within 64 KiB, and the workflow's output stays within its own bound. A call that follows a cursor itself is a later record's (§10). +- **No direct HTTP.** MCP stays the transport; a system without a tool server is reached through one. + +### 7. Words and texts + +**The deferral does not apply**: a call is not deferred, and `interaction_requested` is never recorded for it. + +**The outcome sentence of `run_definition`.** `describeOutput` takes the run's record beside its output, `(output, record)`, since the output alone cannot tell a notification's `{}` from a call that answered `{}` (`interaction-words.ts:47-48`), and the run words take the record from the run they describe: `run_definition` answers the same `RunDetail` as `get_run`, record included (`packages/definitions/src/runs/run.ts:100-111`; `operations/run-definition.ts:37`, which answers `RunSchema` today, against `operations/get-run.ts:25`), built for both by `runOf` (`runs/run-lookup.ts:16-18`), which takes in `detailOf` (`:20-22`), so there is one run shape on the wire and the run words' `DescribedRun` (`run-words.ts:9`) picks `record` beside `output`. A call's run then reads, measured with a copy of `inWords` and the phrasing helpers: + +> Ran the interaction function “thread-replies”. It asked the thread replies tool of chat; its answer: (user: “member-17”, text: “Shipped the fix to staging.”, and ts: “1728380000.000100”) and (user: “member-42”, text: “Thanks, closing the ticket.”, and ts: “1728380100.000200”). + +277 characters, with the list's records in parentheses (§9); "Ran the interaction function “thread-replies”. It asked the thread replies tool of chat, whose answer of 14 items is too long to repeat here; the whole of it is in the details below." when the words pass their 300 (`packages/definitions/src/plain-language/in-words.ts:4,51-54`), 182; "…; its answer: nothing." for `{}`, 109. Past 400 the existing cut keeps the first sentence and "The rest is in the details below." (`word-bounds.ts:15-27`). It names the tool in words and never as `chat/thread_replies`, as the history and the explanation of a refusal do (0003 §2), and counts items only where the answer is a list, since the brain does not know what a row is. A request's sentences do not change: "It delivered its notification." for a notification, read from its request record, and "Its answer: …". + +**The refusal words**, each measured in `get_run`'s outcome sentence, cut at 400 (`run-words.ts:46-49`; `tool-result.ts:56`), and in `run_definition`'s refusal, "Could not run …", cut at 600 (`tool-result.ts:41`), with the name "thread-replies" and with a name of 48 characters, the longest (`definition-fields.ts:13`). A tool's error, for a tool not marked read-only, reads "The run of the interaction function “thread-replies” did not go through: it could not finish after calling a tool that may change something, so whether that happened is not known, because the tool answered an error, which the details below give. It is not run again by itself: a person decides, or a workflow rule that names this kind; its history shows the call.", 363 characters, 397 with the longest name, and 347 and 381 as a refusal; with `server_failed`, 327 and 361; a reasoning run's with `no_answer`, the longest of its becauses, 354 and 388; the run words pass a `conflict`'s because as they pass an `unavailable`'s (`run-words.ts:28-44`). For a read-only tool, "… did not go through: it called tools but could not finish, because the tool answered an error, which the details below give. Every tool it called only reads, by its server's own account, so running it again is safe: a new run, or a workflow's retry, may make it; its history shows what it called.", 348, 382 with the longest name, and 353 and 387 as a refusal, which adds "Nothing was changed."; with `server_failed`, 312 and 346. A tool the server does not list, "… did not go through: this server does not offer a tool it names, because the tool server it names does not have that tool. This can be put right on your side: …", 374, and 408 with the longest name, which the existing cut ends after its first sentence, as it does for a reasoning run today; 379 and 413 as a refusal. An `unworkable` call, "… did not go through: it cannot work as it is written. This can be corrected and tried again; the details below say what to change.", 182 and 216; 187 and 221 as a refusal. + +**The definition's summary** names `call: { server, tool }` in its details (§1); `get_definition` and `list_definitions` are in §8. + +**The instructions.** "An interaction function asks a person or a system and takes the answer later." is wrong for a call, and so is "A run of an interaction function or a workflow answers started". The interaction sentence becomes "An interaction function asks a system and answers at once, or asks a person and answers started.", 96 characters, and says itself what the run sentence said of it; the run sentence names the workflow alone, "A run of a workflow answers started; get_run shows whether it ended or still waits.", 89, the text a server without interaction functions already serves (`instructions.test.ts:33`), so `typesThatFinishLater` holds `workflow` alone (`instructions.ts:126`). Measured by running a copy of `instructionsFor` over the tool lists of `served-instructions.ts`, which reproduces 1,495, 1,950 and 1,990: **1,514** on the org endpoint, **1,942** on the brain endpoint and **1,982** on `/mcp`, each strictly under 2,000, `/mcp` with 17 characters of room where it had 9. The terminology page's longer sentence, "An interaction function asks a system and answers at once, or asks a person and takes the answer later.", would take 1,989 on `/mcp` with the run sentence for workflows alone and say nothing of a run that answers started, so the instructions keep the shorter one and the terminology page the longer. + +**The `interaction-function` guide** gains a section, "Asking a system", after "A function document" and before the amendment's "Sending through a tool", in the agent's order: list the servers and their hints, test the tool and read its `answer`, write `call`, `read` and `output.schema`, save; then the arguments, then the endings and paging. 3,186 bytes, in the appendix; the page's opening names the three shapes, its field table gains `call` and says `to` and `expires` are required without it and refused with it, "How a run ends" and "Bounds" gain the rows of §2 and §4, and "In a workflow" says a call's step takes its output within the call. The page stays near 24 KB served of its 64 KiB: the 17.4 KB the amendment leaves (§6), 0020's subsection and rows, and this section with its rows, about 5 KB. + +**The `give-tools` recipe** routes every tool to a reasoning function, so an agent asked for "the open invoices each morning" would write the prompt this record retires. Its step 5 gains one sentence, 289 bytes with its leading space, before the amendment's: " When the brain only needs what a tool answers, with nothing to reason about, write an interaction function that calls the tool instead: it answers at once with what the tool answered and spends no model call, and the interaction-function guide says how to write it from the test's answer." It is appended where and only where the amendment's sentence is, where the server serves interaction functions, which `recipesFor` knows from the guides of the definition types it is given (`packages/server/src/guides/recipes.ts:70-74`), since both send the agent to the same guide; `recipesFollowedWith`, which filters by the tools a connection lists, is not that predicate. Its step 4, "read the answer: that is what the function's model will see", reads "read its `text`: that is what the function's model will see", since a test now answers `answer` beside `text`, at the same byte count; the recipe then takes 2,052 bytes where interaction functions are not served and 2,601 where they are, of 4,096; it names no new tool, so it is offered where it was. **`first-brain`** needs no clause: its step 4 sends every use of a tool server to `give-tools`, where the sentence is, and a second router in its step 7 would say the same thing twice. + +**The HTTP reference's** sentence that an interaction run "waits until its request is answered, expires or is cancelled; a notification to the inbox succeeds within the request" (`docs/reference/http.md:59`) gains its exception: "…; a notification to the inbox, and a function that calls a tool, answer within the request, the second with what the tool answered as its output or with an ending its format lists." Its `cancel_run` paragraph (`:110`) and the tool's description (`packages/definitions/src/cancellation/cancel-run.ts:23`) name a run that ends within its call: "…; a run that ends within its call, as a reasoning, computation or recall function does or an interaction function that calls a tool, cannot be cancelled.", 683 characters of 800. Its history paragraph already says a tool-using run records the call facts (`:185`). The "Testing a tool" table gains `answer`. **The workflow reference** says a call's run ends within the call and waits 70 s (`workflow-format.md:213`), the task's output is "the answer an interaction function took" (`:209`), and its retry example and prose say that a 503 is safe to try again and a 409 is not, with `effect_unknown` raised at 409 under its own type (§2, §9). **The terminology page** takes the longer sentence (`docs/concepts/terminology.md:72`); its next, "One run makes one request; the answer settles that run…", reads "A run that asks a person makes one request; the answer settles that run…", and the Request row (`:66`) reads "What a run of an interaction function that asks a person asks: …", since a run that asks a system makes none; the Answer row says "the party or the tool" (`:69`), and no row is added for a call; `scripts/docs-interaction.test.ts:43` requires the new sentence. + +### 8. Studio and the API + +A call function's `details` hold `call: { server, tool }`. How `get_definition` and `list_definitions` show a definition's shape without its source is decided by the record on definition fields, which is pending; this record does not depend on it. `get_run` answers a call's run with its output and its record `{ server, tool, read? }`; `get_run_history` its two call facts; `list_brain_events` the run's start and end beside them; a recall function folds the run's start and its ending with the output, as for every run, and never the call facts (`packages/workflow-host/src/follower/followed-events.ts:20-25`), so what a call read yesterday is what a view can remember; an event trigger reacts to `interaction/` ending as to any run's. `list_interactions` never shows a call. `test_tool_call` answers `answer` (§1). + +### 9. Debt met on the way, fixed in the design + +| Debt | What the build does | +| ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `calledOnce` takes a connection its own way, lists nothing, and answers a connection it could not take as a call's `server_failure` or `timed_out` (`packages/mcp/src/delivery/call-once.ts:35-46`), where a run's opening takes, lists and refuses through `connectedTo`, `serverFailed` and `notListed` (`access/server-listing.ts:18-58`; `access/tool-naming.ts:53-60,99-103`) | `callOnce` takes the slot within `takenWithin`, lists the tools as `connectedTo` lists them and finds the tool with `unlistedOn`, answering `unopened` with the `ToolsNotOpened` a run's opening answers, through `serverFailed` and `notListed` (§5), before any hook | +| The start-forward-answer sequence is written in `caller` (`calls/tool-caller.ts:41-53`) and again in the amended `callOnce` | One `journalledCall` in `calls/`, read by both | +| `deliveryBounds.connectionMs` and `callMs` copy `toolBounds.openMs` and `callMs` (`delivery/delivery-bounds.ts:7`, `bounds/call-bounds.ts:11-12`), clamped again at `delivery/connection-bound.ts:15-17` and `call-once.ts:53`; the longest 429 wait is fixed at zero (`call-once.ts:54`) | `callOnce` reads `Timing`; `deliveryBounds` goes whole, its `resultBytes` already gone by the amendment; the caller gives the longest wait, zero for a delivery and `toolBounds.longestRetryWaitMs` for a call | +| The single-call path is named for one caller: the folder `delivery/`, `DeliveryCall`, `deliveredCall` (`delivery-bounds.ts:9-16`, `delivery-call.ts:8`, `index.ts:10`), now the call path of a delivery, a read, a telling and a call | The folder is `one-call/`, the call `OneCall`, `deliveredCall` `oneCall`, beside the `calledOnce` it calls (`call-once.ts:29`), which keeps its name; `CalledOnce` keeps the amendment's name; `takenWithin` and `boundedSlot` stay (`connection-bound.ts:28-55`) | +| The delivery amendment's draft repeats `runIdKey` of `calls/call-meta.ts:1` in a second file of `calls/`, and nothing imports it | Removed | +| `noServerFor` (`capabilities/reasoning/src/tools/tool-opening.ts:14-21`) keeps the subject "The reasoning function names …" that 0018 took out of `namedLinks`, for an adapter given no tool access (`definition-run.ts:16`), which the server never builds (`packages/server/src/composition/served-tools.ts:33-36`) | The adapter's `tools` is required, tests give an access with no servers, and `namedLinks`' "No MCP server named … is configured for this brain" is the one sentence | +| Two readings of one answer: `resultText` (`bounds/result-text.ts:62-67`) for a model and 0020's document for a pointer, in two places; a test shows only the first; 0020's read bounds "the result" at 64 KiB (0020 §6, step 4), which, measured as `result_bytes` is (`calls/recorded-calls.ts:60-67`), counts a structure and its repeated text twice | `answerDocument` beside `resultText` in one module, named for both, `bounds/tool-results.ts`, with one test comparing them; `test_tool_call` answers `answer` beside `text`, amending 0018 §1; 0020's read bounds the document it reads, not the whole result | +| Arguments are text only (`channels/channel-arguments.ts:31-34,44,50`), as 0020's `limit: '15'` shows | Being paid in the delivery's build: the one `renderedArguments` answers typed values for every `with` (§1); this record reads it and adds no renderer | +| Five copies of the output sentence, "Its answer…" or "Its result is too long to repeat here…" (`capabilities/interaction/src/capability/interaction-words.ts:46-54`, `capabilities/reasoning/src/capability/reasoning-function.ts:46-51`, `capabilities/computation/src/capability/computation-function.ts:45-50`, `capabilities/recall/src/capability/recall-function.ts:25-29`, `capabilities/coordination/src/capability/workflow.ts:25-29`), and an output alone cannot tell a notification from a call | `outputInWords(noun)` in the definitions package, read by the five; `describeOutput(output, record)` (§7) | +| A list of records reads as one run of fields, "…, and ts: “…” and user: “…”", because a list's items are not wrapped as an object's nested values are (`in-words.ts:14-16` against `:29-33`) | A list's items are wrapped as `nested` wraps them; the pinned texts that list records change with it | +| A tool is put in words four ways: `inWords` (`names/tool-words.ts:18-20`, "the thread replies tool of chat"), `nameInWords` (`packages/definitions/src/plain-language/event-words.ts:57-61`), the amendment's "through the tool post_message of chat" (§3) and 0018's "The tool “search” of “graph”" (§1) | One `toolInWords(reference)` in `@beonauto/mcp`, which the definitions package reads since 0018, in the form the history and the refusals use; the amendment's `throughWords` and `deliveryStarted` and the test's words read it | +| The server's and the tool's shape words exist only inside `toolReferenceShape` (`names/tool-reference.ts:46-47`), and the per-server and delivery amendments write "Expected a tool name of 1 to 128 …" anew | Two constants, composed into the third, read by the settings, the delivery's and the call's refusals | +| The amendment's `ToolDeliverySchema`, `compiledRoute` and `route` folder are named for a delivery and serve a call too | `ToolUseSchema`, `{ server, tool, with }`, with `DeliverBlockSchema`, the former `ToolDeliverySchema`, adding `sent` and `CallBlockSchema` adding `read`; `compiledToolBlock` for `compiledRoute`; the folder `tool-blocks` for `route`; `routeOf`, a request's, keeps its name | +| `{ input, today, now }` is built at `run/interaction-run.ts:18-20` and again by the amendment's `deliveryVariablesOf` from `requested_at` | One `momentVariables(input, at)` | +| `tools_unfinished` is an `unavailable` at 503 (`packages/operations/src/outcome/unavailable.ts:7`; `packages/api/src/problem/problem.ts:70`; `docs/reference/workflow-format.md:227`), so the reference's own retry on `{ status: 503 }` (`:43-46`) repeats a run whose tools may have acted, where its prose says a `catch` retrying communication errors does not (`:234`) and 0003 §6 meant none to; a `catch` matches by the fields it names (`task-outcomes.ts:56`) | One rule for every run (§2): `tools_unfinished` stays an `unavailable` at 503 and means that every tool the run called is read-only, its remedy saying a run again is safe and its `mayHaveChanged` gone, and still among the kinds resolved only by change (`problem.ts:88-93`), so no `Retry-After`; the new `conflict` kind `effect_unknown` (`conflict.ts:3-11`) answers 409 under a problem type of its own (`problem.ts:69-73`; `packages/operations/src/outcome/problem-types.ts:3-11`), with its words in `explanation.ts`; `Conflict` gains the optional `because`, in every place Build step 1 lists; the workflow's mapping settles `effect_unknown` as a `conflict` with its because (`packages/workflow-engine/src/dsl/raised-error.ts:104,116-126`); `RunTools` answers whether every tool called is marked read-only and `unfinished` builds either (`capabilities/reasoning/src/tools/tool-endings.ts:19-26`); the reference's table gains the row of `effect_unknown` at 409 (`workflow-format.md:227`), and its example and prose (`:43-46,234`) say that a 503 is safe to try again and a 409 is not | +| The becauses are `UnavailableBecauseSchema` (`unavailable.ts:15-30`), named for the one reason that carried them, and serve a `conflict` too | `RejectionBecauseSchema`, read by `Unavailable` and `Conflict` | +| The computation reference and the concepts page teach a reasoning function that passes a tool's rows on unchanged (`docs/reference/computation-format.md:181-214,254-261`; `docs/concepts/functions.md:67`) | Both read figures through an interaction function that calls the tool, `read-campaign-costs` with `call: { server: ads, tool: campaign-costs, with: { month: '{{ input.month }}' }, read: /rows }` and an output schema of the rows, its workflow step `type: interaction` with `output.as: '${ { rows: ., period: $input.period } }'`, since the function answers the rows themselves | +| The descriptions of the run's kinds and becauses (`packages/definitions/src/runs/run.ts:22,28,38`), of `run_definition` (`run-definition.ts:13,17`) and of `cancel_run` (`cancel-run.ts:23`) know a tool call only as a reasoning function's | Each names the call: `tools_unfinished` for a run whose every tool is read-only, `effect_unknown` among the kinds of a `conflict` with the same becauses, `tool_error` among them, an interaction function whose tool's answer gave no output among `unworkable`'s cases, and the two sentences of §3 and §7 | + +### 10. What this record does not decide + +A call that follows a tool's cursor itself; a call of two tools, one resolving what the other takes, which a workflow of two calls or a reasoning function does; whether the operator's `testable`, which vouches that a tool may be tested, should also make a failure after sending safe to retry, where this record reads the server's `readOnlyHint` alone; a workflow retry that waits as a server asked, which needs an error to carry a wait; how `get_definition` and `list_definitions` show a definition's shape, which the pending record on definition fields decides; a call's answer kept between runs; bounds per function; a call that answers binary content or a resource link; the hosted runtime, which offers interaction functions when it offers tool servers (amendment §3); the rename of the wire names (0007), which touches `call` nowhere. + +## Consequences + +- A workflow reads from and acts on a system through a function saved, versioned and checked like every other: one recorded call, its arguments written and typed, its answer read at a named place, checked against a declared shape and bounded, with no model asked, no prompt to trust and no tool output read by a model; the computation reference's reading function becomes this. +- An interaction function has three shapes in one format and one run path: a call answers at once, a request through a tool and a request in the inbox answer later. They share the tool blocks, the renderer, the variables, the answer check, the call path, the recorder and the document of an answer; the request's schedule, projection and route stay the request's. +- Every ending is an outcome that exists but one, with one because added; a retry on status 503 matches what is safe to try again, a run whose tools only read and a call that sent nothing, and never a definition, an input or an answer that is wrong, nor a run whose tool may have acted; a call that reached its tool is never made again under its id. +- One rule for every run that called tools and could not finish, reasoning and call alike: `tools_unfinished`, 503, when every tool it called is read-only, and `effect_unknown`, a `conflict` at 409, when any is not, so no retry on 503, the workflow reference's own example among them, repeats a tool that may already have posted, paid or written. +- The call path becomes one: `callOnce` opens, lists and refuses as a run's opening does, shares the recorded sequence with a run's caller, and is named for what it is. +- `run_definition` answers `RunDetail`, the run with its `record`, as `get_run` does: an additive change of the wire, which the owner of Studio is told of, since a client that reads `run_definition`'s answer meets a field it did not have. +- The cost: `describeOutput` takes the record, so five capabilities change a signature; `callOnce` lists the server's tools at every call, six requests to an `http` server no run holds, and takes an optional hook; `Conflict` takes a `because`, and an HTTP caller or a workflow that read a reasoning run's `tools_unfinished` at 503 reads `effect_unknown` at 409 wherever the run called a tool not marked read-only, which is most tools of today's servers; `test_tool_call` gains a field; the instructions change two sentences and shrink by 8 characters on `/mcp`; the guide grows by 3,186 bytes and `give-tools` by 289. + +## Verification the build must include + +- Parsing: a document with `call` parsed and summarised with `details: { call: { server, tool } }`; every refusal of §1 with its line and words, among them each of the six keys beside `call`, a body beside it, `call` without `output.schema`, a document with neither `call` nor `to`, the server's and the tool's shapes, a template reading `to`, a property the input schema forbids, `${`, a text template rendering a structure without `input`, arguments over 16 KiB without `input`, and `read` empty, without its slash and with a bare `~`; the empty pointer refused with the one sentence in `deliver.sent` and `replies.read` too; a body beside `call` refused and never compiled as a message; the request documents of the amendment and 0020 parsed as before. +- Rendering: a number, a boolean, `null`, a nested object and a list written as values; a text template; a whole-value template giving a number, an object and a text; `| json` rendering text; a missing input property `invalid_input`; a text template rendering a structure at the run `unworkable`; arguments at 16 KiB admitted and one byte more `unworkable`; the delivery's, the read's and the telling's existing tests sending what they sent. +- The document: `answerDocument` and `resultText` over the same answers, structured content with its JSON text, with a summary text, several text blocks, only an image, nothing, and a text that is not JSON, pinned side by side. +- The run, through the real server over HTTP and MCP against the fake MCP server: a call read from structured content beside a summary text, from a JSON text, and from a plain text without `read`; the output the value, the record `{ server, tool, read }`, the start with `calls_tools` and without `finishes_later`, `tool_call_started` appended before the fake server sees the call and `tool_call_answered` after, `com.beonauto/run_id` in the call's metadata, `arguments_json` and `result_json` only on a server with `record_content`, scrubbed and cut to 4 KiB, the history showing them at 2 KiB in the words of §2; every row of §2's table, each with its outcome, kind, because, words, workflow error type and status, and its `Retry-After` over HTTP, among them a server of another org or brain, a tool not allowed, an unreachable server, one answering 401 at its opening, one whose listing times out, and a tool the server does not list, each with no call fact, no `tools/call` reaching the fake server, and the same id running again; a tool answering `isError`, a 5xx, a timeout at 30 s and a 429 for 20 s, each for a tool marked `readOnlyHint: true`, ending `unavailable` `tools_unfinished` at 503 with its because and `retry_after_ms` and no `Retry-After`, and for a tool marked `readOnlyHint: false`, one with no hints and one marked `destructiveHint: false` alone, ending `conflict` `effect_unknown` at 409; each refused under the same id as `tools_called`; a 429 for 2 s waited out and answered; a forgotten session reopened once; a `stdio` process restarted once; a start the ledger refuses, `CalledOnce` `not_sent`, ending `failed` with an incident and no call reaching the fake server; a tool refusing the arguments with -32602 ending `conflict` `unworkable` with its words, for a read-only tool and for any other; a render stopped by a filter `invalid_input` with "cannot be rendered with this input"; the start's `call_id` the run's id; `run_definition` answering `RunDetail` with the record `{ server, tool, read }`, as `get_run` does; a caller that goes away mid-call over HTTP and over MCP with the answer still recorded and the run ended; `cancel_run` of a call in flight `conflict`; `answer_interaction` on a call's run `conflict`; no open request counted. +- A reasoning run, through the real server against the fake MCP server with a scripted model: one that called only tools marked read-only and then met five server failures, its run bound, a model that stopped answering, or a final step that still called tools, each ending `unavailable` `tools_unfinished` at 503 with its because; the same runs having called one tool not so marked, before or after the read-only ones, and whether that call answered or not, each ending `conflict` `effect_unknown` at 409 with the same because and detail; a run that called no tool ending as today; each refused under the same id as `tools_called`. +- In a workflow: the step's output the value; a `catch` on status 503 retrying `mcp_server_failed`, `tool_not_offered` and `tools_unfinished`, and not `unworkable` nor `effect_unknown`; a `catch` naming the type `https://on.auto/problems/effect_unknown`, or a `when` on its kind, retrying it on purpose; a `when` on `because == "tool_error"` for both endings after sending; a reasoning step's `effect_unknown` raised at 409 under its type and the reference's example not retrying it, and its `tools_unfinished` retried; a two-page loop over a paging tool with `export.as` and `then`; a recall function folding a call's output; an event trigger reacting to its ending. +- Words and texts: the outcome sentences of §7 for two items, fourteen and `{}`, the notification's and a request's answer sentences unchanged, the other four capabilities' sentences from the one function; list items in parentheses; the refusal words within 600; the instructions at 1,514, 1,942 and 1,982 with the two sentences, each under 2,000, and a server without interaction functions unchanged; `give-tools` at 2,052 and 2,601 bytes under the predicate, `first-brain` unchanged; the guide's section served and the page under 64 KiB; `test_tool_call`'s `answer` equal to what a call's `answerDocument` reads for the same answer, absent past 64 KiB or without a document, its description under 300; the words of `tools_unfinished` and `effect_unknown` with each because and a 48-character name within 400 in `get_run`'s sentence, no because cut, and within 600 in `run_definition`'s refusal; `server_failed` reading "because a tool server failed"; the operation's words saying "Nothing was changed." for the first and not the second; the problem documents of both, `Retry-After` on neither, and the header, where it is sent, five seconds whatever the server asked; `run_definition`'s description at 789 and `cancel_run`'s at 683; no word naming a tool as `server/tool` but `namedLinks`' existing refusal; the internal-terms check; the leak test over the new facts, outputs and logs. +- The replay test of `speaking-to-agents.test.ts` gains episode 10: asked to put the replies of a thread into the weekly summary, the scripted agent reads the interaction sentence and the `give-tools` sentence, tests `thread_replies`, reads its `answer`, and saves one interaction function with `call`, `read` and `output.schema`, and no reasoning function whose prompt says to answer with the rows. + +## Build + +One pull request based on the delivery amendment's branch once it merges, where `callOnce(call)` answers `CalledOnce`, a delivery records its own start, and `renderedArguments` is typed, in this order, each step passing the gate: + +1. `@beonauto/operations`: the `conflict` kind `effect_unknown` with its problem type and words, the because `tool_error` and its words, `server_failed`'s words "because a tool server failed" (`plain-language/explanation.ts:159`), `tools_unfinished`'s remedy without `mayHaveChanged`, kept among the kinds resolved only by change; `RejectionBecauseSchema`; `Conflict`'s optional `because`, carried by every place a rejection's because travels: `Rejected.because` (`outcome/outcome.ts:20`), the conflict struct of `SettlementSchema` (`outcome/settlement.ts:28`), `Remedies` (`plain-language/plain-language.ts:3`) and `explanationByBecause` (`explanation.ts:147`). `@beonauto/api`: the problem type of `effect_unknown` at 409 and `Problem.because` for a conflict (`problem/problem.ts:41`). `@beonauto/workflow-engine`: `effect_unknown` settled as a `conflict` with its because, reading `reasonOfKind` (`problem-types.ts:21`), the `tools_called` branch keeping none (`dsl/raised-error.ts:104,116-126`). `@beonauto/mcp`: `RunTools` answering whether every tool the run called is marked read-only. `@beonauto/definitions`: the conflict struct of `RunRejectionSchema` with the optional `because`, a change of its schema (`runs/run.ts:32-41`), `replayed()` (`runs/run-lookup.ts:48-49`), `explainedRejectionOf` (`plain-language/run-words.ts:32-35`), `reading/listed-run.ts:22,25` and the conflict branch of `operations/run-attempt.ts:80-88`, each passing a conflict's because; `run_definition` answering `RunDetail` through one `runOf`; `describeOutput(output, record)`, `outputInWords`, list items in parentheses, the history's numbers, the descriptions of §9. `capabilities/reasoning`: `unfinished`, `stoppedEnding` and `unavailableAfter` building `Unavailable` `tools_unfinished` or `Conflict` `effect_unknown` by that answer, its required `tools`; the five capabilities' sentences through `outputInWords`; the reasoning tests of §Verification. +2. `@beonauto/mcp`: the folder `one-call/`, `OneCall` and `oneCall` for `DeliveryCall` and `deliveredCall`, `calledOnce`, `takenWithin` and `boundedSlot` kept; `callOnce(call, recordStart?)` reading `Timing`, taking the slot within `takenWithin`, listing as `connectedTo` lists and finding the tool with `unlistedOn`, `unopened` as the `ToolsNotOpened` of a run's opening, the optional hook after the tool is found listed, `not_sent`, `answered` with the amendment's fields and the listed tool's hints, the caller's longest wait; `journalledCall(slot, call, recording, recordStart?)` read by `caller` and `callOnce`; `deliveryBounds` and the draft's second key file removed; `answerDocument` beside `resultText` in `bounds/tool-results.ts`; `CallReply` and `test_tool_call` with `answer`; `toolInWords` and the two shape constants, read by the history's words in `@beonauto/definitions`; the fake server's tools: structured content beside a summary, a JSON text, a plain text, several text blocks, `isError`, a slow tool, a 429, a paging tool, and each with `readOnlyHint` true, false and absent. The interaction delivery mapping `unopened` to a failed attempt. +3. `@beonauto/interaction`: the renames of the amendment's delivery-specific names, the folder `route` to `tool-blocks`, `ToolDeliverySchema` to `DeliverBlockSchema` on a shared `ToolUseSchema`, `{ server, tool, with }`, and `compiledRoute` to `compiledToolBlock`, `routeOf` kept; `CallBlockSchema` and `momentVariables`; the document's union with `call` and the refusals of §1; the `call` folder with the run and its endings; `callsTools`, `finishesLater` and `longestRunOf` by shape; the summary's details; `describeAnswer` reading the record; the ports with the tool access and the composition giving it. `@beonauto/api`: the two sentences, `typesThatFinishLater`, the pinned counts and texts in `tool-testing-instructions.test.ts` and `instructions.test.ts`, the README's served text and its sentence on `run_definition`. `@beonauto/server`: the `give-tools` sentence under the predicate and its pinned texts, the replay episode, the tests of §Verification on both stores. +4. The documentation: the interaction reference and its served guide, the HTTP and MCP references, the workflow and computation references, the concepts and terminology pages, `give-tools`' step 4, `scripts/docs-interaction.test.ts`, the READMEs of interaction, mcp, definitions, operations and api, the amendments of the appendix appended to 0018 and to 0003, and the decisions index. The texts that state the old rule of `tools_unfinished`, each to the rule of §2: `docs/reference/reasoning-format.md:105`, served as the reasoning guide (`packages/server/src/guides/served-guides.ts:23`); `docs/engineering/reference/reasoning-format.md:111-114,118,318-320`; `docs/concepts/workflows.md:31,78`; `docs/engineering/reference/http.md:111`; `docs/reference/http.md:392,427`; `docs/engineering/reference/workflow-format.md:52`; `docs/reference/workflow-format.md:43-52,227,234,244`; `capabilities/reasoning/README.md:53`; `packages/workflow-engine/README.md:334`; `packages/api/README.md:34`. The edits to the delivery amendment and to 0020, read in their drafts: + - 0020 §4 (its line 188): "and the start is recorded before the call is sent through the caller's hook, by `callStarted`, so a delivery is one recorded call as a run's are" reads "and the delivery records its own `delivery_started` before the call is sent, with the fields `callStarted` builds less `call_id` (0010, amendment, §3)". + - 0020 §2 (its line 129): "otherwise the text of its first text block read as JSON; a tool that gives neither, or text that is not JSON, has no document, and a pointer over none fails as `unreadable`" reads "otherwise its first text block parsed as JSON, else that text; a tool that gives neither has no document, and a pointer that finds nothing, in a text or in no document, fails as `unreadable`", the rule of `answerDocument` (decision 0021, §1). + - 0020 §6, step 4, and §11's row "A read's result" (its lines 240 and 283): "which the result's 64 KiB bounds (`toolBounds.resultBytes`)" and "64 KiB (`toolBounds.resultBytes`), the bound a run's call has" read "which bounds the document the read takes at 64 KiB as JSON (`toolBounds.resultBytes`)" and "The document, 64 KiB as JSON (`toolBounds.resultBytes`)", so a structure repeated as text is not counted twice. + - 0020 §11's row "A read in flight" (its line 291): "10 s to open and 30 s to answer (`deliveryBounds`)" reads "10 s to open and to list, and 30 s to answer (`toolBounds.openMs`, `toolBounds.callMs`)", since `deliveryBounds` goes. + - The amendment's §1 and 0020 §2: a pointer of `deliver.sent` and of `replies.read` is refused when it is not a JSON Pointer, the empty pointer among them, with the one sentence of decision 0021 §1, "Expected a JSON Pointer such as /messages: it starts with a slash, writes ~ as ~0 and a slash within a name as ~1", by the one check in `tool-blocks`. + +## Appendix: the texts + +The section of the interaction reference, served as the `interaction-function` guide, after "A function document": + +> ## Asking a system +> +> A function that asks a system names, in `call`, a tool of a tool server the brain may use and the arguments the call takes, and answers at once with what the tool answered: its run makes one call, takes the part of the answer `read` points at, checks it against `output.schema` and ends with it as its output. It makes no request and sends no message, so it has no `to`, `expires`, `deliver`, `replies` or body, and it asks no model. Write a call from what the tool answers, not from memory: +> +> 1. Call `list_tool_servers` to see the tool servers this brain may use, the tools each offers, the `input_schema` each takes and whether its server marks it read-only. +> 2. Test the tool with `test_tool_call`, with the arguments its `input_schema` takes, and read its `answer`: that is the document `read` points into, the tool's structured content, else the first text block parsed as JSON, else that text. A tool that lists the replies of a thread in a chat answers the replies and a cursor to the next page, for example. +> 3. Write `call` with `server` and `tool` as `list_tool_servers` names them, `with`, the arguments, and `read`, a JSON Pointer to the part of the answer the function answers with, such as `/messages`; leave `read` out to answer with the whole answer. Write `output.schema`, the shape that part has in the test's answer. +> 4. Show the person the whole document and save it with `create_definition`, `type` interaction, once they agree. `run_definition` runs it and answers its output at once. +> +> The arguments in `with` are the tool's own, named as its `input_schema` names them, and may nest. A number, `true`, `false`, `null`, or a list or an object of them, is sent as written. A text is a Liquid template that reads `input`, `today` and `now`, and is sent as the text it renders; a text that is one `{{ }}` and nothing else sends the value it reads as that value is, so `'{{ input.limit }}'` sends a number when the input's `limit` is a number. `read` names a place in the answer and nothing more: shape what a call answers in the workflow, with `output.as`, or with a computation function. +> +> A run whose tool server this brain may not use, whose tool the operator of this server does not allow or the server does not list, or whose tool server cannot be reached, ends `unavailable` before anything is sent, in the words of `list_tool_servers`, and can be run again. Once the call is sent it is in the run's history. When the tool server fails on it, or the tool answers an error, a tool its server marks read-only ends the run `unavailable` as `tools_unfinished`, status 503, which a workflow may try again; any other tool may have changed something, so the run ends `conflict` as `effect_unknown`, status 409, and a workflow tries it again only where its `catch` names that kind. A run whose arguments the tool refused, or whose answer holds nothing where `read` points or does not match `output.schema`, ends `conflict` as `unworkable`, which trying again does not change. A tool that pages answers one page for each run: answer with the part that holds both the items and the cursor, and have the workflow call again with the cursor while there is one. + +The rows of its field table: + +> | `call` | The tool the function asks: `server` and `tool` as `list_tool_servers` names them, `with`, the arguments, and `read`, a JSON Pointer to the part of the answer it answers with; with it, `output.schema` is required and `to`, `expires`, `deliver`, `replies`, `from`, `reply` and the body are refused, as the section on asking a system says | +> | `to`, `expires` | Required without `call`, refused with it | + +The rows of "How a run ends", beside the request's: + +> | `succeeded` | A function with `call`: the tool answered and the part `read` points at matched `output.schema`; it is the output | +> | `conflict`, kind `unworkable` | A function with `call`: an argument renders a structure among text or the arguments pass 16 KiB; the tool refused the arguments; the answer has nothing to read, nothing where `read` points, more than 64 KiB there, or a value `output.schema` refuses | +> | `unavailable`, kind `tool_not_offered` | The tool server does not serve this brain, its operator does not allow the tool, or the server does not list it; nothing was sent | +> | `unavailable`, kind `mcp_server_failed` | The tool server could not be reached or refused the connection or the listing of its tools; nothing was sent | +> | `unavailable`, kind `tools_unfinished` | A function with `call` whose tool its server marks read-only: the call was sent and its server failed on it or did not answer within 30 seconds, `because` `server_failed`, or its tool answered an error, `because` `tool_error`; trying again is safe | +> | `conflict`, kind `effect_unknown` | The same for any other tool, which may have changed something: whether it did is not known, so the run is not made again under its id, no retry on status 503 matches it, and a person or a workflow rule that names the kind decides | + +The sentence of `give-tools`, step 5, before the amendment's, 289 bytes with its leading space: + +> When the brain only needs what a tool answers, with nothing to reason about, write an interaction function that calls the tool instead: it answers at once with what the tool answered and spends no model call, and the interaction-function guide says how to write it from the test's answer. + +The instructions' two sentences: + +> An interaction function asks a system and answers at once, or asks a person and answers started. +> +> A run of a workflow answers started; get_run shows whether it ended or still waits. + +The field `answer` of `test_tool_call`, 237 characters: + +> The document a call's read points into: the structured content, else the first text block parsed as JSON, else that text, scrubbed of the server's secrets; absent when the tool answered neither, and when it takes more than 64 KiB as JSON + +The explanation of the because `tool_error`, 64 characters: + +> because the tool answered an error, which the details below give + +The paragraph the build appends to decision 0018's amendments, amending its section 1: + +> - **The document a call reads (2026-10-08).** `test_tool_call` also answers `answer`, the document a call's `read` points into: the structured content, else the first text block parsed as JSON, else that text, built by the `answerDocument` an interaction function's call reads with, scrubbed as `text` is and left out past 64 KiB ([decision 0021](0021-asking-a-system.md)). Section 1 answered only what a model sees, since the next thing written from a test was a prompt; a call's `read` and `output.schema` are written against the document, which `text` hides where a tool gives structured content beside a summary, so the test answers both. + +The paragraph the build appends to decision 0003, amending its sections 4 and 6: + +> - **A run that called tools and could not finish (2026-10-08).** Which of two endings it meets is read over every call the run started: when every tool it called is one its server marks `readOnlyHint`, it ends `unavailable`, kind `tools_unfinished`, status 503, which says running it again is safe; when any is not, it ends `conflict`, kind `effect_unknown`, status 409, since whether the tool's effect happened is not known and a person or a workflow rule that names the kind decides. Either carries its `because` as section 4 gave it, and a `catch` on status 503 never repeats a call that may have acted, as section 6 meant ([decision 0021](0021-asking-a-system.md)). + +## Amendments from the build + +Built on 2026-10-09 on `origin/main` at b753ded3, after the one-vocabulary rename, in the four steps of the Build, each passing the gate. The texts the record gives held where measured; a count the rename had moved before this record moves by what the record predicted: + +| Count | The record | Measured | Why they differ | +| -------------------------------------------------------------------------------------------------------------------- | ------------------- | -------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Instructions, org, brain and `/mcp` endpoints | 1,514, 1,942, 1,982 | 1,514, 1,842, 1,882 | The rename served 1,495, 1,850 and 1,890 before this record; the two sentences move them by +19, −8 and −8, as predicted | +| Instructions of a key that may only read | — | 1,565, 1,602 with `brain:read` alone, 1,487 on the org endpoint, 1,565 on the brain endpoint | From 1,573, 1,610, 1,468 and 1,573 | +| `give-tools`, without and with interaction functions | 2,052, 2,601 bytes | 2,049, 2,598 | The rename's recipe is 3 bytes shorter; the sentence adds 289 bytes, as predicted, and step 4 none; `first-brain` stays at 1,764 and 1,854 | +| `run_definition`'s description | 789 characters | 765 | The rename's is 774; the sentence that names a function that calls a tool takes 9 fewer | +| `cancel_run`'s description | 683 characters | 671 | The rename's is 609; the run that ends within its call adds 62 | +| The listing of tools on the wire | — | 37,714 bytes on `/mcp`, 8,157 on the org endpoint, 29,618 on the brain endpoint | From 37,661, 8,157 and 29,565, the two descriptions above; under 40,000, 9,000 and 32,000 | +| The `interaction-function` guide served | about 24 KB | 36,618 bytes, the section 3,190, bounded at 64 KiB and not pinned | The guide was 28,959 bytes after 0020; the section takes 4 bytes more in the new words, and the formatter pads each row of a table to its widest, so the wider rows of three tables cost the rest; under 64 KiB | +| Outcome sentences: two records, fourteen items, `{}` | 277, 182, 109 | 277, 182, 109 | — | +| `effect_unknown` with `tool_error` and a 48-character name, as an outcome and a refusal; `tools_unfinished` the same | 397, 381; 382, 387 | 397, 381; 382, 387 | — | +| `test_tool_call`'s `answer` description | 237 characters | 237 | — | + +- **The hook is the run's journal.** The record gives `callOnce` an optional `recordStart`. The build gives it the run's own journal and the call's id, `runCall: { callId, journal }`, and the one `journalledCall(forwarding, recording, journalling?)` records `tool_call_started` before the call is forwarded and `tool_call_answered` after, for a run's calls and a call alike, so neither a caller nor the record builds the facts twice. +- **A start that cannot be recorded is a defect, not an answer.** The record names `not_sent` among `CalledOnce`'s answers for a start the ledger refuses. The build ends the call path with a defect, "The start of the call could not be recorded on its run, so the call was not sent", which fails the run with an incident; nothing reaches the server, as the record asks. A delivery, a read and a telling record their own starts and never meet it. +- **`unopened` says what was refused.** It carries `refused`, `tool_not_offered` or `mcp_server_failed`, with the `because` and words of a run's opening, so a call ends with the outcome a run's opening ends with and a delivery records the failed attempt as `tool_not_offered` or `server_failure`; the opening restarts a `stdio` process that exited before it lists the tools, within the open bound, so the listing never reads a process that is gone. +- **The duplicate key file was never merged.** The debt table's second key file of `calls/` was in the delivery amendment's draft; the amendment merged without it, so there was nothing to remove. +- **The server's words test reads the fake server's `thread` tool.** The fake MCP server's data tools, `thread`, `pages`, `blocks` and `strict`, answer what a call reads; the server's tests say "the thread tool of chat", and the record's sentences for `thread_replies` are pinned beside the capability's words. +- **The amendments are dated the day this record was accepted**, 2026-10-09, in 0018 and 0003, and the edits to 0020 are made at the lines its merged text has, with a line in its status naming this record; the empty pointer was already refused with the one sentence in the delivery amendment as merged. +- **The interaction reference** takes the record's section without an example of its own, its rows of "How a run ends" with the request's rows for a tool that is not offered merged into one, and three rows of "Bounds", for a function's call of its tool, what `read` takes and the calls of one run. The computation reference's reading function is the record's `read-campaign-costs`, and the concepts page on functions says a workflow reads figures through an interaction function that calls a tool. +- **The client sends a call as a request and checks nothing of a server's output schema (review).** The MCP client's `callTool` checks a tool's structured content against the `outputSchema` its server declares, after `tools/call` returns, and throws an invalid-params protocol error when it does not match, or before sending when the declared schema does not compile; the brain read that error as `refused` and ended a call `conflict` `unworkable`, "refused the arguments", whose remedy invites running it again, though the tool had run and may have acted. The connection sends `tools/call` through the client's `request` instead, which takes the call-result schema from the method and neither checks a server's output schema nor lists the tools again to fetch it; the brain checks what it reads itself, with `read` and `output.schema`. The fake server's `promised` tool declares an output schema of a count and answers a count that is not a number: a call of it ends with the brain's own "does not match the output schema", and a reasoning run's model receives its answer as a result. +- **An answer the run cannot use after a tool that may write (review).** After a call of a tool not marked read-only answered, an answer with nothing to read, nothing at `read`, more than 64 KiB or 512 levels, or a value `output.schema` refuses still ends `conflict`, `unworkable`, whose remedy says to correct and try again, and a run again calls the tool again; its detail ends "The tool ran and may have changed something; a run again calls it again", so whoever reads it knows. A read-only tool's detail is unchanged. +- **`read_only` on the start of a call, and `tools_called` that says nothing was changed (review).** `tool_call_started`, and a test's `tool_test_started` built by the same builder, carry `read_only: true` when the server marked the tool `readOnlyHint` when it was listed, so a run's state knows after a restart whether every call it recorded only read. A run that ended without success after only such calls answers the same `run_id` with `conflict` `tools_called`, because the new `only_read`, whose words say "Nothing was changed." and that a new run is the way; a run still started, or one that recorded any other call, answers `tools_called` as before, since it may still call a tool that writes or already did. A workflow step that meets it raises the type of `tools_called` without its because, as before. +- **The test's answer is built by the test (review).** `callReplyOf` stays as it was: a reply carries `scrubbedResult`, which reads and scrubs the tool's answer only when asked, and `test_tool_call` builds `answer` from it, so no run of a reasoning function computes a document it never reads. +- **The record's functions (review).** The opening of one call finds the tool with `unlistedOn` and takes it from `offeredOn`, as a run's opening does, and the workflow engine tells a kind of its own that settles as a `conflict` through `reasonOfKind`, in a type guard that narrows the kind to those a conflict carries. +- **One sentence for an argument that renders a structure among text (review).** The save and the run refuse it in the record's sentence, the argument's words of `argumentsFailureWords` and the one remedy clause `notTextRemedy`, "write | json after it, or write it alone as one {{ }} to send it as it is". +- **A notification is read from its record (review).** `describeAnswer` says "It delivered its notification." when the run's record is a request that takes no answer, an inbox notification's, or a delivery's `{ delivered_at }`, which a notification delivered later settles with in place of its request, and the answer's words otherwise. +- **The server's tests of the endings (review).** Every row of §2's table runs through the real server against the fake one, with `[status, type, kind, because, detail, Retry-After]` pinned: the server takes the timing of its tool access from a test, so the bounds of a listing that does not answer and of a call that takes too long are reached at 300 ms, and the fake server can hold the next request of one method unanswered; the start the ledger refuses is a trigger on the ledger's file that aborts the insert of a `tool_call_started`. A caller that leaves a call in flight is tested over HTTP and over MCP, leaving once the fake server has received the call. +- **The comparison names no product.** The paragraph on how products call a system names each for what it is and leaves out the addresses of their pages, by the repository's rule; the protocol keeps its address. + +## Self-check + +Written on 2026-10-08 against `origin/main` at c4377676, read from a copy of that commit's tree, with the delivery amendment and 0020 read from their drafts as cited; every file and line cited above was read there, and a piece the amendment changes is cited to its section. The instruction counts were measured by running a copy of `instructionsFor` with its one import replaced over the tool lists of `served-instructions.ts`, which reproduced the pinned 1,495, 1,950 and 1,990 before the two sentences were put in, and gave 1,514, 1,942 and 1,982 after; the outcome sentences were measured with copies of `inWords`, `listed`, `quoted` and `asSentence`, and with the longest names a definition, a server and a tool may have the too-long sentence takes 358 characters for fourteen items, and 384 with the longest count the words spell out, five hundred and ninety-nine; the descriptions of `run_definition` and `cancel_run` were measured as joined in their files, 798 and 621 today; the recipe was measured with `wc -c` after the per-server substitution, 2,052, and with the amendment's sentence, 2,312, which matches the amendment, and with both, 2,601; the guide's section with `wc -c`, 3,186; the words of a run's ending with copies of the explanation table's texts, in both the forms they are said in: `get_run`'s outcome sentence, cut at 400, where `effect_unknown` takes 363 with `tool_error` and "thread-replies" and 397 with a 48-character name, and `tools_unfinished` at most 382, every because of either kind kept whole for an interaction or a reasoning function; `run_definition`'s refusal, cut at 600, at most 381 and 387; a tool not listed 374 and 408 in the first form, the second of which the existing cut ends after its first sentence as it does for a reasoning run today, and 379 and 413 in the second; `unworkable` 182 and 216, 187 and 221; `effect_unknown`'s why at 105 characters and its remedy at 117, `tool_error`'s words at 64, `tools_unfinished`'s new remedy at 171; the pointer's sentence at 113; the field `answer` at 237 characters; the two amendments' paragraphs at 642 and 669; the outcome sentence with the made-up members at 277. The delivery is read as the amendment builds it: its own `delivery_started` before `callOnce(call)`, `unopened` among `CalledOnce`'s answers, and the typed renderer. The products were read at their public references and the protocol at the address its paragraph gives, and quoted from the text those pages served; the record names each product for what it is and no product, platform or business by name, and its example is "a chat". `tools_unfinished` is an `unavailable` at 503 on `origin/main` for every run that called tools (`unavailable.ts:7`; `problem.ts:70`); §2 keeps it for a run whose tools only read and gives any other the `conflict` `effect_unknown` at 409, and says what that moves. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 04ea50ac5..5bcc446d8 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -8,18 +8,19 @@ Use a numbered Markdown filename for each decision and link to it from the affec ## Records -| Record | Status | -| ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------ | -| [1. Run workflows on an engine on the ledger, not on Temporal](0001-workflow-engine-on-the-ledger.md) | accepted 2026-10-04, amended 2026-10-05 | -| [2. Reading the runs of a brain and what happened in it](0002-reading-runs-and-brain-events.md) | accepted 2026-10-05, amended 2026-10-05 and 2026-10-07 | -| [3. MCP servers: a brain reaches the outside world through configured MCP servers](0003-mcp-servers.md) | accepted 2026-10-05, amended 2026-10-06, 2026-10-07 and 2026-10-08 | -| [5. Computation functions: a brain computes with a program, deterministically and without the outside world](0005-computation-functions.md) | proposed 2026-10-05, built 2026-10-06 | -| [7. One vocabulary: the product's words are the only words, in text, code, the API and the ledger](0007-one-vocabulary.md) | accepted 2026-10-09 | -| [10. Interaction functions: a brain asks a person or a system and waits for the answer](0010-interaction-functions.md) | accepted 2026-10-07, amended 2026-10-08 | -| [11. Waiting calls: a workflow waits for a run that finishes later, and a run can be cancelled](0011-waiting-calls.md) | accepted 2026-10-07 | -| [14. A warm worker pool: a run costs its own work, in a worker kept between jobs and let go of after any bad one](0014-warm-worker-pool.md) | proposed 2026-10-06, built 2026-10-07 | -| [15. Several triggers: a workflow starts on an event and on its schedules, each trigger kept and matched on its own](0015-several-triggers.md) | accepted 2026-10-08, amended 2026-10-08 | -| [16. Speaking to agents: what the brain says when an agent connects](0016-speaking-to-agents.md) | accepted 2026-10-07, amended 2026-10-09 | -| [18. Testing a tool call: an agent tries one tool of a tool server through the brain, as a run would call it](0018-testing-a-tool-call.md) | accepted 2026-10-08 | -| [19. The answer shape of an open request: the listing shows the schema each request recorded](0019-the-answer-shape-of-an-open-request.md) | accepted 2026-10-08, built 2026-10-08 | -| [20. Replies to what the brain sent: a function that delivers through a tool reads the replies to its message, and the brain takes its party's reply](0020-replies-through-a-channel.md) | proposed 2026-10-08, built 2026-10-08 | +| Record | Status | +| ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------ | +| [1. Run workflows on an engine on the ledger, not on Temporal](0001-workflow-engine-on-the-ledger.md) | accepted 2026-10-04, amended 2026-10-05 | +| [2. Reading the runs of a brain and what happened in it](0002-reading-runs-and-brain-events.md) | accepted 2026-10-05, amended 2026-10-05 and 2026-10-07 | +| [3. MCP servers: a brain reaches the outside world through configured MCP servers](0003-mcp-servers.md) | accepted 2026-10-05, amended 2026-10-06, 2026-10-07, 2026-10-08 and 2026-10-09 | +| [5. Computation functions: a brain computes with a program, deterministically and without the outside world](0005-computation-functions.md) | proposed 2026-10-05, built 2026-10-06 | +| [7. One vocabulary: the product's words are the only words, in text, code, the API and the ledger](0007-one-vocabulary.md) | accepted 2026-10-09 | +| [10. Interaction functions: a brain asks a person or a system and waits for the answer](0010-interaction-functions.md) | accepted 2026-10-07, amended 2026-10-08 | +| [11. Waiting calls: a workflow waits for a run that finishes later, and a run can be cancelled](0011-waiting-calls.md) | accepted 2026-10-07 | +| [14. A warm worker pool: a run costs its own work, in a worker kept between jobs and let go of after any bad one](0014-warm-worker-pool.md) | proposed 2026-10-06, built 2026-10-07 | +| [15. Several triggers: a workflow starts on an event and on its schedules, each trigger kept and matched on its own](0015-several-triggers.md) | accepted 2026-10-08, amended 2026-10-08 | +| [16. Speaking to agents: what the brain says when an agent connects](0016-speaking-to-agents.md) | accepted 2026-10-07, amended 2026-10-09 | +| [18. Testing a tool call: an agent tries one tool of a tool server through the brain, as a run would call it](0018-testing-a-tool-call.md) | accepted 2026-10-08, amended 2026-10-09 | +| [19. The answer shape of an open request: the listing shows the schema each request recorded](0019-the-answer-shape-of-an-open-request.md) | accepted 2026-10-08, built 2026-10-08 | +| [20. Replies to what the brain sent: a function that delivers through a tool reads the replies to its message, and the brain takes its party's reply](0020-replies-through-a-channel.md) | proposed 2026-10-08, built 2026-10-08, amended 2026-10-09 | +| [21. Asking a system: an interaction function calls a tool and answers with its result](0021-asking-a-system.md) | accepted 2026-10-09, built 2026-10-09 | diff --git a/docs/engineering/reference/http.md b/docs/engineering/reference/http.md index 26a94b376..7a8c488a3 100644 --- a/docs/engineering/reference/http.md +++ b/docs/engineering/reference/http.md @@ -108,7 +108,7 @@ A run without a final result may be attempted again after an interruption, an un ## Errors -API errors use RFC 9457 problem documents with `Content-Type: application/problem+json`, whose `type` is `https://on.auto/problems/`, or `https://on.auto/problems/tools_unfinished` for a run that called tools and could not finish, `https://on.auto/problems/tools_called` for a run not run again under its id because its tools may have been called, and `https://on.auto/problems/cancelled` for a run that was cancelled ([the list](../../reference/http.md#responses-and-errors)). Inspect `reason` and `detail`; `invalid_input` includes an `errors` list of JSON pointers. A rejection that has a `kind` carries it, and an `unavailable` one its `because`, as extension members: `tools_unfinished` means a run called tools and could not finish, so its tools may have changed something, and a request with the same `run_id` answers `conflict` with the kind `tools_called`. `Retry-After: 5` comes only with an `unavailable` answer that a retry of the same request may resolve, never with `tools_unfinished`, `tool_not_offered` or `model_not_offered`. +API errors use RFC 9457 problem documents with `Content-Type: application/problem+json`, whose `type` is `https://on.auto/problems/`, or `https://on.auto/problems/tools_unfinished` for a run that called only tools that read and could not finish, `https://on.auto/problems/effect_unknown` for one that called a tool that may change something and could not finish, `https://on.auto/problems/tools_called` for a run not run again under its id because its tools may have been called, and `https://on.auto/problems/cancelled` for a run that was cancelled ([the list](../../reference/http.md#responses-and-errors)). Inspect `reason` and `detail`; `invalid_input` includes an `errors` list of JSON pointers. A rejection that has a `kind` carries it, and an `unavailable` or `conflict` one its `because`, as extension members: `tools_unfinished`, at 503, means a run called tools and could not finish, and every tool it called is one its server marks read-only, so a new run is safe; `effect_unknown`, a `conflict` at 409, means the same of a run that called any other tool, which may have changed something; after either, a request with the same `run_id` answers `conflict` with the kind `tools_called`. `Retry-After: 5` comes only with an `unavailable` answer that a retry of the same request may resolve, never with `tools_unfinished`, `tool_not_offered` or `model_not_offered`. | Status | Common reason | | ------ | ----------------------------------- | diff --git a/docs/engineering/reference/reasoning-format.md b/docs/engineering/reference/reasoning-format.md index 52d3184a1..8f958f32d 100644 --- a/docs/engineering/reference/reasoning-format.md +++ b/docs/engineering/reference/reasoning-format.md @@ -99,23 +99,23 @@ A document is checked without calling a model, and the same document always give The model receives each tool under a unique name derived from `mcp__server__tool`. Characters other than letters, digits and underscores become underscores; names longer than 64 characters or sharing a normalized name receive an 8-character hash. A numeric suffix resolves any remaining collision within the run's tool list. The description is cut to 4 KiB and the server's input schema is unchanged; what in it the provider of the run's model may refuse or not hold the tool's arguments to is added to the run's `warnings`: a root that is not an object, and for Anthropic models the bounds, `oneOf` and the formats they do not enforce or accept, such as `mcp__graph__search inputSchema#/properties/query/maxLength`. Tools are not sent as strict structured outputs, so the rules a JSON output schema has for open objects and optional properties do not apply to them. The model keeps the function's output format and may call tools, several in one step, before it answers, each call forwarded with the run id in the request's metadata under `com.beonauto/run_id`: -| Bound | Value | When it is reached | -| ----------------------------------------------- | ------------------------------------------------------ | ------------------------------------------------------------------------- | -| Calls in one run | 25 | further calls are refused, then one more step without tools | -| Results sent to the model in one run | 256 KiB | further calls are refused, then one more step without tools | -| One result | 64 KiB | cut at a character, with a note asking for fewer rows, fields or depth | -| One call's arguments | 16 KiB | the call is refused | -| The same tool with the same arguments | twice | the third call is refused, naming the earlier answer | -| One call | 30 s | a tool error that counts as a server failure | -| A 429 | waited out when `Retry-After` asks for at most 10 s | otherwise a server failure | -| Server failures in one run | 5 | `unavailable` of the kind `tools_unfinished`, because `server_failed` | -| One model call | 60 s plus 25 ms for every token of `max_output_tokens` | `unavailable` of the kind `tools_unfinished`, because `model_unavailable` | -| The whole run | 10 minutes, or one model call's limit when longer | `unavailable` of the kind `tools_unfinished`, because `run_bound` | -| The last step, without tools, still calls tools | | `unavailable` of the kind `tools_unfinished`, because `no_answer` | +| Bound | Value | When it is reached | +| ----------------------------------------------- | ------------------------------------------------------ | ---------------------------------------------------------------------- | +| Calls in one run | 25 | further calls are refused, then one more step without tools | +| Results sent to the model in one run | 256 KiB | further calls are refused, then one more step without tools | +| One result | 64 KiB | cut at a character, with a note asking for fewer rows, fields or depth | +| One call's arguments | 16 KiB | the call is refused | +| The same tool with the same arguments | twice | the third call is refused, naming the earlier answer | +| One call | 30 s | a tool error that counts as a server failure | +| A 429 | waited out when `Retry-After` asks for at most 10 s | otherwise a server failure | +| Server failures in one run | 5 | `tools_unfinished` or `effect_unknown`, because `server_failed` | +| One model call | 60 s plus 25 ms for every token of `max_output_tokens` | `tools_unfinished` or `effect_unknown`, because `model_unavailable` | +| The whole run | 10 minutes, or one model call's limit when longer | `tools_unfinished` or `effect_unknown`, because `run_bound` | +| The last step, without tools, still calls tools | | `tools_unfinished` or `effect_unknown`, because `no_answer` | A refused call is not sent and not recorded. The step that follows the end of the calls withholds the tools and tells the model the calls so far as text, so every provider takes it, with their answers in a block fenced by a random marker and labelled as the results of the tools it called: data to answer from, not instructions, and not the words of the user. A result's text content reaches the model as text, structured content only when there is no text, other content as a one-line placeholder, and a result the server marks `isError`, such as a policy's denial, as a tool error the model may recover from; a server's own instructions never reach it. Error text is scrubbed of the entry's secrets and minted tokens. -Each call is two events on the run's stream, appended as it happens: `tool_call_started`, recorded before the call is sent, with its number, the id the model gave it, the server, the tool, and the size and SHA-256 digest of its arguments; and `tool_call_answered`, with the outcome (`result`, `tool_error`, `server_failure`, `timed_out` or `cancelled`), the size and digest of the result's content, its `content` and `structuredContent`, the duration, the JSON-RPC id, and the server's own request id where the entry's `request_id` names it. The arguments and the result's content, scrubbed and cut to 4 KiB as they are stored, are added only for an entry with `record_content: true`. `get_run_history` shows them. Once a run has recorded a call, every `unavailable` ending names the tools it called in words and has the kind `tools_unfinished`, and `run_definition` answers the same `run_id` with `conflict` of the kind `tools_called` unless the run succeeded: a tool may have changed something, so start a new run instead. It answers the same while a run of a function with `tools` is started, before any call too, since that run may still be in progress. +Each call is two events on the run's stream, appended as it happens: `tool_call_started`, recorded before the call is sent, with its number, the id the model gave it, the server, the tool, the size and SHA-256 digest of its arguments, and `read_only: true` where the server marks the tool read-only; and `tool_call_answered`, with the outcome (`result`, `tool_error`, `server_failure`, `timed_out` or `cancelled`), the size and digest of the result's content, its `content` and `structuredContent`, the duration, the JSON-RPC id, and the server's own request id where the entry's `request_id` names it. The arguments and the result's content, scrubbed and cut to 4 KiB as they are stored, are added only for an entry with `record_content: true`. `get_run_history` shows them. Once a run has recorded a call, every `unavailable` ending names the tools it called in words and has the kind `tools_unfinished` when every tool it called is one its server marks `readOnlyHint`, which says running it again is safe, and becomes a `conflict` of the kind `effect_unknown`, with the same because and words, when any is not, since whether that tool changed something is not known; `run_definition` answers the same `run_id` with `conflict` of the kind `tools_called` unless the run succeeded: a tool may have changed something, so start a new run instead. It answers the same while a run of a function with `tools` is started, before any call too, since that run may still be in progress. A run that calls tools costs more: every step resends the conversation so far, so a run that uses its whole budget of results sends on the order of a million tokens. @@ -315,7 +315,8 @@ The definition operations answer every rejection as a problem document, and reco | The provider cannot be reached or cannot serve now, or does not answer in time | `unavailable`, saying to try again later | 503 | | A JSON answer does not match the schema | `unavailable`, with the first issues, saying to try again | 503 | | The definition names a tool the brain's MCP servers do not offer, or a server cannot be used | `unavailable` of the kind `tool_not_offered` or `mcp_server_failed`; the model is not called | 503 | -| The run called tools and then could not finish | `unavailable` of the kind `tools_unfinished`, naming the tools it called in words | 503 | +| The run called only tools their servers mark read-only and then could not finish | `unavailable` of the kind `tools_unfinished`, naming the tools it called in words | 503 | +| The run called a tool its server does not mark read-only and then could not finish | `conflict` of the kind `effect_unknown`, naming the tools it called in words; no retry on status 503 repeats it | 409 | | The same `run_id` again, after a run that called tools and did not succeed | `conflict` of the kind `tools_called`; inspect the history and external effects before deliberately starting a new run | 409 | | The same `run_id` again, while a run of a reasoning function with `tools` is started | `conflict` of the kind `tools_called`; the run may still be in progress, so inspect its history before starting another | 409 | | The caller goes away before the answer | the run is interrupted and recorded as `failed`; in-flight tool calls are cancelled and may have external effects | 499 | diff --git a/docs/engineering/reference/workflow-format.md b/docs/engineering/reference/workflow-format.md index e7983ecec..c486a3847 100644 --- a/docs/engineering/reference/workflow-format.md +++ b/docs/engineering/reference/workflow-format.md @@ -49,7 +49,7 @@ A call cancelled, by the timeout of its task, by its `call_deadline` or because When the workflow ends, its last event settles the run through `runSettler` of `@beonauto/definitions`: - **succeeded** with the output of the workflow, when it completes; -- **rejected** when an error is not caught: `invalid_input` for an error of a 4xx status other than 408 and 429 (the input led to it, and retrying the run answers the same), and `unavailable` for every other status (a timeout, a failure to reach a definition, a server error: retrying the run may succeed), except that an error of a kind with a problem type of its own takes that kind's reason, `unavailable` for `tools_unfinished` and `conflict` for `tools_called`. The detail is the title or type, the detail and the instance of the error, and the error's `kind` and `because` are kept on the rejection only for `tools_unfinished` and `tools_called`, whose words hold for the whole workflow, so a workflow whose step ended `tools_unfinished` reads as `tools_unfinished`, with its because, not as a plain `unavailable`; any other kind of a step, such as `mcp_server_failed`, whose words speak for that step alone (they say nothing was called through it, though an earlier step may have called tools), settles as a plain `unavailable` with the step's detail; +- **rejected** when an error is not caught: `invalid_input` for an error of a 4xx status other than 408 and 429 (the input led to it, and retrying the run answers the same), and `unavailable` for every other status (a timeout, a failure to reach a definition, a server error: retrying the run may succeed), except that an error of a kind with a problem type of its own takes that kind's reason, `unavailable` for `tools_unfinished` and `conflict` for `effect_unknown` and `tools_called`. The detail is the title or type, the detail and the instance of the error, and the error's `kind` and `because` are kept on the rejection only for `tools_unfinished`, `effect_unknown` and `tools_called`, whose words hold for the whole workflow, so a workflow whose step ended `tools_unfinished` reads as `tools_unfinished`, with its because, not as a plain `unavailable`, and one whose step ended `effect_unknown` as `effect_unknown`, with its because; any other kind of a step, such as `mcp_server_failed`, whose words speak for that step alone (they say nothing was called through it, though an earlier step may have called tools), settles as a plain `unavailable` with the step's detail; - **rejected** with the reason `cancelled` and the kind of the cancel, when it was cancelled (`requested`, `deadline` or `parent_ended`, with the actor and the reason the cancel carried) or when it has run `WORKFLOW_MAX_DURATION` (`overrun`); - **rejected** with the reason `conflict` and the kind `oversized`, when its output is larger than a run records (1048574 bytes as JSON); - **failed** when the run breaks down. diff --git a/docs/engineering/self-host/configuration.md b/docs/engineering/self-host/configuration.md index 164b73af9..0d44c8a0e 100644 --- a/docs/engineering/self-host/configuration.md +++ b/docs/engineering/self-host/configuration.md @@ -47,7 +47,7 @@ allowed_models: ## MCP servers -A reasoning function that lists `tools` calls the tools of the MCP servers in `mcp_servers` ([decision 0003](../../decisions/0003-mcp-servers.md)), and an interaction function sends its request through the tool it names in `deliver`. The key is the name a function writes in `server/tool`, and each entry is a remote server or a process, in the shape assistants read: +A reasoning function that lists `tools` calls the tools of the MCP servers in `mcp_servers` ([decision 0003](../../decisions/0003-mcp-servers.md)), and an interaction function calls the tool it names in `call`, or sends its request through the tool it names in `deliver`. The key is the name a function writes in `server/tool`, and each entry is a remote server or a process, in the shape assistants read: ```yaml mcp_servers: @@ -96,7 +96,7 @@ The server checks every entry at start and stops, naming the setting and the pla ## Interaction functions -An interaction function sends its request through a tool of a server in `mcp_servers`, which it names in its own document under `deliver`, or leaves it in the brain's inbox, and may read the replies to what it sent through another tool of that server ([decision 0010](../../decisions/0010-interaction-functions.md)). Nothing is configured for it beyond the tool server: the entry's `allowed` names the tools its functions may use, checked again at every attempt and every read, and its `record_content` records what they send and what the tool answers. +An interaction function calls a tool of a server in `mcp_servers` and answers with its result, naming it in its own document under `call` ([decision 0021](../../decisions/0021-asking-a-system.md)), or sends its request through such a tool, named under `deliver`, or leaves it in the brain's inbox, and may read the replies to what it sent through another tool of that server ([decision 0010](../../decisions/0010-interaction-functions.md)). Nothing is configured for it beyond the tool server: the entry's `allowed` names the tools its functions may use, checked again at every call, every attempt and every read, and its `record_content` records what they send and what the tool answers. One setting goes with interaction functions: diff --git a/docs/reference/computation-format.md b/docs/reference/computation-format.md index f8957cab2..8b13d73c4 100644 --- a/docs/reference/computation-format.md +++ b/docs/reference/computation-format.md @@ -178,15 +178,19 @@ A workflow calls a computation function as it calls a reasoning function, with ` A run rejected with `conflict` raises a `runtime` error with status 409, and the error's `kind` is `unworkable`. A retry gives the same result for the same input, so a retry policy should not match it: retry on status 503, which a run that was `unavailable` raises, as the example below does. -This workflow reads a month's campaign costs through an MCP server, computes the pace of each campaign and has a reasoning function write a summary. A workflow reaches an MCP server only through a reasoning function's [tools](reasoning-format.md#tools), so the first function reads the rows and passes them on unchanged; the arithmetic happens in the computation function, and the words in the last reasoning function. +This workflow reads a month's campaign costs through a tool server, computes the pace of each campaign and has a reasoning function write a summary. The first function is an [interaction function that asks a system](interaction-format.md#asking-a-system): it calls the tool and answers the rows it gives, checked against its output schema, with no model to read them; the arithmetic happens in the computation function, and the words in the reasoning function. -The reasoning function `read-campaign-costs`: +The interaction function `read-campaign-costs`: ```markdown --- description: Reads the cost rows of every campaign for a month -model: anthropic/claude-sonnet-4-5 -tools: [ads/campaign-costs] +call: + server: ads + tool: campaign-costs + with: + month: '{{ input.month }}' + read: /rows input: schema: type: object @@ -194,23 +198,16 @@ input: month: { type: string } required: [month] output: - format: json schema: - type: object - properties: - rows: - type: array - items: - type: object - properties: - campaign: { type: string } - cost_cents: { type: integer } - budget_cents: { type: integer } - required: [campaign, cost_cents, budget_cents] - required: [rows] + type: array + items: + type: object + properties: + campaign: { type: string } + cost_cents: { type: integer } + budget_cents: { type: integer } + required: [campaign, cost_cents, budget_cents] --- - -Call ads/campaign-costs for the month {{ input.month }} and answer with the rows it gives, exactly as it gives them. Do not add, round or calculate anything. ``` The computation function `campaign-pace` is the document at the top of this page. The reasoning function `write-pace-summary`: @@ -254,11 +251,11 @@ do: - read: call: run_definition with: - type: reasoning + type: interaction name: read-campaign-costs input: { month: '${ .month }' } output: - as: '${ { rows: .rows, period: $input.period } }' + as: '${ { rows: ., period: $input.period } }' - compute: try: - pace: diff --git a/docs/reference/http.md b/docs/reference/http.md index 05c91888f..db2534402 100644 --- a/docs/reference/http.md +++ b/docs/reference/http.md @@ -56,7 +56,7 @@ Changing a document creates a version. Updating it with identical source records ## Interaction functions -A self-hosted runtime offers interaction functions under the same operations, with `interaction` in place of `reasoning` in each route, such as `POST /definitions/interaction` and `POST /definitions/interaction/{name}/run`. The `source` is an [interaction function document](interaction-format.md), and names, document size, versions and retirement follow the rules for reasoning functions above. A run answers 200 with `status: started` and its `run_id`, and waits until its request is answered, expires or is cancelled; a notification without `deliver` succeeds within the request. A `to` or an argument the function cannot render answers `conflict` with the kind `unworkable`, a server or a tool the brain may not use `unavailable` with the kind `tool_not_offered`, and a brain with as many open requests as the runtime allows `unavailable` with the kind `requests_full`. +A self-hosted runtime offers interaction functions under the same operations, with `interaction` in place of `reasoning` in each route, such as `POST /definitions/interaction` and `POST /definitions/interaction/{name}/run`. The `source` is an [interaction function document](interaction-format.md), and names, document size, versions and retirement follow the rules for reasoning functions above. A run answers 200 with `status: started` and its `run_id`, and waits until its request is answered, expires or is cancelled; a notification to the inbox, and a function that calls a tool, answer within the request, the second with what the tool answered as its output or with an ending its format lists. A `to` or an argument the function cannot render answers `conflict` with the kind `unworkable`, a server or a tool the brain may not use `unavailable` with the kind `tool_not_offered`, and a brain with as many open requests as the runtime allows `unavailable` with the kind `requests_full`. These routes are relative to `/v1/orgs/{org}/brains/{brain}`: @@ -107,7 +107,7 @@ A workflow runs once for each `run_id`. Executing it again with the `run_id` of ### Cancelling a run -`cancel_run` cancels a workflow run that is still `started`, or the run of an interaction function whose request waits, and needs `brain:write`. It takes an optional `reason`, 1 to 1,024 characters with no control character, which the run keeps as the detail of its ending; without one, the detail says who asked. The request is recorded on the run at once, on any server, and it returns 200 with the run as it stands, still `started`. Within a moment the run stops its tasks, cancels each run it waits for, and ends `rejected` with the reason `cancelled` and the kind `requested`; read it with `get_run`. Asking again before it ended records nothing more. It returns `not_found` when the brain has no run with that id, and `conflict` when the run has ended, or when it is a run of a reasoning, computation or recall function, which ends within the request that started it and cannot be interrupted from outside. +`cancel_run` cancels a workflow run that is still `started`, or the run of an interaction function whose request waits, and needs `brain:write`. It takes an optional `reason`, 1 to 1,024 characters with no control character, which the run keeps as the detail of its ending; without one, the detail says who asked. The request is recorded on the run at once, on any server, and it returns 200 with the run as it stands, still `started`. Within a moment the run stops its tasks, cancels each run it waits for, and ends `rejected` with the reason `cancelled` and the kind `requested`; read it with `get_run`. Asking again before it ended records nothing more. It returns `not_found` when the brain has no run with that id, and `conflict` when the run has ended, or when it is a run that ends within the request that started it, as a reasoning, computation or recall function does or an interaction function that calls a tool, which cannot be interrupted from outside. A cancelled run's rejection names the kind of its cancellation: @@ -182,7 +182,7 @@ These routes are relative to `/v1/orgs/{org}/brains/{brain}` and need `brain:rea `list_runs` returns `runs`, newest first by when each run first started. A listed run has the fields `get_run` returns, without `output`, `record` and the detail and issues of a rejection; a rejection shows its `reason`, with `kind` and `because` when the function gave them. `status` keeps the runs whose status is `started`, `succeeded`, `rejected` or `failed`, and `type` and `name` keep the runs of one definition. -`get_run_history` returns the `events` of one run in the order the brain recorded them, oldest first unless `order` is `desc`, so each event comes after the event that caused it: the facts the runtime recorded about the run, each start and how it ended. Tool-using runs also record `tool_call_started` and `tool_call_answered`. The run of an interaction function shows its request as `interaction_requested`, with the delivery, the party, the size of the message and the expiry, each delivery attempt as `delivery_started`, with the tool and the size and digest of its arguments, and `delivery_ended`, with how it ended and the size and digest of what the tool answered, the arguments and the answer shown at 2 KiB only where the operator records the server's content, and each reply taken or refused as `reply_taken` or `reply_refused`, with the reply's identity and never its words; its answer is the run's output, which the history does not show. A run that does not exist in the brain returns `not_found`. For a workflow, the history also holds one `workflow_input_applied` event for each input its run took, followed by one event for each step that input moved, named for how the step ended that input: `step_waiting`, `step_finished`, `step_failed`, `step_skipped`, or `step_started` for a step that runs others, such as a `do` or a `fork`, and was still running. A step that starts and finishes in one input shows only `step_finished`. `list_brain_events` returns the `events` of the whole brain, newest first unless `order` is `asc`: definitions created, updated and retired, runs started and ended, tool calls, tests of a tool, the brain's reads of a conversation that found a reply or failed and its tellings, the inputs workflow runs took and the events published to the brain. `type` keeps one event type. `since`, an ISO 8601 time with its offset such as `2026-10-05T09:00:00Z`, keeps what the brain recorded from that time on, in either order; its date must exist in the calendar and its hour run from 00 to 23, so `T24:00:00Z` is refused. `run_id` keeps what one run and every run it started recorded, such as the functions a workflow called: its whole tree. A run that another run started belongs to the tree of the run at its top, so its own id answers no events; read the tree from the id of the run you started. +`get_run_history` returns the `events` of one run in the order the brain recorded them, oldest first unless `order` is `desc`, so each event comes after the event that caused it: the facts the runtime recorded about the run, each start and how it ended. Tool-using runs, a reasoning function's and an interaction function's that calls a tool, also record `tool_call_started` and `tool_call_answered`, the first with `read_only: true` where the server marks the tool read-only. The run of an interaction function shows its request as `interaction_requested`, with the delivery, the party, the size of the message and the expiry, each delivery attempt as `delivery_started`, with the tool and the size and digest of its arguments, and `delivery_ended`, with how it ended and the size and digest of what the tool answered, the arguments and the answer shown at 2 KiB only where the operator records the server's content, and each reply taken or refused as `reply_taken` or `reply_refused`, with the reply's identity and never its words; its answer is the run's output, which the history does not show. A run that does not exist in the brain returns `not_found`. For a workflow, the history also holds one `workflow_input_applied` event for each input its run took, followed by one event for each step that input moved, named for how the step ended that input: `step_waiting`, `step_finished`, `step_failed`, `step_skipped`, or `step_started` for a step that runs others, such as a `do` or a `fork`, and was still running. A step that starts and finishes in one input shows only `step_finished`. `list_brain_events` returns the `events` of the whole brain, newest first unless `order` is `asc`: definitions created, updated and retired, runs started and ended, tool calls, tests of a tool, the brain's reads of a conversation that found a reply or failed and its tellings, the inputs workflow runs took and the events published to the brain. `type` keeps one event type. `since`, an ISO 8601 time with its offset such as `2026-10-05T09:00:00Z`, keeps what the brain recorded from that time on, in either order; its date must exist in the calendar and its hour run from 00 to 23, so `T24:00:00Z` is refused. `run_id` keeps what one run and every run it started recorded, such as the functions a workflow called: its whole tree. A run that another run started belongs to the tree of the run at its top, so its own id answers no events; read the tree from the id of the run you started. Tool-call events identify the server, tool, argument and result sizes and digests, and how each call ended. Content is omitted unless the operator enables `record_content`. Recorded content is scrubbed and bounded; the history API shows at most 2 KiB of each recorded argument or result. Anyone with read access to the brain can read that history. A started call without an answer may have had an external effect; absence of an answer does not prove it was cancelled before acting. @@ -348,7 +348,7 @@ Authorization: Bearer ### Testing a tool -`test_tool_call` calls one tool of one tool server with the `arguments` given, as a run of a reasoning function would call it, and answers what that run's model would see, so you learn what a tool answers before a function names it, or find an id a prompt needs, without making a function to look. `arguments` is the object the tool's `input_schema` takes, `{}` when left out, and may take at most 16 KiB as JSON; more returns `invalid_input` at `/arguments` before anything is called. +`test_tool_call` calls one tool of one tool server with the `arguments` given, as a run of a reasoning function would call it, and answers what that run's model would see and the document an interaction function's call reads, so you learn what a tool answers before a function names it, or find an id a prompt needs, without making a function to look. `arguments` is the object the tool's `input_schema` takes, `{}` when left out, and may take at most 16 KiB as JSON; more returns `invalid_input` at `/arguments` before anything is called. ```http POST /v1/orgs/acme/brains/sales/tool-servers/graph/tools/search/test @@ -365,6 +365,7 @@ Content-Type: application/json "tool": "search", "outcome": "result", "text": "Found 2 operations: listRenewals and renewalById.", + "answer": "Found 2 operations: listRenewals and renewalById.", "result_bytes": 98, "duration_ms": 312, "tested_at": "2026-10-08T09:00:00.000Z" @@ -376,6 +377,7 @@ Content-Type: application/json | `test_id` | The id of the test, a UUID, under which its two events are recorded | | `outcome` | `result`; `tool_error` when the tool answered an error or refused the arguments, which a run's model may recover from; `server_failure` when the server failed on the call; `timed_out` when it did not answer within 30 seconds | | `text` | What a reasoning function's model would see: text content as text, structured content as JSON only when there is no text, other content as a placeholder, scrubbed of the server's secrets and cut at 64 KiB with a note; for a failure, the words the model gets | +| `answer` | The document a call's `read` points into: the structured content, else the first text block parsed as JSON, else that text, scrubbed of the server's secrets; absent when the tool answered neither, and when it takes more than 64 KiB as JSON | | `result_bytes` | The size of the whole result as the server gave it, before the cut; `null` when there is none | | `duration_ms` | How long the call took | | `server_request_id` | The server's own id of the request, where the operator's `request_id` says where it carries one | @@ -389,7 +391,7 @@ Each test is recorded in the brain's history as `tool_test_started` and `tool_te Successful responses contain JSON with `Cache-Control: no-store`. Create operations return 201; other successful operations generally return 200. -API errors use RFC 9457 problem documents with `Content-Type: application/problem+json`. Inspect `reason` and `detail`; `invalid_input` includes an `errors` list of JSON pointers. A rejection that has a `kind` carries it, and an `unavailable` one its `because`: `tools_unfinished` means a run called tools and could not finish, so its tools may have changed something, and a request with the same run id answers `conflict` with the kind `tools_called`. `Retry-After` comes only with an `unavailable` answer that a retry of the same request may resolve, never with `tools_unfinished`, `tool_not_offered` or `model_not_offered`, and never with `cancelled`. A `conflict` may have the kind `oversized`, for a workflow whose output is larger than a run records. +API errors use RFC 9457 problem documents with `Content-Type: application/problem+json`. Inspect `reason` and `detail`; `invalid_input` includes an `errors` list of JSON pointers. A rejection that has a `kind` carries it, and an `unavailable` or `conflict` one its `because`. A run that called tools and could not finish ends `unavailable` with the kind `tools_unfinished` when every tool it called is one its server marks read-only, so running it again is safe, and `conflict` with the kind `effect_unknown` when any is not, since whether that tool changed something is not known; either way a request with the same run id answers `conflict` with the kind `tools_called`, with the `because` `only_read`, and words that say nothing was changed, when every tool the run called only reads. `Retry-After` comes only with an `unavailable` answer that a retry of the same request may resolve, never with `tools_unfinished`, `tool_not_offered` or `model_not_offered`, and never with `cancelled`. A `conflict` may have the kind `oversized`, for a workflow whose output is larger than a run records. | Status | Common reason | | ------ | ------------------------------------------ | @@ -406,26 +408,27 @@ API errors use RFC 9457 problem documents with `Content-Type: application/proble A problem document's `type` is a URI that names its kind of problem: -| Type | Status | Means | -| ------------------------------------------------- | ------ | -------------------------------------------------------------------------------------------------------- | -| `https://on.auto/problems/bad_request` | 400 | The request is malformed, such as a field given twice or a body that is not a JSON object | -| `https://on.auto/problems/unauthenticated` | 401 | Credentials are missing or invalid | -| `https://on.auto/problems/forbidden` | 403 | The credentials may not do that | -| `https://on.auto/problems/origin_not_allowed` | 403 | A browser sent the request from an origin the server does not allow | -| `https://on.auto/problems/not_found` | 404 | Something the request names does not exist | -| `https://on.auto/problems/method_not_allowed` | 405 | The path does not take that method | -| `https://on.auto/problems/conflict` | 409 | The request clashes with what is there | -| `https://on.auto/problems/tools_called` | 409 | A run that may have called tools is not run again under its id; its `reason` is `conflict` | -| `https://on.auto/problems/cancelled` | 409 | The run was cancelled, with the `kind` of its cancellation; its `reason` is `cancelled` | -| `https://on.auto/problems/unanswered` | 410 | Nobody answered the request of an interaction function, with the `kind` `expired` or `undelivered` | -| `https://on.auto/problems/content_too_large` | 413 | The body is larger than 1 MiB | -| `https://on.auto/problems/unsupported_media_type` | 415 | The body is not sent as `application/json` in UTF-8 | -| `https://on.auto/problems/invalid_input` | 422 | The input does not fit, with the `errors` that point at it | -| `https://on.auto/problems/client_closed_request` | 499 | The client went away before the answer | -| `https://on.auto/problems/internal` | 500 | Something went wrong inside the server | -| `https://on.auto/problems/unavailable` | 503 | Something the server relies on cannot serve now | -| `https://on.auto/problems/tools_unfinished` | 503 | A run called tools and could not finish; its `reason` is `unavailable`, and it is never retried as it is | -| `https://on.auto/problems/rebuilding` | 503 | The view of a recall function is still being built; its `reason` is `unavailable`; try again later | +| Type | Status | Means | +| ------------------------------------------------- | ------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `https://on.auto/problems/bad_request` | 400 | The request is malformed, such as a field given twice or a body that is not a JSON object | +| `https://on.auto/problems/unauthenticated` | 401 | Credentials are missing or invalid | +| `https://on.auto/problems/forbidden` | 403 | The credentials may not do that | +| `https://on.auto/problems/origin_not_allowed` | 403 | A browser sent the request from an origin the server does not allow | +| `https://on.auto/problems/not_found` | 404 | Something the request names does not exist | +| `https://on.auto/problems/method_not_allowed` | 405 | The path does not take that method | +| `https://on.auto/problems/conflict` | 409 | The request clashes with what is there | +| `https://on.auto/problems/tools_called` | 409 | A run that may have called tools is not run again under its id; its `reason` is `conflict` | +| `https://on.auto/problems/effect_unknown` | 409 | A run called a tool that may change something and could not finish, so whether it did is not known; its `reason` is `conflict`, and a person or a workflow rule that names it decides | +| `https://on.auto/problems/cancelled` | 409 | The run was cancelled, with the `kind` of its cancellation; its `reason` is `cancelled` | +| `https://on.auto/problems/unanswered` | 410 | Nobody answered the request of an interaction function, with the `kind` `expired` or `undelivered` | +| `https://on.auto/problems/content_too_large` | 413 | The body is larger than 1 MiB | +| `https://on.auto/problems/unsupported_media_type` | 415 | The body is not sent as `application/json` in UTF-8 | +| `https://on.auto/problems/invalid_input` | 422 | The input does not fit, with the `errors` that point at it | +| `https://on.auto/problems/client_closed_request` | 499 | The client went away before the answer | +| `https://on.auto/problems/internal` | 500 | Something went wrong inside the server | +| `https://on.auto/problems/unavailable` | 503 | Something the server relies on cannot serve now | +| `https://on.auto/problems/tools_unfinished` | 503 | A run called tools that only read and could not finish; its `reason` is `unavailable`, and a new run may make it | +| `https://on.auto/problems/rebuilding` | 503 | The view of a recall function is still being built; its `reason` is `unavailable`; try again later | A 500 response contains an incident reference. Its `instance` and the `x-request-id` response header identify the server log entry. Include that reference when reporting a problem, without sharing credentials or confidential input. Malformed HTTP can return a bare status before the API handles it. diff --git a/docs/reference/interaction-format.md b/docs/reference/interaction-format.md index 69acc6164..15dbdb51c 100644 --- a/docs/reference/interaction-format.md +++ b/docs/reference/interaction-format.md @@ -2,7 +2,7 @@ # Interaction function format -The API stores an interaction function as a definition of the type `interaction`. Its source document names a party and an expiry, holds the message, and may name the tool of a tool server its request is sent through. A run renders the request from its input, sends it through that tool or, for a function that names none, leaves it in the brain's inbox, and waits, holding nothing of the server, until the request is answered, expires or is cancelled. The answer, checked against the answer schema the document gives, is the run's output. Use one wherever a brain asks a person or a system and takes the answer later: an approval, a choice, a figure only someone else has, or a notification that needs no answer. +The API stores an interaction function as a definition of the type `interaction`. A function takes one of three shapes. One that asks a system names, in `call`, a tool of a tool server and answers at once with what the tool answered. One that asks a person names a party and an expiry and holds the message, which a run renders from its input and either sends through the tool of a tool server the document names in `deliver` or, for a function that names none, leaves in the brain's inbox; the run then waits, holding nothing of the server, until the request is answered, expires or is cancelled, and the answer, checked against the answer schema the document gives, is its output. Use a call wherever a brain needs what a system answers now, such as the replies of a thread, and a request wherever it asks a person and takes the answer later: an approval, a choice, a figure only someone else has, or a notification that needs no answer. ## A function document @@ -37,6 +37,19 @@ Please review the brief for {{ input.campaign }}. For this document, `create_definition` takes `type: "interaction"`, a function `name` such as `approve-brief`, and the document as `source`. `run_definition` takes the same type and name, with `campaign`, `owner` and `summary` in the `input` object. Both operations also require the brain id unless the MCP connection is scoped to that brain. The run answers `status: started` with its `run_id`, and the request waits in the brain's inbox until it is answered, since the function names no `deliver`. +## Asking a system + +A function that asks a system names, in `call`, a tool of a tool server the brain may use and the arguments the call takes, and answers at once with what the tool answered: its run makes one call, takes the part of the answer `read` points at, checks it against `output.schema` and ends with it as its output. It makes no request and sends no message, so it has no `to`, `expires`, `deliver`, `replies` or body, and it asks no model. Write a call from what the tool answers, not from memory: + +1. Call `list_tool_servers` to see the tool servers this brain may use, the tools each offers, the `input_schema` each takes and whether its server marks it read-only. +2. Test the tool with `test_tool_call`, with the arguments its `input_schema` takes, and read its `answer`: that is the document `read` points into, the tool's structured content, else the first text block parsed as JSON, else that text. A tool that lists the replies of a thread in a chat answers the replies and a cursor to the next page, for example. +3. Write `call` with `server` and `tool` as `list_tool_servers` names them, `with`, the arguments, and `read`, a JSON Pointer to the part of the answer the function answers with, such as `/messages`; leave `read` out to answer with the whole answer. Write `output.schema`, the shape that part has in the test's answer. +4. Show the person the whole document and save it with `create_definition`, `type` interaction, once they agree. `run_definition` runs it and answers its output at once. + +The arguments in `with` are the tool's own, named as its `input_schema` names them, and may nest. A number, `true`, `false`, `null`, or a list or an object of them, is sent as written. A text is a Liquid template that reads `input`, `today` and `now`, and is sent as the text it renders; a text that is one `{{ }}` and nothing else sends the value it reads as that value is, so `'{{ input.limit }}'` sends a number when the input's `limit` is a number. `read` names a place in the answer and nothing more: shape what a call answers in the workflow, with `output.as`, or with a computation function. + +A run whose tool server this brain may not use, whose tool the operator of this server does not allow or the server does not list, or whose tool server cannot be reached, ends `unavailable` before anything is sent, in the words of `list_tool_servers`, and can be run again. Once the call is sent it is in the run's history. When the tool server fails on it, or the tool answers an error, a tool its server marks read-only ends the run `unavailable` as `tools_unfinished`, status 503, which a workflow may try again; any other tool may have changed something, so the run ends `conflict` as `effect_unknown`, status 409, and a workflow tries it again only where its `catch` names that kind. A run whose arguments the tool refused, or whose answer holds nothing where `read` points or does not match `output.schema`, ends `conflict` as `unworkable`, which trying again does not change. A tool that pages answers one page for each run: answer with the part that holds both the items and the cursor, and have the workflow call again with the cursor while there is one. + ## Sending through a tool A function sends its request through a tool of a tool server the brain may use by naming the tool in `deliver`, with the arguments the call takes; it needs nothing set up beyond the tool server itself. A function without `deliver` waits in the brain's inbox. Write a delivery from what the tool answers, not from memory: @@ -116,9 +129,10 @@ A request sent through a tool is attempted at once, and after a failed attempt a | Field | Purpose | | --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `description` | Optional explanation, 1 to 1,000 characters | -| `to` | Required; a template that renders the party the request goes to, a name the brain's readers filter by, which a delivery's templates may use as the tool's address | +| `call` | The tool the function asks: `server` and `tool` as `list_tool_servers` names them, `with`, the arguments, and `read`, a JSON Pointer to the part of the answer it answers with; with it, `output.schema` is required and `to`, `expires`, `deliver`, `replies`, `from`, `reply` and the body are refused, as the section on asking a system says | +| `to` | Required without `call`, refused with it; a template that renders the party the request goes to, a name the brain's readers filter by, which a delivery's templates may use as the tool's address | | `from` | Optional; a template, as `to` is, rendering the party whose reply counts, the answerer; `to` when left out. A request sent to a room names the person here | -| `expires` | Required; an ISO 8601 duration from one minute to thirty days, such as `PT4H` or `P2D`, after which a request nobody answered ends | +| `expires` | Required without `call`, refused with it; an ISO 8601 duration from one minute to thirty days, such as `PT4H` or `P2D`, after which a request nobody answered ends | | `deliver` | The tool the request is sent through: `server` and `tool` as `list_tool_servers` names them; `with`, the arguments of the call, typed as written, a number, boolean, null, list or object sent as written with the templates in its strings rendered, a string that is one `{{ expression }}` sent as the value it reads, and any other string rendered as text; and `sent`, two JSON Pointers into the tool's answer, to the conversation and the message's identity. Left out, the request waits in the inbox | | `replies` | Only with `deliver`, its `sent` and `output.schema`: how the replies to the message are read, `conversation`, `tool`, `with`, `read`, `wait` and `tell`, as the section on sending through a tool says | | `input.schema` | Optional JSON Schema of the input | @@ -127,7 +141,7 @@ A request sent through a tool is attempted at once, and after a failed attempt a `model`, `config`, `tools`, `language` and every other field are rejected with their line. The body after the front matter is the message. `to`, `from` and the message are Liquid templates that read `input`, `today` and `now` and nothing else, so a template that does not compile, names another value, or names a property the input schema does not allow is rejected with its line when the document is saved. A rendered value is text and is never read again as a template. The saved function has the `media_type` `text/markdown`. -A delivery's templates are checked the same way when the document is saved, each against the names its place reads: `deliver.with` those above; `replies.conversation` `to`, `sent.conversation` and `sent.id`; `replies.with` those and `conversation` and `since`; `tell.with` `to`, `sent.conversation`, `sent.id` and `message`. A structure inside a longer text is rejected with its line. Each pointer of `deliver.sent` and `replies.read` is a JSON Pointer such as `/messages`: it starts with a slash, writes `~` as `~0` and a slash within a name as `~1`. `read.order` is `oldest_first` or `newest_first`, and `wait` lies between `PT5S` and `PT1H`, `PT5S` when left out. The server and the tool are not checked when the document is saved: a run whose server does not serve the brain, or whose tool its operator does not allow, is `unavailable` with the kind `tool_not_offered`. +The templates of a call and a delivery are checked the same way when the document is saved, each against the names its place reads: `call.with` and `deliver.with` those above; `replies.conversation` `to`, `sent.conversation` and `sent.id`; `replies.with` those and `conversation` and `since`; `tell.with` `to`, `sent.conversation`, `sent.id` and `message`. A structure inside a longer text is rejected with its line. Each pointer of `call.read`, `deliver.sent` and `replies.read` is a JSON Pointer such as `/messages`: it starts with a slash, writes `~` as `~0` and a slash within a name as `~1`. `read.order` is `oldest_first` or `newest_first`, and `wait` lies between `PT5S` and `PT1H`, `PT5S` when left out. The server and the tool are not checked when the document is saved: a run whose server does not serve the brain, or whose tool its operator does not allow, is `unavailable` with the kind `tool_not_offered`. ## Answering a request @@ -151,45 +165,54 @@ Where a function reads replies, the person answers by replying to the message th ## How a run ends -| Ending | When | -| ---------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `succeeded` | The request was answered, the answer its output; or a notification was delivered, at once without `deliver`, with the output `{}` | -| `invalid_input` | The input does not match the input schema, or a template names a value the input lacks | -| `conflict`, kind `unworkable` | `to` or `from` rendered empty, as something that is not text, with a control character or longer than 256 bytes; the message longer than 8 KiB; or the arguments of `deliver` over 16 KiB, or one that renders no text | -| `unavailable`, kind `tool_not_offered` | A server the function names does not serve the brain, or its operator does not allow a tool the function names | -| `unavailable`, kind `requests_full` | The brain already has as many open requests as the runtime allows, 10,000 unless its operator sets another number | -| `rejected` as `unanswered`, kind `expired` | Nobody answered the request before it expired | -| `rejected` as `unanswered`, kind `undelivered` | Every attempt to deliver a notification failed | -| `rejected` as `cancelled` | `cancel_run` cancelled the run, or the workflow that waited for it ran out of time or ended first | -| `failed` | The runtime itself broke down | - -A request that has ended is final for its run id: answering it is `conflict`, and running the function again starts a new request under a new id. +| Ending | When | +| ---------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `succeeded` | The request was answered, the answer its output; or a notification was delivered, at once without `deliver`, with the output `{}` | +| `succeeded` | A function with `call`: the tool answered and the part `read` points at matched `output.schema`; it is the output | +| `invalid_input` | The input does not match the input schema, or a template names a value the input lacks | +| `conflict`, kind `unworkable` | `to` or `from` rendered empty, as something that is not text, with a control character or longer than 256 bytes; the message longer than 8 KiB; or the arguments of `deliver` over 16 KiB, or one that renders no text | +| `conflict`, kind `unworkable` | A function with `call`: an argument renders a structure among text or the arguments pass 16 KiB; the tool refused the arguments; the answer has nothing to read, nothing where `read` points, more than 64 KiB there, or a value `output.schema` refuses | +| `unavailable`, kind `tool_not_offered` | A server the function names does not serve the brain, or its operator does not allow a tool the function names; for a function with `call`, also a tool the server does not list; nothing was sent | +| `unavailable`, kind `mcp_server_failed` | A function with `call`: the tool server could not be reached or refused the connection or the listing of its tools; nothing was sent | +| `unavailable`, kind `tools_unfinished` | A function with `call` whose tool its server marks read-only: the call was sent and its server failed on it or did not answer within 30 seconds, `because` `server_failed`, or its tool answered an error, `because` `tool_error`; trying again is safe | +| `conflict`, kind `effect_unknown` | The same for any other tool, which may have changed something: whether it did is not known, so the run is not made again under its id, no retry on status 503 matches it, and a person or a workflow rule that names the kind decides | +| `unavailable`, kind `requests_full` | The brain already has as many open requests as the runtime allows, 10,000 unless its operator sets another number | +| `rejected` as `unanswered`, kind `expired` | Nobody answered the request before it expired | +| `rejected` as `unanswered`, kind `undelivered` | Every attempt to deliver a notification failed | +| `rejected` as `cancelled` | `cancel_run` cancelled the run, or the workflow that waited for it ran out of time or ended first | +| `failed` | The runtime itself broke down | + +After a call of a tool that may change something, an answer the run cannot use still ends `unworkable`, and its detail says that a run again calls the tool again. A request that has ended is final for its run id: answering it is `conflict`, and running the function again starts a new request under a new id. The run's history shows the request as `interaction_requested`, with the delivery, the party, the size of the message, whether it takes an answer and the expiry; then each delivery attempt as `delivery_started`, with the tool and the size and digest of its arguments, and `delivery_ended`, with how it ended, the size and digest of what the tool answered and, where the delivery writes `sent`, what the message was delivered as; each reply taken or refused as `reply_taken` or `reply_refused`, with the reply's identity and never its words; and the run's end. The arguments, the message among them, and what the tool answered are shown at 2 KiB only where the operator records the server's content. An answer is the run's output, which `get_run` returns and the history does not show. `list_brain_events` shows the brain's reads of a conversation that found a reply or failed, and its tellings. ## Bounds -| Bound | Value | When it is reached | -| ------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------ | -| The rendered party, and the answerer | 256 bytes, no control character | `conflict`, `unworkable` | -| The rendered message | 8 KiB | `conflict`, `unworkable` | -| The arguments of a call | 16 KiB as JSON | `conflict`, `unworkable` at the start; a read or telling not sent | -| A call of the tool | 10 seconds to open, 30 seconds to answer | A failed attempt or read | -| What `sent` reads, and the conversation key | 256 bytes of text each, no control character | Delivered without what it was delivered as; the request takes no reply | -| A read's answer | 64 KiB | A failed read; the reading goes on | -| A reply's words | 8 KiB | The reply is refused | -| A reply rule | 16 values, 16 words a value, 64 bytes a word | Refused when saved | -| Refused replies of a request | 10 recorded, 3 told | Further ones leave no trace, or are recorded and nobody told | -| Reads of a conversation | 5 seconds after a request joins, then 10, 20 and 40, every minute, and every 5 minutes, never sooner than `wait` | — | -| `expires` | 1 minute to 30 days | Refused when saved | -| An answer | 64 KiB as JSON, 512 levels | `invalid_input` | -| `claimed_for` | 256 bytes | `invalid_input` | -| Attempts | 5: at once, then after 1, 2, 4 and 8 minutes | A question stays open; a notification ends `unanswered` as `undelivered` | -| Open requests of a brain | 10,000 unless the operator sets another number | `unavailable`, `requests_full` | +| Bound | Value | When it is reached | +| --------------------------------------------- | ---------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------- | +| The rendered party, and the answerer | 256 bytes, no control character | `conflict`, `unworkable` | +| The rendered message | 8 KiB | `conflict`, `unworkable` | +| The arguments of a call | 16 KiB as JSON | `conflict`, `unworkable` at the start; a read or telling not sent | +| The arguments of a call that reads no `input` | 16 KiB as JSON | Refused when saved | +| A call of the tool | 10 seconds to open, 30 seconds to answer | A failed attempt or read | +| A function's call of its tool | 10 seconds to open, 10 to list the tools, 30 to answer | `mcp_server_failed` before sending; then `tools_unfinished` or `effect_unknown` | +| What a function's `read` takes | 64 KiB as JSON, 512 levels | `conflict`, `unworkable` | +| Calls in one run of a function with `call` | 1, within 70 seconds | — | +| What `sent` reads, and the conversation key | 256 bytes of text each, no control character | Delivered without what it was delivered as; the request takes no reply | +| A read's answer | 64 KiB | A failed read; the reading goes on | +| A reply's words | 8 KiB | The reply is refused | +| A reply rule | 16 values, 16 words a value, 64 bytes a word | Refused when saved | +| Refused replies of a request | 10 recorded, 3 told | Further ones leave no trace, or are recorded and nobody told | +| Reads of a conversation | 5 seconds after a request joins, then 10, 20 and 40, every minute, and every 5 minutes, never sooner than `wait` | — | +| `expires` | 1 minute to 30 days | Refused when saved | +| An answer | 64 KiB as JSON, 512 levels | `invalid_input` | +| `claimed_for` | 256 bytes | `invalid_input` | +| Attempts | 5: at once, then after 1, 2, 4 and 8 minutes | A question stays open; a notification ends `unanswered` as `undelivered` | +| Open requests of a brain | 10,000 unless the operator sets another number | `unavailable`, `requests_full` | ## In a workflow -A workflow calls an interaction function as it calls any function, with `call: run_definition` and `type: interaction`. The step waits for the request, as long as the function's `expires` and a minute more, holding nothing of the server, across a restart, and its output is the answer. A request nobody answers raises an error of the [problem type](http.md#responses-and-errors) `https://on.auto/problems/unanswered`, status 410, with the kind `expired` or, for a notification, `undelivered`. No retry policy matches it unless it names that type, so a workflow handles an unanswered request only on purpose, and a workflow that does not catch it ends `rejected` as `unanswered` with the same kind. When the step runs out of time, or the workflow ends first, the request is cancelled. +A workflow calls an interaction function as it calls any function, with `call: run_definition` and `type: interaction`. A function with `call` answers within the call: the step's output is what the tool answered at `read`, and the step waits at most 70 seconds and a minute for it. A 503 it raises, `mcp_server_failed`, `tool_not_offered` or `tools_unfinished`, is safe to try again, and so is matched by a retry on that status; `effect_unknown` raises 409 under the [problem type](http.md#responses-and-errors) `https://on.auto/problems/effect_unknown`, which a `catch` retries only where it names that type or kind. For a function that asks a person, the step waits for the request, as long as the function's `expires` and a minute more, holding nothing of the server, across a restart, and its output is the answer. A request nobody answers raises an error of the [problem type](http.md#responses-and-errors) `https://on.auto/problems/unanswered`, status 410, with the kind `expired` or, for a notification, `undelivered`. No retry policy matches it unless it names that type, so a workflow handles an unanswered request only on purpose, and a workflow that does not catch it ends `rejected` as `unanswered` with the same kind. When the step runs out of time, or the workflow ends first, the request is cancelled. This workflow asks for the approval and, when nobody answers in time, records that the brief went unapproved: diff --git a/docs/reference/mcp.md b/docs/reference/mcp.md index 48a8d77ac..b574776bd 100644 --- a/docs/reference/mcp.md +++ b/docs/reference/mcp.md @@ -62,7 +62,7 @@ Definition operations identify a function or workflow by `type` and `name`. Crea `list_tool_servers` lists the MCP servers set up for the brain, each with the tools a reasoning function may name in `tools` as `server/tool` or `server/*`, so an agent finds the names without being told them; a server that cannot be asked just now shows `unavailable`, in words, in place of its tools. Each tool carries the hints its server gives it, as `annotations`, and `testable`, whether `test_tool_call` may test it. Its optional `server` keeps one server; a name the brain has no server of is refused with `invalid_input`. It asks the servers when it is called, so it is marked as reaching outside the runtime, and it works on a retired brain. On `/mcp` and the org endpoint its `brain` is optional: without it, the tool answers for the whole org, each server with `brains`, the brains whose functions may use it, `["*"]` for every brain of the org, so an agent can see what is set up before a brain exists; with it, the tool keeps the servers that serve that brain. A key limited to some brains gives `brain`. On a brain endpoint it takes no `brain` and lists that brain's servers. See [Tool servers](http.md#tool-servers) for the fields. -`test_tool_call` calls one tool of a tool server with the `arguments` given, as a run of a reasoning function would call it, and answers what that run's model would see, so an agent learns what a tool answers, or finds an id a prompt needs, such as a channel's, without making a function to look. Only a tool its server marks read-only, or that the operator marks testable on its entry, can be tested; any other is refused with words that say so. The call is live and recorded in the brain's history, which `list_brain_events` shows. It needs `brain:write`, so a key that may only read is not offered it, and it is marked as reaching outside the runtime, and as destructive while the operator marks any tool testable. See [Testing a tool](http.md#testing-a-tool) for the fields. +`test_tool_call` calls one tool of a tool server with the `arguments` given, as a run of a reasoning function would call it, and answers what that run's model would see, `text`, and the document an interaction function's call reads, `answer`, so an agent learns what a tool answers, or finds an id a prompt needs, such as a channel's, without making a function to look. Only a tool its server marks read-only, or that the operator marks testable on its entry, can be tested; any other is refused with words that say so. The call is live and recorded in the brain's history, which `list_brain_events` shows. It needs `brain:write`, so a key that may only read is not offered it, and it is marked as reaching outside the runtime, and as destructive while the operator marks any tool testable. See [Testing a tool](http.md#testing-a-tool) for the fields. `list_interactions` lists the brain's open requests, newest first, each with the `run_id` of the interaction function's run that asked, the party, the `delivery`, the tool it is sent through or `null` for the inbox, the message, the expiry, how its delivery stands, `answer_schema`, the shape an answer must have as the request recorded it, `null` for a notification, and, for a request whose function reads replies, the `conversation` read, the `answerer` and how many replies were refused, with optional `to` and `function` filters; it pages with `limit` and `cursor`. `answer_interaction` answers one, with that `run_id`, an `answer` its `answer_schema` takes and an optional `claimed_for`; the run succeeds with the answer as its output, which reaches the workflow that waits for it. An answer that does not match is `invalid_input` with a pointer under `/answer`, and a request that has ended or was answered otherwise is `conflict`. See [Interaction function format](interaction-format.md#answering-a-request). diff --git a/docs/reference/reasoning-format.md b/docs/reference/reasoning-format.md index 4eea42fb6..94c6ac45f 100644 --- a/docs/reference/reasoning-format.md +++ b/docs/reference/reasoning-format.md @@ -102,7 +102,7 @@ To find the names, call `list_tool_servers` (`GET /v1/orgs/{org}/brains/{brain}/ The model receives those tools and can call them before it answers. A run makes at most 25 calls and receives at most 256 KiB of results; a call that would exceed a bound, sends more than 16 KiB of arguments or repeats an earlier call a third time is refused, and the model is told why. Once the calls end, the model answers from what it has, without the tools. Each call appears in the run's history, with the server and tool, the size of its arguments and result, and how it ended. -A run whose function names a tool the brain's servers do not offer, or whose server cannot be reached, is `unavailable` before the model is called. After a tool call, an `unavailable` ending has the kind `tools_unfinished`: a tool may already have changed an external system. The same run id cannot run that work again and returns `tools_called`. A tool-using run still marked `started` also cannot restart under its id, even before its first recorded call. Inspect its history and any external effects before deliberately starting a new run with a new id. Retrying a successful run returns its recorded result without calling tools again. +A run whose function names a tool the brain's servers do not offer, or whose server cannot be reached, is `unavailable` before the model is called. After a tool call, a run that cannot finish ends `unavailable` with the kind `tools_unfinished` when every tool it called is one its server marks read-only, so a new run, or a workflow's retry on status 503, may make it, and `conflict` with the kind `effect_unknown`, status 409, when any is not, since that tool may already have changed an external system. The same run id cannot run that work again and returns `tools_called`. A tool-using run still marked `started` also cannot restart under its id, even before its first recorded call. Inspect its history and any external effects before deliberately starting a new run with a new id. Retrying a successful run returns its recorded result without calling tools again. Tool access is available in a self-hosted runtime whose operator configures MCP servers; Auto Cloud does not offer it yet. The operator configures the servers with `mcp_servers` and narrows the tools of each with its `allowed`, as the repository's [configuration guide](https://github.com/BeOnAuto/auto-brain/blob/main/docs/engineering/self-host/configuration.md#mcp-servers) describes. See [Tool access inside a reasoning function](../concepts/functions.md#tool-access-inside-a-reasoning-function). diff --git a/docs/reference/workflow-format.md b/docs/reference/workflow-format.md index b54f18ec0..e9302569e 100644 --- a/docs/reference/workflow-format.md +++ b/docs/reference/workflow-format.md @@ -91,7 +91,7 @@ do: review: ${ .review } ``` -For this document, `create_definition` takes `type: "workflow"`, a workflow `name` and the document as `source`. `run_definition` takes the same type and name, with `brief` in the `input` object. The run reviews the brief, then waits. Sending it the event `com.example.brief.decided` with `data` of `{"approved": true}` ends it with `approved: true` and the review. A decision of `{"approved": false, "reason": "..."}` ends it with that reason, and no decision within seven days ends it with the reason `No decision within a week`. +For this document, `create_definition` takes `type: "workflow"`, a workflow `name` and the document as `source`. `run_definition` takes the same type and name, with `brief` in the `input` object. The run reviews the brief, retrying the review on status 503, which a function's run raises only where trying again is safe: it could not be reached, a tool server could not be used, or it called only tools that read and could not finish. A run whose tool may have changed something raises `effect_unknown` at 409, which the retry leaves alone. The run then waits. Sending it the event `com.example.brief.decided` with `data` of `{"approved": true}` ends it with `approved: true` and the review. A decision of `{"approved": false, "reason": "..."}` ends it with that reason, and no decision within seven days ends it with the reason `No decision within a week`. ## Document fields @@ -206,11 +206,11 @@ A `switch` tests its cases in order and follows the `then` of the first case who ### Calling a function -`call: run_definition` runs the active latest version of another definition in the same brain: `with.type` names its type, `reasoning` for a reasoning function, `interaction` for an [interaction function](interaction-format.md), `computation` for a [computation function](computation-format.md), `recall` for a [recall function](recall-format.md) or `workflow` for another workflow, and `with.name` the definition. `with.input` is a template for its input, `{}` when left out. The task's output is that run's output: the text or JSON value a reasoning function answered, the answer an interaction function's request took, the value a computation function's program gave, what a recall function answered from its view, or the output of the workflow it ran. +`call: run_definition` runs the active latest version of another definition in the same brain: `with.type` names its type, `reasoning` for a reasoning function, `interaction` for an [interaction function](interaction-format.md), `computation` for a [computation function](computation-format.md), `recall` for a [recall function](recall-format.md) or `workflow` for another workflow, and `with.name` the definition. `with.input` is a template for its input, `{}` when left out. The task's output is that run's output: the text or JSON value a reasoning function answered, the answer an interaction function took from its tool or its request, the value a computation function's program gave, what a recall function answered from its view, or the output of the workflow it ran. Each time the task runs, including on a retry, it starts a separate run of the function, recorded under its own `run_id`. That run acts for the caller who started the workflow, with the permissions that caller had when the workflow started. `run_definition` takes no arguments other than `type`, `name` and `input`. -A run of a reasoning, computation or recall function finishes within the call that starts it. A run of a workflow or of an interaction function finishes later, and the task waits for it: the run records the task it answers, and its ending, whenever it comes and on whichever server it is recorded, answers the task. While the task waits, it holds none of the calls the server runs at once, and a restart of the server does not start the run again. A task waits for a run as long as a run of that definition may take, plus a minute, and never past the longest the workflow itself may still run: a workflow may take the longest a run may last, an interaction function the `expires` of its document, a reasoning function its model's deadline or, when it names tools, the bound of its tool loop, and a computation or recall function ten seconds. That is settled when the workflow starts, for each task that names its definition as written; a task whose `with.type` or `with.name` is an expression, or a definition saved after the workflow started, waits as long as the longest run of any function other than a workflow, plus a minute. When the wait passes, the task raises a `timeout` error, status 408, and the run it waited for is cancelled with the kind `deadline`; an ending that comes later answers nothing. +A run of a reasoning, computation or recall function, or of an interaction function that calls a tool, finishes within the call that starts it. A run of a workflow or of an interaction function that asks a person finishes later, and the task waits for it: the run records the task it answers, and its ending, whenever it comes and on whichever server it is recorded, answers the task. While the task waits, it holds none of the calls the server runs at once, and a restart of the server does not start the run again. A task waits for a run as long as a run of that definition may take, plus a minute, and never past the longest the workflow itself may still run: a workflow may take the longest a run may last, an interaction function the `expires` of its document, or 70 seconds when it calls a tool, a reasoning function its model's deadline or, when it names tools, the bound of its tool loop, and a computation or recall function ten seconds. That is settled when the workflow starts, for each task that names its definition as written; a task whose `with.type` or `with.name` is an expression, or a definition saved after the workflow started, waits as long as the longest run of any function other than a workflow, plus a minute. When the wait passes, the task raises a `timeout` error, status 408, and the run it waited for is cancelled with the kind `deadline`; an ending that comes later answers nothing. Workflows that call workflows reach at most 8 calls deep: the run that would sit a ninth call below the workflow at the top is refused as a `conflict`, which the calling task raises as a `runtime` error. The runs under one workflow at the top of a tree wait for at most 1,000 calls at once, a limit the deployment can change; the next call is refused in the same way. @@ -223,6 +223,7 @@ A run of the function that does not succeed raises an error the workflow can cat | Rejected with `not_found` | `configuration` | 404 | | Rejected with `conflict` | `runtime` | 409 | | Rejected with `conflict` of the kind `tools_called` | `https://on.auto/problems/tools_called` | 409 | +| Rejected with `conflict` of the kind `effect_unknown` | `https://on.auto/problems/effect_unknown` | 409 | | Rejected with `unavailable` | `communication` | 503 | | Rejected with `unavailable` of the kind `tools_unfinished` | `https://on.auto/problems/tools_unfinished` | 503 | | Rejected with `cancelled` | `https://on.auto/problems/cancelled` | 409 | @@ -231,7 +232,7 @@ A run of the function that does not succeed raises an error the workflow can cat | Failed | `runtime` | 500 | | Could not be reached | `communication` | 503 | -The short types are under `https://open-workflow-specification.org/spec/1.0.0/errors/`. A reasoning function that called tools and could not finish has a type of its own, the [problem type](http.md#responses-and-errors) `https://on.auto/problems/tools_unfinished`, never `communication`: its tools may have changed something, so a `catch` that retries communication errors does not run them again under a new id. So does a step that meets a run of a function whose tools may have been called before, as when the server restarted during the step and performs it again: it raises `https://on.auto/problems/tools_called`, never `runtime`, and a workflow that ends with it is rejected as a `conflict` of that kind, which says to check the run's history and start a new run. A workflow that wants another run names one of those types in its `catch` and starts one knowingly. +The short types are under `https://open-workflow-specification.org/spec/1.0.0/errors/`. A function that called tools and could not finish has a type of its own, never `communication`: the [problem type](http.md#responses-and-errors) `https://on.auto/problems/tools_unfinished`, status 503, when every tool it called is one its server marks read-only, so a `catch` on status 503 may run it again safely, and `https://on.auto/problems/effect_unknown`, status 409, when any is not. That tool may have changed something, so no `catch` on status 503 or on communication errors runs it again under a new id; one that names its type, or its kind in a `when`, does so on purpose. So does a step that meets a run of a function whose tools may have been called before, as when the server restarted during the step and performs it again: it raises `https://on.auto/problems/tools_called`, never `runtime`, and a workflow that ends with it is rejected as a `conflict` of that kind, which says to check the run's history and start a new run. A workflow that wants another run names one of those types in its `catch` and starts one knowingly. A computation function's run that is rejected with `conflict` has the kind `unworkable`: its program raised an error, gave no output or more than one, or did more work or nested deeper than a run may. The same input gives the same result every time, so a retry policy should not match it: retry on status 503, which a run that was `unavailable` raises, and leave 409 out. [Computation function format](computation-format.md#in-a-workflow) has a workflow that does so. @@ -241,7 +242,7 @@ A run that was cancelled raises the [problem type](http.md#responses-and-errors) A request of an [interaction function](interaction-format.md) that nobody answered raises the problem type `https://on.auto/problems/unanswered`, status 410, with the kind `expired` when it expired unanswered or `undelivered` when a notification could not be delivered. No retry policy matches it unless it names that type, as with `errors: { with: { type: https://on.auto/problems/unanswered, kind: expired } }`, since asking again is a decision. A workflow that does not catch it is rejected as `unanswered` with the same kind. -The error's `title` names the definition and, for a rejection, its reason; its `detail` carries the detail the run gave. A rejection that has a kind carries it as the error's `kind`, and its cause as `because`, as the [HTTP problem document](http.md#responses-and-errors) does: a reasoning function whose tools are not offered is `tool_not_offered`, one whose tool server cannot be used `mcp_server_failed`, and one that called tools and could not finish `tools_unfinished`, its tools having perhaps changed something. A `catch` reads them in the error it catches, so `when: '${ $error.kind == "tool_not_offered" }'` handles only that, and `${ $error.because }` names why. +The error's `title` names the definition and, for a rejection, its reason; its `detail` carries the detail the run gave. A rejection that has a kind carries it as the error's `kind`, and its cause as `because`, as the [HTTP problem document](http.md#responses-and-errors) does: a reasoning function whose tools are not offered is `tool_not_offered`, one whose tool server cannot be used `mcp_server_failed`, and one that called tools and could not finish `tools_unfinished` or `effect_unknown`, by whether every tool it called only reads, with the because `tool_error` when a tool answered an error and `server_failed` when a tool server failed. A `catch` reads them in the error it catches, so `when: '${ $error.kind == "tool_not_offered" }'` handles only that, and `${ $error.because }` names why. ### Waiting for events diff --git a/packages/api/README.md b/packages/api/README.md index 2d96a48c6..b42a7c6d2 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -23,7 +23,7 @@ A path with no route gets `404` `not_found`, and a method the path does not serv - **Body.** A body must be a JSON object sent as `application/json` in UTF-8, with no `Content-Encoding` other than `identity`, and no larger than 1 MiB. Otherwise it is `413` `content_too_large`, `415` `unsupported_media_type` or `400` `bad_request`. - **Answer.** A success is the operation's output as JSON with the operation's success status (`200`, or `201` where the operation says so) and `Cache-Control: no-store`. -An outcome that is not a success becomes a problem document. Its `type` is `https://on.auto/problems/` and its `title` the reason's, from the registry of problem types in `src/problem/problem.ts`, except for a kind that has a type of its own, `kindsWithTypes` of `@beonauto/operations`: a rejection of the kind `tools_unfinished` is `https://on.auto/problems/tools_unfinished`, titled `Tools unfinished`, with its reason `unavailable` and its status 503, one of the kind `tools_called` is `https://on.auto/problems/tools_called`, titled `Tools called`, with its reason `conflict` and its status 409, and one of the kind `rebuilding`, a recall function whose view is still being built, is `https://on.auto/problems/rebuilding`, titled `Rebuilding`, with its reason `unavailable`, its status 503 and `Retry-After`, since trying again later may succeed. The workflow engine raises the same types for them, never a communication or runtime error, and a workflow that ends with one is rejected with its reason and kind. +An outcome that is not a success becomes a problem document. Its `type` is `https://on.auto/problems/` and its `title` the reason's, from the registry of problem types in `src/problem/problem.ts`, except for a kind that has a type of its own, `kindsWithTypes` of `@beonauto/operations`: a rejection of the kind `tools_unfinished`, a run whose every tool only reads and that could not finish, is `https://on.auto/problems/tools_unfinished`, titled `Tools unfinished`, with its reason `unavailable` and its status 503, one of the kind `effect_unknown`, a run that called a tool that may change something and could not finish, is `https://on.auto/problems/effect_unknown`, titled `Effect unknown`, with its reason `conflict` and its status 409, one of the kind `tools_called` is `https://on.auto/problems/tools_called`, titled `Tools called`, with its reason `conflict` and its status 409, and one of the kind `rebuilding`, a recall function whose view is still being built, is `https://on.auto/problems/rebuilding`, titled `Rebuilding`, with its reason `unavailable`, its status 503 and `Retry-After`, since trying again later may succeed. The workflow engine raises the same types for them, never a communication or runtime error, and a workflow that ends with one is rejected with its reason and kind. | Outcome | Status and reason | | --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | @@ -31,7 +31,7 @@ An outcome that is not a success becomes a problem document. Its `type` is `http | failed | `500` `internal`, saying nothing about the cause; its `instance` is `urn:uuid:`, the incident id under which the server logs the error | | cancelled | `499` `client_closed_request` when the client went away, `503` `unavailable` when the server is stopping | -A `403` `forbidden` also carries `WWW-Authenticate: Bearer error="insufficient_scope"`. A `503` `unavailable` carries `Retry-After: 5` when a retry of the same request may resolve it, so not for the kind `tools_unfinished`, a run that called tools and could not finish, which the same request answers with `tools_called`, nor for the kinds `tool_not_offered` and `model_not_offered`, which only a change of the function or the configuration resolves. A `410` `unanswered` carries none: the same request answers it again. +A `403` `forbidden` also carries `WWW-Authenticate: Bearer error="insufficient_scope"`. A `503` `unavailable` carries `Retry-After: 5` when a retry of the same request may resolve it, so not for the kind `tools_unfinished`, a run that called tools that only read and could not finish, which the same request answers with `tools_called` and a new run may make, nor for the kinds `tool_not_offered` and `model_not_offered`, which only a change of the function or the configuration resolves. A `410` `unanswered` carries none: the same request answers it again. ## Over MCP @@ -92,7 +92,7 @@ The HTTP method decides nothing. The server serves these, which `packages/server | `send_run_event` | false | false | false | false | | `get_guide` | true | false | true | false | -`run_definition` is destructive while a tool server is configured, since a reasoning function's tools or the tool an interaction function sends through may then change something outside, and `test_tool_call` while an entry of `mcp_servers` marks a tool testable, since a tool its server does not mark read-only may then be tested; the table is the server without either, as the test serves it. `list_models` and `run_definition` reach model providers, and `list_tool_servers` and `test_tool_call` reach the tool servers. +`run_definition` is destructive while a tool server is configured, since a reasoning function's tools, or the tool an interaction function calls or sends through, may then change something outside, and `test_tool_call` while an entry of `mcp_servers` marks a tool testable, since a tool its server does not mark read-only may then be tested; the table is the server without either, as the test serves it. `list_models` and `run_definition` reach model providers, and `list_tool_servers` and `test_tool_call` reach the tool servers. **Calls.** On a scoped endpoint the org, and the brain of a brain endpoint, come from the URL, and the arguments are the whole input; on `/mcp` the org is the caller's own and the brain an argument. The input is decoded with the `json` encoding. A call goes through the same dispatcher and the same `settle` as an HTTP request, with the caller in the URL's org. @@ -113,9 +113,9 @@ So invalid arguments are a tool result with `isError` and an `invalid_input` pro **Instructions.** Every endpoint answers `initialize` with instructions that `instructionsFor` in `src/mcp/instructions.ts` builds per request from the tools the key is offered, the definition types of `McpServing`, which the server gives from the capabilities it registers, each with the noun of its definitions and the name of its guide, and the recipes, each with the tools its steps call. So they never name a tool the connection does not list. They are the texts of [decision 0016](../../docs/decisions/0016-speaking-to-agents.md#3-the-instructions-per-endpoint): what a brain is, as [Brain terminology](../../docs/concepts/terminology.md) defines it; a sentence for each type the server runs; what the connection acts on; where the formats and the recipes are; how a reasoning function names its model and its tools, and, where `test_tool_call` is listed, that it shows what a tool answers; that `get_run` shows whether a run that finishes later ended or still waits; that what the person answers to a run that waits goes to its request through `answer_interaction`, and no new run; that runs, history, events and requests come a page at a time; how to answer the person; and what a refusal says. For the server with every type and every tool, `/mcp` reads: -> A brain is the complete system for a business responsibility. It holds the functions that do its work and the workflows that coordinate them, and it keeps every run with its result as its history. A reasoning function has a prompt and calls a language model. An interaction function asks a person or a system and takes the answer later. A computation function runs a program on its input and gives the same output every time. A recall function answers from what it keeps of the brain's own history: every run's start and end, with its result when it succeeded, the definitions saved and the events published to the brain, never a run's input or its tool calls, so nothing has to write into it. A workflow runs functions in steps, waits for input and can start on a schedule or on an event. This connection acts in the caller's own org: list_brains shows its brains and create_brain makes one. Before writing a definition, read its format with get_guide, which also holds the recipes: first-brain, remember, give-tools and schedule. A reasoning function names a model that list_models lists and may name tools that list_tool_servers lists; test_tool_call shows what a tool answers. A run of an interaction function or a workflow answers started; get_run shows whether it ended or still waits. When the person approves, rejects or otherwise answers what a run waits on, answer its request with answer_interaction, in the shape its function's answer takes, and start no new run for it. Runs, history, events and requests come a page at a time; read on only when the person needs more. When you tell the person what happened, say what was done and what they can do next, in the words of brains, functions, workflows and runs, not in the tools' names, fields or rules; give an id or a status only when the person needs it to act. A tool that cannot do what was asked says why and what to change. +> A brain is the complete system for a business responsibility. It holds the functions that do its work and the workflows that coordinate them, and it keeps every run with its result as its history. A reasoning function has a prompt and calls a language model. An interaction function asks a system and answers at once, or asks a person and answers started. A computation function runs a program on its input and gives the same output every time. A recall function answers from what it keeps of the brain's own history: every run's start and end, with its result when it succeeded, the definitions saved and the events published to the brain, never a run's input or its tool calls, so nothing has to write into it. A workflow runs functions in steps, waits for input and can start on a schedule or on an event. This connection acts in the caller's own org: list_brains shows its brains and create_brain makes one. Before writing a definition, read its format with get_guide, which also holds the recipes: first-brain, remember, give-tools and schedule. A reasoning function names a model that list_models lists and may name tools that list_tool_servers lists; test_tool_call shows what a tool answers. A run of a workflow answers started; get_run shows whether it ended or still waits. When the person approves, rejects or otherwise answers what a run waits on, answer its request with answer_interaction, in the shape its function's answer takes, and start no new run for it. Runs, history, events and requests come a page at a time; read on only when the person needs more. When you tell the person what happened, say what was done and what they can do next, in the words of brains, functions, workflows and runs, not in the tools' names, fields or rules; give an id or a status only when the person needs it to act. A tool that cannot do what was asked says why and what to change. -On `/orgs/{org}/mcp` the connection "manages the brains of one org", whose functions and workflows are made on the brain's own connection, `/orgs/{org}/brains/{brain}/mcp`, and `get_guide` "holds what these words mean"; on `/orgs/{org}/brains/{brain}/mcp` it "acts inside one brain", and the recipes it names leave out `first-brain`, which needs `create_brain`. A recipe is named only where the tools its steps call are listed, and the sentence on `get_guide` names the recipes only where `create_definition` is. Each endpoint's instructions stay under 2,000 characters, open with the terminology page's definition of a brain, name as types only those on that page, and use no term of the internal vocabulary once the address of a brain's connection is taken out, which `src/mcp/instructions.test.ts` checks. The server refuses to start when the instructions of a key that may call every tool would be longer. With a fifth type served, the record's texts would pass that bound, so, as its bound says, the sentence on the interaction tools is in their descriptions and the interaction-function guide, the clause that gives a waiting run its event is in `send_run_event`'s description and the workflow guide, the run sentence covers every type whose run finishes later, and the `brain` argument is left to each tool's schema. With every tool they take 1,890 characters on `/mcp`, 1,495 on `/orgs/{org}/mcp` and 1,850 on `/orgs/{org}/brains/{brain}/mcp`; a key that may only read is offered no `test_tool_call`, and its instructions keep 1,573 characters on `/mcp`, 1,610 with `brain:read` alone, 1,468 on the org endpoint and 1,573 on the brain endpoint. The org endpoint lists `list_tool_servers`, the org operation that answers for the whole org, so its sentence on models is the one that also names the tools: 22 characters more than when it listed `list_models` alone. The clause on `test_tool_call` is the short one, 42 characters, so that the closing sentence on refusals keeps its place within the bound. +On `/orgs/{org}/mcp` the connection "manages the brains of one org", whose functions and workflows are made on the brain's own connection, `/orgs/{org}/brains/{brain}/mcp`, and `get_guide` "holds what these words mean"; on `/orgs/{org}/brains/{brain}/mcp` it "acts inside one brain", and the recipes it names leave out `first-brain`, which needs `create_brain`. A recipe is named only where the tools its steps call are listed, and the sentence on `get_guide` names the recipes only where `create_definition` is. Each endpoint's instructions stay under 2,000 characters, open with the terminology page's definition of a brain, name as types only those on that page, and use no term of the internal vocabulary once the address of a brain's connection is taken out, which `src/mcp/instructions.test.ts` checks. The server refuses to start when the instructions of a key that may call every tool would be longer. With a fifth type served, the record's texts would pass that bound, so, as its bound says, the sentence on the interaction tools is in their descriptions and the interaction-function guide, the clause that gives a waiting run its event is in `send_run_event`'s description and the workflow guide, the run sentence covers every type whose run finishes later, and the `brain` argument is left to each tool's schema. With every tool they take 1,882 characters on `/mcp`, 1,514 on `/orgs/{org}/mcp` and 1,842 on `/orgs/{org}/brains/{brain}/mcp`; a key that may only read is offered no `test_tool_call`, and its instructions keep 1,565 characters on `/mcp`, 1,602 with `brain:read` alone, 1,487 on the org endpoint and 1,565 on the brain endpoint. The org endpoint lists `list_tool_servers`, the org operation that answers for the whole org, so its sentence on models is the one that also names the tools: 22 characters more than when it listed `list_models` alone. The clause on `test_tool_call` is the short one, 42 characters, so that the closing sentence on refusals keeps its place within the bound. **Guides.** Every endpoint serves `get_guide`, a read-only tool whose `guide` argument is the enum of the guides' names and whose answer is one guide's whole Markdown as text, and lists the same guides as resources, `guide://`, `text/markdown`, annotated for the assistant, which answer the same text. The server passes the guides as `guides`, a `Guide` each with its `name`, `title`, `description` and `text`, and the recipes as `recipes`. A `Recipe` is a guide with the `arguments` its prompt takes, the `formatGuide` it embeds, the tools its steps `call` and the `request` it makes of the person's words. Each recipe whose steps call only tools the connection lists, the rule that names it in the instructions, is also a prompt, under its name and title, so a brain endpoint offers no `first-brain` and a key that may only read no prompt: `prompts/get` answers one user message, the request with the person's words filled in and then the recipe, followed by its format guide as an embedded resource. An endpoint refuses to mount without the guide of each definition type and the format guide of each recipe. diff --git a/packages/api/src/mcp/instructions.test.ts b/packages/api/src/mcp/instructions.test.ts index 7305a1099..c0ab29b2f 100644 --- a/packages/api/src/mcp/instructions.test.ts +++ b/packages/api/src/mcp/instructions.test.ts @@ -56,7 +56,8 @@ function asServedWithFourTypes(recordText: string): string { return amendments.reduce((text, [recorded, served]) => text.replace(recorded, served), recordText); } -const interactionSentence = 'An interaction function asks a person or a system and takes the answer later.'; +const interactionSentence = + 'An interaction function asks a system and answers at once, or asks a person and answers started.'; const answeringSentence = "When the person approves, rejects or otherwise answers what a run waits on, answer its request with answer_interaction, in the shape its function's answer takes, and start no new run for it."; @@ -69,7 +70,7 @@ function asServedWithFiveTypes(recordText: string): string { ) .replace( 'A run of a workflow answers started; get_run shows whether it ended or still waits.', - `A run of an interaction function or a workflow answers started; get_run shows whether it ended or still waits. ${answeringSentence}`, + `A run of a workflow answers started; get_run shows whether it ended or still waits. ${answeringSentence}`, ) .replace('Runs, history and events come', 'Runs, history, events and requests come'); } diff --git a/packages/api/src/mcp/instructions.ts b/packages/api/src/mcp/instructions.ts index 6177826fa..71c4ce1ab 100644 --- a/packages/api/src/mcp/instructions.ts +++ b/packages/api/src/mcp/instructions.ts @@ -32,7 +32,7 @@ const reasoningFunctionType = 'reasoning'; const purposeByType: Readonly> = { reasoning: 'A reasoning function has a prompt and calls a language model.', - interaction: 'An interaction function asks a person or a system and takes the answer later.', + interaction: 'An interaction function asks a system and answers at once, or asks a person and answers started.', computation: 'A computation function runs a program on its input and gives the same output every time.', recall: "A recall function answers from what it keeps of the brain's own history: every run's start and end, with its result when it succeeded, the definitions saved and the events published to the brain, never a run's input or its tool calls, so nothing has to write into it.", @@ -115,7 +115,7 @@ const modelsAndTools: Sentences = ({ reasoning, listed }) => { ]; }; -const typesThatFinishLater: ReadonlySet = new Set(['interaction', 'workflow']); +const typesThatFinishLater: ReadonlySet = new Set(['workflow']); const runsThatFinishLater: Sentences = ({ listed, definitionTypes }) => { const finishingLater = definitionTypes diff --git a/packages/api/src/mcp/tool-testing-instructions.test.ts b/packages/api/src/mcp/tool-testing-instructions.test.ts index e0cad2bcc..dab418c13 100644 --- a/packages/api/src/mcp/tool-testing-instructions.test.ts +++ b/packages/api/src/mcp/tool-testing-instructions.test.ts @@ -11,10 +11,10 @@ import { import { instructionsFor } from './instructions.ts'; describe('the length of the instructions with every tool', () => { - it('take 1,890 characters on /mcp, 1,495 on the org endpoint and 1,850 on the brain endpoint', () => { + it('take 1,882 characters on /mcp, 1,514 on the org endpoint and 1,842 on the brain endpoint', () => { expect( everyEndpoint.map(([endpoint, served]) => instructionsFor(endpoint, served, definitionTypes, recipes).length), - ).toEqual([1495, 1850, 1890]); + ).toEqual([1514, 1842, 1882]); }); }); @@ -57,6 +57,6 @@ describe('what the instructions say of testing a tool', () => { ]; expect(lengths.filter((instructions) => instructions.includes('test_tool_call'))).toEqual([]); - expect(lengths.map(({ length }) => length)).toEqual([1573, 1610, 1468, 1573]); + expect(lengths.map(({ length }) => length)).toEqual([1565, 1602, 1487, 1565]); }); }); diff --git a/packages/api/src/problem/problem.test.ts b/packages/api/src/problem/problem.test.ts index 0e226a22e..fb73efe59 100644 --- a/packages/api/src/problem/problem.test.ts +++ b/packages/api/src/problem/problem.test.ts @@ -158,3 +158,20 @@ describe('a view still being built', () => { expect(retryAfterOf({ kind: 'rebuilding' })).toBe('5'); }); }); + +describe('a run whose tool may have changed something', () => { + it('is described at 409 under a type of its own, with its because, and asks for no retry', () => { + const unknown = problemOf('conflict', 'It may have posted', { kind: 'effect_unknown', because: 'tool_error' }); + + expect(unknown).toEqual({ + type: 'https://on.auto/problems/effect_unknown', + title: 'Effect unknown', + status: 409, + detail: 'It may have posted', + reason: 'conflict', + kind: 'effect_unknown', + because: 'tool_error', + }); + expect(problemResponse(unknown).headers.get('retry-after')).toBeNull(); + }); +}); diff --git a/packages/api/src/problem/problem.ts b/packages/api/src/problem/problem.ts index 5e080c5e1..64844b865 100644 --- a/packages/api/src/problem/problem.ts +++ b/packages/api/src/problem/problem.ts @@ -3,7 +3,7 @@ import { problemTypeOf, type KindWithType, type RejectionKind, - type UnavailableBecause, + type RejectionBecause, type UnavailableKind, } from '@beonauto/operations'; @@ -38,7 +38,7 @@ export interface Problem { readonly instance?: string; readonly errors?: readonly ProblemIssue[]; readonly kind?: RejectionKind; - readonly because?: UnavailableBecause; + readonly because?: RejectionBecause; } export type OptionalProblemMembers = Pick; @@ -69,6 +69,7 @@ const problemTypes: Readonly> = { const kindProblemTypes: Readonly> = { tools_unfinished: { status: 503, title: 'Tools unfinished' }, tools_called: { status: 409, title: 'Tools called' }, + effect_unknown: { status: 409, title: 'Effect unknown' }, rebuilding: { status: 503, title: 'Rebuilding' }, }; diff --git a/packages/definitions/README.md b/packages/definitions/README.md index 9b5b7433c..4c95f2aec 100644 --- a/packages/definitions/README.md +++ b/packages/definitions/README.md @@ -62,6 +62,7 @@ A capability has: - `type`: 3 to 32 lowercase letters, digits and hyphens, starting with a letter. It is the type of the capability's definitions in routes, in the `type` field and in stream names, so it never changes. - `title`, and `guide`: the `name` of the guide that holds its format, which the MCP layer serves through `get_guide` and as the resource `guide://`, so that no operation description carries a format. The server makes the guide from the public reference page of the type, `docs/reference/-format.md`, and refuses to start without it. A guide may carry `onThisServer`, one sentence of what this server's configuration offers, such as the providers a reasoning function names its model through, which the guide ends with. `create_definition` names each type with the kind of definition it is and its guide, from the registered capabilities. +- `noun`, its definitions' noun for one and for several, and `describeOutput(output, record?)`, the sentence the operations say of a run's output, given the run's record beside it, which tells an interaction function's call that answered `{}` from a notification. `outputInWords('answer' | 'result')` is the sentence of every capability but that call's: the output in words within 300 characters, the records of a list each in parentheses, or that it is too long to repeat. - `mediaType`: the media type of its definition documents, such as `text/markdown`. - `parse(source)`: turns the document into the capability's own value. Parsing is validation: everything that can be checked without running is checked here. It fails with `InvalidInput`, whose issues each say in `detail` where in the document and what is wrong (line and problem). An issue's `pointer` addresses the document as a whole, so it is `''`; the operations answer it under `/source`. - `summarize(parsed)`: what the operations show about a definition without knowing the type: an optional `description`, optional JSON Schemas of the input a run takes (`inputSchema`) and the output it gives (`outputSchema`), `triggers`, the parsed triggers of a definition that starts runs on its own, as a workflow with a schedule does, in the order its document names them, each with its `kind`, `event`, `cron` or `every`, its `reference` in the document, and its rule: an event trigger's `filters`, each a `reference`, a `type` and the `attributes` it matches, a cron's `expression` or an every's `milliseconds` (`TriggerSchema`), and optional `warnings`: what `parse` found that does not stop the definition from being accepted but may not work everywhere, each a line of text that says where in the document (for reasoning: a schema some providers reject or do not enforce); and optional `details`, a JSON object the registry stores with the definition's record and keeps in its state, but never in a definition an operation answers, so that code which reads the record without the capability's parser can act on it (for recall: the fold, `answer`, the filters, `initial` and the view's schema, which the workflow host folds a view by). @@ -110,7 +111,7 @@ The queries need `brain:read` and the commands `brain:write`. `run_definition` i ## Runs -A run carries `run_id`, `type`, `name`, `definition_version`, `status`, `output` when it succeeded, `rejection` (`reason`, `detail`, `issues` for `invalid_input`, for `unavailable` the `kind` and `because` the capability gave, for `conflict` its `kind`, and for `cancelled` the kind of the cancel) when the capability rejected it or the run was cancelled, `started_at`, `started_by`, and `finished_at` once it ended. `get_run` also shows the `record` the capability gave of what it did: of the run that succeeded, of what it did before it rejected the run when its rejection carries a `record`, such as the tokens a model call spent, or of the work it started that finishes later, kept when that work ends rejected or failed and dropped when a retry starts the run again. `run_definition` answers without the record, which can be large, since the output is what its caller asked for. Its status is `started` while it runs, while work it started finishes after the call returned, or when the process ended before it finished; then `succeeded`, `rejected` or `failed`. +A run carries `run_id`, `type`, `name`, `definition_version`, `status`, `output` when it succeeded, `rejection` (`reason`, `detail`, `issues` for `invalid_input`, for `unavailable` and `conflict` the `kind` and `because` the capability gave, and for `cancelled` the kind of the cancel) when the capability rejected it or the run was cancelled, `started_at`, `started_by`, and `finished_at` once it ended. `get_run` also shows the `record` the capability gave of what it did: of the run that succeeded, of what it did before it rejected the run when its rejection carries a `record`, such as the tokens a model call spent, or of the work it started that finishes later, kept when that work ends rejected or failed and dropped when a retry starts the run again. `run_definition` answers the same run, its record included, as `get_run` does, so the run has one shape on the wire, `RunDetailSchema`, and the words of either read the record beside the output. Its status is `started` while it runs, while work it started finishes after the call returned, or when the process ended before it finished; then `succeeded`, `rejected` or `failed`. `run_definition` answers with the run when it succeeded, and in status `started` when the capability started work that finishes later. When the capability rejects it with `invalid_input`, `unavailable` or `conflict`, the operation is rejected with that reason, detail and issues, and the rejection is recorded on the run. When the capability breaks down, the call fails with an incident, as any defect does, and the run is recorded as `failed`; the defect itself goes only to the incident reporter. A rejected operation carries no run id, so a caller that wants to read a rejected run later gives it an id. @@ -171,7 +172,7 @@ A deferred run takes the facts of its own work until it is settled, tool calls a ### The kind of a conflict -A run rejected with `conflict` records the kind the capability gave, so `get_run`, `list_runs`, the history and the run's words all show it. `unworkable` says the definition cannot run as written for this input, which only changing the definition or the input puts right: a computation function's program that raised an error, gave no output or more than one, did more work or nested deeper than a run may, or answered what the output schema refuses; `run_definition` also answers it, without recording a run, for a stored definition its capability no longer parses. `tools_called` says a run of a function that may have called tools is not run again under its id (see [Run ids and retries](#run-ids-and-retries)). `@beonauto/definitions/testing` has a probe whose mishap `unworkable` rejects that way. +A run rejected with `conflict` records the kind the capability gave, so `get_run`, `list_runs`, the history and the run's words all show it. `unworkable` says the definition cannot run as written for this input, which only changing the definition or the input puts right: a computation function's program that raised an error, gave no output or more than one, did more work or nested deeper than a run may, or answered what the output schema refuses; `run_definition` also answers it, without recording a run, for a stored definition its capability no longer parses. `effect_unknown` says a run called a tool its server does not mark read-only and could not finish, so whether that tool changed something is not known; it carries the `because` of its ending, `server_failed`, `tool_error` or one of a reasoning run's, as `tools_unfinished`, its `unavailable` twin for a run whose every tool only reads, does. `tools_called` says a run of a function that may have called tools is not run again under its id (see [Run ids and retries](#run-ids-and-retries)); it carries the because `only_read` when the run ended without success and every call it recorded has `read_only: true`, whose words say that nothing was changed. `@beonauto/definitions/testing` has a probe whose mishap `unworkable` rejects that way. ## Reading function documents diff --git a/packages/definitions/src/cancellation/cancel-run.ts b/packages/definitions/src/cancellation/cancel-run.ts index 73a8ec361..7c1ca447e 100644 --- a/packages/definitions/src/cancellation/cancel-run.ts +++ b/packages/definitions/src/cancellation/cancel-run.ts @@ -20,7 +20,7 @@ const description = [ 'Cancels a run that is still going and finishes later, such as a workflow, at the request of the person:', "the request is recorded on the run at once, and the run ends as cancelled within a moment, its workflow's steps stopped and the runs they wait for cancelled too.", 'It cannot be undone, and what the run did before it ended stays done.', - 'Use it only when the person asks to stop that run; a reasoning, computation or recall function runs within its call and cannot be cancelled.', + 'Use it only when the person asks to stop that run; a run that ends within its call, as a reasoning, computation or recall function does or an interaction function that calls a tool, cannot be cancelled.', "`run_id` is the run's id and `reason`, kept on the run, says why; asking again records nothing more, and get_run shows how it ended.", ].join(' '); diff --git a/packages/definitions/src/capability/capability.test.ts b/packages/definitions/src/capability/capability.test.ts index cf09d9c36..093f451c2 100644 --- a/packages/definitions/src/capability/capability.test.ts +++ b/packages/definitions/src/capability/capability.test.ts @@ -181,7 +181,7 @@ describe('what a capability decides of each document and says of its runs', () = { summary: 'Waiting.', data: { to: 'ada' } }, ]); expect(own.runWords.delivery(deliveryOfWords)).toBe( - 'Delivery attempt 1 of the request started, through the tool post_message of chat.', + 'Delivery attempt 1 of the request started, through the post message tool of chat.', ); }); }); diff --git a/packages/definitions/src/capability/capability.ts b/packages/definitions/src/capability/capability.ts index d637dd3e9..126aa62c4 100644 --- a/packages/definitions/src/capability/capability.ts +++ b/packages/definitions/src/capability/capability.ts @@ -99,7 +99,7 @@ export interface CapabilityDeclaration { readonly title: string; readonly guide: CapabilityGuide; readonly noun: Noun; - readonly describeOutput: (output: Schema.Json) => string; + readonly describeOutput: (output: Schema.Json, record?: Schema.JsonObject) => string; readonly mediaType: string; readonly parse: (source: string) => Effect.Effect; readonly summarize: (parsed: NoInfer) => DefinitionSummary; @@ -161,7 +161,7 @@ export interface Capability { readonly title: string; readonly guide: CapabilityGuide; readonly noun: Noun; - readonly describeOutput: (output: Schema.Json) => string; + readonly describeOutput: (output: Schema.Json, record?: Schema.JsonObject) => string; readonly mediaType: string; readonly longestAnyRunMs: number; readonly reachesOutside: boolean; diff --git a/packages/definitions/src/index.ts b/packages/definitions/src/index.ts index bcf35fc0a..fe4d21912 100644 --- a/packages/definitions/src/index.ts +++ b/packages/definitions/src/index.ts @@ -147,7 +147,7 @@ export { type TriggerFilter, } from './registry/definition-triggers.ts'; export { runStartedOf, type StartedRun } from './runs/run-starts.ts'; -export { inWords, wordsOf } from './plain-language/in-words.ts'; +export { inWords, outputInWords, wordsOf } from './plain-language/in-words.ts'; export { triggerNamed } from './plain-language/event-words.ts'; export { mostInputBytes, mostInputDepth, mostResultBytes } from './runs/recorded-size.ts'; export { diff --git a/packages/definitions/src/operations/deferred-run.test.ts b/packages/definitions/src/operations/deferred-run.test.ts index 65f21e812..b056f0309 100644 --- a/packages/definitions/src/operations/deferred-run.test.ts +++ b/packages/definitions/src/operations/deferred-run.test.ts @@ -13,6 +13,7 @@ const waiting = { status: 'started', started_at: firstMoment, started_by: 'acme-admin', + record: { handed_on: relayedId }, }, }; @@ -23,7 +24,7 @@ describe('a run whose capability finishes it after the call returns', () => { expect(await running()).toStrictEqual(waiting); expect(await reading()).toStrictEqual({ status: 'succeeded', - output: { ...waiting.output, record: { handed_on: relayedId } }, + output: waiting.output, }); }); @@ -42,7 +43,7 @@ describe('a run whose capability finishes it after the call returns', () => { expect(relayer.runs()).toBe(1); expect(await reading()).toStrictEqual({ status: 'succeeded', - output: { ...waiting.output, record: { handed_on: relayedId } }, + output: waiting.output, }); }); diff --git a/packages/definitions/src/operations/run-attempt.ts b/packages/definitions/src/operations/run-attempt.ts index 8e1de7e29..ddfa2f8bb 100644 --- a/packages/definitions/src/operations/run-attempt.ts +++ b/packages/definitions/src/operations/run-attempt.ts @@ -1,4 +1,4 @@ -import type { ConflictKind, UnavailableBecause, UnavailableKind } from '@beonauto/operations'; +import type { ConflictKind, RejectionBecause, UnavailableKind } from '@beonauto/operations'; import { Effect, Schema } from 'effect'; import type { CapabilityAnswer, CapabilityRejection } from '../capability/capability.ts'; @@ -59,7 +59,7 @@ function rejectedForInput(rejection: Rejection & Recorded): Effect.Effect { @@ -78,11 +78,20 @@ function rejectedAsUnavailable(rejection: Unavailability): Effect.Effect { - const { detail, kind } = rejection; - return rejectedWith({ reason: 'conflict', detail, ...(kind === undefined ? {} : { kind }) }, rejection); + const { detail, kind, because } = rejection; + return rejectedWith( + { + reason: 'conflict', + detail, + ...(kind === undefined ? {} : { kind }), + ...(because === undefined ? {} : { because }), + }, + rejection, + ); } export function attempt(running: Effect.Effect): Effect.Effect { diff --git a/packages/definitions/src/operations/run-definition.test.ts b/packages/definitions/src/operations/run-definition.test.ts index 1dbc991e7..fb99ff695 100644 --- a/packages/definitions/src/operations/run-definition.test.ts +++ b/packages/definitions/src/operations/run-definition.test.ts @@ -79,21 +79,17 @@ describe('run_definition', () => { started_at: later, started_by: 'acme-admin', finished_at: later, + record: { greeting: 'Hello' }, }, }); }); - it('answers with the run that get_run reads, without the record that get_run adds', async () => { + it('answers with the run that get_run reads, its record included', async () => { const { call } = await withGreetAndPlain(); + const ran = await call(runDefinition, running('greet', { run_id: runId })); - expect(await call(runDefinition, running('greet', { run_id: runId }))).toStrictEqual({ - status: 'succeeded', - output: greeted, - }); - expect(await call(getRun, toAlpha(acmeAdmin, { run_id: runId }))).toStrictEqual({ - status: 'succeeded', - output: { ...greeted, record: { greeting: 'Hello' } }, - }); + expect(ran).toStrictEqual({ status: 'succeeded', output: { ...greeted, record: { greeting: 'Hello' } } }); + expect(await call(getRun, toAlpha(acmeAdmin, { run_id: runId }))).toStrictEqual(ran); }); }); diff --git a/packages/definitions/src/operations/run-definition.ts b/packages/definitions/src/operations/run-definition.ts index eabbe65f1..baafbcf9a 100644 --- a/packages/definitions/src/operations/run-definition.ts +++ b/packages/definitions/src/operations/run-definition.ts @@ -4,7 +4,7 @@ import { Effect, Schema } from 'effect'; import type { Capability } from '../capability/capability.ts'; import { knownCapabilities } from '../capability/known-capabilities.ts'; import { runPlainLanguage } from '../plain-language/run-words.ts'; -import { RunSchema } from '../runs/run.ts'; +import { RunDetailSchema } from '../runs/run.ts'; import { RunIdInputField, InputField, DefinitionNameField } from './definition-fields.ts'; import { runRequest } from './run-requests.ts'; @@ -14,7 +14,7 @@ const description = [ "Use it to run a saved definition at the person's request; a workflow's steps call it the same way.", '`type` and `name` say which definition, `input` is the value it takes, as the input_schema get_definition shows,', 'and `run_id` is optional: give the same id to retry safely, since a run that ended or waits answers as it stands and one that failed runs again.', - 'A reasoning function that names tools may change something outside the brain, so a run of one that did not succeed is never run again under its id;', + 'A function that calls a tool may change something outside the brain, so a run of one that did not succeed is never run again under its id;', 'get_run_history shows what it called.', ].join(' '); @@ -34,7 +34,7 @@ export function defineRunDefinition(capabilities: readonly Capability[]) { input: Schema.optionalKey(InputField), run_id: Schema.optionalKey(RunIdInputField), }), - outputSchema: RunSchema, + outputSchema: RunDetailSchema, reasons: ['not_found', 'conflict', 'invalid_input', 'unavailable', 'cancelled', 'unanswered'], handle: Effect.fnUntraced(function* ({ type, name, input = {}, run_id: suppliedId }) { const capability = yield* known.capabilityOfType(type); diff --git a/packages/definitions/src/operations/run-failures.test.ts b/packages/definitions/src/operations/run-failures.test.ts index edc0eb3d9..b93c179c1 100644 --- a/packages/definitions/src/operations/run-failures.test.ts +++ b/packages/definitions/src/operations/run-failures.test.ts @@ -120,6 +120,25 @@ describe('a run whose capability finds, by running it, that its definition is un }); }); +describe('a run that could not finish after calling a tool that may change something', () => { + it('is rejected with conflict of that kind and its because, which is recorded', async () => { + const { call, running, getRun, prober } = await withPlain(); + prober.sufferOnNextRun('unknown'); + const rejection = { + reason: 'conflict', + detail: 'The probe could not finish after calling a tool that may post', + kind: 'effect_unknown', + because: 'tool_error', + }; + + expect(await running({})).toEqual({ status: 'rejected', ...rejection }); + expect(await call(getRun, readingTheRun)).toStrictEqual({ + status: 'succeeded', + output: { ...failedRun, status: 'rejected', rejection }, + }); + }); +}); + describe('a run whose capability breaks down', () => { it('fails with an incident that holds the defect, and is recorded as failed', async () => { const { call, running, getRun, prober, reported } = await withPlain(); diff --git a/packages/definitions/src/operations/start-version.test.ts b/packages/definitions/src/operations/start-version.test.ts index 350ecb982..d8ac9f428 100644 --- a/packages/definitions/src/operations/start-version.test.ts +++ b/packages/definitions/src/operations/start-version.test.ts @@ -55,6 +55,7 @@ describe('a start of one version of a definition, once', () => { started_at: firstMoment, started_by: 'brain:alpha', finished_at: firstMoment, + record: { greeting: 'Hello' }, }, }); expect(definitions.ledger.streamNames().filter((stream) => stream.endsWith(runId))).toEqual([ diff --git a/packages/definitions/src/operations/start-version.ts b/packages/definitions/src/operations/start-version.ts index beceffac7..b82052230 100644 --- a/packages/definitions/src/operations/start-version.ts +++ b/packages/definitions/src/operations/start-version.ts @@ -5,7 +5,7 @@ import type { Capability } from '../capability/capability.ts'; import { knownCapabilities } from '../capability/known-capabilities.ts'; import { definitionWordsFor } from '../plain-language/definition-words.ts'; import { runPlainLanguage } from '../plain-language/run-words.ts'; -import { RunSchema } from '../runs/run.ts'; +import { RunDetailSchema } from '../runs/run.ts'; import { RunIdInputField, InputField, DefinitionNameField } from './definition-fields.ts'; import { startVersionOnce } from './run-requests.ts'; @@ -32,7 +32,7 @@ export function defineStartVersion(capabilities: readonly Capability[]) { input: InputField, run_id: RunIdInputField, }), - outputSchema: RunSchema, + outputSchema: RunDetailSchema, reasons: ['not_found', 'conflict', 'invalid_input', 'unavailable', 'cancelled', 'unanswered'], handle: Effect.fnUntraced(function* ({ type, name, version, input, run_id: id }) { const capability = yield* known.capabilityOfType(type); diff --git a/packages/definitions/src/plain-language/event-words.ts b/packages/definitions/src/plain-language/event-words.ts index 1340ffe41..6cdfa6e5b 100644 --- a/packages/definitions/src/plain-language/event-words.ts +++ b/packages/definitions/src/plain-language/event-words.ts @@ -1,11 +1,10 @@ -import { answeredInWords } from '@beonauto/mcp'; +import { answeredInWords, toolInWords } from '@beonauto/mcp'; import { capitalized, explanationOf, plainNumber, quoted } from '@beonauto/operations'; import type { StartingTrigger } from '../registry/definition-triggers.ts'; import type { CancelRequestKind, ToolCallAnswered } from '../runs/run-events.ts'; import type { RunRejection } from '../runs/run.ts'; import type { DefinitionWords } from './definition-words.ts'; -import { wordsOf } from './in-words.ts'; import { explainedRejectionOf } from './run-words.ts'; export const runFinished = 'A run finished.'; @@ -54,14 +53,8 @@ export function definitionRetired(words: DefinitionWords, type: string, name: st return `${capitalized(words.named(type, name))} was retired.`; } -const notLettersOrDigits = /[^A-Za-z0-9]+/gu; - -function nameInWords(name: string): string { - return wordsOf(name.replaceAll(notLettersOrDigits, ' ')); -} - export function toolCalled(number: number, server: string, tool: string): string { - return `A run made tool call ${plainNumber(number)}, to the ${nameInWords(tool)} tool of ${nameInWords(server)}.`; + return `A run made tool call ${plainNumber(number)}, to ${toolInWords({ server, tool })}.`; } export function toolAnswered(number: number, outcome: ToolCallAnswered['outcome']): string { diff --git a/packages/definitions/src/plain-language/in-words.test.ts b/packages/definitions/src/plain-language/in-words.test.ts index 347bc2787..3f085edb0 100644 --- a/packages/definitions/src/plain-language/in-words.test.ts +++ b/packages/definitions/src/plain-language/in-words.test.ts @@ -1,7 +1,7 @@ import type { Schema } from 'effect'; import { describe, expect, it } from 'vitest'; -import { inWords, wordsOf } from '../index.ts'; +import { inWords, outputInWords, wordsOf } from '../index.ts'; const renderings: ReadonlyArray = [ ['Profits rose.', '“Profits rose.”'], @@ -11,6 +11,10 @@ const renderings: ReadonlyArray = [ [null, 'nothing'], [[], 'an empty list'], [['a', 'b'], '“a” and “b”'], + [ + [{ user: 'ada', text: 'Shipped.' }, { user: 'grace', text: 'Thanks.' }, {}], + '(user: “ada” and text: “Shipped.”), (user: “grace” and text: “Thanks.”), and nothing', + ], [{}, 'nothing'], [{ approve: true }, 'approve: yes'], [{ approve: true, reason: 'cheap' }, 'approve: yes and reason: “cheap”'], @@ -34,6 +38,20 @@ describe('inWords', () => { }); }); +describe('outputInWords', () => { + it('says the answer or the result in words, or that it is too long to repeat', () => { + expect([ + outputInWords('answer')({ approve: true }), + outputInWords('result')([{ verdict: 'approve' }]), + outputInWords('result')('a'.repeat(299)), + ]).toEqual([ + 'Its answer: approve: yes.', + 'Its result: (verdict: “approve”).', + 'Its result is too long to repeat here; the whole of it is in the details below.', + ]); + }); +}); + describe('wordsOf', () => { it.each([ ['customer_name', 'customer name'], diff --git a/packages/definitions/src/plain-language/in-words.ts b/packages/definitions/src/plain-language/in-words.ts index 36e0e2ca8..d4e11aab7 100644 --- a/packages/definitions/src/plain-language/in-words.ts +++ b/packages/definitions/src/plain-language/in-words.ts @@ -1,4 +1,4 @@ -import { listed, quoted } from '@beonauto/operations'; +import { asSentence, listed, quoted } from '@beonauto/operations'; import { Predicate, type Schema } from 'effect'; const mostCharacters = 300; @@ -12,7 +12,7 @@ export function wordsOf(name: string): string { } function renderedArray(items: readonly Schema.Json[]): string { - return items.length === 0 ? 'an empty list' : listed(items.map((item) => rendered(item))); + return items.length === 0 ? 'an empty list' : listed(items.map((item) => nested(item))); } function renderedObject(fields: Readonly>): string { @@ -52,3 +52,14 @@ export function inWords(value: Schema.Json): string | undefined { const words = rendered(value); return words.length <= mostCharacters ? words : undefined; } + +type OutputNoun = 'answer' | 'result'; + +export function outputInWords(noun: OutputNoun): (output: Schema.Json) => string { + return (output) => { + const words = inWords(output); + return words === undefined + ? `Its ${noun} is too long to repeat here; the whole of it is in the details below.` + : asSentence(`Its ${noun}: ${words}`); + }; +} diff --git a/packages/definitions/src/plain-language/run-words.ts b/packages/definitions/src/plain-language/run-words.ts index 580a3116e..3ee0fd604 100644 --- a/packages/definitions/src/plain-language/run-words.ts +++ b/packages/definitions/src/plain-language/run-words.ts @@ -1,12 +1,12 @@ import { capitalized, explanationOf, type ExplainedRejection, type PlainLanguage } from '@beonauto/operations'; import type { Capability } from '../capability/capability.ts'; -import type { Run } from '../runs/run.ts'; +import type { RunDetail } from '../runs/run.ts'; import { definitionWordsFor } from './definition-words.ts'; export type RunMoment = 'just started' | 'looked up'; -type DescribedRun = Pick; +type DescribedRun = Pick; interface RunContext { readonly named: string; @@ -21,26 +21,24 @@ interface DefinitionAddress { const outputBeyondWords = 'Its result is in the details below.'; -function describedOutput(capability: Capability | undefined, { output }: DescribedRun): string { - return capability === undefined || output === undefined ? outputBeyondWords : capability.describeOutput(output); +function describedOutput(capability: Capability | undefined, { output, record }: DescribedRun): string { + return capability === undefined || output === undefined + ? outputBeyondWords + : capability.describeOutput(output, record); } export function explainedRejectionOf(rejection: DescribedRun['rejection']): ExplainedRejection { if (rejection === undefined) { return { reason: 'conflict', kind: 'unworkable' }; } - if (rejection.reason === 'conflict') { - const { kind } = rejection; - return kind === undefined ? { reason: 'conflict' } : { reason: 'conflict', kind }; - } if (rejection.reason === 'cancelled' || rejection.reason === 'unanswered') { return { reason: rejection.reason, kind: rejection.kind }; } - if (rejection.reason !== 'unavailable' || rejection.kind === undefined) { + if (rejection.reason === 'invalid_input' || rejection.kind === undefined) { return { reason: rejection.reason }; } - const { kind, because } = rejection; - return because === undefined ? { reason: 'unavailable', kind } : { reason: 'unavailable', kind, because }; + const { reason, kind, because } = rejection; + return because === undefined ? { reason, kind } : { reason, kind, because }; } function rejectionWords({ named }: RunContext, { rejection }: DescribedRun): string { @@ -48,7 +46,7 @@ function rejectionWords({ named }: RunContext, { rejection }: DescribedRun): str return `The run of ${named} did not go through: ${why}. ${remedy}`; } -const wordsByStatus: Readonly string>> = { +const wordsByStatus: Readonly string>> = { started: ({ named, moment }) => moment === 'just started' ? `${capitalized(named)} has started and is still running. It carries on by itself, and how it ends can be looked up later.` diff --git a/packages/definitions/src/presenting/run-presenter.test.ts b/packages/definitions/src/presenting/run-presenter.test.ts index dcb051d8e..dc0d7f4e5 100644 --- a/packages/definitions/src/presenting/run-presenter.test.ts +++ b/packages/definitions/src/presenting/run-presenter.test.ts @@ -228,6 +228,10 @@ describe('the presenter of the start of a tool call', () => { }); }); + it('shows that its server marks the tool read-only, where the start recorded it', () => { + expect(presented({ ...started, read_only: true })).toMatchObject({ data: { read_only: true } }); + }); + it('keeps nothing of a name its server gave but letters and digits, so no slash or dot reaches the summary', () => { expect(presented({ ...started, tool: 'files/read.v2', server: 'graph-eu' })).toMatchObject({ summary: 'A run made tool call 3, to the files read v2 tool of graph eu.', diff --git a/packages/definitions/src/presenting/run-presenter.ts b/packages/definitions/src/presenting/run-presenter.ts index 5c56ea7ca..33cb18e6b 100644 --- a/packages/definitions/src/presenting/run-presenter.ts +++ b/packages/definitions/src/presenting/run-presenter.ts @@ -50,10 +50,6 @@ function rejectionShown(rejection: RunRejection) { if (rejection.reason === 'invalid_input') { return { reason: rejection.reason, detail, ...issuesShown(rejection.issues) }; } - if (rejection.reason === 'conflict') { - const { reason, kind } = rejection; - return { reason, detail, ...(kind === undefined ? {} : { kind }) }; - } if (rejection.reason === 'cancelled' || rejection.reason === 'unanswered') { const { reason, kind } = rejection; return { reason, detail, kind }; @@ -97,6 +93,7 @@ function callStartedAccount(event: ToolCallStarted, fact: Fact): Account { tool, arguments_bytes: event.arguments_bytes, arguments_sha256: cutAtCodePoint(event.arguments_sha256, mostDigestBytes), + ...(event.read_only === true ? { read_only: true } : {}), ...contentShown('arguments_json', event.arguments_json), }, }; diff --git a/packages/definitions/src/reading/listed-run.ts b/packages/definitions/src/reading/listed-run.ts index f379ab743..01584ee30 100644 --- a/packages/definitions/src/reading/listed-run.ts +++ b/packages/definitions/src/reading/listed-run.ts @@ -2,7 +2,7 @@ import { CancelledKindSchema, ConflictKindSchema, UnansweredKindSchema, - UnavailableBecauseSchema, + RejectionBecauseSchema, UnavailableKindSchema, type RecordedEvent, } from '@beonauto/operations'; @@ -19,10 +19,10 @@ const ListedRejectionSchema = Schema.Struct({ kind: Schema.optionalKey( Schema.Union([UnavailableKindSchema, ConflictKindSchema, CancelledKindSchema, UnansweredKindSchema]), ), - because: Schema.optionalKey(UnavailableBecauseSchema), + because: Schema.optionalKey(RejectionBecauseSchema), }).annotate({ description: - 'Why the run was rejected: its reason, the kind it gave for unavailable or conflict, the kind of a cancellation or of a request nobody answered, and for unavailable the because it gave', + 'Why the run was rejected: its reason, the kind it gave for unavailable or conflict, the kind of a cancellation or of a request nobody answered, and for unavailable or conflict the because it gave', }); export const ListedRunSchema = Schema.Struct({ @@ -53,10 +53,6 @@ function listedRejectionOf(rejection: RunRejection): ListedRejection { if (rejection.reason === 'invalid_input') { return { reason: rejection.reason }; } - if (rejection.reason === 'conflict') { - const { reason, kind } = rejection; - return { reason, ...(kind === undefined ? {} : { kind }) }; - } if (rejection.reason === 'cancelled' || rejection.reason === 'unanswered') { const { reason, kind } = rejection; return { reason, kind }; diff --git a/packages/definitions/src/run-work/delivery-words.test.ts b/packages/definitions/src/run-work/delivery-words.test.ts index c1cd9d7f4..ec1dbc79b 100644 --- a/packages/definitions/src/run-work/delivery-words.test.ts +++ b/packages/definitions/src/run-work/delivery-words.test.ts @@ -12,8 +12,8 @@ describe('the words of a delivery', () => { throughTheTool({ server: 'chat', tool: 'post_message' }), deliveryStarted({ number: 1, server: 'chat', tool: 'post_message' }), ]).toEqual([ - 'through the tool post_message of chat', - 'Delivery attempt 1 of the request started, through the tool post_message of chat.', + 'through the post message tool of chat', + 'Delivery attempt 1 of the request started, through the post message tool of chat.', ]); }); diff --git a/packages/definitions/src/run-work/delivery-words.ts b/packages/definitions/src/run-work/delivery-words.ts index 88ffe5636..1f0a4aebe 100644 --- a/packages/definitions/src/run-work/delivery-words.ts +++ b/packages/definitions/src/run-work/delivery-words.ts @@ -1,3 +1,4 @@ +import { toolInWords } from '@beonauto/mcp'; import { plainNumber } from '@beonauto/operations'; import type { DeliveryBecause, DeliveryEnded, DeliveryStarted } from '../runs/run-events.ts'; @@ -13,7 +14,7 @@ const becauseWords: Readonly> = { }; export function throughTheTool({ server, tool }: Pick): string { - return `through the tool ${tool} of ${server}`; + return `through ${toolInWords({ server, tool })}`; } export function deliveryStarted(fact: Pick): string { diff --git a/packages/definitions/src/run-work/run-work-presenter.test.ts b/packages/definitions/src/run-work/run-work-presenter.test.ts index 65630a672..0c070333a 100644 --- a/packages/definitions/src/run-work/run-work-presenter.test.ts +++ b/packages/definitions/src/run-work/run-work-presenter.test.ts @@ -112,7 +112,7 @@ describe('the start of a delivery', () => { expect(presented(started)).toEqual([ { type: 'delivery_started', - summary: 'Delivery attempt 1 of the request started, through the tool post_message of chat.', + summary: 'Delivery attempt 1 of the request started, through the post message tool of chat.', data: { run_id: runId, by: 'brain:alpha', @@ -238,7 +238,7 @@ describe('a reply the run took or refused', () => { ).toEqual([ { type: 'reply_taken', - summary: 'A reply from the party answered the request, read through the tool thread_replies of chat.', + summary: 'A reply from the party answered the request, read through the thread replies tool of chat.', data: { run_id: runId, by: 'brain:alpha', ...shownReply, answer_bytes: 20 }, }, ]); diff --git a/packages/definitions/src/runs/run-calls.test.ts b/packages/definitions/src/runs/run-calls.test.ts index 8c44c1ea8..3868369b5 100644 --- a/packages/definitions/src/runs/run-calls.test.ts +++ b/packages/definitions/src/runs/run-calls.test.ts @@ -90,6 +90,15 @@ const toolsWereCalled = new Conflict({ kind: 'tools_called', }); +const toolsOnlyRead = new Conflict({ + detail: + "The run called tools and did not succeed, so it is not run again under its id; every tool it called only reads, by its server's own account, so nothing was changed: start a new run with another run id, and read with get_run_history what it called", + kind: 'tools_called', + because: 'only_read', +}); + +const readCalled: RunEvent = { ...called, read_only: true, number: 1, ...during }; + const noMoreWork = new Conflict({ detail: 'The run has ended, so it records no more of its work' }); describe('a tool call of a run', () => { @@ -159,3 +168,26 @@ describe('the calls a run recorded', () => { ); }); }); + +describe('the calls of a run whose every tool only reads', () => { + it('keep the run from running again under its id, saying that nothing was changed once it ended', () => { + const readAgain = { ...readCalled, number: 2 }; + + expect([ + decided(starting(), started, readCalled, readAgain, unavailable), + decided(starting(), started, readCalled, failed), + decided(starting(), started, readCalled, callStarted, unavailable), + decided(starting(), started, readCalled), + ]).toEqual([ + Result.fail(toolsOnlyRead), + Result.fail(toolsOnlyRead), + Result.fail(toolsWereCalled), + Result.fail(toolsWereCalled), + ]); + }); + + it('are known after a start recorded again, since each start keeps what the calls before it recorded', () => { + expect(stateAfter(started, readCalled, failed, started)).toMatchObject({ calledOnlyReadOnly: true }); + expect(stateAfter(started, callStarted, failed, started)).toMatchObject({ calledOnlyReadOnly: false }); + }); +}); diff --git a/packages/definitions/src/runs/run-decider.test.ts b/packages/definitions/src/runs/run-decider.test.ts index cdb63c032..d4a6457ab 100644 --- a/packages/definitions/src/runs/run-decider.test.ts +++ b/packages/definitions/src/runs/run-decider.test.ts @@ -157,6 +157,7 @@ describe('a run', () => { callsTools: false, lastCall: 0, mayHaveChanged: false, + calledOnlyReadOnly: true, deliveryInFlight: null, broughtAnswer: null, deliveredAt: null, @@ -188,6 +189,7 @@ describe('a run started again', () => { callsTools: false, lastCall: 0, mayHaveChanged: false, + calledOnlyReadOnly: true, deliveryInFlight: null, broughtAnswer: null, deliveredAt: null, diff --git a/packages/definitions/src/runs/run-decisions.ts b/packages/definitions/src/runs/run-decisions.ts index 2424e95b3..29de10843 100644 --- a/packages/definitions/src/runs/run-decisions.ts +++ b/packages/definitions/src/runs/run-decisions.ts @@ -17,6 +17,7 @@ import type { RunEvent } from './run-events.ts'; import { awaitsSettlement, cancelBeforeStartOf, + changedNothing, hasFinalResult, isRunning, mayHaveChangedSomething, @@ -27,6 +28,7 @@ import { type RecordedRunState, } from './run-state.ts'; import { settlementKeyOf, succeedsWith } from './settlement-keys.ts'; +import { startedCallingTools, toolsOnlyRead, toolsWereCalled } from './tool-refusals.ts'; type Decision = Result.Result>; @@ -34,18 +36,6 @@ export type Claim = 'run' | 'answer'; const nothingToRecord: Decision = Result.succeed([]); -const toolsWereCalled = new Conflict({ - detail: - 'The run called tools and did not succeed, so it is not run again under its id, since a tool may have changed something; start a new run with another run id, and read with get_run_history what it called', - kind: 'tools_called', -}); - -const startedCallingTools = new Conflict({ - detail: - 'The run has started and its definition calls tools, so it is not run again under its id: it may still be in progress, or have stopped without recording how it ended, and its tools may have changed something; start a new run with another run id, and read with get_run_history what it has called so far', - kind: 'tools_called', -}); - const noSuchRun = new NotFound({ detail: 'There is no such run in this brain' }); export const runTaken = new Conflict({ @@ -74,6 +64,9 @@ function claimOfRecorded(state: RecordedRunState): Result.Result { +function answerWith(run: RunDetail): Effect.Effect { return run.rejection === undefined ? Effect.succeed(run) : Effect.fail(replayed(run.rejection)); } -export function answerOf(id: string, state: RunStreamState): Effect.Effect { - return runOf(id, state).pipe(Effect.flatMap(answerWith)); +export function answerOf(id: string, state: RunStreamState): Effect.Effect { + return runDetailOf(id, state).pipe(Effect.flatMap(answerWith)); } diff --git a/packages/definitions/src/runs/run-settler.test.ts b/packages/definitions/src/runs/run-settler.test.ts index d7d9aa1a7..1cf514a3d 100644 --- a/packages/definitions/src/runs/run-settler.test.ts +++ b/packages/definitions/src/runs/run-settler.test.ts @@ -35,7 +35,7 @@ describe('settling a deferred run', () => { }); expect(await running()).toStrictEqual({ status: 'succeeded', - output: { ...settled, status: 'succeeded', output: 'handed on' }, + output: { ...settled, status: 'succeeded', output: 'handed on', record: { steps: 3 } }, }); expect(relayer.runs()).toBe(1); }); @@ -72,6 +72,21 @@ describe('settling a deferred run as rejected or failed', () => { expect(relayer.runs()).toBe(1); }); + it('records a conflict with its kind and because', async () => { + const { running, settling } = await withHandOn(); + await running(); + const unknown = { + reason: 'conflict', + detail: 'It may have posted', + kind: 'effect_unknown', + because: 'tool_error', + } as const; + + expect(await settling({ status: 'rejected', ...unknown })).toMatchObject( + Result.succeed({ status: 'rejected', rejection: unknown }), + ); + }); + it('as unavailable or failed lets a call with its id run it again', async () => { const { running, relayer, settling } = await withHandOn(); await running(); diff --git a/packages/definitions/src/runs/run-settler.ts b/packages/definitions/src/runs/run-settler.ts index 8e423a942..c19577029 100644 --- a/packages/definitions/src/runs/run-settler.ts +++ b/packages/definitions/src/runs/run-settler.ts @@ -65,8 +65,8 @@ function rejectionOf(settlement: SettledRejection): RunRejection { return { reason, detail, ...(kind === undefined ? {} : { kind }), ...(because === undefined ? {} : { because }) }; } if (settlement.reason === 'conflict') { - const { reason, detail, kind } = settlement; - return { reason, detail, ...(kind === undefined ? {} : { kind }) }; + const { reason, detail, kind, because } = settlement; + return { reason, detail, ...(kind === undefined ? {} : { kind }), ...(because === undefined ? {} : { because }) }; } if (settlement.reason === 'unanswered') { const { reason, detail, kind } = settlement; diff --git a/packages/definitions/src/runs/run-state.ts b/packages/definitions/src/runs/run-state.ts index 737a7851e..ecd660c4a 100644 --- a/packages/definitions/src/runs/run-state.ts +++ b/packages/definitions/src/runs/run-state.ts @@ -34,6 +34,7 @@ export interface RecordedRunState { readonly callsTools: boolean; readonly lastCall: number; readonly mayHaveChanged: boolean; + readonly calledOnlyReadOnly: boolean; readonly deliveryInFlight: number | null; readonly broughtAnswer: BroughtAnswer | null; readonly deliveredAt: string | null; @@ -64,6 +65,20 @@ export function cancelBeforeStartOf(state: RunStreamState): AskedCancel | undefi return state !== undefined && 'cancelledBeforeStart' in state ? state.cancelledBeforeStart : undefined; } +type CallsBefore = Pick; + +const noCallsBefore: CallsBefore = { lastCall: 0, mayHaveChanged: false, calledOnlyReadOnly: true }; + +function callsBefore(earlier: RunState): CallsBefore { + return earlier === undefined + ? noCallsBefore + : { + lastCall: earlier.lastCall, + mayHaveChanged: earlier.mayHaveChanged, + calledOnlyReadOnly: earlier.calledOnlyReadOnly, + }; +} + function startedRun(event: RunStarted, earlier: RunState): RecordedRunState { const { definition_type: type, @@ -83,8 +98,7 @@ function startedRun(event: RunStarted, earlier: RunState): RecordedRunState { finishesLater: finishes_later === true, deferred: false, callsTools: calls_tools === true, - lastCall: earlier?.lastCall ?? 0, - mayHaveChanged: earlier?.mayHaveChanged ?? false, + ...callsBefore(earlier), deliveryInFlight: null, broughtAnswer: null, deliveredAt: null, @@ -152,7 +166,12 @@ function evolveStarted(state: RecordedRunState, event: Exclude Result.Result; -export type StartOf = (call: Pick) => StartedFields; +export type StartOf = (call: Pick) => StartedFields; export function namingOf(servers: readonly McpServerSettings[]): NamingCheck { const entries = new Map(servers.map((settings) => [settings.name, { settings }])); diff --git a/packages/mcp/src/access/linked-access.ts b/packages/mcp/src/access/linked-access.ts index 714fd2816..f19cac730 100644 --- a/packages/mcp/src/access/linked-access.ts +++ b/packages/mcp/src/access/linked-access.ts @@ -3,7 +3,7 @@ import { Effect } from 'effect'; import { defaultTiming } from '../bounds/call-bounds.ts'; import { secretsOfServers } from '../bounds/secrets.ts'; import { serverLink, type LinkOptions } from '../connections/server-links.ts'; -import { deliveredCall } from '../delivery/delivery-call.ts'; +import { oneCall } from '../one-call/one-call.ts'; import type { McpSettings } from '../settings/mcp-settings.ts'; import { untestableNoting } from '../tool-tests/testing-guard.ts'; import { openedRun } from './run-opening.ts'; @@ -29,7 +29,7 @@ export function linkedAccess(settings: McpSettings, options: ToolAccessOptions): configured: settings.servers.length > 0, testing: settings.servers, open: (context, references) => openedRun({ context, references, links, secrets, timing, toolsListed, report }), - callOnce: (call) => deliveredCall(call, { links, secrets, timing }), + callOnce: (call, runCall) => oneCall(call, { links, secrets, timing, toolsListed }, runCall), listServers: (scope, named) => Effect.promise(() => toolServersOf({ scope, named }, { links, secrets, timing, toolsListed })), close: async () => { diff --git a/packages/mcp/src/access/server-listing.ts b/packages/mcp/src/access/server-listing.ts index 43b05e561..a7d8d3e10 100644 --- a/packages/mcp/src/access/server-listing.ts +++ b/packages/mcp/src/access/server-listing.ts @@ -1,9 +1,9 @@ import { Result } from 'effect'; import type { Timing } from '../bounds/call-bounds.ts'; -import type { ListedTool } from '../bounds/result-text.ts'; import type { Secrets } from '../bounds/secrets.ts'; -import { takenSlot } from '../calls/server-slot.ts'; +import type { ListedTool } from '../bounds/tool-results.ts'; +import { takenSlot, type ServerSlot } from '../calls/server-slot.ts'; import { failureOf, type FailureKind, type ServerFailure } from '../connections/server-failures.ts'; import type { ServerLink } from '../connections/server-links.ts'; import type { McpServerSettings } from '../settings/mcp-settings.ts'; @@ -27,37 +27,43 @@ const failedBecause: Readonly> = { key_refused: 'key_refused', }; -function serverFailed(link: ServerLink, { kind, message }: ServerFailure, { scrub }: Secrets): McpServerFailed { +export function serverFailed( + { settings }: Pick, + { kind, message }: ServerFailure, + { scrub }: Secrets, +): McpServerFailed { return new McpServerFailed({ because: failedBecause[kind], - detail: `The MCP server ${link.settings.name} could not be used: ${scrub(message)}`, + detail: `The MCP server ${settings.name} could not be used: ${scrub(message)}`, }); } -export async function connectedTo( - link: ServerLink, +export function listedThrough( + slot: ServerSlot, { secrets, timing, toolsListed }: Listing, ): Promise> { - const taken = await takenSlot(link); - if (Result.isFailure(taken)) { - return Result.fail(serverFailed(link, taken.failure, secrets)); - } - const slot = taken.success; return slot .connection() .listTools(timing.openMs) .then( (tools) => { - toolsListed(link.settings, tools); + toolsListed(slot.settings, tools); return Result.succeed({ slot, tools }); }, async (error: unknown) => { await slot.release(); - return Result.fail(serverFailed(link, failureOf(error), secrets)); + return Result.fail(serverFailed(slot, failureOf(error), secrets)); }, ); } +export async function connectedTo(link: ServerLink, listing: Listing): Promise> { + const taken = await takenSlot(link); + return Result.isFailure(taken) + ? Result.fail(serverFailed(link, taken.failure, listing.secrets)) + : listedThrough(taken.success, listing); +} + export async function released(listed: readonly Listed[]): Promise { await Promise.all(listed.map(({ slot }) => slot.release())); } diff --git a/packages/mcp/src/access/tool-access.test.ts b/packages/mcp/src/access/tool-access.test.ts index 7569631ab..e9db179cd 100644 --- a/packages/mcp/src/access/tool-access.test.ts +++ b/packages/mcp/src/access/tool-access.test.ts @@ -180,7 +180,7 @@ describe('a server that cannot be used', () => { const { access } = accessTo({ graph: graphOf(fake) }); const holding = await offeredBy(access, 'graph/search'); - fake.answerNextWith(503); + fake.answerNextOf('tools/list', 503); const failing = await refusalOf(access, 'graph/search'); const reply = await holding.offered[0]?.call({ callId: 'call-1', input: { query: 'acme' } }, controlledSignals()); diff --git a/packages/mcp/src/access/tool-access.ts b/packages/mcp/src/access/tool-access.ts index a00c3abda..13865cb55 100644 --- a/packages/mcp/src/access/tool-access.ts +++ b/packages/mcp/src/access/tool-access.ts @@ -3,9 +3,9 @@ import { Effect } from 'effect'; import type { Timing } from '../bounds/call-bounds.ts'; import type { RunTools } from '../calls/run-tools.ts'; import type { LinkOptions } from '../connections/server-links.ts'; -import type { CalledOnce, DeliveryCall } from '../delivery/delivery-bounds.ts'; import type { ToolServer } from '../listing/tool-server.ts'; import type { ToolReference } from '../names/tool-reference.ts'; +import type { CallOnce } from '../one-call/one-call.ts'; import { brainsServedBy, isListedFor, @@ -33,7 +33,7 @@ export interface ToolAccess { ) => Effect.Effect; readonly named: NamingCheck; readonly startOf: StartOf; - readonly callOnce: (call: DeliveryCall) => Effect.Effect; + readonly callOnce: CallOnce; readonly listServers: (scope: ServersScope, named?: string) => Effect.Effect; readonly brainsServedBy: (server: string) => readonly string[]; readonly close: () => Promise; @@ -58,7 +58,7 @@ export function makeToolAccess(settings: McpSettings, options: ToolAccessOptions open: (context, references) => Effect.flatMap(Effect.promise(loaded), (access) => access.open(context, references)), named: namingOf(settings.servers), startOf: startsOf(settings.servers), - callOnce: (call) => Effect.flatMap(Effect.promise(loaded), (access) => access.callOnce(call)), + callOnce: (call, runCall) => Effect.flatMap(Effect.promise(loaded), (access) => access.callOnce(call, runCall)), listServers: (scope, named) => settings.servers.some((server) => isListedFor(server, scope, named)) ? Effect.flatMap(Effect.promise(loaded), (access) => access.listServers(scope, named)) diff --git a/packages/mcp/src/access/tool-naming.ts b/packages/mcp/src/access/tool-naming.ts index cebb35338..cfed478ee 100644 --- a/packages/mcp/src/access/tool-naming.ts +++ b/packages/mcp/src/access/tool-naming.ts @@ -2,7 +2,7 @@ import { servesBrain } from '@beonauto/config'; import { capitalized, type BrainAddress } from '@beonauto/operations'; import { Result } from 'effect'; -import type { ListedTool } from '../bounds/result-text.ts'; +import type { ListedTool } from '../bounds/tool-results.ts'; import type { OfferedOnServer } from '../calls/run-parts.ts'; import type { ServerSlot } from '../calls/server-slot.ts'; import type { ServerLink } from '../connections/server-links.ts'; diff --git a/packages/mcp/src/bounds/result-text.test.ts b/packages/mcp/src/bounds/tool-results.test.ts similarity index 74% rename from packages/mcp/src/bounds/result-text.test.ts rename to packages/mcp/src/bounds/tool-results.test.ts index cbb02c195..b7c015bf7 100644 --- a/packages/mcp/src/bounds/result-text.test.ts +++ b/packages/mcp/src/bounds/tool-results.test.ts @@ -1,7 +1,10 @@ import { Redacted } from 'effect'; import { describe, expect, it } from 'vitest'; +import { secretsOf } from './secrets.ts'; +import { bytesOf, canonicalJson, cutAsStored, cutAtCodePoint } from './text-bytes.ts'; import { + answerDocument, answerOf, decodeListedTools, decodeToolResult, @@ -9,9 +12,7 @@ import { errorTextForOperator, metaValueOf, resultText, -} from './result-text.ts'; -import { secretsOf } from './secrets.ts'; -import { bytesOf, canonicalJson, cutAsStored, cutAtCodePoint } from './text-bytes.ts'; +} from './tool-results.ts'; const scrub = secretsOf([Redacted.make('graph-api-key-4f1d9a7c2b')]).scrub; @@ -125,3 +126,51 @@ describe('the answer of a tool as its caller reads it', () => { ]).toEqual([{ content: [{ type: 'text', text: 'Posted.' }], structuredContent: { ts: '1.1' } }, { content: [] }]); }); }); + +const answers: ReadonlyArray = [ + [ + 'structured content with its JSON text, which the two read alike', + { content: [{ type: 'text', text: '{"rows":2}' }], structuredContent: { rows: 2 } }, + '{"rows":2}', + { rows: 2 }, + ], + [ + 'structured content beside a summary, where the model sees the summary and the document is the structure', + { content: [{ type: 'text', text: '2 rows.' }], structuredContent: { rows: 2 } }, + '2 rows.', + { rows: 2 }, + ], + [ + 'several text blocks, which the model reads joined and the document reads the first of', + { + content: [ + { type: 'text', text: '{"page":1}' }, + { type: 'text', text: '{"page":2}' }, + ], + }, + '{"page":1}\n{"page":2}', + { page: 1 }, + ], + [ + 'only an image, which the model sees as a placeholder and the document does not have', + { content: [{ type: 'image', data: 'AA==', mimeType: 'image/png' }] }, + '[image content (image/png), not shown]', + undefined, + ], + ['nothing', { content: [] }, '(The tool answered with no content.)', undefined], + [ + 'a text that is not JSON, which both read as that text', + { content: [{ type: 'text', text: 'Done.' }] }, + 'Done.', + 'Done.', + ], +]; + +describe('the two readings of one answer', () => { + it.each(answers)('of %s', (_case, answer, seen, document) => { + expect([resultText(decodeToolResult(answer)), answerDocument(answerOf(JSON.stringify(answer)))]).toEqual([ + seen, + document, + ]); + }); +}); diff --git a/packages/mcp/src/bounds/result-text.ts b/packages/mcp/src/bounds/tool-results.ts similarity index 85% rename from packages/mcp/src/bounds/result-text.ts rename to packages/mcp/src/bounds/tool-results.ts index bff28942f..5dd6a78a0 100644 --- a/packages/mcp/src/bounds/result-text.ts +++ b/packages/mcp/src/bounds/tool-results.ts @@ -40,6 +40,10 @@ export type ListedTool = typeof ListedToolSchema.Type; export type ToolAnnotations = typeof ToolAnnotationsSchema.Type; +export function isReadOnly(annotations: ToolAnnotations | undefined): boolean { + return annotations?.readOnlyHint === true; +} + type ContentBlock = typeof ContentBlockSchema.Type; export type AnswerBlock = ContentBlock; @@ -54,6 +58,8 @@ export function answerOf(resultJson: string): ToolAnswer { return Option.getOrElse(decodeAnswer(resultJson), () => noAnswer); } +const decodeJsonText = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Json)); + export const decodeToolResult = Schema.decodeUnknownSync(ToolResultSchema); export const decodeListedTools = Schema.decodeUnknownSync(Schema.Struct({ tools: Schema.Array(ListedToolSchema) })); @@ -83,6 +89,14 @@ export function resultText({ content, structuredContent }: ToolResult): string { return lines.length === 0 ? nothing : lines.join('\n'); } +export function answerDocument({ content, structuredContent }: ToolAnswer): Schema.Json | undefined { + if (structuredContent !== undefined) { + return structuredContent; + } + const text = content.find((block) => isText(block))?.text; + return text === undefined ? undefined : Option.getOrElse(decodeJsonText(text), () => text); +} + export function errorTextForModel(text: string, scrub: (text: string) => string): string { return scrub(text.trim()).slice(0, mostErrorCharacters); } diff --git a/packages/mcp/src/calls/call-facts.ts b/packages/mcp/src/calls/call-facts.ts index 7fd4bbe40..dbcca3dd4 100644 --- a/packages/mcp/src/calls/call-facts.ts +++ b/packages/mcp/src/calls/call-facts.ts @@ -10,6 +10,7 @@ export const CallStartedSchema = Schema.Struct({ arguments_bytes: Schema.Int, arguments_sha256: Schema.String, arguments_json: Schema.optionalKey(Schema.String), + read_only: Schema.optionalKey(Schema.Literal(true)), }); export const CallAnsweredSchema = Schema.Struct({ diff --git a/packages/mcp/src/calls/call-replies.test.ts b/packages/mcp/src/calls/call-replies.test.ts index 7431c0561..a2b6185bb 100644 --- a/packages/mcp/src/calls/call-replies.test.ts +++ b/packages/mcp/src/calls/call-replies.test.ts @@ -47,6 +47,8 @@ async function opened(meta: Readonly>, ...tools: readonly return { call, journal, messages }; } +const aReader: unknown = expect.any(Function); + describe('what a call answers beside what the model sees', () => { it('is the outcome, the size of the whole result, how long it took and the id the server gave it, as recorded', async () => { const { call, journal } = await opened({ 'com.beonauto/run_id': testId }, 'search', 'denied'); @@ -61,6 +63,7 @@ describe('what a call answers beside what the model sees', () => { resultBytes: Buffer.byteLength(JSON.stringify({ content: [{ type: 'text', text: 'Found 2 rows for acme.' }] })), durationMs: aDuration, serverRequestId: 'call-1', + scrubbedResult: aReader, }, expect.objectContaining({ isError: true, outcome: 'tool_error', serverRequestId: 'call-2' }), ]); diff --git a/packages/mcp/src/calls/call-replies.ts b/packages/mcp/src/calls/call-replies.ts index 345cfd340..9bc4d6854 100644 --- a/packages/mcp/src/calls/call-replies.ts +++ b/packages/mcp/src/calls/call-replies.ts @@ -1,8 +1,14 @@ import type { ServerMessage } from '../access/caller-context.ts'; import type { CallsEndedBecause } from '../access/mcp-server-failed.ts'; import { failedOnce, failuresEnded, shownResult, type CallTally } from '../bounds/call-bounds.ts'; -import { errorTextForModel, errorTextForOperator, resultText } from '../bounds/result-text.ts'; import { bytesOf } from '../bounds/text-bytes.ts'; +import { + answerOf, + errorTextForModel, + errorTextForOperator, + resultText, + type ToolAnswer, +} from '../bounds/tool-results.ts'; import type { CallOutcome } from './call-facts.ts'; import { runIdKey, toolTestIdKey } from './call-meta.ts'; import type { Forwarded } from './tool-calls.ts'; @@ -19,6 +25,7 @@ export interface CallReply extends ModelWords { readonly resultBytes: number | null; readonly durationMs: number; readonly serverRequestId: string | null; + readonly scrubbedResult?: () => ToolAnswer; } export interface Replying { @@ -78,6 +85,10 @@ export function callReplyOf(words: ModelWords, done: Forwarded, durationMs: numb }; } +export function readableReply(reply: CallReply, { resultJson }: Forwarded, scrub: (text: string) => string): CallReply { + return resultJson === null ? reply : { ...reply, scrubbedResult: () => answerOf(scrub(resultJson)) }; +} + export function unsentReply(text: string): CallReply { return { text, isError: true, outcome: 'not_sent', resultBytes: null, durationMs: 0, serverRequestId: null }; } diff --git a/packages/mcp/src/calls/recorded-calls.test.ts b/packages/mcp/src/calls/recorded-calls.test.ts index 3f62e59a3..c5d37bda7 100644 --- a/packages/mcp/src/calls/recorded-calls.test.ts +++ b/packages/mcp/src/calls/recorded-calls.test.ts @@ -97,7 +97,7 @@ async function recordedCalls( } describe('the events of a call', () => { - it('records the size and digest of the arguments and the result, and no content', async () => { + it('records the size and digest of the arguments and the result, that its server marks the tool read-only, and no content', async () => { expect(await recordedCalls({}, [['graph/search', { query: 'acme' }]])).toEqual([ { type: 'tool_call_started', @@ -107,6 +107,7 @@ describe('the events of a call', () => { tool: 'search', arguments_bytes: 16, arguments_sha256: digest('{"query":"acme"}'), + read_only: true, }, { type: 'tool_call_answered', diff --git a/packages/mcp/src/calls/recorded-calls.ts b/packages/mcp/src/calls/recorded-calls.ts index 9582e7ff2..46d694e4b 100644 --- a/packages/mcp/src/calls/recorded-calls.ts +++ b/packages/mcp/src/calls/recorded-calls.ts @@ -4,6 +4,7 @@ import type { Effect } from 'effect'; import { toolBounds } from '../bounds/call-bounds.ts'; import { bytesOf, cutAsStored } from '../bounds/text-bytes.ts'; +import type { McpServerSettings } from '../settings/mcp-settings.ts'; import type { CallAnswered, CallOutcome, CallStarted } from './call-facts.ts'; export type NumberedAnswer = CallAnswered & { readonly number: number }; @@ -27,6 +28,7 @@ export interface StartingCall { export interface StartedCall extends StartingCall { readonly callId: string; + readonly readOnly: boolean; } export interface CallResult { @@ -47,6 +49,13 @@ export interface Recording { readonly scrub: (text: string) => string; } +export function recordingOf( + settings: Pick, + scrub: (text: string) => string, +): Recording { + return { content: settings.record_content, requestId: settings.request_id !== null, scrub }; +} + function digestOf(text: string): string { return createHash('sha256').update(text, 'utf8').digest('hex'); } @@ -66,7 +75,12 @@ export function startedFields(call: StartingCall, recording: Recording): Started } export function callStarted(call: StartedCall, recording: Recording): CallStarted { - return { type: 'tool_call_started', call_id: call.callId, ...startedFields(call, recording) }; + return { + type: 'tool_call_started', + call_id: call.callId, + ...startedFields(call, recording), + ...(call.readOnly ? { read_only: true } : {}), + }; } function resultOf(resultJson: string | null, recording: Recording) { diff --git a/packages/mcp/src/calls/run-parts.ts b/packages/mcp/src/calls/run-parts.ts index 65b5c1686..701a2ece8 100644 --- a/packages/mcp/src/calls/run-parts.ts +++ b/packages/mcp/src/calls/run-parts.ts @@ -3,8 +3,8 @@ import type { Effect } from 'effect'; import type { CallerContext, ServerMessage } from '../access/caller-context.ts'; import type { CallsEndedBecause } from '../access/mcp-server-failed.ts'; import type { CallTally, Timing } from '../bounds/call-bounds.ts'; -import type { ListedTool } from '../bounds/result-text.ts'; import type { Secrets } from '../bounds/secrets.ts'; +import type { ListedTool } from '../bounds/tool-results.ts'; import type { ToolReference } from '../names/tool-reference.ts'; import type { ServerSlot } from './server-slot.ts'; @@ -41,6 +41,6 @@ export interface RunToolsParts { export interface RunState { readonly tally: () => CallTally; readonly tallied: (tally: CallTally) => void; - readonly used: (reference: ToolReference) => void; + readonly used: (offer: OfferedOnServer) => void; readonly ended: (because: CallsEndedBecause) => void; } diff --git a/packages/mcp/src/calls/run-tools.test.ts b/packages/mcp/src/calls/run-tools.test.ts index 2c64466c4..933ffb594 100644 --- a/packages/mcp/src/calls/run-tools.test.ts +++ b/packages/mcp/src/calls/run-tools.test.ts @@ -16,6 +16,18 @@ async function runWith(tools: readonly string[]) { return run; } +describe('whether every tool a run called only reads', () => { + it('holds before any call and while each tool called is one its server marks read-only, and not once another is called', async () => { + const { tools, call } = await runWith(['search', 'echo']); + const before = tools.calledOnlyReadOnly(); + await call('search', { query: 'acme' }); + const afterReading = tools.calledOnlyReadOnly(); + await call('echo', {}); + + expect([before, afterReading, tools.calledOnlyReadOnly(), tools.calledAny()]).toEqual([true, true, false, true]); + }); +}); + describe('what the model sees of a tool', () => { it('sees its description cut to 4 KiB', async () => { const { tools, call } = await runWith(['verbose']); diff --git a/packages/mcp/src/calls/run-tools.ts b/packages/mcp/src/calls/run-tools.ts index af10c46e3..882a9d6a2 100644 --- a/packages/mcp/src/calls/run-tools.ts +++ b/packages/mcp/src/calls/run-tools.ts @@ -2,12 +2,11 @@ import type { Schema } from 'effect'; import type { CallsEndedBecause } from '../access/mcp-server-failed.ts'; import { callsEnded, cutToDescriptionBound, noCalls } from '../bounds/call-bounds.ts'; -import type { ToolAnnotations } from '../bounds/result-text.ts'; +import { isReadOnly, type ToolAnnotations } from '../bounds/tool-results.ts'; import { modelFacingNames } from '../names/model-facing-names.ts'; -import type { ToolReference } from '../names/tool-reference.ts'; import { toolsInWords } from '../names/tool-words.ts'; import type { CallReply } from './call-replies.ts'; -import type { CallSignals, RunState, RunToolsParts, ToolCallRequest } from './run-parts.ts'; +import type { CallSignals, OfferedOnServer, RunState, RunToolsParts, ToolCallRequest } from './run-parts.ts'; import { caller } from './tool-caller.ts'; export interface OfferedTool { @@ -28,6 +27,7 @@ export interface RunTools { readonly ended: Readonly; readonly ending: () => ToolsEnding | undefined; readonly calledAny: () => boolean; + readonly calledOnlyReadOnly: () => boolean; readonly usedInWords: () => string; readonly close: () => Promise; } @@ -35,15 +35,15 @@ export interface RunTools { export function runTools(parts: RunToolsParts): RunTools { let tally = noCalls; let ending: ToolsEnding | undefined; - const used: ToolReference[] = []; + const used: OfferedOnServer[] = []; const stop = new AbortController(); const state: RunState = { tally: () => tally, tallied: (next) => { tally = next; }, - used: (reference) => { - used.push(reference); + used: (offer) => { + used.push(offer); }, ended: (because) => { ending ??= { because }; @@ -63,7 +63,8 @@ export function runTools(parts: RunToolsParts): RunTools { ended: stop.signal, ending: () => ending, calledAny: () => used.length > 0, - usedInWords: () => toolsInWords(used), + calledOnlyReadOnly: () => used.every(({ tool }) => isReadOnly(tool.annotations)), + usedInWords: () => toolsInWords(used.map(({ reference }) => reference)), close: async () => { await Promise.all(parts.slots.map((slot) => slot.release())); }, diff --git a/packages/mcp/src/calls/tool-caller.test.ts b/packages/mcp/src/calls/tool-caller.test.ts index 1b327e307..0883321bd 100644 --- a/packages/mcp/src/calls/tool-caller.test.ts +++ b/packages/mcp/src/calls/tool-caller.test.ts @@ -90,7 +90,7 @@ describe('a run that ends while it calls', () => { const { call, fake, journal } = await runWith(['search'], 10_000); const signals = controlledSignals(); - fake.answerNextWith(429, 1, { 'retry-after': '5' }); + fake.answerNextOf('tools/call', 429, 1, { 'retry-after': '5' }); const replied = call('search', { query: 'waiting' }, signals); await setTimeout(100); signals.end(); diff --git a/packages/mcp/src/calls/tool-caller.ts b/packages/mcp/src/calls/tool-caller.ts index 59250ab13..a409a07ed 100644 --- a/packages/mcp/src/calls/tool-caller.ts +++ b/packages/mcp/src/calls/tool-caller.ts @@ -1,13 +1,69 @@ import { admission } from '../bounds/call-bounds.ts'; -import { callReplyOf, failureCounted, replyOf, unsentReply, type CallReply, type Replying } from './call-replies.ts'; -import { callAnswered, callStarted } from './recorded-calls.ts'; +import { isReadOnly } from '../bounds/tool-results.ts'; +import type { CallStarted } from './call-facts.ts'; +import { + callReplyOf, + failureCounted, + readableReply, + replyOf, + unsentReply, + type CallReply, + type Replying, +} from './call-replies.ts'; +import { callAnswered, callStarted, recordingOf, type NumberedAnswer, type Recording } from './recorded-calls.ts'; import type { CallSignals, NamedOffer, RunState, RunToolsParts, ToolCallRequest } from './run-parts.ts'; -import { forwarded, type Forwarded } from './tool-calls.ts'; +import { forwarded, type Forwarded, type Forwarding } from './tool-calls.ts'; const notRecorded = unsentReply('This call could not be recorded on its run, so it was not sent; answer without it.'); const notSent = unsentReply('The run has ended, so this call was not sent.'); +export interface CallJournalling { + readonly callId: string; + readonly readOnly: boolean; + readonly started: (fact: CallStarted) => Promise; + readonly answered: (fact: NumberedAnswer) => Promise; +} + +interface SentCall { + readonly sent: true; + readonly done: Forwarded; + readonly durationMs: number; +} + +export type JournalledCall = SentCall | { readonly sent: false }; + +const unsent: JournalledCall = { sent: false }; + +async function sentCall(forwarding: Forwarding): Promise { + const began = performance.now(); + const done = await forwarded(forwarding); + return { sent: true, done, durationMs: Math.round(performance.now() - began) }; +} + +function startOf(forwarding: Forwarding, recording: Recording, { callId, readOnly }: CallJournalling): CallStarted { + const { slot, tool, input } = forwarding; + const argumentsJson = JSON.stringify(input); + return callStarted({ callId, readOnly, server: slot.settings.name, tool, argumentsJson }, recording); +} + +export async function journalledCall( + forwarding: Forwarding, + recording: Recording, + journalling?: CallJournalling, +): Promise { + if (journalling === undefined) { + return sentCall(forwarding); + } + const number = await journalling.started(startOf(forwarding, recording, journalling)); + if (number === undefined) { + return unsent; + } + const sent = await sentCall(forwarding); + await journalling.answered(callAnswered({ ...sent.done, number, durationMs: sent.durationMs }, recording)); + return sent; +} + function replied(state: RunState, done: Forwarded, replying: Replying, durationMs: number): CallReply { const failure = failureCounted(state.tally(), done, replying); state.tallied(failure.tally); @@ -16,7 +72,7 @@ function replied(state: RunState, done: Forwarded, replying: Replying, durationM } const reply = replyOf(state.tally(), done, replying); state.tallied(reply.tally); - return callReplyOf(reply.value, done, durationMs); + return readableReply(callReplyOf(reply.value, done, durationMs), done, replying.scrub); } export function caller( @@ -24,11 +80,11 @@ export function caller( state: RunState, offered: NamedOffer, ): (request: ToolCallRequest, signals: CallSignals) => Promise { - const { slot, reference } = offered; + const { slot } = offered; const { context, timing } = parts; const { scrub } = parts.secrets; - const recording = { content: slot.settings.record_content, requestId: slot.settings.request_id !== null, scrub }; - const replying = { server: reference.server, meta: context.meta, scrub, report: parts.report }; + const recording = recordingOf(slot.settings, scrub); + const replying = { server: offered.reference.server, meta: context.meta, scrub, report: parts.report }; return async ({ callId, input }, signals) => { if (signals.signal.aborted) { return notSent; @@ -38,19 +94,23 @@ export function caller( if (!admitted.admitted) { return unsentReply(admitted.refusal); } - const start = callStarted({ callId, ...reference, argumentsJson: JSON.stringify(input) }, recording); - const number = await parts.run(context.journal.started(start)); - if (number === undefined) { - return notRecorded; - } - state.used(reference); - const began = performance.now(); const forwarding = { slot, tool: offered.tool.name, input, meta: context.meta, signal: signals.signal }; - const done = await forwarded({ ...forwarding, ...timing }); - const durationMs = Math.round(performance.now() - began); - if (!signals.cancelled.aborted) { - await parts.run(context.journal.answered(callAnswered({ ...done, number, durationMs }, recording))); - } - return replied(state, done, replying, durationMs); + const journey = await journalledCall({ ...forwarding, ...timing }, recording, { + callId, + readOnly: isReadOnly(offered.tool.annotations), + started: async (fact) => { + const number = await parts.run(context.journal.started(fact)); + if (number !== undefined) { + state.used(offered); + } + return number; + }, + answered: async (fact) => { + if (!signals.cancelled.aborted) { + await parts.run(context.journal.answered(fact)); + } + }, + }); + return journey.sent ? replied(state, journey.done, replying, journey.durationMs) : notRecorded; }; } diff --git a/packages/mcp/src/calls/tool-calls.test.ts b/packages/mcp/src/calls/tool-calls.test.ts index adad5b012..fe30129ca 100644 --- a/packages/mcp/src/calls/tool-calls.test.ts +++ b/packages/mcp/src/calls/tool-calls.test.ts @@ -1,7 +1,15 @@ import { afterEach, describe, expect, it } from 'vitest'; import type { Timing } from '../bounds/call-bounds.ts'; -import { deniedText, fakeApiKey, fakeChannels, inTurn, openFakeToolRun, toolRunId } from '../testing/index.ts'; +import { + brokenPromise, + deniedText, + fakeApiKey, + fakeChannels, + inTurn, + openFakeToolRun, + toolRunId, +} from '../testing/index.ts'; const quick: Timing = { callMs: 2000, openMs: 2000, longestRetryWaitMs: 1000 }; @@ -54,6 +62,18 @@ describe('the result of a call', () => { expect(tools.ended.aborted).toBe(false); }); + it('gives the model what a tool answered against its own output schema, and never says the arguments were refused', async () => { + const run = await openFakeToolRun(['promised'], quick, { data: true }); + closing.push(run.close); + + expect(await run.call('promised', {})).toMatchObject({ + text: JSON.stringify(brokenPromise), + isError: false, + outcome: 'result', + }); + expect(run.fake.received()).toHaveLength(1); + }); + it('cuts a result over 64 KiB at a code point, with the note', async () => { const { call } = await runWith(['large']); @@ -150,7 +170,7 @@ describe('a server that fails a call', () => { it('fails a call its server answers HTTP 403 as a failure of that call, counted as one, and never as a refused key', async () => { const { call, fake, journal, tools } = await runWith(['search']); - fake.answerNextWith(403, 5); + fake.answerNextOf('tools/call', 403, 5); const replies = await inTurn([1, 2, 3, 4, 5], (attempt) => call('search', { query: `denied ${attempt}` })); expect(replies[0]).toMatchObject({ @@ -176,9 +196,9 @@ describe('a server that asks to slow down or forgets a session', () => { it('waits out a 429 within the longest wait, and fails one asking longer, ending the calls as rate limited after five', async () => { const { call, fake, tools } = await runWith(['search'], quick); - fake.answerNextWith(429, 1, { 'retry-after': '0' }); + fake.answerNextOf('tools/call', 429, 1, { 'retry-after': '0' }); const waited = await call('search', { query: 'waited' }); - fake.answerNextWith(429, 5, { 'retry-after': '5' }); + fake.answerNextOf('tools/call', 429, 5, { 'retry-after': '5' }); const [limited] = await inTurn(['limited', 'b', 'c', 'd', 'e'], (query) => call('search', { query })); expect(waited).toMatchObject({ text: 'Found 2 rows for waited.', isError: false }); diff --git a/packages/mcp/src/calls/tool-calls.ts b/packages/mcp/src/calls/tool-calls.ts index e30457290..b57bcfd50 100644 --- a/packages/mcp/src/calls/tool-calls.ts +++ b/packages/mcp/src/calls/tool-calls.ts @@ -1,6 +1,6 @@ import { setTimeout } from 'node:timers/promises'; -import { metaValueOf, type ToolResult } from '../bounds/result-text.ts'; +import { metaValueOf, type ToolResult } from '../bounds/tool-results.ts'; import { ignored } from '../connections/ignored.ts'; import type { Observed } from '../connections/observed-requests.ts'; import { failureOf, type FailureKind } from '../connections/server-failures.ts'; diff --git a/packages/mcp/src/connections/mcp-connection.ts b/packages/mcp/src/connections/mcp-connection.ts index 576bb7e13..8a43ccadf 100644 --- a/packages/mcp/src/connections/mcp-connection.ts +++ b/packages/mcp/src/connections/mcp-connection.ts @@ -1,7 +1,7 @@ import { Client } from '@modelcontextprotocol/client'; import { Predicate } from 'effect'; -import { decodeListedTools, decodeToolResult, type ListedTool, type ToolResult } from '../bounds/result-text.ts'; +import { decodeListedTools, decodeToolResult, type ListedTool, type ToolResult } from '../bounds/tool-results.ts'; import { observations, type Observed, type Observations } from './observed-requests.ts'; interface ToolCalling { @@ -24,7 +24,7 @@ export interface McpConnection { } interface ToolClient { - readonly callTool: Client['callTool']; + readonly request: Client['request']; readonly listTools: Client['listTools']; } @@ -106,8 +106,8 @@ export function connectionOver( call: async ({ tool, input, meta, signal, timeoutMs }) => { const marker = observed.mark(); const settled = await client - .callTool( - { name: tool, arguments: { ...input }, [metaField]: { ...meta } }, + .request( + { method: 'tools/call', params: { name: tool, arguments: { ...input }, [metaField]: { ...meta } } }, { signal, timeout: timeoutMs, onresumptiontoken: marker }, ) .then(settledOf, (error: unknown) => ({ error })); diff --git a/packages/mcp/src/delivery/call-once.ts b/packages/mcp/src/delivery/call-once.ts deleted file mode 100644 index 3820a30b2..000000000 --- a/packages/mcp/src/delivery/call-once.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { cutToFailureBound } from '../bounds/call-bounds.ts'; -import { answerOf } from '../bounds/result-text.ts'; -import { answeredFields, type Recording } from '../calls/recorded-calls.ts'; -import { forwarded, type Forwarded } from '../calls/tool-calls.ts'; -import type { ServerLink } from '../connections/server-links.ts'; -import { boundedSlot, connectionBoundOf, takenWithin } from './connection-bound.ts'; -import { - deliveryBounds, - type AnsweredOnce, - type DeliveryAccess, - type DeliveryCall, - type UnopenedOnce, -} from './delivery-bounds.ts'; - -export function answeredOnce(done: Forwarded, durationMs: number, recording: Recording): AnsweredOnce { - const { scrub } = recording; - const answered = { kind: 'answered', fields: answeredFields(done, recording), durationMs } as const; - const { outcome, message, retryAfterMs } = done; - const shown = scrub(done.resultJson ?? ''); - if (outcome === 'result') { - return { ...answered, outcome, answer: answerOf(shown), detail: '', retryAfterMs }; - } - return { ...answered, outcome, detail: cutToFailureBound(message === '' ? shown : scrub(message)), retryAfterMs }; -} - -function unopened(detail: string): UnopenedOnce { - return { kind: 'unopened', because: 'mcp_server_failed', detail: cutToFailureBound(detail) }; -} - -export async function calledOnce( - call: DeliveryCall, - link: ServerLink, - access: DeliveryAccess, - signal: Readonly, -): Promise { - const connectionMs = connectionBoundOf(access.timing); - const taken = await takenWithin(link, connectionMs); - if ('late' in taken) { - return unopened(`The MCP server ${call.reference.server} did not open a connection within ${connectionMs} ms`); - } - if ('failure' in taken) { - return unopened(access.secrets.scrub(taken.failure.message)); - } - const slot = boundedSlot(taken.slot, connectionMs); - try { - return await forwarded({ - slot, - tool: call.reference.tool, - input: call.input, - meta: call.meta, - callMs: Math.min(access.timing.callMs, deliveryBounds.callMs), - longestRetryWaitMs: 0, - signal, - }); - } finally { - await slot.release(); - } -} diff --git a/packages/mcp/src/delivery/delivery-bounds.ts b/packages/mcp/src/delivery/delivery-bounds.ts deleted file mode 100644 index 5cbbb074a..000000000 --- a/packages/mcp/src/delivery/delivery-bounds.ts +++ /dev/null @@ -1,54 +0,0 @@ -import type { Schema } from 'effect'; - -import type { NotOfferedBecause } from '../access/tool-not-offered.ts'; -import type { Timing } from '../bounds/call-bounds.ts'; -import type { AnswerBlock } from '../bounds/result-text.ts'; -import type { Secrets } from '../bounds/secrets.ts'; -import type { CallOutcome } from '../calls/call-facts.ts'; -import type { AnsweredFields } from '../calls/recorded-calls.ts'; -import type { ServerLink } from '../connections/server-links.ts'; -import type { ToolReference } from '../names/tool-reference.ts'; - -export const deliveryBounds = { connectionMs: 10_000, callMs: 30_000 } as const; - -export interface DeliveryCall { - readonly org: string; - readonly brain: string; - readonly reference: ToolReference; - readonly input: Readonly>; - readonly meta: Readonly>; -} - -export interface CallAnswer { - readonly content: readonly AnswerBlock[]; - readonly structuredContent?: Schema.Json; -} - -interface AnsweredCall { - readonly kind: 'answered'; - readonly fields: AnsweredFields; - readonly durationMs: number; - readonly detail: string; - readonly retryAfterMs: number | null; -} - -export type AnsweredOnce = - | (AnsweredCall & { readonly outcome: 'result'; readonly answer: CallAnswer }) - | (AnsweredCall & { readonly outcome: Exclude }); - -export interface UnopenedOnce { - readonly kind: 'unopened'; - readonly because: 'mcp_server_failed'; - readonly detail: string; -} - -export type CalledOnce = - | { readonly kind: 'not_offered'; readonly because: NotOfferedBecause; readonly detail: string } - | UnopenedOnce - | AnsweredOnce; - -export interface DeliveryAccess { - readonly links: ReadonlyMap; - readonly secrets: Secrets; - readonly timing: Timing; -} diff --git a/packages/mcp/src/delivery/delivery-call.test.ts b/packages/mcp/src/delivery/delivery-call.test.ts deleted file mode 100644 index 4e4c36efc..000000000 --- a/packages/mcp/src/delivery/delivery-call.test.ts +++ /dev/null @@ -1,129 +0,0 @@ -import { Buffer } from 'node:buffer'; - -import { describe, expect, it } from 'vitest'; - -import { - calledOnce, - delivery, - deliveredRunId, - deliveryAccess, - deliveryId, - deliveryKey, - deliveryServer, - failedWith, -} from '../testing/delivery-calls.ts'; -import { deliveryBounds } from './delivery-bounds.ts'; - -const largeText = '\u{1F600}'.repeat(8 * 256); - -const largeAnswerBytes = Buffer.byteLength(JSON.stringify({ content: [{ type: 'text', text: largeText }] })); - -const aDigest: unknown = expect.stringMatching(/^[0-9a-f]{64}$/u); - -const aNumber: unknown = expect.any(Number); - -const scrubbed = JSON.stringify({ token: '[redacted]' }); - -describe('one call of a tool for a delivery', () => { - it('calls the tool with its arguments and the metadata its caller gives, and answers the call whole', async () => { - const fake = await deliveryServer(); - const access = deliveryAccess(fake.url); - - expect(await calledOnce(access)).toEqual({ - kind: 'answered', - outcome: 'result', - fields: { result_bytes: 95, result_sha256: aDigest, jsonrpc_id: aNumber }, - answer: { content: [{ type: 'text', text: JSON.stringify(delivery.input) }] }, - durationMs: aNumber, - detail: '', - retryAfterMs: null, - }); - expect(access.startOf(delivery)).toEqual({ - server: 'graph', - tool: 'echo', - arguments_bytes: 48, - arguments_sha256: aDigest, - }); - expect(fake.received()).toEqual([ - { - tool: 'echo', - arguments: delivery.input, - meta: { 'com.beonauto/run_id': deliveredRunId, 'com.beonauto/delivery_id': deliveryId }, - }, - ]); - expect(fake.seen().map(({ rpc }) => rpc)).not.toContain('tools/list'); - }); - - it('records the arguments and the answer, scrubbed, where the server records its content', async () => { - const fake = await deliveryServer(); - const access = deliveryAccess(fake.url, { record_content: true }); - const call = { ...delivery, input: { token: deliveryKey } }; - - const called = await calledOnce(access, call); - - expect([ - access.startOf(call), - access.startOf({ ...call, reference: { server: 'wiki', tool: 'echo' } }), - ]).toMatchObject([{ arguments_json: scrubbed }, { server: 'wiki', arguments_bytes: 36 }]); - expect(access.startOf({ ...call, reference: { server: 'wiki', tool: 'echo' } })).not.toHaveProperty( - 'arguments_json', - ); - expect(called).toMatchObject({ - fields: { result_json: JSON.stringify({ content: [{ type: 'text', text: scrubbed }] }) }, - }); - }); -}); - -describe('a call for a delivery that fails', () => { - it('is a tool error when the tool answers with one, or the server does not have the tool', async () => { - const fake = await deliveryServer(); - const access = deliveryAccess(fake.url); - - expect(await calledOnce(access, { reference: { server: 'graph', tool: 'denied' } })).toMatchObject( - failedWith('tool_error', expect.stringContaining('The field salary is denied by the policy')), - ); - expect(await calledOnce(access, { reference: { server: 'graph', tool: 'gone' } })).toMatchObject( - failedWith('tool_error', expect.stringContaining('Tool gone not found')), - ); - }); - - it('ends at its call bound, waiting no longer than a delivery may', async () => { - const fake = await deliveryServer(); - const sleeping = { reference: { server: 'graph', tool: 'sleep' }, input: { ms: 5000 } }; - - expect(await calledOnce(deliveryAccess(fake.url, {}, 200), sleeping)).toMatchObject({ - ...failedWith('timed_out', 'The MCP server did not answer within 200 ms'), - fields: { result_bytes: null, result_sha256: null }, - }); - expect(deliveryBounds).toMatchObject({ connectionMs: 10_000, callMs: 30_000 }); - }); -}); - -describe('what a tool answers a delivery', () => { - it('is carried whole, its content and its structured content, and counted', async () => { - const fake = await deliveryServer(); - const access = deliveryAccess(fake.url); - - const large = await calledOnce(access, { reference: { server: 'graph', tool: 'large' }, input: { kib: 8 } }); - const structured = await calledOnce(access, { reference: { server: 'graph', tool: 'profile' }, input: {} }); - - expect(large).toMatchObject({ - outcome: 'result', - answer: { content: [{ type: 'text', text: largeText }] }, - fields: { result_bytes: largeAnswerBytes }, - }); - expect(structured).toMatchObject({ - outcome: 'result', - answer: { content: [], structuredContent: { name: 'Ada', rows: 2 } }, - }); - }); - - it('is scrubbed of the secrets of the server before the caller reads it', async () => { - const fake = await deliveryServer(); - - expect(await calledOnce(deliveryAccess(fake.url), { input: { token: deliveryKey } })).toMatchObject({ - outcome: 'result', - answer: { content: [{ type: 'text', text: scrubbed }] }, - }); - }); -}); diff --git a/packages/mcp/src/delivery/delivery-call.ts b/packages/mcp/src/delivery/delivery-call.ts deleted file mode 100644 index 56f58d35e..000000000 --- a/packages/mcp/src/delivery/delivery-call.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { Array as Arr, Effect, Option, Result } from 'effect'; - -import { namedLinks } from '../access/tool-naming.ts'; -import type { ToolNotOffered } from '../access/tool-not-offered.ts'; -import type { Recording } from '../calls/recorded-calls.ts'; -import type { ServerLink } from '../connections/server-links.ts'; -import { answeredOnce, calledOnce } from './call-once.ts'; -import type { CalledOnce, DeliveryAccess, DeliveryCall } from './delivery-bounds.ts'; - -function recordingOn({ settings }: ServerLink, { secrets }: DeliveryAccess): Recording { - return { content: settings.record_content, requestId: settings.request_id !== null, scrub: secrets.scrub }; -} - -function called(call: DeliveryCall, link: ServerLink, access: DeliveryAccess): Effect.Effect { - return Effect.gen(function* () { - const began = performance.now(); - const done = yield* Effect.promise((signal) => calledOnce(call, link, access, signal)); - return 'kind' in done ? done : answeredOnce(done, Math.round(performance.now() - began), recordingOn(link, access)); - }); -} - -export function deliveredCall(call: DeliveryCall, access: DeliveryAccess): Effect.Effect { - return Result.match(namedLinks(call, { references: [call.reference], links: access.links }), { - onFailure: ({ because, detail }: Pick) => - Effect.succeed({ kind: 'not_offered', because, detail }), - onSuccess: (links) => called(call, Option.getOrThrow(Arr.head(links)), access), - }); -} diff --git a/packages/mcp/src/delivery/delivery-refusals.test.ts b/packages/mcp/src/delivery/delivery-refusals.test.ts deleted file mode 100644 index de91a6256..000000000 --- a/packages/mcp/src/delivery/delivery-refusals.test.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { once } from 'node:events'; -import { createServer } from 'node:http'; - -import { Effect, Function, Result, Schema } from 'effect'; -import { describe, expect, it } from 'vitest'; - -import { toolBounds } from '../bounds/call-bounds.ts'; -import { - calledOnce, - closedAfter, - deliveryAccess, - deliveryKey, - deliveryServer, - failedWith, - unopenedWith, -} from '../testing/delivery-calls.ts'; -import { patientTiming, recordingCallJournal, reportingAccess, toolRun } from '../testing/index.ts'; -import { deliveryBounds } from './delivery-bounds.ts'; - -const portOf = Schema.decodeUnknownSync(Schema.Struct({ port: Schema.Number })); - -const alpha = { org: 'acme', brain: 'alpha' }; - -describe('a server that cannot take a delivery now', () => { - it('hands on the wait a 429 asks for to the schedule of the delivery, without waiting it out in the call', async () => { - const fake = await deliveryServer(); - const access = deliveryAccess(fake.url); - const tools = Result.getOrThrow( - await Effect.runPromise( - Effect.result(access.open(toolRun(recordingCallJournal()), [{ server: 'graph', tool: 'search' }])), - ), - ); - closedAfter(tools.close); - - fake.answerNextWith(429, 1, { 'retry-after': '120' }); - - expect(await calledOnce(access)).toMatchObject( - failedWith('server_failure', 'The MCP server answered HTTP 429', 120_000), - ); - }); - - it('is unopened when the server cannot be reached, so nothing was sent', async () => { - const fake = await deliveryServer(); - const { url } = fake; - await fake.close(); - - expect(await calledOnce(deliveryAccess(url))).toEqual(unopenedWith('The MCP server could not be reached')); - }); - - it('keeps the words of a failure to 1 KiB, the bound every reader of a failure shares', async () => { - const fake = await deliveryServer(); - const refusal = 'The gateway refused the request. '.repeat(100); - const { access } = reportingAccess( - { graph: { url: fake.url, headers: { Authorization: 'Bearer ${GRAPH_API_KEY}' }, org: 'acme' } }, - { environment: { GRAPH_API_KEY: deliveryKey }, fetch: () => Promise.reject(new Error(refusal)) }, - ); - closedAfter(access.close); - - expect(await calledOnce(access)).toEqual(unopenedWith(refusal.slice(0, toolBounds.failureBytes))); - }); -}); - -describe('a delivery through a tool this brain is not offered', () => { - it('is refused before anything is sent or recorded, for a server it does not have, of another org, or a tool not allowed', async () => { - const fake = await deliveryServer(); - const access = deliveryAccess(fake.url); - - expect([ - await calledOnce(access, { reference: { server: 'wiki', tool: 'echo' } }), - await calledOnce(deliveryAccess(fake.url, { org: 'globex' })), - await calledOnce(access, { reference: { server: 'graph', tool: 'environment' } }), - ]).toEqual([ - { - kind: 'not_offered', - because: 'mcp_server_not_configured', - detail: 'No MCP server named wiki is configured for this brain', - }, - { - kind: 'not_offered', - because: 'mcp_server_not_configured', - detail: 'No MCP server named graph is configured for this brain', - }, - { - kind: 'not_offered', - because: 'tool_not_allowed', - detail: 'The operator of this server does not allow graph/environment', - }, - ]); - expect(fake.received()).toEqual([]); - }); - - it('is told by name alone, before any connection, as a run asks before it records anything', async () => { - const fake = await deliveryServer(); - const access = deliveryAccess(fake.url); - - expect([ - Result.isSuccess(access.named(alpha, [{ server: 'graph', tool: 'echo' }])), - access.named(alpha, [{ server: 'graph', tool: 'environment' }]), - ]).toMatchObject([true, Result.fail({ because: 'tool_not_allowed' })]); - expect(fake.seen()).toEqual([]); - }); -}); - -describe('a delivery to a server that opens no connection', () => { - it( - 'fails at the bound of a delivery, though the shared opening of tool calls may wait longer', - async () => { - const silent = createServer(Function.constVoid); - silent.listen(0, '127.0.0.1'); - await once(silent, 'listening'); - closedAfter(async () => { - silent.closeAllConnections(); - silent.close(); - await once(silent, 'close'); - }); - const startedAt = Date.now(); - - const ended = await calledOnce(deliveryAccess(`http://127.0.0.1:${String(portOf(silent.address()).port)}/mcp`)); - - expect(patientTiming.openMs).toBeGreaterThan(deliveryBounds.connectionMs); - expect(ended).toEqual( - unopenedWith(`The MCP server graph did not open a connection within ${deliveryBounds.connectionMs} ms`), - ); - expect(Date.now() - startedAt).toBeLessThan(patientTiming.openMs); - }, - 2 * deliveryBounds.connectionMs, - ); -}); diff --git a/packages/mcp/src/index.ts b/packages/mcp/src/index.ts index c845cdecc..c587aca79 100644 --- a/packages/mcp/src/index.ts +++ b/packages/mcp/src/index.ts @@ -8,14 +8,9 @@ export type { CallJournal, RecordedCall } from './calls/recorded-calls.ts'; export type { CallSignals, ToolCallRequest } from './calls/run-parts.ts'; export type { OfferedTool, RunTools, ToolsEnding } from './calls/run-tools.ts'; export type { AnsweredFields, StartedFields } from './calls/recorded-calls.ts'; -export { - deliveryBounds, - type AnsweredOnce, - type CallAnswer, - type CalledOnce, - type DeliveryCall, - type UnopenedOnce, -} from './delivery/delivery-bounds.ts'; +export type { AnsweredOnce, CallAnswer, CalledOnce, UnopenedOnce } from './one-call/called-once.ts'; +export type { OneCall, RunCall } from './one-call/one-call.ts'; +export { answerDocument, isReadOnly, type ToolAnnotations } from './bounds/tool-results.ts'; export type { NamingCheck, StartOf } from './access/entry-reads.ts'; export { ConversationCallEventSchema, @@ -39,7 +34,7 @@ export { } from './own-calls/conversation-calls.ts'; export { defineListToolServers } from './listing/list-tool-servers.ts'; export { defineListToolServersInOrg, type BrainLookup } from './listing/list-tool-servers-in-org.ts'; -export { answeredInWords } from './names/tool-words.ts'; +export { answeredInWords, toolInWords } from './names/tool-words.ts'; export { defineTestToolCall } from './tool-tests/test-tool-call.ts'; export { type ToolTestEvent } from './tool-tests/tool-test-events.ts'; export { toolTestPresenter } from './tool-tests/tool-test-presenter.ts'; diff --git a/packages/mcp/src/listing/shown-tools.ts b/packages/mcp/src/listing/shown-tools.ts index b8efa6eab..07ea76b90 100644 --- a/packages/mcp/src/listing/shown-tools.ts +++ b/packages/mcp/src/listing/shown-tools.ts @@ -1,8 +1,8 @@ import { Predicate, Schema } from 'effect'; import { cutToDescriptionBound } from '../bounds/call-bounds.ts'; -import type { ListedTool } from '../bounds/result-text.ts'; import type { Secrets } from '../bounds/secrets.ts'; +import type { ListedTool } from '../bounds/tool-results.ts'; import type { ToolLists } from '../settings/mcp-settings.ts'; import { canBeTested } from '../tool-tests/testing-guard.ts'; import type { ServerTool } from './tool-server.ts'; diff --git a/packages/mcp/src/listing/tool-server.ts b/packages/mcp/src/listing/tool-server.ts index 9e513fd0f..6c2b40d68 100644 --- a/packages/mcp/src/listing/tool-server.ts +++ b/packages/mcp/src/listing/tool-server.ts @@ -1,6 +1,6 @@ import { Schema } from 'effect'; -import { ToolAnnotationsSchema } from '../bounds/result-text.ts'; +import { ToolAnnotationsSchema } from '../bounds/tool-results.ts'; const serverFields = { name: Schema.String.annotate({ diff --git a/packages/mcp/src/names/tool-words.ts b/packages/mcp/src/names/tool-words.ts index 4e9a5838c..1db4c75a8 100644 --- a/packages/mcp/src/names/tool-words.ts +++ b/packages/mcp/src/names/tool-words.ts @@ -25,6 +25,10 @@ function toolsOfServer(server: string, tools: readonly string[]): string { return `the ${named} ${distinct.length === 1 ? 'tool' : 'tools'} of ${inWords(server)}`; } +export function toolInWords({ server, tool }: ToolReference): string { + return toolsOfServer(server, [tool]); +} + export function toolsInWords(tools: readonly ToolReference[]): string { const byServer = Map.groupBy(tools, ({ server }) => server); const parts = [...byServer].map(([server, used]: readonly [string, readonly ToolReference[]]) => diff --git a/packages/mcp/src/one-call/call-opening.test.ts b/packages/mcp/src/one-call/call-opening.test.ts new file mode 100644 index 000000000..99b8653c5 --- /dev/null +++ b/packages/mcp/src/one-call/call-opening.test.ts @@ -0,0 +1,73 @@ +import { Function, Redacted } from 'effect'; +import { describe, expect, it } from 'vitest'; + +import { defaultTiming } from '../bounds/call-bounds.ts'; +import { secretsOf } from '../bounds/secrets.ts'; +import type { McpConnection } from '../connections/mcp-connection.ts'; +import type { ServerLink } from '../connections/server-links.ts'; +import type { StdioServerSettings } from '../settings/mcp-settings.ts'; +import { openedFor } from './call-opening.ts'; + +const settings: StdioServerSettings = { + type: 'stdio', + name: 'notes', + org: 'acme', + brains: null, + allowed: null, + testable: [], + record_content: false, + request_id: null, + secrets: [], + command: '/usr/local/bin/notes-mcp-server', + args: [], + env: new Map(), +}; + +const hanging: ServerLink = { + settings, + take: () => Promise.withResolvers().promise, + renew: () => Promise.withResolvers().promise, + release: () => Promise.resolve(), + stop: () => Promise.resolve(), +}; + +const listing = { + secrets: secretsOf([Redacted.make('notes-key')]), + timing: { ...defaultTiming, openMs: 10 }, + toolsListed: Function.constVoid, +}; + +const exited: McpConnection = { + call: () => Promise.reject(new Error('No call is made')), + listTools: () => Promise.reject(new Error('No listing is made')), + closed: Promise.resolve(), + end: () => Promise.resolve(), +}; + +const notStartingAgain: ServerLink = { + ...hanging, + take: () => Promise.resolve(exited), + renew: () => Promise.reject(new Error('The notes process could not start')), +}; + +describe('the opening of one call to a process that exited', () => { + it('starts the process again before it lists its tools, and is unopened when it cannot', async () => { + expect(await openedFor({ server: 'notes', tool: 'search' }, notStartingAgain, listing)).toEqual({ + kind: 'unopened', + refused: 'mcp_server_failed', + because: 'failing', + detail: 'The MCP server notes could not be used: The notes process could not start', + }); + }); +}); + +describe('the opening of one call', () => { + it('gives up on a connection that does not open within the open bound, as a server that could not be reached', async () => { + expect(await openedFor({ server: 'notes', tool: 'search' }, hanging, listing)).toEqual({ + kind: 'unopened', + refused: 'mcp_server_failed', + because: 'unreachable', + detail: 'The MCP server notes could not be used: The MCP server did not open a connection within 10 ms', + }); + }); +}); diff --git a/packages/mcp/src/one-call/call-opening.ts b/packages/mcp/src/one-call/call-opening.ts new file mode 100644 index 000000000..79d0a788f --- /dev/null +++ b/packages/mcp/src/one-call/call-opening.ts @@ -0,0 +1,56 @@ +import { Array as Arr, Option, Result } from 'effect'; + +import { listedThrough, serverFailed, type Listing } from '../access/server-listing.ts'; +import { notListed, offeredOn, unlistedOn } from '../access/tool-naming.ts'; +import { isReadOnly, type ListedTool } from '../bounds/tool-results.ts'; +import type { ServerSlot } from '../calls/server-slot.ts'; +import { failureOf, type ServerFailure } from '../connections/server-failures.ts'; +import type { ServerLink } from '../connections/server-links.ts'; +import type { ToolReference } from '../names/tool-reference.ts'; +import { failedToOpenOnce, notOfferedOnce, type UnopenedOnce } from './called-once.ts'; +import { boundedSlot, takenWithin } from './connection-bound.ts'; + +export interface Opened { + readonly slot: ServerSlot; + readonly tool: ListedTool; + readonly readOnly: boolean; +} + +async function restartedIfExited(slot: ServerSlot): Promise { + const restart = await slot.restartIfExited().catch((error: unknown) => failureOf(error)); + return typeof restart === 'string' ? undefined : restart; +} + +export async function openedFor( + reference: ToolReference, + link: ServerLink, + listing: Listing, +): Promise { + const { openMs } = listing.timing; + const taken = await takenWithin(link, openMs); + if ('late' in taken) { + const message = `The MCP server did not open a connection within ${openMs} ms`; + return failedToOpenOnce(serverFailed(link, { kind: 'timed_out', message }, listing.secrets)); + } + if ('failure' in taken) { + return failedToOpenOnce(serverFailed(link, taken.failure, listing.secrets)); + } + const bounded = boundedSlot(taken.slot, openMs); + const exited = await restartedIfExited(bounded); + if (exited !== undefined) { + await bounded.release(); + return failedToOpenOnce(serverFailed(link, exited, listing.secrets)); + } + const listed = await listedThrough(bounded, listing); + if (Result.isFailure(listed)) { + return failedToOpenOnce(listed.failure); + } + const naming = { references: [reference] }; + const unlisted = unlistedOn(listed.success, naming); + if (unlisted.length > 0) { + await listed.success.slot.release(); + return notOfferedOnce(notListed(unlisted)); + } + const { slot, tool } = Option.getOrThrow(Arr.head(offeredOn(listed.success, naming))); + return { slot, tool, readOnly: isReadOnly(tool.annotations) }; +} diff --git a/packages/mcp/src/one-call/called-once.ts b/packages/mcp/src/one-call/called-once.ts new file mode 100644 index 000000000..0635974c7 --- /dev/null +++ b/packages/mcp/src/one-call/called-once.ts @@ -0,0 +1,67 @@ +import type { Schema } from 'effect'; + +import type { McpServerFailed, ServerFailedBecause } from '../access/mcp-server-failed.ts'; +import type { NotOfferedBecause, ToolNotOffered } from '../access/tool-not-offered.ts'; +import { cutToFailureBound } from '../bounds/call-bounds.ts'; +import { answerOf, resultText, type AnswerBlock, type ToolAnnotations } from '../bounds/tool-results.ts'; +import type { CallOutcome } from '../calls/call-facts.ts'; +import { answeredFields, type AnsweredFields, type Recording } from '../calls/recorded-calls.ts'; +import type { Forwarded } from '../calls/tool-calls.ts'; + +export interface CallAnswer { + readonly content: readonly AnswerBlock[]; + readonly structuredContent?: Schema.Json; +} + +interface AnsweredCall { + readonly kind: 'answered'; + readonly fields: AnsweredFields; + readonly durationMs: number; + readonly detail: string; + readonly retryAfterMs: number | null; + readonly annotations: ToolAnnotations | undefined; +} + +export type AnsweredOnce = + | (AnsweredCall & { readonly outcome: 'result'; readonly answer: CallAnswer }) + | (AnsweredCall & { readonly outcome: Exclude }); + +interface Unopened { + readonly kind: 'unopened'; + readonly refused: Refused; + readonly because: Because; + readonly detail: string; +} + +export type UnopenedOnce = + | Unopened<'tool_not_offered', NotOfferedBecause> + | Unopened<'mcp_server_failed', ServerFailedBecause>; + +export type CalledOnce = UnopenedOnce | AnsweredOnce; + +type Refusal = Pick; + +export function notOfferedOnce({ because, detail }: Refusal): UnopenedOnce { + return { kind: 'unopened', refused: 'tool_not_offered', because, detail }; +} + +export function failedToOpenOnce({ because, detail }: Refusal): UnopenedOnce { + return { kind: 'unopened', refused: 'mcp_server_failed', because, detail: cutToFailureBound(detail) }; +} + +export interface Answering { + readonly durationMs: number; + readonly recording: Recording; + readonly annotations: ToolAnnotations | undefined; +} + +export function answeredOnce(done: Forwarded, { durationMs, recording, annotations }: Answering): AnsweredOnce { + const answered = { kind: 'answered', fields: answeredFields(done, recording), durationMs, annotations } as const; + const { outcome, message, retryAfterMs } = done; + const answer = answerOf(recording.scrub(done.resultJson ?? '')); + if (outcome === 'result') { + return { ...answered, outcome, answer, detail: '', retryAfterMs }; + } + const said = message === '' ? resultText(answer) : recording.scrub(message); + return { ...answered, outcome, detail: cutToFailureBound(said), retryAfterMs }; +} diff --git a/packages/mcp/src/delivery/connection-bound.test.ts b/packages/mcp/src/one-call/connection-bound.test.ts similarity index 80% rename from packages/mcp/src/delivery/connection-bound.test.ts rename to packages/mcp/src/one-call/connection-bound.test.ts index 92d19129d..cd099f785 100644 --- a/packages/mcp/src/delivery/connection-bound.test.ts +++ b/packages/mcp/src/one-call/connection-bound.test.ts @@ -2,13 +2,11 @@ import { setTimeout as sleep } from 'node:timers/promises'; import { describe, expect, it } from 'vitest'; -import { defaultTiming } from '../bounds/call-bounds.ts'; import type { ServerSlot } from '../calls/server-slot.ts'; import type { McpConnection } from '../connections/mcp-connection.ts'; import type { ServerLink } from '../connections/server-links.ts'; import type { StdioServerSettings } from '../settings/mcp-settings.ts'; -import { boundedSlot, connectionBoundOf, takenWithin } from './connection-bound.ts'; -import { deliveryBounds } from './delivery-bounds.ts'; +import { boundedSlot, takenWithin } from './connection-bound.ts'; const connection: McpConnection = { call: () => Promise.reject(new Error('No call is made')), @@ -53,15 +51,7 @@ function opening(take: () => Promise): Opening { }; } -describe('the opening of a connection for a delivery', () => { - it('waits as long as the shared opening may, but never longer than a delivery may', () => { - expect([ - connectionBoundOf(defaultTiming), - connectionBoundOf({ openMs: 60_000 }), - connectionBoundOf({ openMs: 250 }), - ]).toEqual([deliveryBounds.connectionMs, deliveryBounds.connectionMs, 250]); - }); - +describe('the opening of a connection for one call', () => { it('takes a connection that opens in time, and the failure of one that cannot open', async () => { const quick = opening(() => Promise.resolve(connection)); const failing = opening(() => Promise.reject(new Error('connect ECONNREFUSED 127.0.0.1:1'))); @@ -89,7 +79,7 @@ describe('the opening of a connection for a delivery', () => { }); }); -describe('the session of a delivery, opened again or its process started again', () => { +describe('the session of one call, opened again or its process started again', () => { const hanging: ServerSlot = { settings, connection: () => connection, @@ -103,7 +93,7 @@ describe('the session of a delivery, opened again or its process started again', restartIfExited: () => Promise.resolve('restarted'), }; - it('waits no longer than the bound of a delivery, and answers as the slot does when it is in time', async () => { + it('waits no longer than the open bound, and answers as the slot does when it is in time', async () => { const bounded = boundedSlot(hanging, 10); expect(await bounded.reopenOnce()).toBe(false); diff --git a/packages/mcp/src/delivery/connection-bound.ts b/packages/mcp/src/one-call/connection-bound.ts similarity index 87% rename from packages/mcp/src/delivery/connection-bound.ts rename to packages/mcp/src/one-call/connection-bound.ts index 8f904a948..5e60f268f 100644 --- a/packages/mcp/src/delivery/connection-bound.ts +++ b/packages/mcp/src/one-call/connection-bound.ts @@ -2,20 +2,14 @@ import { setTimeout as sleep } from 'node:timers/promises'; import { Result } from 'effect'; -import type { Timing } from '../bounds/call-bounds.ts'; import { takenSlot, type ServerSlot } from '../calls/server-slot.ts'; import type { ServerFailure } from '../connections/server-failures.ts'; import type { ServerLink } from '../connections/server-links.ts'; -import { deliveryBounds } from './delivery-bounds.ts'; export type Taken = { readonly slot: ServerSlot } | { readonly failure: ServerFailure } | { readonly late: true }; const late: Taken = { late: true }; -export function connectionBoundOf({ openMs }: Pick): number { - return Math.min(openMs, deliveryBounds.connectionMs); -} - function releasedIfTaken(after: Taken): Promise { return 'slot' in after ? after.slot.release() : Promise.resolve(); } diff --git a/packages/mcp/src/one-call/one-call-refusals.test.ts b/packages/mcp/src/one-call/one-call-refusals.test.ts new file mode 100644 index 000000000..860648e6c --- /dev/null +++ b/packages/mcp/src/one-call/one-call-refusals.test.ts @@ -0,0 +1,144 @@ +import { once } from 'node:events'; +import { createServer } from 'node:http'; + +import { Function, Result, Schema } from 'effect'; +import { describe, expect, it } from 'vitest'; + +import { toolBounds } from '../bounds/call-bounds.ts'; +import { reportingAccess } from '../testing/index.ts'; +import { + calledOnce, + closedAfter, + oneCallAccess, + oneCallKey, + oneCallServer, + failedWith, + unopenedWith, +} from '../testing/one-calls.ts'; + +const portOf = Schema.decodeUnknownSync(Schema.Struct({ port: Schema.Number })); + +const alpha = { org: 'acme', brain: 'alpha' }; + +const couldNotBeUsed = 'The MCP server graph could not be used: '; + +describe('a server that cannot take one call now', () => { + it('hands on the wait a 429 asks for to its caller, without waiting it out in the call', async () => { + const fake = await oneCallServer(); + const access = oneCallAccess(fake.url); + + fake.answerNextOf('tools/call', 429, 1, { 'retry-after': '120' }); + + expect(await calledOnce(access)).toMatchObject( + failedWith('server_failure', 'The MCP server answered HTTP 429', 120_000), + ); + }); + + it('is unopened when the server cannot be reached, or refuses to list its tools, so nothing was sent', async () => { + const fake = await oneCallServer(); + fake.answerNextOf('tools/list', 503); + const listingRefused = await calledOnce(oneCallAccess(fake.url)); + const { url } = fake; + await fake.close(); + + expect([listingRefused, await calledOnce(oneCallAccess(url))]).toEqual([ + unopenedWith('mcp_server_failed', 'failing', `${couldNotBeUsed}The MCP server answered HTTP 503`), + unopenedWith('mcp_server_failed', 'unreachable', `${couldNotBeUsed}The MCP server could not be reached`), + ]); + expect(fake.received()).toEqual([]); + }); + + it('keeps the words of a failure to 1 KiB, the bound every reader of a failure shares', async () => { + const fake = await oneCallServer(); + const refusal = 'The gateway refused the request. '.repeat(100); + const { access } = reportingAccess( + { graph: { url: fake.url, headers: { Authorization: 'Bearer ${GRAPH_API_KEY}' }, org: 'acme' } }, + { environment: { GRAPH_API_KEY: oneCallKey }, fetch: () => Promise.reject(new Error(refusal)) }, + ); + closedAfter(access.close); + + expect(await calledOnce(access)).toEqual( + unopenedWith('mcp_server_failed', 'failing', `${couldNotBeUsed}${refusal}`.slice(0, toolBounds.failureBytes)), + ); + }); +}); + +describe('one call through a tool this brain is not offered', () => { + it('is refused before anything is sent or recorded, for a server it does not have, of another org, or a tool not allowed', async () => { + const fake = await oneCallServer(); + const access = oneCallAccess(fake.url); + + expect([ + await calledOnce(access, { reference: { server: 'wiki', tool: 'echo' } }), + await calledOnce(oneCallAccess(fake.url, { org: 'globex' })), + await calledOnce(access, { reference: { server: 'graph', tool: 'environment' } }), + ]).toEqual([ + unopenedWith( + 'tool_not_offered', + 'mcp_server_not_configured', + 'No MCP server named wiki is configured for this brain', + ), + unopenedWith( + 'tool_not_offered', + 'mcp_server_not_configured', + 'No MCP server named graph is configured for this brain', + ), + unopenedWith( + 'tool_not_offered', + 'tool_not_allowed', + 'The operator of this server does not allow graph/environment', + ), + ]); + expect(fake.seen()).toEqual([]); + }); + + it('is told by name alone, before any connection, as a run asks before it records anything', async () => { + const fake = await oneCallServer(); + const access = oneCallAccess(fake.url); + + expect([ + Result.isSuccess(access.named(alpha, [{ server: 'graph', tool: 'echo' }])), + access.named(alpha, [{ server: 'graph', tool: 'environment' }]), + ]).toMatchObject([true, Result.fail({ because: 'tool_not_allowed' })]); + expect(fake.seen()).toEqual([]); + }); +}); + +describe('one call to a server that opens no connection', () => { + it('fails at the open bound, as unreachable, with nothing sent', async () => { + const silent = createServer(Function.constVoid); + silent.listen(0, '127.0.0.1'); + await once(silent, 'listening'); + closedAfter(async () => { + silent.closeAllConnections(); + silent.close(); + await once(silent, 'close'); + }); + const startedAt = Date.now(); + + const ended = await calledOnce( + oneCallAccess(`http://127.0.0.1:${String(portOf(silent.address()).port)}/mcp`, {}, { openMs: 300 }), + ); + + expect(ended).toEqual( + unopenedWith( + 'mcp_server_failed', + 'unreachable', + `${couldNotBeUsed}The MCP server did not open a connection within 300 ms`, + ), + ); + expect(Date.now() - startedAt).toBeLessThan(5000); + }); +}); + +describe('one call to a server that does not list its tools in time', () => { + it('fails at the open bound, as unreachable, with nothing sent', async () => { + const fake = await oneCallServer(); + fake.holdNextOf('tools/list'); + + expect(await calledOnce(oneCallAccess(fake.url, {}, { openMs: 300 }))).toEqual( + unopenedWith('mcp_server_failed', 'unreachable', `${couldNotBeUsed}The MCP server did not answer in time`), + ); + expect(fake.received()).toEqual([]); + }); +}); diff --git a/packages/mcp/src/one-call/one-call-runs.test.ts b/packages/mcp/src/one-call/one-call-runs.test.ts new file mode 100644 index 000000000..06b0206b2 --- /dev/null +++ b/packages/mcp/src/one-call/one-call-runs.test.ts @@ -0,0 +1,113 @@ +import { Effect } from 'effect'; +import { describe, expect, it } from 'vitest'; + +import type { CallJournal } from '../calls/recorded-calls.ts'; +import { recordingCallJournal } from '../testing/index.ts'; +import { calledOnce, echoCall, oneCallAccess, oneCallServer, unopenedWith } from '../testing/one-calls.ts'; + +const callId = '0199a3c4-7d2e-7c1a-9b3f-2f1e0d9c8b7a'; + +const aDigest: unknown = expect.stringMatching(/^[0-9a-f]{64}$/u); + +const aNumber: unknown = expect.any(Number); + +function seenWhenStarted(journal: CallJournal, onStart: () => void): CallJournal { + return { ...journal, started: (fact) => Effect.sync(onStart).pipe(Effect.andThen(journal.started(fact))) }; +} + +describe('a call recorded on its run', () => { + it('records its start under the id it is given once its tool is listed and before it is sent, and its answer after', async () => { + const fake = await oneCallServer(); + const journal = recordingCallJournal(); + const receivedAtTheStart: number[] = []; + const runCall = { + callId, + journal: seenWhenStarted(journal, () => { + receivedAtTheStart.push(fake.received().length); + }), + }; + + const called = await calledOnce(oneCallAccess(fake.url), {}, runCall); + + expect(receivedAtTheStart).toEqual([0]); + expect(journal.facts()).toEqual([ + { + type: 'tool_call_started', + call_id: callId, + server: 'graph', + tool: 'echo', + arguments_bytes: 48, + arguments_sha256: aDigest, + number: 1, + }, + { + type: 'tool_call_answered', + number: 1, + outcome: 'result', + result_bytes: 95, + result_sha256: aDigest, + jsonrpc_id: aNumber, + duration_ms: aNumber, + }, + ]); + expect(called).toMatchObject({ kind: 'answered', outcome: 'result' }); + }); +}); + +describe('a call its run records nothing of', () => { + it('records nothing for a tool that is not offered or not listed, or a server that cannot be reached', async () => { + const fake = await oneCallServer(); + const journal = recordingCallJournal(); + const runCall = { callId, journal }; + const unreachable = await oneCallServer(); + await unreachable.close(); + + expect([ + await calledOnce(oneCallAccess(fake.url), { reference: { server: 'graph', tool: 'environment' } }, runCall), + await calledOnce(oneCallAccess(fake.url), { reference: { server: 'graph', tool: 'gone' } }, runCall), + await calledOnce(oneCallAccess(unreachable.url), {}, runCall), + ]).toEqual([ + unopenedWith( + 'tool_not_offered', + 'tool_not_allowed', + 'The operator of this server does not allow graph/environment', + ), + unopenedWith('tool_not_offered', 'tool_not_listed', 'The MCP server graph does not list the tool gone'), + unopenedWith( + 'mcp_server_failed', + 'unreachable', + 'The MCP server graph could not be used: The MCP server could not be reached', + ), + ]); + expect([journal.facts(), fake.received()]).toEqual([[], []]); + }); + + it('dies, sending nothing, when its run records no start, and lets its session go', async () => { + const fake = await oneCallServer(); + const journal = recordingCallJournal(); + journal.refuseStartsFromNowOn(); + + await expect(calledOnce(oneCallAccess(fake.url), {}, { callId, journal })).rejects.toThrow( + 'The start of the call could not be recorded on its run, so the call was not sent', + ); + expect([fake.received(), fake.openSessions(), fake.endedSessions()]).toEqual([[], 0, 1]); + }); +}); + +describe('the wait a 429 asks for in one call', () => { + it('is waited out within the wait its caller gives, and handed on when its caller gives none', async () => { + const fake = await oneCallServer(); + const access = oneCallAccess(fake.url); + const waiting = { ...echoCall, longestRetryWaitMs: 10_000 }; + + fake.answerNextOf('tools/call', 429, 1, { 'retry-after': '1' }); + const waited = await calledOnce(access, waiting); + fake.answerNextOf('tools/call', 429, 1, { 'retry-after': '1' }); + const handedOn = await calledOnce(access); + + expect([waited, handedOn]).toMatchObject([ + { outcome: 'result' }, + { outcome: 'server_failure', detail: 'The MCP server answered HTTP 429', retryAfterMs: 1000 }, + ]); + }); +}); diff --git a/packages/mcp/src/one-call/one-call.test.ts b/packages/mcp/src/one-call/one-call.test.ts new file mode 100644 index 000000000..bf885cb27 --- /dev/null +++ b/packages/mcp/src/one-call/one-call.test.ts @@ -0,0 +1,176 @@ +import { Buffer } from 'node:buffer'; + +import { describe, expect, it } from 'vitest'; + +import { brokenPromise, deniedText } from '../testing/index.ts'; +import { + calledOnce, + echoCall, + callingRunId, + oneCallAccess, + deliveryId, + oneCallKey, + oneCallServer, + failedWith, + unopenedWith, +} from '../testing/one-calls.ts'; + +const largeText = '\u{1F600}'.repeat(8 * 256); + +const largeAnswerBytes = Buffer.byteLength(JSON.stringify({ content: [{ type: 'text', text: largeText }] })); + +const aDigest: unknown = expect.stringMatching(/^[0-9a-f]{64}$/u); + +const aNumber: unknown = expect.any(Number); + +const scrubbed = JSON.stringify({ token: '[redacted]' }); + +describe('one call of a tool', () => { + it('lists the tools of its server, calls the tool with the metadata its caller gives, and answers the call whole', async () => { + const fake = await oneCallServer(); + const access = oneCallAccess(fake.url); + + expect(await calledOnce(access)).toEqual({ + kind: 'answered', + outcome: 'result', + fields: { result_bytes: 95, result_sha256: aDigest, jsonrpc_id: aNumber }, + answer: { content: [{ type: 'text', text: JSON.stringify(echoCall.input) }] }, + durationMs: aNumber, + detail: '', + retryAfterMs: null, + annotations: undefined, + }); + expect(access.startOf(echoCall)).toEqual({ + server: 'graph', + tool: 'echo', + arguments_bytes: 48, + arguments_sha256: aDigest, + }); + expect(fake.received()).toEqual([ + { + tool: 'echo', + arguments: echoCall.input, + meta: { 'com.beonauto/run_id': callingRunId, 'com.beonauto/delivery_id': deliveryId }, + }, + ]); + }); + + it('lists the tools of its server before it calls one', async () => { + const fake = await oneCallServer(); + await calledOnce(oneCallAccess(fake.url)); + const asked = fake.seen().map(({ rpc }) => rpc); + + expect(asked.indexOf('tools/list')).toBeGreaterThan(-1); + expect(asked.indexOf('tools/list')).toBeLessThan(asked.indexOf('tools/call')); + }); +}); + +describe('what one call answers of its tool', () => { + it('answers the hints the server gives the tool it listed', async () => { + const fake = await oneCallServer(); + + expect(await calledOnce(oneCallAccess(fake.url), { reference: { server: 'graph', tool: 'search' } })).toMatchObject( + { outcome: 'result', annotations: { readOnlyHint: true, openWorldHint: true } }, + ); + }); + + it('records the arguments and the answer, scrubbed, where the server records its content', async () => { + const fake = await oneCallServer(); + const access = oneCallAccess(fake.url, { record_content: true }); + const call = { ...echoCall, input: { token: oneCallKey } }; + + const called = await calledOnce(access, call); + + expect([ + access.startOf(call), + access.startOf({ ...call, reference: { server: 'wiki', tool: 'echo' } }), + ]).toMatchObject([{ arguments_json: scrubbed }, { server: 'wiki', arguments_bytes: 36 }]); + expect(access.startOf({ ...call, reference: { server: 'wiki', tool: 'echo' } })).not.toHaveProperty( + 'arguments_json', + ); + expect(called).toMatchObject({ + fields: { result_json: JSON.stringify({ content: [{ type: 'text', text: scrubbed }] }) }, + }); + }); +}); + +describe('a call that fails', () => { + it('is a tool error in the words of the tool when it answers with one, and in the words of the server when it refuses the arguments', async () => { + const fake = await oneCallServer({ data: true }); + const access = oneCallAccess(fake.url, { allowed: ['denied', 'strict'] }); + + expect(await calledOnce(access, { reference: { server: 'graph', tool: 'denied' } })).toMatchObject({ + ...failedWith('tool_error', deniedText), + fields: { result_bytes: aNumber }, + }); + expect( + await calledOnce(access, { reference: { server: 'graph', tool: 'strict' }, input: { limit: '15' } }), + ).toMatchObject({ + ...failedWith('tool_error', expect.stringContaining('limit must be a whole number')), + fields: { result_bytes: null }, + }); + }); + + it('is not opened for a tool its server does not list, and nothing is sent', async () => { + const fake = await oneCallServer(); + + expect(await calledOnce(oneCallAccess(fake.url), { reference: { server: 'graph', tool: 'gone' } })).toEqual( + unopenedWith('tool_not_offered', 'tool_not_listed', 'The MCP server graph does not list the tool gone'), + ); + expect(fake.received()).toEqual([]); + }); + + it('ends at its call bound', async () => { + const fake = await oneCallServer(); + const sleeping = { reference: { server: 'graph', tool: 'sleep' }, input: { ms: 5000 } }; + + expect(await calledOnce(oneCallAccess(fake.url, {}, { callMs: 200 }), sleeping)).toMatchObject({ + ...failedWith('timed_out', 'The MCP server did not answer within 200 ms'), + fields: { result_bytes: null, result_sha256: null }, + }); + }); +}); + +describe('what a tool answers', () => { + it('is carried whole, its content and its structured content, and counted', async () => { + const fake = await oneCallServer(); + const access = oneCallAccess(fake.url); + + const large = await calledOnce(access, { reference: { server: 'graph', tool: 'large' }, input: { kib: 8 } }); + const structured = await calledOnce(access, { reference: { server: 'graph', tool: 'profile' }, input: {} }); + + expect(large).toMatchObject({ + outcome: 'result', + answer: { content: [{ type: 'text', text: largeText }] }, + fields: { result_bytes: largeAnswerBytes }, + }); + expect(structured).toMatchObject({ + outcome: 'result', + answer: { content: [], structuredContent: { name: 'Ada', rows: 2 } }, + }); + }); + + it('is scrubbed of the secrets of the server before the caller reads it', async () => { + const fake = await oneCallServer(); + + expect(await calledOnce(oneCallAccess(fake.url), { input: { token: oneCallKey } })).toMatchObject({ + outcome: 'result', + answer: { content: [{ type: 'text', text: scrubbed }] }, + }); + }); +}); + +describe('one call of a tool that declares an output schema', () => { + it('answers what a tool answered against its own output schema, as it answered it', async () => { + const fake = await oneCallServer({ data: true }); + + expect( + await calledOnce(oneCallAccess(fake.url), { reference: { server: 'graph', tool: 'promised' } }), + ).toMatchObject({ + kind: 'answered', + outcome: 'result', + answer: { structuredContent: brokenPromise }, + }); + expect(fake.received()).toHaveLength(1); + }); +}); diff --git a/packages/mcp/src/one-call/one-call.ts b/packages/mcp/src/one-call/one-call.ts new file mode 100644 index 000000000..5b809cee8 --- /dev/null +++ b/packages/mcp/src/one-call/one-call.ts @@ -0,0 +1,91 @@ +import { Array as Arr, Effect, Option, Result } from 'effect'; + +import type { Listing } from '../access/server-listing.ts'; +import { namedLinks } from '../access/tool-naming.ts'; +import type { ToolNotOffered } from '../access/tool-not-offered.ts'; +import { recordingOf, type CallJournal } from '../calls/recorded-calls.ts'; +import { journalledCall, type CallJournalling } from '../calls/tool-caller.ts'; +import type { ServerLink } from '../connections/server-links.ts'; +import type { ToolReference } from '../names/tool-reference.ts'; +import { openedFor } from './call-opening.ts'; +import { answeredOnce, notOfferedOnce, type CalledOnce } from './called-once.ts'; + +export interface OneCall { + readonly org: string; + readonly brain: string; + readonly reference: ToolReference; + readonly input: Readonly>; + readonly meta: Readonly>; + readonly longestRetryWaitMs?: number; +} + +export interface RunCall { + readonly callId: string; + readonly journal: CallJournal; +} + +export type CallOnce = (call: OneCall, runCall?: RunCall) => Effect.Effect; + +export interface OneCallAccess extends Listing { + readonly links: ReadonlyMap; +} + +type Journalling = (readOnly: boolean) => CallJournalling; + +interface Calling { + readonly access: OneCallAccess; + readonly journalling: Journalling | undefined; + readonly signal: Readonly; +} + +const notRecorded = 'The start of the call could not be recorded on its run, so the call was not sent'; + +async function calledOnce(call: OneCall, link: ServerLink, { access, journalling, signal }: Calling) { + const opened = await openedFor(call.reference, link, access); + if ('kind' in opened) { + return opened; + } + const { slot, tool, readOnly } = opened; + const recording = recordingOf(link.settings, access.secrets.scrub); + const { callMs } = access.timing; + const longestRetryWaitMs = Math.min(call.longestRetryWaitMs ?? 0, access.timing.longestRetryWaitMs); + const forwarding = { slot, tool: tool.name, input: call.input, meta: call.meta, callMs, longestRetryWaitMs, signal }; + try { + const journey = await journalledCall(forwarding, recording, journalling?.(readOnly)); + if (!journey.sent) { + throw new Error(notRecorded); + } + return answeredOnce(journey.done, { durationMs: journey.durationMs, recording, annotations: tool.annotations }); + } finally { + await slot.release(); + } +} + +type Awaiting = (effect: Effect.Effect) => Promise; + +function journallingOf(runCall: RunCall | undefined, awaited: Awaiting): Journalling | undefined { + return runCall === undefined + ? undefined + : (readOnly) => ({ + callId: runCall.callId, + readOnly, + started: (fact) => awaited(runCall.journal.started(fact)), + answered: async (fact) => { + await awaited(runCall.journal.answered(fact)); + }, + }); +} + +function called(call: OneCall, link: ServerLink, access: OneCallAccess, runCall?: RunCall): Effect.Effect { + return Effect.gen(function* () { + const journalling = journallingOf(runCall, Effect.runPromiseWith(yield* Effect.context())); + return yield* Effect.promise((signal) => calledOnce(call, link, { access, journalling, signal })); + }); +} + +export function oneCall(call: OneCall, access: OneCallAccess, runCall?: RunCall): Effect.Effect { + return Result.match(namedLinks(call, { references: [call.reference], links: access.links }), { + onFailure: (refusal: Pick) => Effect.succeed(notOfferedOnce(refusal)), + onSuccess: (links) => called(call, Option.getOrThrow(Arr.head(links)), access, runCall), + }); +} diff --git a/packages/mcp/src/own-calls/conversation-call-presenter.test.ts b/packages/mcp/src/own-calls/conversation-call-presenter.test.ts index 565f605ab..c818591da 100644 --- a/packages/mcp/src/own-calls/conversation-call-presenter.test.ts +++ b/packages/mcp/src/own-calls/conversation-call-presenter.test.ts @@ -64,7 +64,7 @@ describe('a read the brain made in a conversation, as the brain events show it', at: recorded.at, type: 'replies_read', summary: - 'The brain looked for new replies in the conversation “C0123/1699.1” through the tool thread_replies of chat and found 2, took 1 as an answer and refused 1.', + 'The brain looked for new replies in the conversation “C0123/1699.1” through the thread replies tool of chat and found 2, took 1 as an answer and refused 1.', data: { call_id: 'call-1', by: 'brain:alpha', @@ -103,13 +103,13 @@ describe('a read the brain could not make in a conversation, as the brain events }); expect([notOffered?.summary, timedOut?.summary]).toEqual([ - 'The brain could not read the replies of the conversation “C0123/1699.1” through the tool thread_replies of chat: the tool server no longer offers the tool to this brain.', - 'The brain could not read the replies of the conversation “C0123/1699.1” through the tool thread_replies of chat: the tool server did not answer within 30 seconds.', + 'The brain could not read the replies of the conversation “C0123/1699.1” through the thread replies tool of chat: the tool server no longer offers the tool to this brain.', + 'The brain could not read the replies of the conversation “C0123/1699.1” through the thread replies tool of chat: the tool server did not answer within 30 seconds.', ]); expect(notOffered?.data).not.toHaveProperty('duration_ms'); expect(notSent).toMatchObject({ summary: - 'The brain could not read the replies of the conversation “C0123/1699.1” through the tool thread_replies of chat: its arguments could not be rendered, so nothing was sent.', + 'The brain could not read the replies of the conversation “C0123/1699.1” through the thread replies tool of chat: its arguments could not be rendered, so nothing was sent.', data: { detail: 'The argument ts of the read cannot be rendered' }, }); expect(notSent?.data).not.toHaveProperty('arguments_bytes'); @@ -139,7 +139,7 @@ describe('a telling the brain made in a conversation, as the brain events show i const refused = shown({ type: 'telling_ended', call_id: 'call-3', outcome: 'tool_not_offered', ...recorded }); expect([started?.summary, ended?.summary, refused?.summary]).toEqual([ - 'The brain told the party how to answer through the tool post_message of chat.', + 'The brain told the party how to answer through the post message tool of chat.', 'The tool that told the party answered.', 'The tool that told the party was no longer offered by its server.', ]); diff --git a/packages/mcp/src/own-calls/conversation-call-presenter.ts b/packages/mcp/src/own-calls/conversation-call-presenter.ts index 21e780d23..7c97f32e5 100644 --- a/packages/mcp/src/own-calls/conversation-call-presenter.ts +++ b/packages/mcp/src/own-calls/conversation-call-presenter.ts @@ -3,7 +3,7 @@ import { Schema } from 'effect'; import { toolBounds } from '../bounds/call-bounds.ts'; import { cutAsStored } from '../bounds/text-bytes.ts'; -import { answeredInWords } from '../names/tool-words.ts'; +import { answeredInWords, toolInWords } from '../names/tool-words.ts'; import { ConversationCallEventSchema, conversationCallsKind, @@ -53,7 +53,7 @@ function contentShown(name: string, content: string | undefined, mostBytes: numb } function throughTheTool({ server, tool }: Pick): string { - return `through the tool ${named(tool)} of ${named(server)}`; + return `through ${toolInWords({ server: named(server), tool: named(tool) })}`; } function argumentsShown(bytes: number | undefined, digest: string | undefined) { diff --git a/packages/mcp/src/own-calls/conversation-calls.test.ts b/packages/mcp/src/own-calls/conversation-calls.test.ts index 3e3d655c8..4669f4a25 100644 --- a/packages/mcp/src/own-calls/conversation-calls.test.ts +++ b/packages/mcp/src/own-calls/conversation-calls.test.ts @@ -1,7 +1,7 @@ import { Result } from 'effect'; import { describe, expect, it } from 'vitest'; -import type { AnsweredOnce } from '../delivery/delivery-bounds.ts'; +import type { AnsweredOnce, CalledOnce } from '../one-call/called-once.ts'; import { conversationCallStreamOf, type ConversationCallEvent } from './conversation-call-events.ts'; import { conversationCallDecider, repliesReadOf, tellingEndedOf, tellingStartedOf } from './conversation-calls.ts'; @@ -19,9 +19,15 @@ const answered: AnsweredOnce = { durationMs: 7, detail: '', retryAfterMs: null, + annotations: { readOnlyHint: true }, }; -const notOffered = { kind: 'not_offered', because: 'tool_not_allowed', detail: 'Not allowed' } as const; +const notOffered: CalledOnce = { + kind: 'unopened', + refused: 'tool_not_offered', + because: 'tool_not_allowed', + detail: 'Not allowed', +}; const reading = { callId: 'call-1', @@ -66,7 +72,11 @@ describe('a telling the brain makes in a conversation', () => { it('records no answer when the server no longer offers the tool, or its connection could not be opened', () => { expect([ tellingEndedOf('call-3', notOffered, recorded), - tellingEndedOf('call-4', { kind: 'unopened', because: 'mcp_server_failed', detail: 'Unreachable' }, recorded), + tellingEndedOf( + 'call-4', + { kind: 'unopened', refused: 'mcp_server_failed', because: 'unreachable', detail: 'Unreachable' }, + recorded, + ), ]).toEqual([ { type: 'telling_ended', call_id: 'call-3', outcome: 'tool_not_offered', ...recorded }, { type: 'telling_ended', call_id: 'call-4', outcome: 'server_failure', ...recorded }, diff --git a/packages/mcp/src/own-calls/conversation-calls.ts b/packages/mcp/src/own-calls/conversation-calls.ts index 3c2c24b18..af832ddd3 100644 --- a/packages/mcp/src/own-calls/conversation-calls.ts +++ b/packages/mcp/src/own-calls/conversation-calls.ts @@ -1,6 +1,6 @@ import type { StartedFields } from '../calls/recorded-calls.ts'; -import type { CalledOnce } from '../delivery/delivery-bounds.ts'; import type { ToolReference } from '../names/tool-reference.ts'; +import type { CalledOnce } from '../one-call/called-once.ts'; import { ConversationCallEventSchema, type ConversationCallEvent, @@ -33,7 +33,7 @@ function tellingOutcomeOf(end: CalledOnce): TellingOutcome { if (end.kind === 'answered') { return end.outcome; } - return end.kind === 'unopened' ? 'server_failure' : 'tool_not_offered'; + return end.refused === 'tool_not_offered' ? 'tool_not_offered' : 'server_failure'; } export interface Telling { diff --git a/packages/mcp/src/testing/fake-chat.test.ts b/packages/mcp/src/testing/fake-chat.test.ts index 882423465..500672c99 100644 --- a/packages/mcp/src/testing/fake-chat.test.ts +++ b/packages/mcp/src/testing/fake-chat.test.ts @@ -1,10 +1,10 @@ import { describe, expect, it } from 'vitest'; -import { calledOnce, closedAfter, deliveryAccess, deliveryKey } from './delivery-calls.ts'; import { serveFakeMcp } from './fake-mcp-server.ts'; +import { calledOnce, closedAfter, oneCallAccess, oneCallKey } from './one-calls.ts'; async function chatServer() { - const fake = await serveFakeMcp({ bearer: deliveryKey, chat: true }); + const fake = await serveFakeMcp({ bearer: oneCallKey, chat: true }); closedAfter(fake.close); return fake; } @@ -14,7 +14,7 @@ const posting = { reference: { server: 'graph', tool: 'post_message' } }; describe('the chat of the fake MCP server', () => { it('posts a message, in a thread when asked, and answers where it landed', async () => { const fake = await chatServer(); - const access = deliveryAccess(fake.url, { allowed: ['post_message'] }); + const access = oneCallAccess(fake.url, { allowed: ['post_message'] }); const posted = await calledOnce(access, { ...posting, input: { channel: '#approvals', text: 'Approve?' } }); await calledOnce(access, { @@ -38,7 +38,7 @@ describe('the chat of the fake MCP server', () => { describe('the replies in the chat of the fake MCP server', () => { it('are listed for a thread after a point, or for the whole conversation, oldest first', async () => { const fake = await chatServer(); - const access = deliveryAccess(fake.url, { allowed: ['post_message', 'thread_replies'] }); + const access = oneCallAccess(fake.url, { allowed: ['post_message', 'thread_replies'] }); await calledOnce(access, { ...posting, input: { channel: '#approvals', text: 'Approve?' } }); fake.chat.reply({ channel: '#approvals', thread: '1699.000001', user: 'ada', text: 'approve' }); fake.chat.reply({ channel: '#approvals', user: 'ada', text: 'Thanks' }); @@ -61,9 +61,9 @@ describe('the replies in the chat of the fake MCP server', () => { describe('the chat of a fake MCP server that echoes a value', () => { it('carries it in every answer of its chat tools, as a server that echoes its key would', async () => { - const fake = await serveFakeMcp({ bearer: deliveryKey, chat: true, echoes: 'echoed-value-5c1d' }); + const fake = await serveFakeMcp({ bearer: oneCallKey, chat: true, echoes: 'echoed-value-5c1d' }); closedAfter(fake.close); - const access = deliveryAccess(fake.url, { allowed: ['post_message', 'thread_replies'] }); + const access = oneCallAccess(fake.url, { allowed: ['post_message', 'thread_replies'] }); const answers = await Promise.all([ calledOnce(access, { ...posting, input: { channel: '#approvals', text: 'Approve?' } }), @@ -76,14 +76,14 @@ describe('the chat of a fake MCP server that echoes a value', () => { describe('the chat tools of the fake MCP server', () => { it('is offered only by a server asked to hold one', async () => { - const plain = await serveFakeMcp({ bearer: deliveryKey }); + const plain = await serveFakeMcp({ bearer: oneCallKey }); closedAfter(plain.close); expect( - await calledOnce(deliveryAccess(plain.url, { allowed: ['post_message'] }), { + await calledOnce(oneCallAccess(plain.url, { allowed: ['post_message'] }), { ...posting, input: { channel: '#approvals', text: 7 }, }), - ).toMatchObject({ outcome: 'tool_error' }); + ).toMatchObject({ kind: 'unopened', because: 'tool_not_listed' }); }); }); diff --git a/packages/mcp/src/testing/fake-data-tools.ts b/packages/mcp/src/testing/fake-data-tools.ts new file mode 100644 index 000000000..9cfdc7247 --- /dev/null +++ b/packages/mcp/src/testing/fake-data-tools.ts @@ -0,0 +1,77 @@ +import { ProtocolError, ProtocolErrorCode, type CallToolResult } from '@modelcontextprotocol/server'; + +import type { FakeTool } from './fake-tools.ts'; + +export const threadReplies = [ + { user: 'member-17', text: 'Shipped the fix to staging.', ts: '1728380000.000100' }, + { user: 'member-42', text: 'Thanks, closing the ticket.', ts: '1728380100.000200' }, +]; + +const firstPage = { items: [1, 2], next: 'p2' }; + +const lastPage = { items: [3], next: null }; + +export const brokenPromise = { count: 'many' }; + +function argument(input: unknown, name: string): unknown { + return Reflect.get(new Object(input), name); +} + +function texts(...lines: readonly string[]): CallToolResult { + return { content: lines.map((line) => ({ type: 'text', text: line })) }; +} + +export const dataTools: readonly FakeTool[] = [ + { + name: 'thread', + description: 'Lists the replies of a thread, oldest first, and says how many there are.', + inputSchema: { + type: 'object', + properties: { thread: { type: 'string' }, limit: { type: 'number' } }, + required: [], + }, + annotations: { readOnlyHint: true }, + answer: () => ({ + ...texts('2 replies in the thread.'), + structuredContent: { ok: true, messages: threadReplies, has_more: false }, + }), + }, + { + name: 'pages', + description: 'Lists one page of items and the cursor of the next page, null after the last.', + inputSchema: { type: 'object', properties: { cursor: { type: 'string' } }, required: [] }, + annotations: { readOnlyHint: true }, + answer: (input) => { + const page = argument(input, 'cursor') === firstPage.next ? lastPage : firstPage; + return { ...texts(`${page.items.length} items.`), structuredContent: page }; + }, + }, + { + name: 'blocks', + description: 'Answers in two text blocks.', + inputSchema: { type: 'object', properties: {}, required: [] }, + annotations: { readOnlyHint: true }, + answer: () => texts('{"page":1}', '{"page":2}'), + }, + { + name: 'promised', + description: 'Declares an output schema of a count, and answers a count that is not a number.', + inputSchema: { type: 'object', properties: {}, required: [] }, + outputSchema: { type: 'object', properties: { count: { type: 'number' } }, required: ['count'] }, + annotations: { readOnlyHint: true }, + answer: () => ({ ...texts(JSON.stringify(brokenPromise)), structuredContent: brokenPromise }), + }, + { + name: 'strict', + description: 'Counts to the limit it is given, a whole number, and refuses any other.', + inputSchema: { type: 'object', properties: { limit: { type: 'number' } }, required: ['limit'] }, + annotations: { readOnlyHint: true }, + answer: (input) => { + const limit = argument(input, 'limit'); + if (!Number.isInteger(limit)) { + throw new ProtocolError(ProtocolErrorCode.InvalidParams, 'limit must be a whole number'); + } + return texts(JSON.stringify({ limit })); + }, + }, +]; diff --git a/packages/mcp/src/testing/fake-mcp-server.ts b/packages/mcp/src/testing/fake-mcp-server.ts index 0ddbc5813..6183ddca0 100644 --- a/packages/mcp/src/testing/fake-mcp-server.ts +++ b/packages/mcp/src/testing/fake-mcp-server.ts @@ -4,7 +4,7 @@ import { Option, Schema } from 'effect'; import { fakeAuthorization, type ClientRegistration, type FakeAuthorization } from './fake-authorization.ts'; import { fakeChat, type FakeChat } from './fake-chat.ts'; import { fakeSessions, type FakeSessions } from './fake-sessions.ts'; -import type { ReceivedCall } from './fake-tools.ts'; +import type { FakeHints, ReceivedCall } from './fake-tools.ts'; import { serveOnLoopback, type FailedRequest, type FetchHandler } from './loopback-server.ts'; export interface FakeMcpOptions { @@ -13,7 +13,9 @@ export interface FakeMcpOptions { readonly requestIdHeader?: string; readonly issuesSessionIds?: boolean; readonly annotated?: boolean; + readonly hints?: FakeHints; readonly chat?: boolean; + readonly data?: boolean; readonly echoes?: string; } @@ -35,7 +37,14 @@ export interface FakeMcpServer { readonly endedSessions: () => number; readonly tokenRequests: () => number; readonly answerNextWith: (status: number, times?: number, headers?: Readonly>) => void; + readonly answerNextOf: ( + rpc: string, + status: number, + times?: number, + headers?: Readonly>, + ) => void; readonly answerNextCallAfterNoise: (messages: number) => void; + readonly holdNextOf: (rpc: string) => void; readonly forgetSessions: () => void; readonly revokeTokens: () => void; readonly removeTool: (name: string) => void; @@ -49,24 +58,40 @@ interface Programmed { } interface NextAnswers { - readonly programmed: () => Programmed | undefined; + readonly programmed: (rpc: string | undefined) => Programmed | undefined; readonly noise: () => number; readonly answerWith: FakeMcpServer['answerNextWith']; + readonly answerOf: FakeMcpServer['answerNextOf']; readonly answerAfterNoise: FakeMcpServer['answerNextCallAfterNoise']; + readonly held: (rpc: string | undefined) => boolean; + readonly hold: FakeMcpServer['holdNextOf']; +} + +function programmedAnswers(status: number, times: number, headers: Readonly>): Programmed[] { + return Array.from({ length: times }, () => ({ status, headers })); } function nextAnswers(): NextAnswers { const programmed: Programmed[] = []; + const programmedOf = new Map(); + const holding = new Set(); let noise = 0; return { - programmed: () => programmed.shift(), + held: (rpc) => holding.delete(String(rpc)), + hold: (rpc) => { + holding.add(rpc); + }, + programmed: (rpc) => programmedOf.get(String(rpc))?.shift() ?? programmed.shift(), noise: () => { const messages = noise; noise = 0; return messages; }, answerWith: (status, times = 1, headers = {}) => { - programmed.push(...Array.from({ length: times }, () => ({ status, headers }))); + programmed.push(...programmedAnswers(status, times, headers)); + }, + answerOf: (rpc, status, times = 1, headers = {}) => { + programmedOf.set(rpc, [...(programmedOf.get(rpc) ?? []), ...programmedAnswers(status, times, headers)]); }, answerAfterNoise: (messages) => { noise = messages; @@ -132,7 +157,10 @@ function mcpEndpoint({ sessions, see, next, requestIdHeader }: Endpoint): FetchH session: request.headers.get('mcp-session-id'), authorization: request.headers.get('authorization'), }); - const programmed = next.programmed(); + if (next.held(rpc)) { + return Promise.withResolvers().promise; + } + const programmed = next.programmed(rpc); if (programmed !== undefined) { return new Response(JSON.stringify({ error: `answered ${programmed.status}` }), programmed); } @@ -184,7 +212,20 @@ function recordsOf(options: FakeMcpOptions, removed: ReadonlySet, chat: return { isRemoved: (tool: string) => removed.has(tool), annotated: options.annotated ?? true, + hints: options.hints ?? {}, chat: options.chat === true ? chat : undefined, + data: options.data === true, + }; +} + +function tokensOf(authorizations: readonly FakeAuthorization[]): Pick { + return { + tokenRequests: () => authorizations.reduce((total, authorization) => total + authorization.tokenRequests(), 0), + revokeTokens: () => { + for (const authorization of authorizations) { + authorization.revokeEveryToken(); + } + }, }; } @@ -219,15 +260,12 @@ export async function serveFakeMcp(options: FakeMcpOptions = {}): Promise authorizations.reduce((total, authorization) => total + authorization.tokenRequests(), 0), + ...tokensOf(authorizations), answerNextWith: next.answerWith, + answerNextOf: next.answerOf, answerNextCallAfterNoise: next.answerAfterNoise, + holdNextOf: next.hold, forgetSessions: sessions.forget, - revokeTokens: () => { - for (const authorization of authorizations) { - authorization.revokeEveryToken(); - } - }, removeTool: (name) => { removed.add(name); }, diff --git a/packages/mcp/src/testing/fake-stdio-server.ts b/packages/mcp/src/testing/fake-stdio-server.ts index b021a84c3..709521c2f 100644 --- a/packages/mcp/src/testing/fake-stdio-server.ts +++ b/packages/mcp/src/testing/fake-stdio-server.ts @@ -48,6 +48,8 @@ const server = fakeToolServer({ receive: (call) => received.push(call), isRemoved: () => false, annotated: true, + hints: {}, + data: false, exit: () => { process.exit(exitCode); }, diff --git a/packages/mcp/src/testing/fake-tool-run.ts b/packages/mcp/src/testing/fake-tool-run.ts index e07e44f03..ab7fa6e72 100644 --- a/packages/mcp/src/testing/fake-tool-run.ts +++ b/packages/mcp/src/testing/fake-tool-run.ts @@ -4,7 +4,7 @@ import type { ServerMessage } from '../access/caller-context.ts'; import type { Timing } from '../bounds/call-bounds.ts'; import type { CallReply } from '../calls/call-replies.ts'; import type { RunTools } from '../calls/run-tools.ts'; -import { serveFakeMcp, type FakeMcpServer } from './fake-mcp-server.ts'; +import { serveFakeMcp, type FakeMcpOptions, type FakeMcpServer } from './fake-mcp-server.ts'; import { reportingAccess } from './reporting-access.ts'; import { controlledSignals, @@ -29,8 +29,12 @@ export interface FakeToolRun { readonly close: () => Promise; } -export async function openFakeToolRun(tools: readonly string[], timing?: Timing): Promise { - const fake = await serveFakeMcp({ bearer: fakeApiKey }); +export async function openFakeToolRun( + tools: readonly string[], + timing?: Timing, + options: FakeMcpOptions = {}, +): Promise { + const fake = await serveFakeMcp({ ...options, bearer: fakeApiKey }); const { access, messages } = reportingAccess( { graph: { url: fake.url, headers: { Authorization: 'Bearer ${GRAPH_API_KEY}' }, org: 'acme' } }, { environment: { GRAPH_API_KEY: fakeApiKey }, ...(timing === undefined ? {} : { timing }) }, diff --git a/packages/mcp/src/testing/fake-tools.ts b/packages/mcp/src/testing/fake-tools.ts index 8d0ee7c20..4f2768047 100644 --- a/packages/mcp/src/testing/fake-tools.ts +++ b/packages/mcp/src/testing/fake-tools.ts @@ -3,6 +3,7 @@ import { setTimeout } from 'node:timers/promises'; import { McpServer, ProtocolError, ProtocolErrorCode, type CallToolResult } from '@modelcontextprotocol/server'; import type { FakeChat } from './fake-chat.ts'; +import { dataTools } from './fake-data-tools.ts'; export interface ReceivedCall { readonly tool: string; @@ -10,11 +11,15 @@ export interface ReceivedCall { readonly meta: unknown; } +export type FakeHints = Readonly>>>; + export interface FakeToolState { readonly receive: (call: ReceivedCall) => number; readonly isRemoved: (tool: string) => boolean; readonly annotated: boolean; + readonly hints: FakeHints; readonly chat?: FakeChat | undefined; + readonly data: boolean; readonly exit: () => void; } @@ -45,11 +50,12 @@ interface CallContext { readonly mcpReq: { readonly signal: Readonly }; } -interface FakeTool { +export interface FakeTool { readonly name: string; readonly title?: string; readonly description?: string; readonly inputSchema: InputSchema; + readonly outputSchema?: Readonly>; readonly annotations?: Readonly>; readonly answer: Answer; } @@ -248,8 +254,12 @@ function chatTools(chat: FakeChat): readonly FakeTool[] { ]; } -function toolsOf({ chat }: FakeToolState): readonly FakeTool[] { - return chat === undefined ? fakeTools : [...fakeTools, ...chatTools(chat)]; +function toolsOf({ chat, data }: FakeToolState): readonly FakeTool[] { + return [...fakeTools, ...(chat === undefined ? [] : chatTools(chat)), ...(data ? dataTools : [])]; +} + +function hintsOf({ hints }: FakeToolState, tool: FakeTool) { + return hints[tool.name] ?? tool.annotations; } export function fakeToolServer(state: FakeToolState): McpServer { @@ -268,7 +278,8 @@ export function fakeToolServer(state: FakeToolState): McpServer { properties: { ...tool.inputSchema.properties }, required: [...tool.inputSchema.required], }, - annotations: state.annotated ? tool.annotations : undefined, + outputSchema: tool.outputSchema, + annotations: state.annotated ? hintsOf(state, tool) : undefined, })), })); mcp.server.setRequestHandler('tools/call', async ({ params }: CallRequest, { mcpReq }: CallContext) => { diff --git a/packages/mcp/src/testing/index.ts b/packages/mcp/src/testing/index.ts index ef0227696..7216b2ee3 100644 --- a/packages/mcp/src/testing/index.ts +++ b/packages/mcp/src/testing/index.ts @@ -12,13 +12,15 @@ export { fakeToolNames, longToolName, verboseDescription, + type FakeHints, type ReceivedCall, } from './fake-tools.ts'; +export { brokenPromise, threadReplies } from './fake-data-tools.ts'; export { fetchWithDeletion } from './deleting-fetch.ts'; export { fakeApiKey, openFakeToolRun, type FakeToolRun } from './fake-tool-run.ts'; export { recordingTimer, type RecordingTimer, type Wait } from './recording-timer.ts'; export { toolTester, toolTests, type TestAsking, type ToolTests, type ToolTestsOptions } from './tool-test-runs.ts'; -export { reportingAccess, type AccessOptions, type ReportingAccess } from './reporting-access.ts'; +export { noToolServers, reportingAccess, type AccessOptions, type ReportingAccess } from './reporting-access.ts'; export { controlledSignals, inTurn, diff --git a/packages/mcp/src/testing/delivery-calls.ts b/packages/mcp/src/testing/one-calls.ts similarity index 51% rename from packages/mcp/src/testing/delivery-calls.ts rename to packages/mcp/src/testing/one-calls.ts index 7195deab9..04f652e9b 100644 --- a/packages/mcp/src/testing/delivery-calls.ts +++ b/packages/mcp/src/testing/one-calls.ts @@ -2,14 +2,16 @@ import { Effect } from 'effect'; import { afterEach } from 'vitest'; import type { ToolAccess } from '../access/tool-access.ts'; -import type { CalledOnce, DeliveryCall } from '../delivery/delivery-bounds.ts'; -import { serveFakeMcp, type FakeMcpServer } from './fake-mcp-server.ts'; +import type { Timing } from '../bounds/call-bounds.ts'; +import type { CalledOnce } from '../one-call/called-once.ts'; +import type { OneCall, RunCall } from '../one-call/one-call.ts'; +import { serveFakeMcp, type FakeMcpOptions, type FakeMcpServer } from './fake-mcp-server.ts'; import { patientTiming } from './index.ts'; import { reportingAccess } from './reporting-access.ts'; -export const deliveryKey = 'graph-api-key-4f1d9a7c2b'; +export const oneCallKey = 'graph-api-key-4f1d9a7c2b'; -export const deliveredRunId = '0199a3c4-7d2e-7c1a-9b3f-2f1e0d9c8b7a'; +export const callingRunId = '0199a3c4-7d2e-7c1a-9b3f-2f1e0d9c8b7a'; export const deliveryId = '5d0e9f6a-1b2c-5d3e-8f4a-6b7c8d9e0f1a'; @@ -23,18 +25,18 @@ export function closedAfter(close: () => Promise): void { closing.push(close); } -export async function deliveryServer(): Promise { - const fake = await serveFakeMcp({ bearer: deliveryKey }); +export async function oneCallServer(options: FakeMcpOptions = {}): Promise { + const fake = await serveFakeMcp({ ...options, bearer: oneCallKey }); closedAfter(fake.close); return fake; } -const delivered = ['echo', 'denied', 'sleep', 'large', 'search', 'gone', 'profile']; +const allowedTools = ['echo', 'denied', 'sleep', 'large', 'search', 'gone', 'profile', 'promised']; -export function deliveryAccess( +export function oneCallAccess( url: string, changes: Readonly> = {}, - callMs = patientTiming.callMs, + timing: Partial = {}, ): ToolAccess { const { access } = reportingAccess( { @@ -42,35 +44,36 @@ export function deliveryAccess( url, headers: { Authorization: 'Bearer ${GRAPH_API_KEY}' }, org: 'acme', - allowed: delivered, + allowed: allowedTools, ...changes, }, }, - { environment: { GRAPH_API_KEY: deliveryKey }, timing: { ...patientTiming, callMs } }, + { environment: { GRAPH_API_KEY: oneCallKey }, timing: { ...patientTiming, ...timing } }, ); closedAfter(access.close); return access; } -export const delivery: DeliveryCall = { +export const echoCall: OneCall = { org: 'acme', brain: 'alpha', reference: { server: 'graph', tool: 'echo' }, input: { channel: '#approvals', text: 'Please approve' }, - meta: { 'com.beonauto/run_id': deliveredRunId, 'com.beonauto/delivery_id': deliveryId }, + meta: { 'com.beonauto/run_id': callingRunId, 'com.beonauto/delivery_id': deliveryId }, }; export function calledOnce( access: Pick, - changes: Readonly> = {}, + changes: Readonly> = {}, + runCall?: RunCall, ): Promise { - return Effect.runPromise(access.callOnce({ ...delivery, ...changes })); + return Effect.runPromise(access.callOnce({ ...echoCall, ...changes }, runCall)); } export function failedWith(outcome: string, detail: unknown, retryAfterMs: number | null = null) { return { kind: 'answered', outcome, detail, retryAfterMs }; } -export function unopenedWith(detail: string) { - return { kind: 'unopened', because: 'mcp_server_failed', detail }; +export function unopenedWith(refused: string, because: string, detail: string) { + return { kind: 'unopened', refused, because, detail }; } diff --git a/packages/mcp/src/testing/reporting-access.ts b/packages/mcp/src/testing/reporting-access.ts index 6b516c065..075d73c4f 100644 --- a/packages/mcp/src/testing/reporting-access.ts +++ b/packages/mcp/src/testing/reporting-access.ts @@ -3,6 +3,7 @@ import { Effect } from 'effect'; import type { ServerMessage } from '../access/caller-context.ts'; import { makeToolAccess, type ToolAccess, type ToolAccessOptions } from '../access/tool-access.ts'; import type { Timing } from '../bounds/call-bounds.ts'; +import { ignored } from '../connections/ignored.ts'; import { readMcpSettings, type Environment } from '../settings/settings-reading.ts'; export interface AccessOptions { @@ -38,3 +39,8 @@ export function reportingAccess( }); return { access, messages: () => [...messages], untestable: () => [...untestable] }; } + +export const noToolServers: ToolAccess = makeToolAccess( + { servers: [] }, + { reportServerMessage: ignored, reportUntestable: ignored }, +); diff --git a/packages/mcp/src/tool-tests/test-answers.test.ts b/packages/mcp/src/tool-tests/test-answers.test.ts new file mode 100644 index 000000000..8be639c2b --- /dev/null +++ b/packages/mcp/src/tool-tests/test-answers.test.ts @@ -0,0 +1,77 @@ +import { Effect, Schema } from 'effect'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { answerDocument } from '../bounds/tool-results.ts'; +import { + fakeApiKey, + patientTiming, + reportingAccess, + serveFakeMcp, + threadReplies, + toolTests, +} from '../testing/index.ts'; + +const closing: (() => Promise)[] = []; + +afterEach(async () => { + await Promise.all(closing.splice(0).map((close) => close())); +}); + +const decodeTested = Schema.decodeUnknownSync( + Schema.Struct({ output: Schema.Struct({ text: Schema.String, answer: Schema.optionalKey(Schema.Json) }) }), +); + +async function testedAndCalled(tool: string, input: Readonly> = {}) { + const fake = await serveFakeMcp({ bearer: fakeApiKey, data: true }); + closing.push(fake.close); + const graph = { url: fake.url, headers: { Authorization: 'Bearer ${GRAPH_API_KEY}' }, org: 'acme' }; + const { access } = reportingAccess({ graph }, { timing: patientTiming, environment: { GRAPH_API_KEY: fakeApiKey } }); + closing.push(access.close); + const tested = decodeTested(await toolTests(access).test({ server: 'graph', tool, arguments: input })).output; + const call = { org: 'acme', brain: 'alpha', reference: { server: 'graph', tool }, input, meta: {} }; + const called = await Effect.runPromise(access.callOnce(call)); + return { + tested, + document: called.kind === 'answered' && called.outcome === 'result' ? answerDocument(called.answer) : undefined, + }; +} + +describe('the answer of a test', () => { + it('is the document a call reads, the structure beside a summary the model sees', async () => { + const { tested, document } = await testedAndCalled('thread'); + + expect(tested).toEqual({ + text: '2 replies in the thread.', + answer: { ok: true, messages: threadReplies, has_more: false }, + }); + expect(tested.answer).toEqual(document); + }); + + it('is the first of several text blocks read as JSON, and a text that is not JSON as that text', async () => { + const blocks = await testedAndCalled('blocks'); + const plain = await testedAndCalled('search', { query: 'acme' }); + + expect([blocks.tested.answer, plain.tested.answer]).toEqual([{ page: 1 }, 'Found 2 rows for acme.']); + expect([blocks.tested.answer, plain.tested.answer]).toEqual([blocks.document, plain.document]); + }); + + it('is the structure of each page of a tool that pages, and the JSON of a text', async () => { + const first = await testedAndCalled('pages'); + const last = await testedAndCalled('pages', { cursor: 'p2' }); + const counted = await testedAndCalled('strict', { limit: 3 }); + + expect([first.tested.answer, last.tested.answer, counted.tested.answer]).toEqual([ + { items: [1, 2], next: 'p2' }, + { items: [3], next: null }, + { limit: 3 }, + ]); + }); + + it('is left out when the tool answered neither structured content nor text, and when it takes more than 64 KiB', async () => { + const photo = await testedAndCalled('photo'); + const large = await testedAndCalled('large', { kib: 70 }); + + expect(['answer' in photo.tested, 'answer' in large.tested]).toEqual([false, false]); + expect([photo.document, typeof large.document]).toEqual([undefined, 'string']); + }); +}); diff --git a/packages/mcp/src/tool-tests/test-calling.test.ts b/packages/mcp/src/tool-tests/test-calling.test.ts index cd1ed9452..f5d1a9276 100644 --- a/packages/mcp/src/tool-tests/test-calling.test.ts +++ b/packages/mcp/src/tool-tests/test-calling.test.ts @@ -17,6 +17,7 @@ function runToolsOffering(offered: readonly OfferedTool[]): RunTools { ended: new AbortController().signal, ending: noEnding, calledAny: () => false, + calledOnlyReadOnly: () => true, usedInWords: () => 'no tool', close: () => Promise.resolve(), }; diff --git a/packages/mcp/src/tool-tests/test-tool-call.test.ts b/packages/mcp/src/tool-tests/test-tool-call.test.ts index 0d5d0b346..0e989c498 100644 --- a/packages/mcp/src/tool-tests/test-tool-call.test.ts +++ b/packages/mcp/src/tool-tests/test-tool-call.test.ts @@ -85,6 +85,7 @@ describe('a test of a tool its server marks read-only', () => { tool: 'search', outcome: 'result', text: 'Found 2 rows for acme.', + answer: 'Found 2 rows for acme.', result_bytes: Buffer.byteLength( JSON.stringify({ content: [{ type: 'text', text: 'Found 2 rows for acme.' }] }), ), diff --git a/packages/mcp/src/tool-tests/test-tool-call.ts b/packages/mcp/src/tool-tests/test-tool-call.ts index 20756484f..befbde12a 100644 --- a/packages/mcp/src/tool-tests/test-tool-call.ts +++ b/packages/mcp/src/tool-tests/test-tool-call.ts @@ -1,12 +1,12 @@ -import { BrainContext, Unavailable, defineCommand, randomUUIDv7 } from '@beonauto/operations'; +import { Unavailable, defineCommand, randomUUIDv7 } from '@beonauto/operations'; import { Effect } from 'effect'; import type { ToolsNotOpened } from '../access/caller-context.ts'; import type { ToolAccess } from '../access/tool-access.ts'; import { cutToFailureBound } from '../bounds/call-bounds.ts'; -import { toolTestIdKey } from '../calls/call-meta.ts'; import { calledWhenTestable, stopped, testedOutcomeOf } from './test-calling.ts'; -import { toolTestJournal } from './tool-test-journal.ts'; +import { testedAnswer } from './tested-answer.ts'; +import { toolTestContext } from './tool-test-journal.ts'; import { TestToolCallInputSchema, ToolTestedSchema } from './tool-test-schemas.ts'; import { toolTestAttempted, toolTestRemedies, toolTestTask, toolTested } from './tool-test-words.ts'; @@ -34,10 +34,8 @@ export function defineTestToolCall(access: Pick) outputSchema: ToolTestedSchema, reasons: ['invalid_input', 'unavailable'], handle: Effect.fnUntraced(function* ({ server, tool, arguments: input = {} }) { - const { org, brain } = yield* BrainContext; const testId = randomUUIDv7(); - const journal = yield* toolTestJournal(testId); - const context = { id: testId, org, brain, journal, meta: { [toolTestIdKey]: testId } }; + const context = yield* toolTestContext(testId); const tested = { reference: { server, tool }, testId, input }; return yield* Effect.uninterruptibleMask((restore) => Effect.gen(function* () { @@ -52,6 +50,7 @@ export function defineTestToolCall(access: Pick) tool, outcome: yield* testedOutcomeOf(reply), text: reply.text, + ...testedAnswer(reply), result_bytes: reply.resultBytes, duration_ms: reply.durationMs, ...(reply.serverRequestId === null ? {} : { server_request_id: reply.serverRequestId }), diff --git a/packages/mcp/src/tool-tests/tested-answer.ts b/packages/mcp/src/tool-tests/tested-answer.ts new file mode 100644 index 000000000..6cedb3895 --- /dev/null +++ b/packages/mcp/src/tool-tests/tested-answer.ts @@ -0,0 +1,11 @@ +import { toolBounds } from '../bounds/call-bounds.ts'; +import { bytesOf } from '../bounds/text-bytes.ts'; +import { answerDocument } from '../bounds/tool-results.ts'; +import type { CallReply } from '../calls/call-replies.ts'; + +export function testedAnswer({ scrubbedResult }: CallReply) { + const document = scrubbedResult === undefined ? undefined : answerDocument(scrubbedResult()); + return document === undefined || bytesOf(JSON.stringify(document)) > toolBounds.resultBytes + ? {} + : { answer: document }; +} diff --git a/packages/mcp/src/tool-tests/testing-guard.ts b/packages/mcp/src/tool-tests/testing-guard.ts index 9b8348a4a..38f29f657 100644 --- a/packages/mcp/src/tool-tests/testing-guard.ts +++ b/packages/mcp/src/tool-tests/testing-guard.ts @@ -1,15 +1,15 @@ -import type { ListedTool, ToolAnnotations } from '../bounds/result-text.ts'; +import { isReadOnly, type ListedTool, type ToolAnnotations } from '../bounds/tool-results.ts'; import { allowsTool } from '../names/tool-reference.ts'; import type { McpServerSettings, ToolLists } from '../settings/mcp-settings.ts'; export function canBeTested(tool: string, annotations: ToolAnnotations | undefined, lists: ToolLists): boolean { - return allowsTool(lists.allowed, tool) && (annotations?.readOnlyHint === true || lists.testable.includes(tool)); + return allowsTool(lists.allowed, tool) && (isReadOnly(annotations) || lists.testable.includes(tool)); } function offersNothingTestable({ allowed, testable }: ToolLists, tools: readonly ListedTool[]): boolean { return ( testable.length === 0 && - !tools.some(({ name, annotations }) => allowsTool(allowed, name) && annotations?.readOnlyHint === true) + !tools.some(({ name, annotations }) => allowsTool(allowed, name) && isReadOnly(annotations)) ); } diff --git a/packages/mcp/src/tool-tests/tool-test-journal.ts b/packages/mcp/src/tool-tests/tool-test-journal.ts index 9d1ec1a7d..c2a649d72 100644 --- a/packages/mcp/src/tool-tests/tool-test-journal.ts +++ b/packages/mcp/src/tool-tests/tool-test-journal.ts @@ -1,14 +1,21 @@ import { BrainContext, BrainWriter, Caller, noLineage, streamPrefixOfBrain } from '@beonauto/operations'; import { Effect } from 'effect'; +import { toolTestIdKey } from '../calls/call-meta.ts'; import { ownCallJournal } from '../own-calls/own-call-journal.ts'; import { toolTestCalls } from './tool-test-decider.ts'; import { toolTestStreamOf } from './tool-test-events.ts'; -export const toolTestJournal = Effect.fnUntraced(function* (testId: string) { +const toolTestJournal = Effect.fnUntraced(function* (testId: string) { const writer = yield* BrainWriter; const stream = toolTestStreamOf(testId); const inTheBrain = `${streamPrefixOfBrain(yield* BrainContext)}${stream}`; const { id: by } = yield* Caller; return ownCallJournal(toolTestCalls(testId), { writer, stream, inTheBrain, by, lineage: noLineage }); }); + +export const toolTestContext = Effect.fnUntraced(function* (testId: string) { + const { org, brain } = yield* BrainContext; + const journal = yield* toolTestJournal(testId); + return { id: testId, org, brain, journal, meta: { [toolTestIdKey]: testId } }; +}); diff --git a/packages/mcp/src/tool-tests/tool-test-recording.test.ts b/packages/mcp/src/tool-tests/tool-test-recording.test.ts index eeed31112..0160c7ee8 100644 --- a/packages/mcp/src/tool-tests/tool-test-recording.test.ts +++ b/packages/mcp/src/tool-tests/tool-test-recording.test.ts @@ -67,6 +67,7 @@ describe('what a test records', () => { tool: 'search', arguments_bytes: Buffer.byteLength(argumentsJson), arguments_sha256: digestOf(argumentsJson), + read_only: true, by: 'acme-builder', at: aTime, }, diff --git a/packages/mcp/src/tool-tests/tool-test-schemas.ts b/packages/mcp/src/tool-tests/tool-test-schemas.ts index 18be089f7..7319e43a8 100644 --- a/packages/mcp/src/tool-tests/tool-test-schemas.ts +++ b/packages/mcp/src/tool-tests/tool-test-schemas.ts @@ -42,6 +42,12 @@ export const ToolTestedSchema = Schema.Struct({ description: "What a reasoning function's model would see of the answer, scrubbed of the server's secrets and cut at 64 KiB with the note a run adds; for a failure, the words a run's model gets", }), + answer: Schema.optionalKey( + Schema.Json.annotate({ + description: + "The document a call's read points into: the structured content, else the first text block parsed as JSON, else that text, scrubbed of the server's secrets; absent when the tool answered neither, and when it takes more than 64 KiB as JSON", + }), + ), result_bytes: Schema.NullOr(Schema.Int).annotate({ description: 'The size of the whole result as its server gave it, before the cut; null when there is none', }), diff --git a/packages/mcp/src/tool-tests/tool-test-words.test.ts b/packages/mcp/src/tool-tests/tool-test-words.test.ts index 8ce90c9c2..c620b29d7 100644 --- a/packages/mcp/src/tool-tests/tool-test-words.test.ts +++ b/packages/mcp/src/tool-tests/tool-test-words.test.ts @@ -1,4 +1,4 @@ -import { unsuccessfulWords, type UnavailableBecause } from '@beonauto/operations'; +import { unsuccessfulWords, type RejectionBecause } from '@beonauto/operations'; import { Effect } from 'effect'; import { describe, expect, it } from 'vitest'; @@ -26,16 +26,16 @@ const answer = { describe('the plain words of test_tool_call', () => { it('say which tool of which server it tried to test, or what it tried when the input does not hold', () => { - expect(words?.attempt({ server: 'graph', tool: 'search' })).toBe('test the tool “search” of “graph”'); + expect(words?.attempt({ server: 'graph', tool: 'search' })).toBe('test the search tool of graph'); expect(words?.attempt({ server: 'Graph/', tool: 'search' })).toBe('test a tool of a tool server'); }); it('say what the tool answered, how long it took and how much, and where the model’s view of it is', () => { expect(words?.outcome(answer, asked)).toBe( - "The tool “search” of “graph” answered in 312 ms with 1,204 bytes; what a reasoning function's model would see is in the details.", + "The search tool of graph answered in 312 ms with 1,204 bytes; what a reasoning function's model would see is in the details.", ); expect(words?.outcome({ ...answer, result_bytes: null }, asked)).toBe( - "The tool “search” of “graph” answered in 312 ms with nothing; what a reasoning function's model would see is in the details.", + "The search tool of graph answered in 312 ms with nothing; what a reasoning function's model would see is in the details.", ); }); @@ -50,7 +50,7 @@ describe('the plain words of test_tool_call', () => { const timedOut = { ...answer, outcome: 'timed_out', result_bytes: null }; expect(words?.outcome(toolError, asked)).toBe( - "The tool “search” of “graph” answered an error, as a run's model would see it; the details show what it said.", + "The search tool of graph answered an error, as a run's model would see it; the details show what it said.", ); expect(words?.outcome(serverFailure, asked)).toBe( 'The tool server “graph” failed to answer the test: The MCP server answered HTTP 503. It may or may not have received the call.', @@ -69,10 +69,10 @@ describe('the plain words of test_tool_call', () => { }); }); -const notOffered = (because: UnavailableBecause) => +const notOffered = (because: RejectionBecause) => ({ status: 'rejected', reason: 'unavailable', detail: 'x', kind: 'tool_not_offered', because }) as const; -const remedies: ReadonlyArray = [ +const remedies: ReadonlyArray = [ [ 'mcp_server_not_configured', 'This can be put right on your side: list_tool_servers shows the tool servers this brain may use, so a test that names one of those can be tried.', @@ -92,7 +92,7 @@ describe('the plain words of a test refused a tool it is not offered', () => { 'end, for %s, with what list_tool_servers shows, in place of the words for a function', (because, remedy) => { const refused = unsuccessfulWords( - 'test the tool “search” of “wiki”', + 'test the search tool of wiki', 'command', notOffered(because), words?.remedies, @@ -106,13 +106,13 @@ describe('the plain words of a test refused a tool it is not offered', () => { it('say why and what to do in one message for a tool server not set up for the brain', () => { expect( unsuccessfulWords( - 'test the tool “search” of “wiki”', + 'test the search tool of wiki', 'command', notOffered('mcp_server_not_configured'), words?.remedies, ), ).toBe( - 'Could not test the tool “search” of “wiki”: this server does not offer a tool it names, because whoever runs the server has not set up a tool server of that name for this brain. Nothing was changed. This can be put right on your side: list_tool_servers shows the tool servers this brain may use, so a test that names one of those can be tried.', + 'Could not test the search tool of wiki: this server does not offer a tool it names, because whoever runs the server has not set up a tool server of that name for this brain. Nothing was changed. This can be put right on your side: list_tool_servers shows the tool servers this brain may use, so a test that names one of those can be tried.', ); }); }); diff --git a/packages/mcp/src/tool-tests/tool-test-words.ts b/packages/mcp/src/tool-tests/tool-test-words.ts index 0352c5958..42cd53cf8 100644 --- a/packages/mcp/src/tool-tests/tool-test-words.ts +++ b/packages/mcp/src/tool-tests/tool-test-words.ts @@ -1,7 +1,8 @@ -import { quoted, type Remedies } from '@beonauto/operations'; +import { capitalized, quoted, type Remedies } from '@beonauto/operations'; import { toolBounds } from '../bounds/call-bounds.ts'; import { serverFailedText } from '../calls/call-replies.ts'; +import { toolInWords } from '../names/tool-words.ts'; export interface TestAsked { readonly server: string; @@ -22,7 +23,7 @@ const measured = new Intl.NumberFormat('en'); export const toolTestTask = 'test a tool of a tool server'; export function toolTestAttempted({ server, tool }: TestAsked): string { - return `test the tool ${quoted(tool)} of ${quoted(server)}`; + return `test ${toolInWords({ server, tool })}`; } export const toolTestRemedies: Remedies = { @@ -44,9 +45,9 @@ function failureOf({ server, text }: TestAnswer): string { const wordsOf: Readonly string>> = { result: (answer) => - `The tool ${quoted(answer.tool)} of ${quoted(answer.server)} answered in ${measured.format(answer.duration_ms)} ms with ${bytesInWords(answer.result_bytes)}; what a reasoning function's model would see is in the details.`, + `${capitalized(toolInWords(answer))} answered in ${measured.format(answer.duration_ms)} ms with ${bytesInWords(answer.result_bytes)}; what a reasoning function's model would see is in the details.`, tool_error: (answer) => - `The tool ${quoted(answer.tool)} of ${quoted(answer.server)} answered an error, as a run's model would see it; the details show what it said.`, + `${capitalized(toolInWords(answer))} answered an error, as a run's model would see it; the details show what it said.`, server_failure: (answer) => `The tool server ${quoted(answer.server)} failed to answer the test: ${failureOf(answer)}. It may or may not have received the call.`, timed_out: (answer) => diff --git a/packages/operations/README.md b/packages/operations/README.md index a8d72e2e2..c56017bab 100644 --- a/packages/operations/README.md +++ b/packages/operations/README.md @@ -71,20 +71,20 @@ A definition sets `reachesOutside: true` when its handler calls a system outside A command sets `irreversible: true` when what it does cannot be undone, such as retiring a brain or cancelling a run, and `repeatable: true` when a repeat of the same call records nothing more, such as an update that changes nothing or retiring what is already retired. The registration carries both, `false` when left out; the MCP tools take `destructiveHint` from `irreversible` or `mayChangeOutside`, and `idempotentHint` from a query or `repeatable`. -`Unavailable` may carry a `kind` and with it a `because`. Both reach the rejected outcome, where `explanationOf` and `unsuccessfulWords` turn them into plain words, and the problem document, as its extension members `kind` and `because`. An operation's `plainLanguage` may give `remedies` by `because`, which `unsuccessfulWords` puts in place of the shared remedy for that operation alone. +`Unavailable` and `Conflict` may carry a `kind` and with it a `because`, one of `RejectionBecauseSchema`. Both reach the rejected outcome, where `explanationOf` and `unsuccessfulWords` turn them into plain words, and the problem document, as its extension members `kind` and `because`. An operation's `plainLanguage` may give `remedies` by `because`, which `unsuccessfulWords` puts in place of the shared remedy for that operation alone. -| `kind` | What could not be used | `because` | -| ------------------- | ------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `model_not_offered` | a model the server does not offer | `provider_not_configured`, its provider is not set up while others are; `model_not_allowed`, it is outside what the operator allows | -| `tool_not_offered` | a tool the server does not offer | `mcp_server_not_configured`, its server is not configured for the org and brain; `tool_not_allowed`, it is outside what the operator allows; `tool_not_listed`, its server does not list it | -| `mcp_server_failed` | a tool server, before any call was recorded | `failing`, it kept failing; `rate_limited`, it asked to wait longer than a run waits; `unreachable`, it could not be reached in time; `key_refused`, it did not accept the key the server gives it, whose words say to check that key rather than to try again | -| `tools_unfinished` | anything, after a tool call was recorded, so that a tool may have changed something | `server_failed`, a tool server kept failing; `model_unavailable`, the model could not be called; `run_bound`, the run ran out of time; `no_answer`, the model still called tools when it had to answer | -| `rebuilding` | the view a recall function keeps, which is still being built from the brain's history | none; trying again later may succeed, and the kind has a problem type of its own, `kindsWithTypes` | -| `requests_full` | room for one more open request in the brain | none; trying again once some are answered, expire or are cancelled may succeed | +| `kind` | What could not be used | `because` | +| ------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `model_not_offered` | a model the server does not offer | `provider_not_configured`, its provider is not set up while others are; `model_not_allowed`, it is outside what the operator allows | +| `tool_not_offered` | a tool the server does not offer | `mcp_server_not_configured`, its server is not configured for the org and brain; `tool_not_allowed`, it is outside what the operator allows; `tool_not_listed`, its server does not list it | +| `mcp_server_failed` | a tool server, before any call was recorded | `failing`, it kept failing; `rate_limited`, it asked to wait longer than a run waits; `unreachable`, it could not be reached in time; `key_refused`, it did not accept the key the server gives it, whose words say to check that key rather than to try again | +| `tools_unfinished` | anything, after a tool call was recorded, where every tool the run called is one its server marks `readOnlyHint`, so that running it again is safe | `server_failed`, a tool server failed; `tool_error`, the tool answered an error; `model_unavailable`, the model could not be called; `run_bound`, the run ran out of time; `no_answer`, the model still called tools when it had to answer | +| `rebuilding` | the view a recall function keeps, which is still being built from the brain's history | none; trying again later may succeed, and the kind has a problem type of its own, `kindsWithTypes` | +| `requests_full` | room for one more open request in the brain | none; trying again once some are answered, expire or are cancelled may succeed | Plain words have bounds: an outcome takes at most `mostOutcomeCharacters`, 400, and the words of a refusal at most `mostRefusalCharacters`, 600. `withinCharacters(text, most)` keeps the whole sentences that fit and ends with "The rest is in the details below.", since the detail is in the structured result beside the words; a first sentence longer than the bound is cut at a word with an ellipsis. -`Conflict` may carry a `kind`: `taken`, `retired`, `concurrent_change`, `unworkable`, `stalled`, the view a recall function keeps stopped at an event its fold could not take, which only a corrected version puts right, or `tools_called`, a run that called tools and did not succeed, or that started and calls tools and may still be in progress, and so is not run again under its id. The words of `tools_unfinished` and `tools_called` never say that nothing was changed: `Explanation` carries `mayHaveChanged` for them, since the tools of the run may have changed something even when the refused request ran nothing. +`Conflict` may carry a `kind`: `taken`, `retired`, `concurrent_change`, `unworkable`, `stalled`, the view a recall function keeps stopped at an event its fold could not take, which only a corrected version puts right, `effect_unknown`, a run that called a tool its server does not mark read-only and could not finish, so that whether the tool changed something is not known, with the becauses of `tools_unfinished` and a problem type of its own at 409, or `tools_called`, a run that called tools and did not succeed, or that started and calls tools and may still be in progress, and so is not run again under its id. The words of `effect_unknown` and `tools_called` never say that nothing was changed: `Explanation` carries `mayHaveChanged` for them, since the tools of the run may have changed something even when the refused request ran nothing; but `tools_called` with the because `only_read`, a run that ended without success after calling only tools that read, has words of its own, which do. Those of `tools_unfinished` do, since every tool the run called only reads, and say that a new run, or a workflow's retry, may make it. `InvalidInput` is for input that matches the input schema but that the handler finds wrong, such as a document it parses. It carries a detail and its issues, each a `detail` and a JSON Pointer `pointer` into the input. A handler that declares `invalid_input` and fails with it is rejected with reason `invalid_input` and those issues, the same rejection the dispatcher gives input that breaks the schema. Every rejection carries at most 100 issues, and each issue only its `detail` and `pointer`. diff --git a/packages/operations/src/index.ts b/packages/operations/src/index.ts index 09b7a3e4a..34a229ade 100644 --- a/packages/operations/src/index.ts +++ b/packages/operations/src/index.ts @@ -151,12 +151,7 @@ export { type RunOutcomeWindow, type RunStream, } from './run-outcomes/run-outcomes.ts'; -export { - Unavailable, - UnavailableBecauseSchema, - UnavailableKindSchema, - type UnavailableBecause, - type UnavailableKind, -} from './outcome/unavailable.ts'; +export { RejectionBecauseSchema, type RejectionBecause } from './outcome/rejection-because.ts'; +export { Unavailable, UnavailableKindSchema, type UnavailableKind } from './outcome/unavailable.ts'; export { uuidV5 } from './uuid/uuid-v5.ts'; export { randomUUIDv7 } from './uuid/uuid-v7.ts'; diff --git a/packages/operations/src/outcome/conflict.ts b/packages/operations/src/outcome/conflict.ts index a87965714..2efd632bc 100644 --- a/packages/operations/src/outcome/conflict.ts +++ b/packages/operations/src/outcome/conflict.ts @@ -1,5 +1,7 @@ import { Data, Schema } from 'effect'; +import type { RejectionBecause } from './rejection-because.ts'; + export const ConflictKindSchema = Schema.Literals([ 'taken', 'retired', @@ -7,6 +9,7 @@ export const ConflictKindSchema = Schema.Literals([ 'unworkable', 'stalled', 'tools_called', + 'effect_unknown', 'oversized', ]); @@ -15,5 +18,6 @@ export type ConflictKind = typeof ConflictKindSchema.Type; export class Conflict extends Data.TaggedError('conflict')<{ readonly detail: string; readonly kind?: ConflictKind; + readonly because?: RejectionBecause; readonly record?: Schema.JsonObject; }> {} diff --git a/packages/operations/src/outcome/outcome.ts b/packages/operations/src/outcome/outcome.ts index ac9168c99..e02f7f988 100644 --- a/packages/operations/src/outcome/outcome.ts +++ b/packages/operations/src/outcome/outcome.ts @@ -1,8 +1,8 @@ import type { Schema } from 'effect'; import { cappedIssues, type Issue } from './issue.ts'; +import type { RejectionBecause } from './rejection-because.ts'; import type { DeclarableReason, RejectionKind } from './rejection.ts'; -import type { UnavailableBecause } from './unavailable.ts'; export type RejectionReason = 'forbidden' | DeclarableReason; @@ -17,7 +17,7 @@ export interface Rejected { readonly detail: string; readonly issues?: readonly Issue[]; readonly kind?: RejectionKind; - readonly because?: UnavailableBecause; + readonly because?: RejectionBecause; } export interface Failed { diff --git a/packages/operations/src/outcome/problem-types.test.ts b/packages/operations/src/outcome/problem-types.test.ts index ea12f6649..22d85a255 100644 --- a/packages/operations/src/outcome/problem-types.test.ts +++ b/packages/operations/src/outcome/problem-types.test.ts @@ -7,20 +7,26 @@ describe('the types of problems', () => { expect(problemTypeOf('unavailable')).toBe('https://on.auto/problems/unavailable'); }); - it('give tools_unfinished, tools_called and rebuilding types of their own, and no other kind', () => { - expect(kindsWithTypes).toEqual(['tools_unfinished', 'tools_called', 'rebuilding']); + it('give tools_unfinished, tools_called, effect_unknown and rebuilding types of their own, and no other kind', () => { + expect(kindsWithTypes).toEqual(['tools_unfinished', 'tools_called', 'effect_unknown', 'rebuilding']); expect([ isKindWithType('tools_unfinished'), isKindWithType('tools_called'), + isKindWithType('effect_unknown'), isKindWithType('rebuilding'), isKindWithType('stalled'), isKindWithType('tool_not_offered'), isKindWithType('toString'), isKindWithType(), - ]).toEqual([true, true, true, false, false, false, false]); + ]).toEqual([true, true, true, true, false, false, false, false]); }); it('keep the reason of each kind with a type of its own', () => { - expect(kindsWithTypes.map((kind) => reasonOfKind(kind))).toEqual(['unavailable', 'conflict', 'unavailable']); + expect(kindsWithTypes.map((kind) => reasonOfKind(kind))).toEqual([ + 'unavailable', + 'conflict', + 'conflict', + 'unavailable', + ]); }); }); diff --git a/packages/operations/src/outcome/problem-types.ts b/packages/operations/src/outcome/problem-types.ts index b752d5c8c..6738c37d9 100644 --- a/packages/operations/src/outcome/problem-types.ts +++ b/packages/operations/src/outcome/problem-types.ts @@ -3,12 +3,18 @@ const problemTypes = 'https://on.auto/problems/'; const reasonsOfKinds = { tools_unfinished: 'unavailable', tools_called: 'conflict', + effect_unknown: 'conflict', rebuilding: 'unavailable', } as const; export type KindWithType = keyof typeof reasonsOfKinds; -export const kindsWithTypes: readonly KindWithType[] = ['tools_unfinished', 'tools_called', 'rebuilding']; +export const kindsWithTypes: readonly KindWithType[] = [ + 'tools_unfinished', + 'tools_called', + 'effect_unknown', + 'rebuilding', +]; export function problemTypeOf(name: string): string { return `${problemTypes}${name}`; diff --git a/packages/operations/src/outcome/rejection-because.ts b/packages/operations/src/outcome/rejection-because.ts new file mode 100644 index 000000000..17414f11a --- /dev/null +++ b/packages/operations/src/outcome/rejection-because.ts @@ -0,0 +1,22 @@ +import { Schema } from 'effect'; + +export const RejectionBecauseSchema = Schema.Literals([ + 'provider_not_configured', + 'model_not_allowed', + 'mcp_server_not_configured', + 'tool_not_allowed', + 'tool_not_listed', + 'not_testable', + 'failing', + 'rate_limited', + 'unreachable', + 'key_refused', + 'server_failed', + 'tool_error', + 'model_unavailable', + 'run_bound', + 'no_answer', + 'only_read', +]); + +export type RejectionBecause = typeof RejectionBecauseSchema.Type; diff --git a/packages/operations/src/outcome/settlement.ts b/packages/operations/src/outcome/settlement.ts index d29e9fee7..2052ded34 100644 --- a/packages/operations/src/outcome/settlement.ts +++ b/packages/operations/src/outcome/settlement.ts @@ -3,8 +3,9 @@ import { Schema } from 'effect'; import { CancelledKindSchema } from './cancelled-run.ts'; import { ConflictKindSchema } from './conflict.ts'; import { IssueSchema } from './issue.ts'; +import { RejectionBecauseSchema } from './rejection-because.ts'; import { UnansweredKindSchema } from './unanswered-run.ts'; -import { UnavailableBecauseSchema, UnavailableKindSchema } from './unavailable.ts'; +import { UnavailableKindSchema } from './unavailable.ts'; const settledBy = { by: Schema.optionalKey(Schema.NonEmptyString) }; @@ -23,9 +24,14 @@ export const SettlementSchema = Schema.Union([ ...rejected, reason: Schema.Literal('unavailable'), kind: Schema.optionalKey(UnavailableKindSchema), - because: Schema.optionalKey(UnavailableBecauseSchema), + because: Schema.optionalKey(RejectionBecauseSchema), + }), + Schema.Struct({ + ...rejected, + reason: Schema.Literal('conflict'), + kind: Schema.optionalKey(ConflictKindSchema), + because: Schema.optionalKey(RejectionBecauseSchema), }), - Schema.Struct({ ...rejected, reason: Schema.Literal('conflict'), kind: Schema.optionalKey(ConflictKindSchema) }), Schema.Struct({ ...rejected, reason: Schema.Literal('cancelled'), kind: CancelledKindSchema }), Schema.Struct({ ...rejected, reason: Schema.Literal('unanswered'), kind: UnansweredKindSchema }), Schema.Struct({ status: Schema.Literal('failed'), incident: Schema.optionalKey(Schema.String), ...settledBy }), diff --git a/packages/operations/src/outcome/unavailable.ts b/packages/operations/src/outcome/unavailable.ts index 3bce02392..1a7c7490a 100644 --- a/packages/operations/src/outcome/unavailable.ts +++ b/packages/operations/src/outcome/unavailable.ts @@ -1,5 +1,7 @@ import { Data, Schema } from 'effect'; +import type { RejectionBecause } from './rejection-because.ts'; + export const UnavailableKindSchema = Schema.Literals([ 'model_not_offered', 'tool_not_offered', @@ -11,28 +13,9 @@ export const UnavailableKindSchema = Schema.Literals([ export type UnavailableKind = typeof UnavailableKindSchema.Type; -export const UnavailableBecauseSchema = Schema.Literals([ - 'provider_not_configured', - 'model_not_allowed', - 'mcp_server_not_configured', - 'tool_not_allowed', - 'tool_not_listed', - 'not_testable', - 'failing', - 'rate_limited', - 'unreachable', - 'key_refused', - 'server_failed', - 'model_unavailable', - 'run_bound', - 'no_answer', -]); - -export type UnavailableBecause = typeof UnavailableBecauseSchema.Type; - export class Unavailable extends Data.TaggedError('unavailable')<{ readonly detail: string; readonly kind?: UnavailableKind; - readonly because?: UnavailableBecause; + readonly because?: RejectionBecause; readonly record?: Schema.JsonObject; }> {} diff --git a/packages/operations/src/plain-language/explanation.test.ts b/packages/operations/src/plain-language/explanation.test.ts index 707338a9e..16b236e24 100644 --- a/packages/operations/src/plain-language/explanation.test.ts +++ b/packages/operations/src/plain-language/explanation.test.ts @@ -151,6 +151,19 @@ describe('explanationOf a run that names no kind', () => { }); }); +describe('unsuccessfulWords of a run whose tools only read', () => { + it('says that nothing changed', () => { + expect( + unsuccessfulWords('run the reasoning function “summary”', 'command', { + ...rejected('unavailable', 'x', undefined, 'tools_unfinished'), + because: 'run_bound', + }), + ).toBe( + "Could not run the reasoning function “summary”: it called tools but could not finish, because it ran out of time. Nothing was changed. Every tool it called only reads, by its server's own account, so running it again is safe: a new run, or a workflow's retry, may make it; its history shows what it called.", + ); + }); +}); + describe('unsuccessfulWords', () => { it('says what could not be done, why, that nothing changed, and what to do, for a command', () => { expect( @@ -163,11 +176,11 @@ describe('unsuccessfulWords', () => { it('does not say that nothing changed for a command whose tool calls may have changed something', () => { expect( unsuccessfulWords('run the reasoning function “summary”', 'command', { - ...rejected('unavailable', 'x', undefined, 'tools_unfinished'), + ...rejected('conflict', 'x', undefined, 'effect_unknown'), because: 'run_bound', }), ).toBe( - 'Could not run the reasoning function “summary”: it called tools but could not finish, because it ran out of time. What it called may have changed something, so it is not run again by itself: check what its history shows it called, then start a new run if it is still needed.', + 'Could not run the reasoning function “summary”: it could not finish after calling a tool that may change something, so whether that happened is not known, because it ran out of time. It is not run again by itself: a person decides, or a workflow rule that names this kind; its history shows the call.', ); }); diff --git a/packages/operations/src/plain-language/explanation.ts b/packages/operations/src/plain-language/explanation.ts index 907efb4bf..72628725a 100644 --- a/packages/operations/src/plain-language/explanation.ts +++ b/packages/operations/src/plain-language/explanation.ts @@ -1,7 +1,7 @@ import type { OperationKind } from '../caller/operation-scope.ts'; import type { Cancelled, Failed, Rejected, RejectionReason } from '../outcome/outcome.ts'; +import type { RejectionBecause } from '../outcome/rejection-because.ts'; import type { RejectionKind } from '../outcome/rejection.ts'; -import type { UnavailableBecause } from '../outcome/unavailable.ts'; import type { Remedies } from './plain-language.ts'; export interface Explanation { @@ -73,6 +73,12 @@ const explanationByKind: Readonly> = { 'So it was not run again: start a new run instead, after checking what its history shows it has called so far.', mayHaveChanged: true, }, + effect_unknown: { + why: 'it could not finish after calling a tool that may change something, so whether that happened is not known', + remedy: + 'It is not run again by itself: a person decides, or a workflow rule that names this kind; its history shows the call.', + mayHaveChanged: true, + }, model_not_offered: { why: 'this server does not offer the model named', remedy: @@ -134,12 +140,11 @@ const explanationByKind: Readonly> = { tools_unfinished: { why: 'it called tools but could not finish', remedy: - 'What it called may have changed something, so it is not run again by itself: check what its history shows it called, then start a new run if it is still needed.', - mayHaveChanged: true, + "Every tool it called only reads, by its server's own account, so running it again is safe: a new run, or a workflow's retry, may make it; its history shows what it called.", }, }; -const explanationByBecause: Readonly> = { +const explanationByBecause: Readonly> = { provider_not_configured: 'because its provider is not set up on this server, though others are', model_not_allowed: 'because it is not among the models whoever runs the server allows', mcp_server_not_configured: 'because whoever runs the server has not set up a tool server of that name for this brain', @@ -151,10 +156,12 @@ const explanationByBecause: Readonly> = { rate_limited: 'because the tool server asked it to slow down for longer than a run waits', unreachable: 'because the tool server could not be reached in time', key_refused: 'because the tool server did not accept the key this server gives it', - server_failed: 'because a tool server kept failing', + server_failed: 'because a tool server failed', + tool_error: 'because the tool answered an error, which the details below give', model_unavailable: 'because the model stopped answering', run_bound: 'because it ran out of time', no_answer: 'because the model kept calling tools instead of answering', + only_read: "because every tool it called only reads, by its server's own account", }; const remedyByBecause: Remedies = { @@ -163,6 +170,17 @@ const remedyByBecause: Remedies = { key_refused: 'Trying again will not help until whoever runs the server checks the key it gives that tool server.', }; +const explanationByKindAndBecause: Readonly< + Partial>>> +> = { + tools_called: { + only_read: { + why: "an attempt of this run under the same id did not succeed, and every tool it called only reads, by its server's own account", + remedy: 'So it was not run again under its id: start a new run instead; its history shows what it called.', + }, + }, +}; + const noRemedies: Remedies = {}; export function explanationOf( @@ -173,6 +191,10 @@ export function explanationOf( return explanationByReason[reason]; } const explanation = explanationByKind[kind]; + const ofBoth = because === undefined ? undefined : explanationByKindAndBecause[kind]?.[because]; + if (ofBoth !== undefined) { + return ofBoth; + } return because === undefined ? explanation : { diff --git a/packages/operations/src/plain-language/plain-language.ts b/packages/operations/src/plain-language/plain-language.ts index 56d39ff5c..24d10678a 100644 --- a/packages/operations/src/plain-language/plain-language.ts +++ b/packages/operations/src/plain-language/plain-language.ts @@ -1,6 +1,6 @@ -import type { UnavailableBecause } from '../outcome/unavailable.ts'; +import type { RejectionBecause } from '../outcome/rejection-because.ts'; -export type Remedies = Readonly>>; +export type Remedies = Readonly>>; export interface PlainLanguage { readonly task: string; diff --git a/packages/operations/src/plain-language/tool-explanations.test.ts b/packages/operations/src/plain-language/tool-explanations.test.ts index 51bc76d53..b9d793ef7 100644 --- a/packages/operations/src/plain-language/tool-explanations.test.ts +++ b/packages/operations/src/plain-language/tool-explanations.test.ts @@ -8,8 +8,24 @@ const toolsNamed = const serverFailed = 'Nothing was called through it, so it can be tried again later; if it keeps happening, whoever runs the server can look into that tool server.'; -const toolsMayHaveWritten = - 'What it called may have changed something, so it is not run again by itself: check what its history shows it called, then start a new run if it is still needed.'; +const toolsOnlyRead = + "Every tool it called only reads, by its server's own account, so running it again is safe: a new run, or a workflow's retry, may make it; its history shows what it called."; + +const effectUnknown = + 'It is not run again by itself: a person decides, or a workflow rule that names this kind; its history shows the call.'; + +const calledAndUnfinished = 'it called tools but could not finish'; + +const mayHaveChanged = + 'it could not finish after calling a tool that may change something, so whether that happened is not known'; + +const unfinishedBecauses = [ + ['server_failed', 'a tool server failed', 'because a tool server failed'], + ['tool_error', 'a tool answered an error', 'because the tool answered an error, which the details below give'], + ['model_unavailable', 'the model did not finish', 'because the model stopped answering'], + ['run_bound', 'ran out of time', 'because it ran out of time'], + ['no_answer', 'never came to an answer', 'because the model kept calling tools instead of answering'], +] as const; const toolEndings: ReadonlyArray = [ [ @@ -60,34 +76,26 @@ const toolEndings: ReadonlyArray = + unfinishedBecauses.flatMap(([because, ended, words]) => [ + [ + `calls of tools that only read, where ${ended}`, + { reason: 'unavailable', kind: 'tools_unfinished', because }, + `${calledAndUnfinished}, ${words}`, + toolsOnlyRead, + ], + [ + `calls of a tool that may change something, where ${ended}`, + { reason: 'conflict', kind: 'effect_unknown', because }, + `${mayHaveChanged}, ${words}`, + effectUnknown, + ], + ]); + describe('explanationOf a run that needs tools', () => { - it.each(toolEndings)('explains %s', (_case, rejection, why, remedy) => { + it.each([...toolEndings, ...unfinishedEndings])('explains %s', (_case, rejection, why, remedy) => { expect(explanationOf(rejection)).toMatchObject({ why, remedy }); }); @@ -98,12 +106,24 @@ describe('explanationOf a run that needs tools', () => { expect(remedy).toHaveLength(214); }); - it('says only of tool calls that could not finish that something may have changed', () => { - const changing = toolEndings.filter(([, rejection]) => explanationOf(rejection).mayHaveChanged === true); - - expect(changing.map(([, rejection]) => rejection.kind)).toEqual( - Array.from({ length: 4 }, () => 'tools_unfinished'), + it('says only of tool calls whose effect is not known that something may have changed', () => { + const changing = [...toolEndings, ...unfinishedEndings].filter( + ([, rejection]) => explanationOf(rejection).mayHaveChanged === true, ); + + expect(changing.map(([, rejection]) => rejection.kind)).toEqual(Array.from({ length: 5 }, () => 'effect_unknown')); + }); + + it('says why the effect is not known in 105 characters and what to do in 117, and that tools that only read may run again in 171', () => { + const unknown = explanationOf({ reason: 'conflict', kind: 'effect_unknown' }); + const onlyRead = explanationOf({ reason: 'unavailable', kind: 'tools_unfinished' }); + + expect([unknown.why.length, unknown.remedy.length, onlyRead.remedy.length]).toEqual([105, 117, 171]); + expect( + explanationOf({ reason: 'conflict', kind: 'effect_unknown', because: 'tool_error' }).why.slice( + unknown.why.length + 2, + ), + ).toHaveLength(64); }); }); @@ -132,3 +152,16 @@ describe('the remedies an operation gives of its own', () => { ); }); }); + +const calledTools = { status: 'rejected', reason: 'conflict', detail: 'x', kind: 'tools_called' } as const; + +const calledOnlyReading = { ...calledTools, because: 'only_read' } as const; + +describe('a run not made again under its id after calling tools that only read', () => { + it('says that nothing was changed, which a run whose tools may have changed something never says', () => { + expect(unsuccessfulWords('run the function', 'command', calledOnlyReading)).toBe( + "Could not run the function: an attempt of this run under the same id did not succeed, and every tool it called only reads, by its server's own account. Nothing was changed. So it was not run again under its id: start a new run instead; its history shows what it called.", + ); + expect(unsuccessfulWords('run the function', 'command', calledTools)).not.toContain('Nothing was changed.'); + }); +}); diff --git a/packages/server/src/asking-a-system/calls.test.ts b/packages/server/src/asking-a-system/calls.test.ts new file mode 100644 index 000000000..fb00959e3 --- /dev/null +++ b/packages/server/src/asking-a-system/calls.test.ts @@ -0,0 +1,81 @@ +import { threadReplies } from '@beonauto/mcp/testing'; +import { describe, expect, it } from 'vitest'; + +import { chatKey } from '../testing/servers/chat-deliveries.ts'; +import { alpha } from '../testing/servers/reasoning-server.ts'; +import { askingASystem, systemRunId, threadInput, type HistoryEvent } from '../testing/servers/system-calls.ts'; +import { workflowTestTimeoutMs } from '../testing/servers/workflow-server.ts'; + +const aNumber: unknown = expect.any(Number); + +const aTime: unknown = expect.any(String); + +const record = { server: 'chat', tool: 'thread', read: '/messages' }; + +function keysOf(event: HistoryEvent | undefined): readonly string[] { + return Object.keys(event?.data ?? {}); +} + +describe('an interaction function that asks a system, over HTTP', { timeout: workflowTestTimeoutMs }, () => { + it('calls the tool once and answers at once with what it read, as get_run answers the run', async () => { + const server = await askingASystem(); + await server.define('thread-replies'); + + const ran = await server.runCall('thread-replies'); + const read = await server.call('GET', `${alpha}/runs/${systemRunId}`); + + expect(ran).toMatchObject({ + status: 200, + body: { + run_id: systemRunId, + type: 'interaction', + name: 'thread-replies', + status: 'succeeded', + output: threadReplies, + record, + started_at: aTime, + finished_at: aTime, + }, + }); + expect(read.body).toEqual(ran.body); + expect(server.fake.received()).toEqual([ + { + tool: 'thread', + arguments: { channel: 'C0123', ts: '1728379900.000050', limit: 100, inclusive: false }, + meta: { 'com.beonauto/run_id': systemRunId }, + }, + ]); + }); +}); + +describe('the history of a run that asks a system', { timeout: workflowTestTimeoutMs }, () => { + it('records the call on the run, its start under the id of the run, in the words of the history', async () => { + const server = await askingASystem(); + await server.define('thread-replies'); + await server.runCall('thread-replies'); + + const history = await server.history(systemRunId); + + expect(history.map(({ type, summary }) => [type, summary])).toEqual([ + ['run_started', 'A run of the interaction function “thread-replies” started.'], + ['tool_call_started', 'A run made tool call 1, to the thread tool of chat.'], + ['tool_call_answered', 'Tool call 1 answered.'], + ['run_succeeded', 'A run finished.'], + ]); + expect(history[1]?.data).toMatchObject({ call_id: systemRunId, server: 'chat', tool: 'thread' }); + expect(history[2]?.data).toMatchObject({ outcome: 'result', duration_ms: aNumber }); + expect([...keysOf(history[1]), ...keysOf(history[2])]).not.toContain('arguments_json'); + }); + + it('keeps the arguments and the answer, scrubbed and shown at 2 KiB, only where its server records content', async () => { + const server = await askingASystem({ entry: { record_content: true } }); + await server.define('thread-replies'); + await server.runCall('thread-replies', { ...threadInput, channel: `${chatKey} ${'x'.repeat(5000)}` }); + + const [, started, answered] = await server.history(systemRunId); + + expect(String(started?.data['arguments_json'])).toMatch(/^\{"channel":"\[redacted\] x+/u); + expect(String(started?.data['arguments_json']).length).toBeLessThanOrEqual(2048); + expect(String(answered?.data['result_json'])).toMatch(/^\{"content":\[\{"type":"text","text":"2 replies/u); + }); +}); diff --git a/packages/server/src/asking-a-system/ending-rows.ts b/packages/server/src/asking-a-system/ending-rows.ts new file mode 100644 index 000000000..8c026c221 --- /dev/null +++ b/packages/server/src/asking-a-system/ending-rows.ts @@ -0,0 +1,215 @@ +import type { CallOptions } from '@beonauto/interaction/testing'; +import { patientTiming } from '@beonauto/mcp/testing'; +import { Schema } from 'effect'; +import { expect } from 'vitest'; + +import type { TestResponse } from '../testing/servers/http-client.ts'; +import { + askingASystem, + calling, + openInput, + type SystemOptions, + type SystemServer, +} from '../testing/servers/system-calls.ts'; + +const quickCalls = { ...patientTiming, openMs: 300, callMs: 300 }; + +const someDetail: unknown = expect.any(String); + +export const mayChange = { readOnlyHint: false }; + +export const problems = 'https://on.auto/problems/'; + +export const mayHaveActed = '. The tool ran and may have changed something; a run again calls it again'; + +export const refusingTheArguments: unknown = expect.stringMatching( + /^The strict tool of chat refused the arguments: .*limit must be a whole number/u, +); + +export const failedAfterCalling = (tool: string): unknown => + expect.stringMatching( + new RegExp(`^The MCP server chat failed: .*, after the run called the ${tool} tool of chat$`, 'u'), + ); + +export const notAnsweredInTime: unknown = expect.stringMatching( + /^The MCP server chat failed: .*did not answer within 300 ms.*, after the run called the sleep tool of chat$/u, +); + +export interface Ending { + readonly status: number; + readonly type: string; + readonly kind?: string; + readonly because?: string; + readonly detail: unknown; + readonly retryAfter: string | null; +} + +export interface Row { + readonly server?: SystemOptions; + readonly document?: CallOptions; + readonly input?: unknown; + readonly before?: (server: SystemServer) => Promise | void; +} + +const decodeProblem = Schema.decodeUnknownSync( + Schema.Struct({ + type: Schema.String, + kind: Schema.optionalKey(Schema.String), + because: Schema.optionalKey(Schema.String), + detail: Schema.Unknown, + }), +); + +export function endingOf({ status, body, headers }: TestResponse): Ending { + return { status, ...decodeProblem(body), retryAfter: headers.get('retry-after') }; +} + +export async function endedFor({ server = {}, document = {}, input, before }: Row): Promise { + const asking = await askingASystem({ timing: quickCalls, ...server }); + await asking.define('asking', document); + await before?.(asking); + return endingOf(await asking.runCall('asking', input)); +} + +const sentNothing = (because: string, detail: unknown, kind = 'tool_not_offered'): Ending => ({ + status: 503, + type: `${problems}unavailable`, + kind, + because, + detail, + retryAfter: kind === 'tool_not_offered' ? null : '5', +}); + +export const unworkable = (detail: unknown): Ending => ({ + status: 409, + type: `${problems}conflict`, + kind: 'unworkable', + detail, + retryAfter: null, +}); + +export const unfinished = (because: string, detail: unknown): Ending => ({ + status: 503, + type: `${problems}tools_unfinished`, + kind: 'tools_unfinished', + because, + detail, + retryAfter: null, +}); + +export const unknown = (because: string, detail: unknown): Ending => ({ + status: 409, + type: `${problems}effect_unknown`, + kind: 'effect_unknown', + because, + detail, + retryAfter: null, +}); + +export const beforeAnything: ReadonlyArray = [ + [ + 'an input that does not match', + { input: { channel: 7 } }, + { status: 422, type: `${problems}invalid_input`, detail: someDetail, retryAfter: null }, + ], + [ + 'an argument that reads what the input lacks', + { document: { input: openInput }, input: { channel: 'C0123' } }, + { + status: 422, + type: `${problems}invalid_input`, + detail: 'The definition reads a field the input does not have', + retryAfter: null, + }, + ], + [ + 'a render a filter stopped', + { document: { with: [" channel: '{{ input.channel | divided_by: 0 }}'"] } }, + { + status: 422, + type: `${problems}invalid_input`, + detail: 'The argument channel of the call cannot be rendered with this input', + retryAfter: null, + }, + ], + [ + 'an argument that renders a structure among text', + { document: { with: [" channel: 'In {{ input.channel }}'"], input: openInput }, input: { channel: ['C0123'] } }, + unworkable( + 'The argument channel of the call renders a value that is not text among text; write | json after it, or write it alone as one {{ }} to send it as it is', + ), + ], + [ + 'arguments past 16 KiB', + { + document: { with: [" a: '{{ input.a }}'", " b: '{{ input.b }}'"], input: openInput }, + input: { a: 'x'.repeat(8200), b: 'x'.repeat(8200) }, + }, + unworkable('The arguments of the call take 16415 bytes, more than the 16384 a call may send'), + ], + [ + 'a server that does not serve the brain', + { server: { entry: { brains: ['beta'] } } }, + sentNothing('mcp_server_not_configured', 'No MCP server named chat is configured for this brain'), + ], + [ + 'a tool the operator does not allow', + { server: { entry: { allowed: ['search'] } } }, + sentNothing('tool_not_allowed', 'The operator of this server does not allow chat/thread'), + ], + [ + 'a server that cannot be reached', + { before: (server) => server.fake.close() }, + sentNothing('unreachable', expect.stringContaining('could not be reached'), 'mcp_server_failed'), + ], + [ + 'a server that refuses the key at its opening', + { + server: { + entry: { headers: { Authorization: 'Bearer ${STALE_CHAT_KEY}' } }, + environment: { STALE_CHAT_KEY: 'chat-api-key-retired' }, + }, + }, + sentNothing( + 'key_refused', + 'The MCP server chat could not be used: The MCP server answered HTTP 401', + 'mcp_server_failed', + ), + ], + [ + 'a listing of the tools the server limits', + { + before: (server) => { + server.fake.answerNextOf('tools/list', 429, 1, { 'retry-after': '120' }); + }, + }, + sentNothing('rate_limited', expect.stringContaining('could not be used'), 'mcp_server_failed'), + ], + [ + 'a listing of the tools that fails', + { + before: (server) => { + server.fake.answerNextOf('tools/list', 500); + }, + }, + sentNothing('failing', expect.stringContaining('answered HTTP 500'), 'mcp_server_failed'), + ], + [ + 'a listing of the tools that does not answer in time', + { + before: (server) => { + server.fake.holdNextOf('tools/list'); + }, + }, + sentNothing( + 'unreachable', + 'The MCP server chat could not be used: The MCP server did not answer in time', + 'mcp_server_failed', + ), + ], + [ + 'a tool the server does not list', + { document: calling('gone') }, + sentNothing('tool_not_listed', 'The MCP server chat does not list the tool gone'), + ], +]; diff --git a/packages/server/src/asking-a-system/endings.test.ts b/packages/server/src/asking-a-system/endings.test.ts new file mode 100644 index 000000000..eb13b2a59 --- /dev/null +++ b/packages/server/src/asking-a-system/endings.test.ts @@ -0,0 +1,130 @@ +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; + +import { describe, expect, it, onTestFinished } from 'vitest'; + +import { alpha } from '../testing/servers/reasoning-server.ts'; +import { + askingASystem, + calling, + problemWith, + systemRunId, + type SystemServer, +} from '../testing/servers/system-calls.ts'; +import { workflowTestTimeoutMs } from '../testing/servers/workflow-server.ts'; +import { beforeAnything, endedFor, endingOf, problems } from './ending-rows.ts'; +import { afterSending, unreadable } from './sent-ending-rows.ts'; + +const anotherRunId = '0199a3c4-7d2e-7c1a-9b3f-2f1e0d9c8b7b'; + +const someDetail: unknown = expect.any(String); + +const anIncident: unknown = expect.stringMatching(/^urn:uuid:/u); + +describe( + 'a run that asks a system and ends before anything is sent, over HTTP', + { timeout: workflowTestTimeoutMs }, + () => { + it.each(beforeAnything)('ends as the table says after %s', async (_case, row, ending) => { + expect(await endedFor(row)).toEqual(ending); + }); + }, +); + +describe( + 'a run that asks a system and ends after its call was sent, over HTTP', + { timeout: workflowTestTimeoutMs }, + () => { + it.each(afterSending)('ends as the table says after %s', async (_case, row, ending) => { + expect(await endedFor(row)).toEqual(ending); + }); + }, +); + +describe( + 'a run that asks a system and cannot use what the tool answered, over HTTP', + { timeout: workflowTestTimeoutMs }, + () => { + it.each(unreadable)('ends as the table says after %s', async (_case, row, ending) => { + expect(await endedFor(row)).toEqual(ending); + }); + }, +); + +async function refusingCallStarts(): Promise { + const directory = mkdtempSync(join(tmpdir(), 'asking-a-system-')); + const ledgerFile = join(directory, 'ledger.db'); + onTestFinished(() => { + rmSync(directory, { recursive: true, force: true }); + }); + const server = await askingASystem({ environment: { LEDGER_FILE: ledgerFile } }); + const database = new DatabaseSync(ledgerFile); + database.exec( + "CREATE TRIGGER refuse_call_starts BEFORE INSERT ON emt_messages WHEN NEW.message_type = 'tool_call_started' BEGIN SELECT RAISE(ABORT, 'refused'); END", + ); + database.close(); + return server; +} + +describe( + 'a run that asks a system whose start of the call the ledger refuses, over HTTP', + { timeout: workflowTestTimeoutMs }, + () => { + it('fails with an incident, sends nothing, and is recorded failed', async () => { + const server = await refusingCallStarts(); + await server.define('asking'); + + const ended = await server.runCall('asking'); + const read = await server.call('GET', `${alpha}/runs/${systemRunId}`); + + expect(endingOf(ended)).toEqual({ + status: 500, + type: `${problems}internal`, + detail: someDetail, + retryAfter: null, + }); + expect(ended.body).toMatchObject({ instance: anIncident }); + expect([read.body, server.fake.received()]).toEqual([problemWith({ status: 'failed' }), []]); + }); + }, +); + +describe('a run that asks a system, run again under its id, over HTTP', { timeout: workflowTestTimeoutMs }, () => { + it('runs again when nothing was sent, and is tools_called once its call was sent', async () => { + const server = await askingASystem({ entry: { allowed: ['search', 'gone', 'denied'] } }); + await server.define('asking'); + await server.define('denying', calling('denied')); + + const unsent = [await server.runCall('asking'), await server.runCall('asking')]; + const sent = [ + await server.runCall('denying', undefined, anotherRunId), + await server.runCall('denying', undefined, anotherRunId), + ]; + + expect([...unsent, ...sent].map(({ status }) => status)).toEqual([503, 503, 503, 409]); + expect([unsent[1]?.body, sent[1]?.body]).toEqual([ + problemWith({ kind: 'tool_not_offered' }), + problemWith({ type: `${problems}tools_called`, kind: 'tools_called', because: 'only_read' }), + ]); + expect(server.fake.received()).toHaveLength(1); + }); + + it('keeps the wait a 429 asked for past the longest a run waits in its record, and waits out a shorter one', async () => { + const server = await askingASystem(); + await server.define('asking', calling('search', [' query: acme'])); + server.fake.answerNextOf('tools/call', 429, 1, { 'retry-after': '20' }); + + const limited = await server.runCall('asking'); + server.fake.answerNextOf('tools/call', 429, 1, { 'retry-after': '2' }); + const waited = await server.runCall('asking', undefined, anotherRunId); + const read = await server.call('GET', `${alpha}/runs/${systemRunId}`); + + expect([limited.status, limited.headers.get('retry-after'), waited.status]).toEqual([503, null, 200]); + expect(read.body).toMatchObject({ + rejection: { kind: 'tools_unfinished', because: 'server_failed' }, + record: { retry_after_ms: 20_000 }, + }); + }); +}); diff --git a/packages/server/src/asking-a-system/in-workflows.test.ts b/packages/server/src/asking-a-system/in-workflows.test.ts new file mode 100644 index 000000000..946d6df8e --- /dev/null +++ b/packages/server/src/asking-a-system/in-workflows.test.ts @@ -0,0 +1,119 @@ +import { inTurn, serveFakeMcp, threadReplies } from '@beonauto/mcp/testing'; +import { Schema } from 'effect'; +import { describe, expect, it, onTestFinished } from 'vitest'; + +import { chatKey } from '../testing/servers/chat-deliveries.ts'; +import { alpha } from '../testing/servers/reasoning-server.ts'; +import { askingASystem, calling, threadInput, type SystemServer } from '../testing/servers/system-calls.ts'; +import { workflowTestTimeoutMs } from '../testing/servers/workflow-server.ts'; + +const decodeRuns = Schema.decodeUnknownSync(Schema.Struct({ runs: Schema.Array(Schema.Unknown) })); + +async function runsOf(server: SystemServer, name: string): Promise { + const listed = await server.call('GET', `${alpha}/runs?type=interaction&name=${name}&limit=100`); + return decodeRuns(listed.body).runs.length; +} + +function retried(name: string, caught: string): string { + return `do: + - ask: + try: + - call-it: { call: run_definition, with: { type: interaction, name: ${name}, input: {} } } + catch: + ${caught} + retry: + delay: PT0.01S + limit: + attempt: { count: 2 } +`; +} + +const onStatus503 = 'errors: { with: { status: 503 } }'; + +function catching(name: string): string { + return `do: + - ask: + try: + - call-it: { call: run_definition, with: { type: interaction, name: ${name}, input: {} } } + catch: + as: failure + when: '\${ $failure.because == "tool_error" }' + do: + - note: { set: { kind: '\${ $failure.kind }', because: '\${ $failure.because }' } } +`; +} + +async function endedWorkflow(server: SystemServer, name: string, steps: string) { + return server.settled(await server.workflow(name, steps)); +} + +describe('a workflow that asks a system in a step', { timeout: workflowTestTimeoutMs }, () => { + it('takes what the tool answered at read as the output of the step', async () => { + const server = await askingASystem(); + await server.define('thread-replies'); + const input = `{ channel: ${threadInput.channel}, thread: '${threadInput.thread}' }`; + + const ended = await endedWorkflow( + server, + 'reading', + `do:\n - read: { call: run_definition, with: { type: interaction, name: thread-replies, input: ${input} } }\n`, + ); + + expect(ended).toMatchObject({ status: 'succeeded', output: threadReplies }); + }); + + it('retries on status 503 what is safe to try again, and neither what is wrong nor what may have changed something', async () => { + const server = await askingASystem({ + hints: { broken: { readOnlyHint: false } }, + servers: { gone: { url: 'http://127.0.0.1:1/mcp', org: 'acme' } }, + }); + await server.define('unreachable', { ...calling('search'), server: 'gone' }); + await server.define('unlisted', calling('missing')); + await server.define('unfinished', calling('denied')); + await server.define('unworkable', calling('strict', [" limit: '15'"])); + await server.define('unknown', calling('broken')); + + const names = ['unreachable', 'unlisted', 'unfinished', 'unworkable', 'unknown']; + await inTurn(names, (name) => endedWorkflow(server, `retrying-${name}`, retried(name, onStatus503))); + + expect(await Promise.all(names.map((name) => runsOf(server, name)))).toEqual([3, 3, 3, 1, 1]); + }); +}); + +describe('a workflow that names effect_unknown on purpose', { timeout: workflowTestTimeoutMs }, () => { + it('retries it when its catch names the type of effect_unknown, or the kind in a when', async () => { + const server = await askingASystem({ hints: { broken: { readOnlyHint: false } } }); + await server.define('by-type', calling('broken')); + await server.define('by-kind', calling('broken')); + + await endedWorkflow( + server, + 'typed', + retried('by-type', 'errors: { with: { type: https://on.auto/problems/effect_unknown } }'), + ); + await endedWorkflow( + server, + 'kinded', + retried('by-kind', `errors: { with: { status: 409 } }\n when: '\${ $error.kind == "effect_unknown" }'`), + ); + + expect([await runsOf(server, 'by-type'), await runsOf(server, 'by-kind')]).toEqual([3, 3]); + }); + + it('tells a tool that answered an error from a server that failed by because, after either ending', async () => { + const writer = await serveFakeMcp({ bearer: chatKey, data: true, hints: { denied: { readOnlyHint: false } } }); + onTestFinished(writer.close); + const server = await askingASystem({ + servers: { writer: { url: writer.url, headers: { Authorization: 'Bearer ${CHAT_KEY}' }, org: 'acme' } }, + }); + await server.define('only-reads', calling('denied')); + await server.define('may-write', { ...calling('denied'), server: 'writer' }); + expect([ + await endedWorkflow(server, 'reading-errors', catching('only-reads')), + await endedWorkflow(server, 'writing-errors', catching('may-write')), + ]).toMatchObject([ + { status: 'succeeded', output: { kind: 'tools_unfinished', because: 'tool_error' } }, + { status: 'succeeded', output: { kind: 'effect_unknown', because: 'tool_error' } }, + ]); + }); +}); diff --git a/packages/server/src/asking-a-system/pages.test.ts b/packages/server/src/asking-a-system/pages.test.ts new file mode 100644 index 000000000..0d4fa1ef4 --- /dev/null +++ b/packages/server/src/asking-a-system/pages.test.ts @@ -0,0 +1,99 @@ +import { threadReplies } from '@beonauto/mcp/testing'; +import { Schema } from 'effect'; +import { describe, expect, it } from 'vitest'; + +import { alpha } from '../testing/servers/reasoning-server.ts'; +import { askingASystem } from '../testing/servers/system-calls.ts'; +import { until } from '../testing/servers/workflow-calls.ts'; +import { workflowSource, workflowTestTimeoutMs } from '../testing/servers/workflow-server.ts'; + +const paging = { + tool: 'pages', + read: null, + with: [" cursor: '{{ input.cursor }}'"], + input: ['input:', ' schema: { type: object, properties: { cursor: { type: string } } }'], + output: ['output:', ' schema: { type: object, required: [items, next] }'], +}; + +const throughEveryPage = `do: + - page: + call: run_definition + with: { type: interaction, name: paging, input: { cursor: '\${ $context.next // "" }' } } + export: + as: '\${ { items: (($context.items // []) + .items), next: .next } }' + - more: + switch: + - again: + when: '\${ $context.next != null }' + then: page + - done: + then: continue + - all: + set: '\${ $context.items }' +`; + +const keptReplies = [ + '---', + 'description: What the thread-replies function read, the last ten times', + 'language: jq', + 'source:', + ' events:', + ' - type: run_succeeded', + ' subject: interaction/thread-replies', + 'view:', + ' initial: []', + ' schema: { type: array, maxItems: 10 }', + "answer: '.'", + '---', + '(. + [$event.data.output]) | .[-10:]', +].join('\n'); + +const onReplies = workflowSource( + 'on-replies', + `schedule: + on: { one: { with: { type: run_succeeded, subject: interaction/thread-replies } } } +do: + - noted: { set: { replies: '\${ .[0].data.output | length }' } } +`, +); + +const decodeRuns = Schema.decodeUnknownSync( + Schema.Struct({ runs: Schema.Array(Schema.Struct({ run_id: Schema.String, status: Schema.String })) }), +); + +describe('a workflow that reads every page of a tool that pages', { timeout: workflowTestTimeoutMs }, () => { + it('calls again with the cursor while there is one, each page a run of its own, keeping what it read', async () => { + const server = await askingASystem(); + await server.define('paging', paging); + + const ended = await server.settled(await server.workflow('every-page', throughEveryPage)); + + expect(ended).toMatchObject({ status: 'succeeded', output: [1, 2, 3] }); + expect(server.fake.received().map(({ arguments: given }) => given)).toEqual([{ cursor: '' }, { cursor: 'p2' }]); + }); +}); + +describe('what a call read, kept and reacted to', { timeout: workflowTestTimeoutMs }, () => { + it('is folded by a recall function from the run that ended, and starts a workflow that listens for that ending', async () => { + const server = await askingASystem(); + await server.define('thread-replies'); + await server.call('POST', `${alpha}/definitions/recall`, { body: { name: 'kept-replies', source: keptReplies } }); + await server.call('POST', `${alpha}/definitions/workflow`, { body: { name: 'on-replies', source: onReplies } }); + + await server.runCall('thread-replies'); + const kept = await until( + () => server.call('POST', `${alpha}/definitions/recall/kept-replies/run`, { body: { input: {} } }), + ({ text }) => text.includes('member-17'), + ); + const [reaction] = await until( + async () => decodeRuns((await server.call('GET', `${alpha}/runs?type=workflow&name=on-replies`)).body).runs, + (runs) => runs.some(({ status }) => status !== 'started'), + ); + + expect(kept.body).toMatchObject({ status: 'succeeded', output: [threadReplies] }); + expect(await server.settled(String(reaction?.run_id))).toMatchObject({ + status: 'succeeded', + output: { replies: 2 }, + }); + }); +}); diff --git a/packages/server/src/asking-a-system/runs.test.ts b/packages/server/src/asking-a-system/runs.test.ts new file mode 100644 index 000000000..b015231b0 --- /dev/null +++ b/packages/server/src/asking-a-system/runs.test.ts @@ -0,0 +1,143 @@ +import { callDocument } from '@beonauto/interaction/testing'; +import { fakeStdioServerPath, stdioTestTimeoutMs } from '@beonauto/mcp/testing'; +import { describe, expect, it } from 'vitest'; + +import { interactionServerOn } from '../testing/servers/interaction-server.ts'; +import { alpha } from '../testing/servers/reasoning-server.ts'; +import { + askingASystem, + calling, + problemWith, + systemRunId, + type SystemServer, +} from '../testing/servers/system-calls.ts'; +import { until } from '../testing/servers/workflow-calls.ts'; +import { settledRun, workflowTestTimeoutMs } from '../testing/servers/workflow-server.ts'; + +const measured = { NODE_V8_COVERAGE: process.env['NODE_V8_COVERAGE'] ?? '' }; + +const notARequest: unknown = expect.stringContaining('The run is not a request'); + +const withinItsCall = + 'The run takes place within the call that started it, which no server can interrupt from outside, so it cannot be cancelled; it ends when that call does'; + +const callFacts = ['run_started', 'tool_call_started', 'tool_call_answered', 'run_succeeded']; + +describe('a run that asks a system whose session or process is gone', { timeout: workflowTestTimeoutMs }, () => { + it('opens a session its server forgot once more, and answers', async () => { + const server = await askingASystem(); + await server.define('asking'); + server.fake.answerNextOf('tools/call', 404); + + expect(await server.runCall('asking')).toMatchObject({ status: 200, body: { status: 'succeeded' } }); + }); + + it( + 'starts a process that exited once more, after a call that ended it could not say whether it changed something', + { timeout: stdioTestTimeoutMs }, + async () => { + const server = await interactionServerOn({ + MCP_SERVERS: JSON.stringify({ + notes: { + command: process.execPath, + args: [fakeStdioServerPath], + env: { NODE_V8_COVERAGE: '${NODE_V8_COVERAGE:-}' }, + org: 'acme', + }, + }), + ...measured, + }); + await server.call('POST', '/v1/orgs/acme/brains', { body: { brain: 'alpha', name: 'Alpha' } }); + const define = (name: string, tool: string, written: readonly string[] = []) => + server.call('POST', `${alpha}/definitions/interaction`, { + body: { name, source: callDocument({ server: 'notes', ...calling(tool, written) }) }, + }); + await define('exiting', 'exit'); + await define('searching', 'search', [' query: acme']); + + const exited = await server.call('POST', `${alpha}/definitions/interaction/exiting/run`, { body: { input: {} } }); + const restarted = await server.call('POST', `${alpha}/definitions/interaction/searching/run`, { + body: { input: {} }, + }); + + expect([exited.status, exited.body]).toEqual([ + 409, + problemWith({ kind: 'effect_unknown', because: 'server_failed' }), + ]); + expect(restarted.body).toMatchObject({ status: 'succeeded', output: 'Found 2 rows for acme.' }); + }, + ); +}); + +async function sleeping(): Promise { + const server = await askingASystem(); + await server.define('sleeping', calling('sleep', [' ms: 1000'])); + return server; +} + +interface Posting { + readonly headers: Readonly>; + readonly body: string; +} + +async function leftOnceCalled(server: SystemServer, url: string, { headers, body }: Posting): Promise { + const leaving = new AbortController(); + const gone: Promise = fetch(url, { method: 'POST', headers, body, signal: leaving.signal }).catch( + (error: unknown) => error, + ); + await until( + () => Promise.resolve(server.fake.received().length), + (received) => received === 1, + ); + leaving.abort(); + return gone; +} + +describe('a run that asks a system while its call is in flight, over HTTP', { timeout: workflowTestTimeoutMs }, () => { + it('goes on to its end when its caller goes away, records the answer, and is no request anyone can answer', async () => { + const server = await sleeping(); + await leftOnceCalled(server, `${server.origin}${alpha}/definitions/interaction/sleeping/run`, { + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ input: {}, run_id: systemRunId }), + }); + + const cancelling = await server.call('POST', `${alpha}/runs/${systemRunId}/cancel`, { body: { reason: 'Stop' } }); + const answering = await server.call('POST', `${alpha}/runs/${systemRunId}/answer`, { body: { answer: {} } }); + const ended = await settledRun(server, `${alpha}/runs/${systemRunId}`); + const listed = await server.call('GET', `${alpha}/interactions`); + + expect([cancelling.status, cancelling.body]).toEqual([409, problemWith({ detail: withinItsCall })]); + expect([answering.status, answering.body]).toEqual([409, problemWith({ detail: notARequest })]); + expect(ended.body).toMatchObject({ status: 'succeeded', output: 'Slept.' }); + expect((await server.history(systemRunId)).map(({ type }) => type)).toEqual(callFacts); + expect(listed.body).toMatchObject({ interactions: [] }); + }); +}); + +describe('a run that asks a system while its call is in flight, over MCP', { timeout: workflowTestTimeoutMs }, () => { + it('goes on to its end when its caller goes away, records the answer, calls once and lets the session go', async () => { + const server = await sleeping(); + const call = { + name: 'run_definition', + arguments: { type: 'interaction', name: 'sleeping', input: {}, run_id: systemRunId }, + }; + await leftOnceCalled(server, `${server.origin}/orgs/acme/brains/alpha/mcp`, { + headers: { + 'content-type': 'application/json', + accept: 'application/json, text/event-stream', + 'mcp-protocol-version': '2025-11-25', + }, + body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/call', params: call }), + }); + + const ended = await settledRun(server, `${alpha}/runs/${systemRunId}`); + const sessionsEnded = await until( + () => Promise.resolve(server.fake.endedSessions()), + (count) => count === 1, + ); + + expect(ended.body).toMatchObject({ status: 'succeeded', output: 'Slept.' }); + expect((await server.history(systemRunId)).map(({ type }) => type)).toEqual(callFacts); + expect([server.fake.received().length, sessionsEnded]).toEqual([1, 1]); + }); +}); diff --git a/packages/server/src/asking-a-system/sent-ending-rows.ts b/packages/server/src/asking-a-system/sent-ending-rows.ts new file mode 100644 index 000000000..6b223b5c3 --- /dev/null +++ b/packages/server/src/asking-a-system/sent-ending-rows.ts @@ -0,0 +1,96 @@ +import { expect } from 'vitest'; + +import { calling } from '../testing/servers/system-calls.ts'; +import { + failedAfterCalling, + mayChange, + mayHaveActed, + notAnsweredInTime, + refusingTheArguments, + unfinished, + unknown, + unworkable, + type Ending, + type Row, +} from './ending-rows.ts'; + +export const afterSending: ReadonlyArray = [ + [ + 'arguments a read-only tool refused', + { document: calling('strict', [" limit: '15'"]) }, + unworkable(refusingTheArguments), + ], + [ + 'arguments any other tool refused', + { server: { hints: { strict: mayChange } }, document: calling('strict', [" limit: '15'"]) }, + unworkable(refusingTheArguments), + ], + [ + 'an error of a read-only tool', + { document: calling('denied') }, + unfinished('tool_error', expect.stringMatching(/^The denied tool of chat answered an error: /u)), + ], + [ + 'a failure of the server on a read-only tool', + { document: calling('broken') }, + unfinished('server_failed', failedAfterCalling('broken')), + ], + [ + 'a read-only tool that took longer than a call may', + { document: calling('sleep', [' ms: 5000']) }, + unfinished('server_failed', notAnsweredInTime), + ], + [ + 'an error of any other tool', + { server: { hints: { denied: mayChange } }, document: calling('denied') }, + unknown('tool_error', expect.stringMatching(/^The denied tool of chat answered an error: /u)), + ], + [ + 'a failure of the server on any other tool', + { server: { hints: { broken: mayChange } }, document: calling('broken') }, + unknown('server_failed', failedAfterCalling('broken')), + ], + [ + 'any other tool that took longer than a call may', + { server: { hints: { sleep: mayChange } }, document: calling('sleep', [' ms: 5000']) }, + unknown('server_failed', notAnsweredInTime), + ], +]; + +export const unreadable: ReadonlyArray = [ + [ + 'an answer with no document', + { document: calling('photo') }, + unworkable('The photo tool of chat answered neither structured content nor text, so there is nothing to read'), + ], + [ + 'an answer with nothing at read', + { document: { read: '/nothing' } }, + unworkable('The answer of the thread tool of chat holds nothing at /nothing'), + ], + [ + 'an answer in text that is not JSON', + { document: { ...calling('search', [' query: acme']), read: '/rows' } }, + unworkable('The search tool of chat answered text that is not JSON, which holds nothing at /rows'), + ], + [ + 'an answer past 64 KiB', + { document: calling('large', [' kib: 70']) }, + unworkable('What the large tool of chat answered takes 71682 bytes as JSON, more than the 65536 an answer may'), + ], + [ + 'an answer nested past 512 levels, of a tool that may change something', + { + document: calling('echo', [" deep: { inner: '{{ input.deep }}' }"]), + input: { deep: Array.from({ length: 511 }).reduce((inner) => ({ n: inner }), 'leaf') }, + }, + unworkable(`What the echo tool of chat answered nests deeper than the 512 levels a value may${mayHaveActed}`), + ], + [ + 'an answer the output schema refuses', + { document: { output: ['output:', ' schema: { type: string }'] } }, + unworkable( + expect.stringMatching(/^What the thread tool of chat answered at \/messages does not match the output schema: /u), + ), + ], +]; diff --git a/packages/server/src/asking-a-system/words.test.ts b/packages/server/src/asking-a-system/words.test.ts new file mode 100644 index 000000000..bb683c5d7 --- /dev/null +++ b/packages/server/src/asking-a-system/words.test.ts @@ -0,0 +1,89 @@ +import { internalTermsIn, plainTextIn, withMcpSession, type McpSession } from '@beonauto/api/testing'; +import { describe, expect, it } from 'vitest'; + +import { chatKey } from '../testing/servers/chat-deliveries.ts'; +import { + askingASystem, + calling, + systemRunId, + threadInput, + type SystemServer, +} from '../testing/servers/system-calls.ts'; +import { workflowTestTimeoutMs } from '../testing/servers/workflow-server.ts'; + +const longestName = `thread-${'x'.repeat(41)}`; + +function onAlpha(server: SystemServer, use: (session: McpSession) => Promise): Promise { + return withMcpSession('current revision', { url: `${server.origin}/orgs/acme/brains/alpha/mcp`, headers: {} }, use); +} + +function saidOf(server: SystemServer, name: string, input: unknown = {}) { + return onAlpha(server, async (session) => { + const ran = await session.callTool('run_definition', { type: 'interaction', name, input, run_id: systemRunId }); + const read = await session.callTool('get_run', { run_id: systemRunId }); + return { ran: plainTextIn(ran), read: plainTextIn(read) }; + }); +} + +describe('the words of a run that asks a system, over MCP', { timeout: workflowTestTimeoutMs }, () => { + it('say what the tool answered, naming the tool in words', async () => { + const server = await askingASystem(); + await server.define('thread-replies'); + + expect((await saidOf(server, 'thread-replies', threadInput)).ran).toBe( + 'Ran the interaction function “thread-replies”. It asked the thread tool of chat; its answer: (user: “member-17”, text: “Shipped the fix to staging.”, and ts: “1728380000.000100”) and (user: “member-42”, text: “Thanks, closing the ticket.”, and ts: “1728380100.000200”).', + ); + }); + + it('say why a run whose tool may have changed something did not go through, whole, with the longest name', async () => { + const server = await askingASystem({ hints: { denied: { readOnlyHint: false } } }); + await server.define(longestName, calling('denied')); + + const { ran, read } = await saidOf(server, longestName); + + expect(read).toBe( + `The run of the interaction function “${longestName}” did not go through: it could not finish after calling a tool that may change something, so whether that happened is not known, because the tool answered an error, which the details below give. It is not run again by itself: a person decides, or a workflow rule that names this kind; its history shows the call.`, + ); + expect(ran).toBe( + `Could not run the interaction function “${longestName}”: it could not finish after calling a tool that may change something, so whether that happened is not known, because the tool answered an error, which the details below give. It is not run again by itself: a person decides, or a workflow rule that names this kind; its history shows the call.`, + ); + expect([read.length, ran.length]).toEqual([397, 381]); + }); +}); + +describe('the words of a run that asks a system, kept plain', { timeout: workflowTestTimeoutMs }, () => { + it('name no internal term, and an output carries no secret of the server', async () => { + const server = await askingASystem({ hints: { denied: { readOnlyHint: false } } }); + await server.define('thread-replies'); + await server.define('denying', calling('denied')); + await server.define('echoing', { ...calling('echo', [" said: '{{ input.said }}'"]), read: '/said' }); + const said = await saidOf(server, 'thread-replies', threadInput); + const refused = await onAlpha(server, (session) => + session.callTool('run_definition', { type: 'interaction', name: 'denying', input: {} }), + ); + const echoed = await server.runCall( + 'echoing', + { said: `the key ${chatKey}` }, + '0199a3c4-7d2e-7c1a-9b3f-2f1e0d9c8b7c', + ); + const summaries = (await server.history(systemRunId)).map(({ summary }) => summary); + + expect( + [said.ran, said.read, plainTextIn(refused), ...summaries].flatMap((words) => internalTermsIn(words)), + ).toEqual([]); + expect(echoed.body).toMatchObject({ output: 'the key [redacted]' }); + expect(echoed.text).not.toContain(chatKey); + }); + + it('say that running again is safe for a run whose tools only read, and that nothing was changed', async () => { + const server = await askingASystem(); + await server.define(longestName, calling('denied')); + + const { ran, read } = await saidOf(server, longestName); + + expect(ran).toBe( + `Could not run the interaction function “${longestName}”: it called tools but could not finish, because the tool answered an error, which the details below give. Nothing was changed. Every tool it called only reads, by its server's own account, so running it again is safe: a new run, or a workflow's retry, may make it; its history shows what it called.`, + ); + expect([read.length, ran.length]).toEqual([382, 387]); + }); +}); diff --git a/packages/server/src/composition/composition-root.ts b/packages/server/src/composition/composition-root.ts index d4433da46..1be1d264c 100644 --- a/packages/server/src/composition/composition-root.ts +++ b/packages/server/src/composition/composition-root.ts @@ -20,8 +20,9 @@ export function compositionRootWith( modelAccessOf: FunctionWiring['modelAccessOf'], programPoolOf?: FunctionWiring['programPoolOf'], clock?: HostClock, + toolTiming?: FunctionWiring['toolTiming'], ): ServerOptions { - const wiring = functionWiringOf(modelAccessOf, programPoolOf); + const wiring = functionWiringOf(modelAccessOf, programPoolOf, toolTiming); return { ...defaultServerOptions, runtimeLayer: ({ ledger }) => applicationLayer(ledgerLayerOf(ledger, wiring.recall.appends)), diff --git a/packages/server/src/composition/served-functions.ts b/packages/server/src/composition/served-functions.ts index 13c264d96..b6567f850 100644 --- a/packages/server/src/composition/served-functions.ts +++ b/packages/server/src/composition/served-functions.ts @@ -8,9 +8,9 @@ import type { WorkflowParts } from '../workflows/workflows.ts'; import { computationServedBy, workerPool, type ProgramPoolOf } from './served-computation.ts'; import type { ModelAccessOf } from './served-reasoning.ts'; import { recallWiring, type RecallWiring } from './served-recall.ts'; -import { toolUsersServedBy } from './served-tools.ts'; +import { toolUsersServedBy, type TimedTools } from './served-tools.ts'; -export interface FunctionWiring { +export interface FunctionWiring extends TimedTools { readonly modelAccessOf: ModelAccessOf; readonly programPoolOf: ProgramPoolOf; readonly recall: RecallWiring; @@ -24,16 +24,22 @@ export interface ServedFunctions { export function functionWiringOf( modelAccessOf: ModelAccessOf, programPoolOf: ProgramPoolOf = workerPool, + toolTiming?: FunctionWiring['toolTiming'], ): FunctionWiring { - return { modelAccessOf, programPoolOf, recall: recallWiring() }; + return { modelAccessOf, programPoolOf, recall: recallWiring(), ...(toolTiming === undefined ? {} : { toolTiming }) }; } export async function functionsServedBy( runtime: AppRuntime, settings: Settings, - { modelAccessOf, programPoolOf, recall: wiring }: FunctionWiring, + { modelAccessOf, programPoolOf, recall: wiring, toolTiming }: FunctionWiring, ): Promise { - const { reasoning, interaction, withToolsClosed } = await toolUsersServedBy(runtime, settings, modelAccessOf); + const { reasoning, interaction, withToolsClosed } = await toolUsersServedBy( + runtime, + settings, + modelAccessOf, + toolTiming, + ); const computation = computationServedBy(settings.computation, programPoolOf); const recall = await wiring.served(runtime, settings, computation.pool); return { diff --git a/packages/server/src/composition/served-tools.ts b/packages/server/src/composition/served-tools.ts index 191b454ce..f94188d54 100644 --- a/packages/server/src/composition/served-tools.ts +++ b/packages/server/src/composition/served-tools.ts @@ -1,5 +1,5 @@ import type { AppRuntime } from '@beonauto/api'; -import { makeToolAccess, type ToolAccess } from '@beonauto/mcp'; +import { makeToolAccess, type Timing, type ToolAccess } from '@beonauto/mcp'; import type { DispatcherServices } from '@beonauto/operations'; import type { Served } from '../lifecycle/lifecycle.ts'; @@ -8,14 +8,23 @@ import type { Settings } from '../settings/settings.ts'; import { interactionServedBy, type ServedInteraction } from './served-interaction.ts'; import { reasoningServedBy, type ModelAccessOf, type ServedReasoning } from './served-reasoning.ts'; +export interface TimedTools { + readonly toolTiming?: Timing; +} + export interface ServedToolUsers { readonly reasoning: ServedReasoning; readonly interaction: ServedInteraction; readonly withToolsClosed: (served: Served) => Served; } -function toolAccessOf(runtime: AppRuntime, { mcp }: Pick): ToolAccess { +function toolAccessOf( + runtime: AppRuntime, + { mcp }: Pick, + timing: Timing | undefined, +): ToolAccess { return makeToolAccess(mcp, { + ...(timing === undefined ? {} : { timing }), reportServerMessage: (report) => { void runtime.run(logServerMessage(report)); }, @@ -29,8 +38,9 @@ export async function toolUsersServedBy( runtime: AppRuntime, settings: Settings, modelAccessOf: ModelAccessOf, + timing?: Timing, ): Promise { - const tools = toolAccessOf(runtime, settings); + const tools = toolAccessOf(runtime, settings, timing); return { reasoning: await reasoningServedBy(runtime, settings, modelAccessOf, tools), interaction: interactionServedBy(runtime, settings, tools), diff --git a/packages/server/src/guides/recipes.ts b/packages/server/src/guides/recipes.ts index babb00bdb..3bc0f508a 100644 --- a/packages/server/src/guides/recipes.ts +++ b/packages/server/src/guides/recipes.ts @@ -73,8 +73,10 @@ const interactionGuide = 'interaction-function'; const whereInteractionIsServed: Readonly> = { 'give-tools': { after: 'and which of them it needs.', - words: + words: [ + " When the brain only needs what a tool answers, with nothing to reason about, write an interaction function that calls the tool instead: it answers at once with what the tool answered and spends no model call, and the interaction-function guide says how to write it from the test's answer.", ' When the brain should send a person a message through a tool and take their answer, write an interaction function instead: the interaction-function guide says how it names the tool it sends through and the tool it reads replies with, each tested the same way.', + ].join(''), }, 'first-brain': { after: 'give it tools with the give-tools recipe', diff --git a/packages/server/src/guides/recipes/give-tools.md b/packages/server/src/guides/recipes/give-tools.md index e7c3f0570..fe3d56ee9 100644 --- a/packages/server/src/guides/recipes/give-tools.md +++ b/packages/server/src/guides/recipes/give-tools.md @@ -5,7 +5,7 @@ A reasoning function can call the tools of the tool servers that whoever runs th 1. Call list_tool_servers to see the tool servers this brain may use and the tools each offers; when the person named a server, give its name as `server`. 2. When no server is listed, tell the person that whoever runs this server sets tool servers up in its settings file, `auto-brain.yaml`, under `mcp_servers`: the server's address, a header that takes its key from an environment variable, and the org it serves, with `allowed` naming the tools a function may call. The server reads them when it starts again. Nothing more can be done here until then. 3. When a server says it cannot be asked just now, tell the person why, in its words; whoever runs the server can look into it. -4. To learn what a tool answers, test it with test_tool_call, with the arguments its input_schema takes, and read the answer: that is what the function's model will see. Never make a function to look. When a prompt needs an id, such as a channel's, test the tool that lists them and take the id from its answer, confirming the choice with the person. A tool that cannot be tested may change something; list_tool_servers says which can. +4. To learn what a tool answers, test it with test_tool_call, with the arguments its input_schema takes, and read its `text`: that is what the function's model will see. Never make a function to look. When a prompt needs an id, such as a channel's, test the tool that lists them and take the id from its answer, confirming the choice with the person. A tool that cannot be tested may change something; list_tool_servers says which can. 5. Ask the person what the function should do with the tools, and which of them it needs. 6. Write a reasoning function whose `tools` name those tools in the format of the reasoning-function guide, show it to the person, and save it with create_definition, `type` reasoning, once they agree. 7. Run it with run_definition. A run that called tools and did not succeed is not run again under its id, since a tool may have changed something: read what it called with get_run_history, and start a new run only if the person still wants one. diff --git a/packages/server/src/guides/served-guides.test.ts b/packages/server/src/guides/served-guides.test.ts index 7cc9912ad..1f0725449 100644 --- a/packages/server/src/guides/served-guides.test.ts +++ b/packages/server/src/guides/served-guides.test.ts @@ -175,7 +175,7 @@ describe('the recipes that give a brain tools', () => { const texts = Object.fromEntries(servedGuidesOf(everyType).recipes.map(({ name, text }) => [name, text])); expect(String(texts['give-tools'])).toContain( - "4. To learn what a tool answers, test it with test_tool_call, with the arguments its input_schema takes, and read the answer: that is what the function's model will see. Never make a function to look. When a prompt needs an id, such as a channel's, test the tool that lists them and take the id from its answer, confirming the choice with the person. A tool that cannot be tested may change something; list_tool_servers says which can.", + "4. To learn what a tool answers, test it with test_tool_call, with the arguments its input_schema takes, and read its `text`: that is what the function's model will see. Never make a function to look. When a prompt needs an id, such as a channel's, test the tool that lists them and take the id from its answer, confirming the choice with the person. A tool that cannot be tested may change something; list_tool_servers says which can.", ); expect(String(texts['give-tools'])).toContain( 'the org it serves, with `allowed` naming the tools a function may call. The server reads them when it starts again.', @@ -194,7 +194,7 @@ describe('the recipes of a server that serves interaction functions', () => { const texts = recipeTextsOf([...everyType, interaction]); expect(texts['give-tools']).toContain( - '5. Ask the person what the function should do with the tools, and which of them it needs. When the brain should send a person a message through a tool and take their answer, write an interaction function instead: the interaction-function guide says how it names the tool it sends through and the tool it reads replies with, each tested the same way.\n', + "5. Ask the person what the function should do with the tools, and which of them it needs. When the brain only needs what a tool answers, with nothing to reason about, write an interaction function that calls the tool instead: it answers at once with what the tool answered and spends no model call, and the interaction-function guide says how to write it from the test's answer. When the brain should send a person a message through a tool and take their answer, write an interaction function instead: the interaction-function guide says how it names the tool it sends through and the tool it reads replies with, each tested the same way.\n", ); expect(texts['first-brain']).toContain( 'give it tools with the give-tools recipe, send someone a message through a tool and take their answer with an interaction function, make the brain remember its answers', @@ -203,7 +203,7 @@ describe('the recipes of a server that serves interaction functions', () => { bytesOf(texts, 'first-brain', 'remember', 'give-tools', 'schedule'), bytesOf(recipeTextsOf(everyType), 'first-brain', 'remember', 'give-tools', 'schedule'), ]).toEqual([ - [1854, 1762, 2309, 1285], + [1854, 1762, 2598, 1285], [1764, 1762, 2049, 1285], ]); }); diff --git a/packages/server/src/guides/terminology-guide.test.ts b/packages/server/src/guides/terminology-guide.test.ts index e390e044b..f05c3d886 100644 --- a/packages/server/src/guides/terminology-guide.test.ts +++ b/packages/server/src/guides/terminology-guide.test.ts @@ -4,6 +4,8 @@ import { describe, expect, it } from 'vitest'; import { terminologyGuideOf } from './terminology-guide.ts'; +const theRequestTerm = 'Interaction:\n\n- Request: What a run of an interaction function that asks a person asks'; + const page = readFileSync(new URL('../../../../docs/concepts/terminology.md', import.meta.url), 'utf8'); describe('the terminology guide', () => { @@ -30,7 +32,7 @@ describe('the terminology guide', () => { const interacting = terminologyGuideOf(page, ['interaction function', 'workflow']).text; const notInteracting = terminologyGuideOf(page, ['workflow']).text; - expect(interacting).toContain('Interaction:\n\n- Request: What a run of an interaction function asks'); + expect(interacting).toContain(theRequestTerm); expect(notInteracting).not.toContain('Interaction:'); expect(notInteracting).not.toContain('- Inbox:'); }); diff --git a/packages/server/src/interaction/replies-over-http.test.ts b/packages/server/src/interaction/replies-over-http.test.ts index 02ad3ce83..c9fa5d561 100644 --- a/packages/server/src/interaction/replies-over-http.test.ts +++ b/packages/server/src/interaction/replies-over-http.test.ts @@ -51,7 +51,7 @@ describe( }, ]); expect(events).toContain( - 'The brain looked for new replies in the conversation “#approvals-ada/1699.000001” through the tool thread_replies of chat and found 2, took 1 as an answer and refused 0.', + 'The brain looked for new replies in the conversation “#approvals-ada/1699.000001” through the thread replies tool of chat and found 2, took 1 as an answer and refused 0.', ); }); diff --git a/packages/server/src/interaction/tool-disallowed.test.ts b/packages/server/src/interaction/tool-disallowed.test.ts index d43778251..479f994dd 100644 --- a/packages/server/src/interaction/tool-disallowed.test.ts +++ b/packages/server/src/interaction/tool-disallowed.test.ts @@ -55,7 +55,7 @@ describe('a tool its operator disallowed after a request asked through it', { ti number: 1, outcome: 'failed', because: 'server_failure', - detail: 'The MCP server could not be reached', + detail: 'The MCP server chat could not be used: The MCP server could not be reached', duration_ms: aNumber, }); expect(facts.at(-1)).toMatchObject({ diff --git a/packages/server/src/interaction/tool-refusals.test.ts b/packages/server/src/interaction/tool-refusals.test.ts index ccc653fb5..27a6efd2a 100644 --- a/packages/server/src/interaction/tool-refusals.test.ts +++ b/packages/server/src/interaction/tool-refusals.test.ts @@ -12,8 +12,6 @@ const decodeHistory = Schema.decodeUnknownSync( Schema.Struct({ events: Schema.Array(Schema.Struct({ type: Schema.String })) }), ); -const namingTheTool: unknown = expect.stringContaining('post_message'); - function asked(server: Awaited>) { return server.call('POST', `${alpha}/definitions/interaction/approve-brief/run`, { body: { input: brief, run_id: '0199a3c4-7d2e-7c1a-9b3f-2f1e0d9c8b7a' }, @@ -54,7 +52,7 @@ describe('a delivery through a tool this brain may not use, over HTTP', { timeou expect(chat.received()).toEqual([]); }); - it('fails an attempt as a tool error when the server no longer lists the tool, and tries again', async () => { + it('fails an attempt as a tool no longer offered when the server no longer lists the tool, sending nothing, and tries again', async () => { const chat = await chatServer(); chat.removeTool('post_message'); const server = await servingInteractions(chatDelivery, chatEnvironment(chat.url)); @@ -65,7 +63,12 @@ describe('a delivery through a tool this brain may not use, over HTTP', { timeou (seen) => seen.length >= 2, ); - expect(facts.at(-1)).toMatchObject({ outcome: 'failed', because: 'tool_error', detail: namingTheTool }); + expect(facts.at(-1)).toMatchObject({ + outcome: 'failed', + because: 'tool_not_offered', + detail: 'The MCP server chat does not list the tool post_message', + }); + expect(chat.received()).toEqual([]); expect((await server.call('GET', `${alpha}/interactions`)).body).toMatchObject({ interactions: [{ standing: 'retrying' }], }); diff --git a/packages/server/src/mcp/answering-what-runs-wait-on.test.ts b/packages/server/src/mcp/answering-what-runs-wait-on.test.ts index 9a18824af..d6fb0750e 100644 --- a/packages/server/src/mcp/answering-what-runs-wait-on.test.ts +++ b/packages/server/src/mcp/answering-what-runs-wait-on.test.ts @@ -163,7 +163,7 @@ describe( "When the person approves, rejects or otherwise answers what a run waits on, answer its request with answer_interaction, in the shape its function's answer takes, and start no new run for it.", ); expect(sentenceNaming(instructions, 'get_run')).toBe( - 'A run of an interaction function or a workflow answers started; get_run shows whether it ended or still waits.', + 'A run of a workflow answers started; get_run shows whether it ended or still waits.', ); expect(instructions).not.toMatch(/\bpoll|\buntil (?:it ends|its status changes)/iu); expect(descriptionIn(meetings.surfaces, 'answer_interaction')).toContain( diff --git a/packages/server/src/mcp/asking-in-a-chat.test.ts b/packages/server/src/mcp/asking-in-a-chat.test.ts index e65698323..203935f24 100644 --- a/packages/server/src/mcp/asking-in-a-chat.test.ts +++ b/packages/server/src/mcp/asking-in-a-chat.test.ts @@ -13,6 +13,8 @@ const namingTheMessage: unknown = expect.stringContaining('"ts":"1699.000001"'); const listingTheMessages: unknown = expect.stringContaining('"messages":['); +const someMessages: unknown = expect.any(Array); + const inMeetings = { brain: 'meetings' }; const approvingInTheChat = [ @@ -118,7 +120,7 @@ describe( const seen = await meetings.onMcp(askedToSendADraft); expect(sentenceNaming(meetings.surfaces.instructions, 'An interaction function')).toBe( - 'An interaction function asks a person or a system and takes the answer later.', + 'An interaction function asks a system and answers at once, or asks a person and answers started.', ); expect([seen.sent.structuredContent, seen.read.structuredContent]).toMatchObject([ { outcome: 'result', text: namingTheMessage }, @@ -139,3 +141,65 @@ describe( }); }, ); + +const readingAThread = [ + '---', + 'description: Read the replies of a thread in the team chat, oldest first', + 'call:', + ' server: chat', + ' tool: thread_replies', + ' with:', + " channel: '{{ input.channel }}'", + " ts: '{{ input.thread }}'", + ' read: /messages', + 'input:', + ' schema: { type: object, required: [channel, thread], properties: { channel: { type: string }, thread: { type: string } } }', + 'output:', + ' schema:', + ' type: array', + ' items: { type: object, required: [user, ts], properties: { user: { type: string }, text: { type: string }, ts: { type: string } } }', + '---', + '', +].join('\n'); + +async function askedForTheRepliesOfAThread(session: McpSession) { + const recipe = textOf(await session.callTool('get_guide', { guide: 'give-tools' })); + const tested = await session.callTool('test_tool_call', { + ...inMeetings, + server: 'chat', + tool: 'thread_replies', + arguments: { channel: '#drafts', ts: '1699.000001' }, + }); + const saved = await session.callTool('create_definition', { + ...inMeetings, + type: 'interaction', + name: 'thread-replies', + source: readingAThread, + }); + const reasoning = await session.callTool('list_definitions', { ...inMeetings, type: 'reasoning' }); + return { recipe, tested, saved, reasoning }; +} + +describe( + 'episode 10: asked to put the replies of a thread into the weekly summary', + { timeout: workflowTestTimeoutMs }, + () => { + it('reads the interaction sentence and the give-tools sentence, tests the tool, reads its answer, and saves one interaction function that calls it', async () => { + const seen = await meetings.onMcp(askedForTheRepliesOfAThread); + + expect(sentenceNaming(meetings.surfaces.instructions, 'An interaction function')).toBe( + 'An interaction function asks a system and answers at once, or asks a person and answers started.', + ); + expect(seen.recipe).toContain( + 'When the brain only needs what a tool answers, with nothing to reason about, write an interaction function that calls the tool instead', + ); + expect(seen.tested.structuredContent).toMatchObject({ + outcome: 'result', + text: listingTheMessages, + answer: { messages: someMessages }, + }); + expect(seen.saved.isError).not.toBe(true); + expect(seen.reasoning.structuredContent).toEqual({ definitions: [] }); + }); + }, +); diff --git a/packages/server/src/mcp/mcp-endpoint-workflows.test.ts b/packages/server/src/mcp/mcp-endpoint-workflows.test.ts index ac9812ede..77e8e8844 100644 --- a/packages/server/src/mcp/mcp-endpoint-workflows.test.ts +++ b/packages/server/src/mcp/mcp-endpoint-workflows.test.ts @@ -87,7 +87,7 @@ describe('/mcp with workflows', { timeout: workflowTestTimeoutMs }, () => { expect(served.tools).toHaveLength(25); expect(served.tools.slice(-2)).toEqual(['send_run_event', 'get_guide']); expect(served.instructions).toContain( - 'or a workflow answers started; get_run shows whether it ended or still waits.', + 'A run of a workflow answers started; get_run shows whether it ended or still waits.', ); }); diff --git a/packages/server/src/mcp/served-tools.test.ts b/packages/server/src/mcp/served-tools.test.ts index 6d5047a94..2e3da7517 100644 --- a/packages/server/src/mcp/served-tools.test.ts +++ b/packages/server/src/mcp/served-tools.test.ts @@ -105,6 +105,14 @@ describe('the descriptions of the tools of /mcp', () => { tools.filter((tool) => descriptionOf(tool.name).length > description.length).map(({ name }) => name), ).toEqual([]); }); + + it('say that a function that calls a tool may change something, in run_definition at 765 characters, and that a run that ends within its call cannot be cancelled, in cancel_run at 671', () => { + const [running, cancelling] = [descriptionOf('run_definition'), descriptionOf('cancel_run')]; + + expect(running).toContain('A function that calls a tool may change something outside the brain'); + expect(cancelling).toContain('or an interaction function that calls a tool, cannot be cancelled.'); + expect([running.length, cancelling.length]).toEqual([765, 671]); + }); }); function everyArgument(): readonly Argument[] { diff --git a/packages/server/src/reasoning/tool-tests-over-mcp.test.ts b/packages/server/src/reasoning/tool-tests-over-mcp.test.ts index 515e79dac..4c309b2fe 100644 --- a/packages/server/src/reasoning/tool-tests-over-mcp.test.ts +++ b/packages/server/src/reasoning/tool-tests-over-mcp.test.ts @@ -72,7 +72,7 @@ describe('test_tool_call over MCP', () => { expect([tested.structuredContent, onTheBrain.structuredContent]).toMatchObject([answered, answered]); expect(plainTextIn(tested)).toMatch( - /^The tool “search” of “graph” answered in [\d,]+ ms with \d+ bytes; what a reasoning function's model would see is in the details\.$/u, + /^The search tool of graph answered in [\d,]+ ms with \d+ bytes; what a reasoning function's model would see is in the details\.$/u, ); expect(listedTools(onTheOrg).map(({ name }) => name)).not.toContain('test_tool_call'); }); @@ -104,7 +104,7 @@ describe('test_tool_call refused over MCP', () => { because: 'not_testable', }); expect(plainTextIn(refused)).toBe( - "Could not test the tool “echo” of “graph”: this server does not offer a tool it names, because by its server's own account it may change something, and whoever runs the server has not listed it as safe to test. Nothing was changed. A tool that may change something is called only by a function the person asked to run; whoever runs the server can mark it testable on its tool server's entry, and list_tool_servers shows which tools can be tested.", + "Could not test the echo tool of graph: this server does not offer a tool it names, because by its server's own account it may change something, and whoever runs the server has not listed it as safe to test. Nothing was changed. A tool that may change something is called only by a function the person asked to run; whoever runs the server can mark it testable on its tool server's entry, and list_tool_servers shows which tools can be tested.", ); expect(graph.received()).toEqual([]); }); @@ -118,7 +118,7 @@ describe('test_tool_call refused over MCP', () => { expect(problemIn(refused)).toMatchObject({ kind: 'tool_not_offered', because: 'mcp_server_not_configured' }); expect(plainTextIn(refused)).toBe( - 'Could not test the tool “search” of “wiki”: this server does not offer a tool it names, because whoever runs the server has not set up a tool server of that name for this brain. Nothing was changed. This can be put right on your side: list_tool_servers shows the tool servers this brain may use, so a test that names one of those can be tried.', + 'Could not test the search tool of wiki: this server does not offer a tool it names, because whoever runs the server has not set up a tool server of that name for this brain. Nothing was changed. This can be put right on your side: list_tool_servers shows the tool servers this brain may use, so a test that names one of those can be tried.', ); expect(graph.seen()).toEqual([]); }); diff --git a/packages/server/src/reasoning/tools-over-mcp.test.ts b/packages/server/src/reasoning/tools-over-mcp.test.ts index dc059aa5e..5d9d107af 100644 --- a/packages/server/src/reasoning/tools-over-mcp.test.ts +++ b/packages/server/src/reasoning/tools-over-mcp.test.ts @@ -135,15 +135,33 @@ describe('a reasoning function with tools over MCP', () => { }); }, ); +}); - it('says in plain words that a run that called tools may have changed something, and is not run again', async () => { +describe('the words of a reasoning run over MCP that could not finish after calling tools', () => { + it('says in plain words that a run whose tools only read may run again, though not under its id', async () => { const fake = await fakeGraph(); const { first, again } = await executedTwice( await serving(fake, [callingTools([['mcp__graph__search', { query: 'acme' }]], timedOut)]), ); expect(plainTextIn(first)).toBe( - 'Could not run the reasoning function “summary”: it called tools but could not finish, because the model stopped answering. What it called may have changed something, so it is not run again by itself: check what its history shows it called, then start a new run if it is still needed.', + "Could not run the reasoning function “summary”: it called tools but could not finish, because the model stopped answering. Nothing was changed. Every tool it called only reads, by its server's own account, so running it again is safe: a new run, or a workflow's retry, may make it; its history shows what it called.", + ); + expect(plainTextIn(again)).toBe( + "Could not run the reasoning function “summary”: an attempt of this run under the same id did not succeed, and every tool it called only reads, by its server's own account. Nothing was changed. So it was not run again under its id: start a new run instead; its history shows what it called.", + ); + expect(fake.received()).toHaveLength(1); + }); + + it('says in plain words that a run whose tool may have changed something is not run again by itself', async () => { + const fake = await fakeGraph(); + const { first, again } = await executedTwice( + await serving(fake, [callingTools([['mcp__graph__echo', { said: 'acme' }]], timedOut)]), + source.replace('graph/search', 'graph/echo'), + ); + + expect(plainTextIn(first)).toBe( + 'Could not run the reasoning function “summary”: it could not finish after calling a tool that may change something, so whether that happened is not known, because the model stopped answering. It is not run again by itself: a person decides, or a workflow rule that names this kind; its history shows the call.', ); expect(plainTextIn(again)).toMatch( /^Could not run the reasoning function “summary”: this run calls tools, and an attempt of it under the same id may still be in progress or did not succeed, so its tools may have changed something\. So it was not run again/u, diff --git a/packages/server/src/reasoning/unfinished-tools-over-http.test.ts b/packages/server/src/reasoning/unfinished-tools-over-http.test.ts new file mode 100644 index 000000000..094433ba8 --- /dev/null +++ b/packages/server/src/reasoning/unfinished-tools-over-http.test.ts @@ -0,0 +1,165 @@ +import { setTimeout } from 'node:timers/promises'; + +import { serveFakeMcp, type FakeHints } from '@beonauto/mcp/testing'; +import { TimedOut, ToolsStopped, type ToolsStoppedBecause } from '@beonauto/reasoning'; +import { answers, callingTools, textResult, type ScriptedCall, type ScriptedReply } from '@beonauto/reasoning/testing'; +import { Effect } from 'effect'; +import { describe, expect, it, onTestFinished } from 'vitest'; + +import { alpha, servingReasoning, type ReasoningServer } from '../testing/servers/reasoning-server.ts'; +import { runIdIn, settledRun, workflowSource, workflowTestTimeoutMs } from '../testing/servers/workflow-server.ts'; + +const apiKey = 'graph-api-key-4f1d9a7c2b'; + +const runId = '0199a3c4-7d2e-7c1a-9b3f-2f1e0d9c8b7a'; + +const source = ['---', 'model: anthropic/claude-sonnet-4-5', 'tools: [graph/*]', '---', 'Summarize acme.'].join('\n'); + +const stoppedBy = + (because: ToolsStoppedBecause): ScriptedReply => + () => + Effect.fail(new ToolsStopped({ detail: 'The run stopped', provider: 'anthropic', because })); + +const timedOut: ScriptedReply = () => + Effect.fail( + new TimedOut({ detail: 'anthropic did not answer within 60000 ms', provider: 'anthropic', timeout_ms: 60_000 }), + ); + +const searched: ScriptedCall = ['mcp__graph__search', { query: 'acme' }]; + +const echoed: ScriptedCall = ['mcp__graph__echo', { said: 'acme' }]; + +const broken = [1, 2, 3, 4, 5].map((attempt): ScriptedCall => ['mcp__graph__broken', { attempt }]); + +async function serving(replies: readonly ScriptedReply[], hints: FakeHints = {}): Promise { + const fake = await serveFakeMcp({ bearer: apiKey, hints }); + onTestFinished(fake.close); + const server = await servingReasoning(replies, { + LOCAL_MODE: 'true', + GRAPH_API_KEY: apiKey, + MCP_SERVERS: JSON.stringify({ + graph: { url: fake.url, headers: { Authorization: 'Bearer ${GRAPH_API_KEY}' }, org: 'acme' }, + }), + }); + onTestFinished(server.stop); + await server.call('POST', '/v1/orgs/acme/brains', { body: { brain: 'alpha', name: 'Alpha' } }); + await server.call('POST', `${alpha}/definitions/reasoning`, { body: { name: 'summary', source } }); + return server; +} + +async function endedTwice(reply: ScriptedReply, hints?: FakeHints) { + const server = await serving([reply], hints); + const running = { body: { input: {}, run_id: runId } }; + const first = await server.call('POST', `${alpha}/definitions/reasoning/summary/run`, running); + const again = await server.call('POST', `${alpha}/definitions/reasoning/summary/run`, running); + return [first.status, first.body, again.status]; +} + +function endedAs(status: number, kind: string, because: string): readonly unknown[] { + return [status, expect.objectContaining({ kind, because }), 409]; +} + +describe('a reasoning run that called only tools that read and could not finish, over HTTP', () => { + it.each([ + ['five server failures', callingTools(broken, stoppedBy('server_failed')), 'server_failed'], + ['its run bound', callingTools([searched], stoppedBy('run_bound')), 'run_bound'], + ['a model that stopped answering', callingTools([searched], timedOut), 'model_unavailable'], + ['a final step that still called tools', callingTools([searched], stoppedBy('no_answer')), 'no_answer'], + ])('is tools_unfinished, 503, after %s, and tools_called under its id', async (_case, reply, because) => { + expect(await endedTwice(reply)).toEqual(endedAs(503, 'tools_unfinished', because)); + }); +}); + +async function untilCalled(calledAny: () => boolean): Promise { + if (!calledAny()) { + await setTimeout(10); + await untilCalled(calledAny); + } +} + +const unanswered: ScriptedReply = (request) => + Effect.promise(async () => { + const sleeping = request.tools?.offered.find(({ name }) => name === 'mcp__graph__sleep'); + void sleeping?.call( + { callId: 'call-1', input: { ms: 5000 } }, + { signal: AbortSignal.timeout(10_000), cancelled: new AbortController().signal }, + ); + await untilCalled(() => request.tools?.calledAny() !== false); + }).pipe(Effect.andThen(stoppedBy('run_bound')(request))); + +const mayChange: ReadonlyArray = [ + ['before the tools that read', callingTools([echoed, searched], timedOut), 'model_unavailable', {}], + ['after the tools that read', callingTools([searched, echoed], stoppedBy('no_answer')), 'no_answer', {}], + [ + 'whose servers then kept failing', + callingTools([echoed, ...broken], stoppedBy('server_failed')), + 'server_failed', + {}, + ], + ['and is still waiting for it', unanswered, 'run_bound', { sleep: { idempotentHint: true } }], +]; + +describe('a reasoning run that called a tool that may change something and could not finish, over HTTP', () => { + it.each(mayChange)( + 'is effect_unknown, 409, when it called one %s, and tools_called under its id', + async (_case, reply, because, hints) => { + expect(await endedTwice(reply, hints)).toEqual(endedAs(409, 'effect_unknown', because)); + }, + ); + + it('ends as before when it called no tool', async () => { + const [status, body] = await endedTwice(timedOut); + + expect(status).toBe(503); + expect(body).not.toHaveProperty('kind'); + }); +}); + +const retryingOn503 = workflowSource( + 'reviewing', + `do: + - review: + try: + - summarize: { call: run_definition, with: { type: reasoning, name: summary, input: {} } } + catch: + errors: + with: { status: 503 } + retry: + delay: PT0.01S + limit: + attempt: { count: 3 } +`, +); + +async function reviewed(replies: readonly ScriptedReply[]) { + const server = await serving(replies); + await server.call('POST', `${alpha}/definitions/workflow`, { body: { name: 'reviewing', source: retryingOn503 } }); + const started = await server.call('POST', `${alpha}/definitions/workflow/reviewing/run`, { body: { input: {} } }); + const ended = await settledRun(server, `${alpha}/runs/${runIdIn(started.body)}`); + return { ended: ended.body, modelCalls: server.modelCalls() }; +} + +describe( + 'a workflow whose reasoning step called tools and could not finish', + { timeout: workflowTestTimeoutMs }, + () => { + it('never retries on status 503 a step whose tool may have changed something, raised at 409 under its own type', async () => { + expect(await reviewed([callingTools([echoed], timedOut)])).toMatchObject({ + ended: { + status: 'rejected', + rejection: { reason: 'conflict', kind: 'effect_unknown', because: 'model_unavailable' }, + }, + modelCalls: 1, + }); + }); + + it('retries on status 503 a step whose tools only read', async () => { + const unfinished = callingTools([searched], timedOut); + + expect(await reviewed([unfinished, unfinished, answers(textResult('Acme has 2 rows.'))])).toMatchObject({ + ended: { status: 'succeeded', output: 'Acme has 2 rows.' }, + modelCalls: 3, + }); + }); + }, +); diff --git a/packages/server/src/testing/servers/interaction-server.ts b/packages/server/src/testing/servers/interaction-server.ts index b867e4576..f1299c8a1 100644 --- a/packages/server/src/testing/servers/interaction-server.ts +++ b/packages/server/src/testing/servers/interaction-server.ts @@ -2,7 +2,7 @@ import { approvalDocument, notificationDocument } from '@beonauto/interaction/te import { Schema } from 'effect'; import { onTestFinished } from 'vitest'; -import { alpha, servingReasoning, type ReasoningServer } from './reasoning-server.ts'; +import { alpha, noNetwork, servingReasoning, type ReasoningServer, type ServedParts } from './reasoning-server.ts'; import { until } from './workflow-calls.ts'; import { runIdIn, settledRun, workflowSource } from './workflow-server.ts'; @@ -45,8 +45,11 @@ function once(stop: () => Promise): () => Promise { }; } -export async function interactionServerOn(environment: Readonly>): Promise { - const server = await servingReasoning([], { LOCAL_MODE: 'true', ...environment }); +export async function interactionServerOn( + environment: Readonly>, + parts: ServedParts = {}, +): Promise { + const server = await servingReasoning([], { LOCAL_MODE: 'true', ...environment }, noNetwork, parts); const stop = once(server.stop); onTestFinished(stop); return { diff --git a/packages/server/src/testing/servers/reasoning-server.ts b/packages/server/src/testing/servers/reasoning-server.ts index 016d4d2bc..e1711fc2c 100644 --- a/packages/server/src/testing/servers/reasoning-server.ts +++ b/packages/server/src/testing/servers/reasoning-server.ts @@ -1,3 +1,4 @@ +import type { Timing } from '@beonauto/mcp'; import { makeModelAccess } from '@beonauto/reasoning'; import { scriptedLanguageModel, type ScriptedReply } from '@beonauto/reasoning/testing'; import type { HostClock } from '@beonauto/workflow-host'; @@ -20,6 +21,7 @@ export interface ReasoningServer { export interface ServedParts { readonly programPoolOf?: ProgramPoolOf; readonly clock?: HostClock; + readonly toolTiming?: Timing; } export const alpha = '/v1/orgs/acme/brains/alpha'; @@ -28,13 +30,13 @@ const localMode: Readonly> = { LOCAL_MODE: 'true' }; type Fetch = typeof globalThis.fetch; -const noNetwork: Fetch = () => Promise.reject(new TypeError('fetch failed: the tests reach no network')); +export const noNetwork: Fetch = () => Promise.reject(new TypeError('fetch failed: the tests reach no network')); export async function servingReasoning( replies: readonly ScriptedReply[], environment: Readonly> = localMode, fetch: Fetch = noNetwork, - { programPoolOf = workerPool, clock }: ServedParts = {}, + { programPoolOf = workerPool, clock, toolTiming }: ServedParts = {}, ): Promise { const ledger = temporaryLedger(); const scripted = scriptedLanguageModel(...replies); @@ -48,6 +50,7 @@ export async function servingReasoning( })), programPoolOf, clock, + toolTiming, ), ); return { diff --git a/packages/server/src/testing/servers/system-calls.ts b/packages/server/src/testing/servers/system-calls.ts new file mode 100644 index 000000000..247e25ead --- /dev/null +++ b/packages/server/src/testing/servers/system-calls.ts @@ -0,0 +1,77 @@ +import { callDocument, type CallOptions } from '@beonauto/interaction/testing'; +import type { Timing } from '@beonauto/mcp'; +import { serveFakeMcp, type FakeHints, type FakeMcpServer } from '@beonauto/mcp/testing'; +import { Schema } from 'effect'; +import { expect, onTestFinished } from 'vitest'; + +import { chatKey } from './chat-deliveries.ts'; +import type { TestResponse } from './http-client.ts'; +import { interactionServerOn, type InteractionServer } from './interaction-server.ts'; +import { alpha } from './reasoning-server.ts'; + +export const threadInput = { channel: 'C0123', thread: '1728379900.000050' }; + +export const systemRunId = '0199a3c4-7d2e-7c1a-9b3f-2f1e0d9c8b7a'; + +const anyOutput: readonly string[] = ['output:', ' schema: {}']; + +export const openInput: readonly string[] = ['input:', ' schema: { type: object }']; + +export function calling(tool: string, written: readonly string[] = []): CallOptions { + return { tool, read: null, with: written, input: openInput, output: anyOutput }; +} + +export function problemWith(fields: Readonly>): unknown { + return expect.objectContaining(fields); +} + +export interface SystemOptions { + readonly hints?: FakeHints; + readonly entry?: Readonly>; + readonly servers?: Readonly>; + readonly timing?: Timing; + readonly environment?: Readonly>; +} + +export interface SystemServer extends InteractionServer { + readonly fake: FakeMcpServer; + readonly define: (name: string, document?: CallOptions) => Promise; + readonly runCall: (name: string, input?: unknown, runId?: string) => Promise; + readonly history: (runId: string) => Promise; +} + +const HistorySchema = Schema.Struct({ + events: Schema.Array( + Schema.Struct({ type: Schema.String, summary: Schema.String, data: Schema.Record(Schema.String, Schema.Unknown) }), + ), +}); + +export type HistoryEvent = (typeof HistorySchema.Type)['events'][number]; + +const decodeHistory = Schema.decodeUnknownSync(HistorySchema); + +export async function askingASystem({ + hints = {}, + entry = {}, + servers = {}, + timing, + environment = {}, +}: SystemOptions = {}): Promise { + const fake = await serveFakeMcp({ bearer: chatKey, data: true, hints }); + onTestFinished(fake.close); + const chat = { url: fake.url, headers: { Authorization: 'Bearer ${CHAT_KEY}' }, org: 'acme', ...entry }; + const server = await interactionServerOn( + { CHAT_KEY: chatKey, MCP_SERVERS: JSON.stringify({ chat, ...servers }), ...environment }, + timing === undefined ? {} : { toolTiming: timing }, + ); + await server.call('POST', '/v1/orgs/acme/brains', { body: { brain: 'alpha', name: 'Alpha' } }); + return { + ...server, + fake, + define: (name, document = {}) => + server.call('POST', `${alpha}/definitions/interaction`, { body: { name, source: callDocument(document) } }), + runCall: (name, input = threadInput, runId = systemRunId) => + server.call('POST', `${alpha}/definitions/interaction/${name}/run`, { body: { input, run_id: runId } }), + history: async (runId) => decodeHistory((await server.call('GET', `${alpha}/runs/${runId}/history`)).body).events, + }; +} diff --git a/packages/server/src/workflows/workflows.test.ts b/packages/server/src/workflows/workflows.test.ts index 4751b1e05..3cd290952 100644 --- a/packages/server/src/workflows/workflows.test.ts +++ b/packages/server/src/workflows/workflows.test.ts @@ -1,3 +1,4 @@ +import { noToolServers } from '@beonauto/mcp/testing'; import { makeReasoningFunctionAdapter } from '@beonauto/reasoning'; import { scriptedLanguageModel } from '@beonauto/reasoning/testing'; import { describe, expect, it } from 'vitest'; @@ -9,6 +10,7 @@ describe('the longest a nested run of a workflow may run', () => { const reasoning = makeReasoningFunctionAdapter({ languageModel: scriptedLanguageModel().languageModel, offered: { providers: [], aliases: [] }, + tools: noToolServers, }); expect(longestCallOf([reasoning])).toBe(1_720_000); diff --git a/packages/workflow-engine/README.md b/packages/workflow-engine/README.md index a83a42246..a7127dfb2 100644 --- a/packages/workflow-engine/README.md +++ b/packages/workflow-engine/README.md @@ -334,7 +334,7 @@ Each sentence is something a reviewer can check against the code or a test. **[e - A run is a tree of frames, one for each task that waits, under the root list. An input becomes a signal, a timer fired, a call answered or events arrived, passed down the tree; the frame whose timer, call or listen it is resumes, and the others stay as they are. - A task starts as soon as its list reaches it, and finishes in the same input unless it waits: on a timer (a wait, a retry's delay, an attempt's limit, a timeout), on a call or on events. A list that ran 100 tasks, or 8,000,000 units of expression work, in one input arms a timer due at once and goes on when it fires (`src/runner/list-runner.ts`). - `started` arms the run's deadline, due when the run has run the most it may, `mostDurationMs`; its fire ends the run `overran`, settled as `cancelled` of the kind `overrun`. A call arms its `call_deadline` and keeps its id. A cancel, an end or a timeout cancels what the frames under it wait for, and a task that raises before it waits disarms its own timeout. A call cancelled by a timeout, an attempt's limit or its own deadline is cancelled with the reason `deadline`, and one cancelled because its run ended or its branch lost a race with the reason `parent_ended`. -- A run ends `completed`, settled `succeeded`; `raised`, settled `rejected` by the error's status, or as `conflict` of the kind `tools_called` or `unavailable` of the kind `tools_unfinished` for those types; an uncaught error of the type `cancelled`, raised by a call whose run was cancelled, settles by its status too, 409, since the workflow itself was not cancelled; an uncaught error of the type `unanswered`, raised by a call whose request nobody answered, status 410, settles `rejected` as `unanswered` with its kind, `expired` or `undelivered`, as the request it waited for ended; `cancelled`, settled `rejected` as `cancelled` with the cancel's kind, reason and actor; `oversized`, settled `rejected` as a `conflict` of the kind `oversized`; and `broken`, settled `failed`. +- A run ends `completed`, settled `succeeded`; `raised`, settled `rejected` by the error's status, or as `conflict` of the kind `tools_called` or `effect_unknown`, or `unavailable` of the kind `tools_unfinished`, for those types, the last two with the error's `because`; an uncaught error of the type `cancelled`, raised by a call whose run was cancelled, settles by its status too, 409, since the workflow itself was not cancelled; an uncaught error of the type `unanswered`, raised by a call whose request nobody answered, status 410, settles `rejected` as `unanswered` with its kind, `expired` or `undelivered`, as the request it waited for ended; `cancelled`, settled `rejected` as `cancelled` with the cancel's kind, reason and actor; `oversized`, settled `rejected` as a `conflict` of the kind `oversized`; and `broken`, settled `failed`. - An event leaves out a timer or a call that the same input opened and closed (invariant 26), and its other outputs keep the order the session emitted them in. - A run ends with its outcome, settled once in its last event. An input that would pass a bound (inputs, history, held data, the size of one event) ends the run, raised, in a small event of its own. - The machine's tests run it through the memory driver of `src/testing`; each piece of the design has one that fails without it, the step entries and their causes in `src/steps`. The workflow adapter runs its tests of how a workflow runs through this driver (`capabilities/coordination/src/workflows`), and replays its 15 recorded input logs through the machine (`capabilities/coordination/input-logs/`). diff --git a/packages/workflow-engine/src/dsl/raised-error.test.ts b/packages/workflow-engine/src/dsl/raised-error.test.ts index 074b54706..1a25f51f7 100644 --- a/packages/workflow-engine/src/dsl/raised-error.test.ts +++ b/packages/workflow-engine/src/dsl/raised-error.test.ts @@ -47,6 +47,35 @@ describe('the rejection of an uncaught error of a kind with a type of its own', }); }); +describe('the rejection of an uncaught error whose effect is unknown', () => { + it('settles effect_unknown as a conflict with its because, raised at 409 under its own type', () => { + const unknown = callErrorOf( + { status: 'rejected', reason: 'conflict', detail: 'Called', kind: 'effect_unknown', because: 'tool_error' }, + { function: 'notify', label: 'the function notify', reference: '/do/0/ask' }, + ); + + expect(unknown).toMatchObject({ type: 'https://on.auto/problems/effect_unknown', status: 409 }); + expect([ + settlementOf({ kind: 'raised', error: unknown }), + settlementOf({ kind: 'raised', error: { ...unknown, because: 'not a because' } }), + ]).toEqual([ + { + status: 'rejected', + reason: 'conflict', + detail: 'The function notify rejected the run with conflict: Called (at /do/0/ask)', + kind: 'effect_unknown', + because: 'tool_error', + }, + { + status: 'rejected', + reason: 'conflict', + detail: 'The function notify rejected the run with conflict: Called (at /do/0/ask)', + kind: 'effect_unknown', + }, + ]); + }); +}); + describe('the error of a call that did not succeed', () => { const site = { function: 'notify', label: 'the function notify', reference: '/do/0/ask' }; diff --git a/packages/workflow-engine/src/dsl/raised-error.ts b/packages/workflow-engine/src/dsl/raised-error.ts index 65e4d241b..e277118cb 100644 --- a/packages/workflow-engine/src/dsl/raised-error.ts +++ b/packages/workflow-engine/src/dsl/raised-error.ts @@ -1,8 +1,10 @@ import { isKindWithType, problemTypeOf, + reasonOfKind, + RejectionBecauseSchema, UnansweredKindSchema, - UnavailableBecauseSchema, + type ConflictKind, type KindWithType, type Settlement, } from '@beonauto/operations'; @@ -101,7 +103,7 @@ export function describeError({ type, title, detail, instance }: DslError): stri const retryableStatuses: ReadonlySet = new Set([408, 429]); -const isUnavailableBecause = Schema.is(UnavailableBecauseSchema); +const isRejectionBecause = Schema.is(RejectionBecauseSchema); const isUnansweredKind = Schema.is(UnansweredKindSchema); @@ -113,16 +115,21 @@ export function reasonOfStatus(status: number): 'invalid_input' | 'unavailable' return status >= 400 && status < 500 && !retryableStatuses.has(status) ? 'invalid_input' : 'unavailable'; } +type ConflictKindWithType = Extract; + +function isConflictKind(kind: KindWithType): kind is ConflictKindWithType { + return reasonOfKind(kind) === 'conflict'; +} + +function becauseOf(kind: KindWithType, because: string | undefined) { + return kind === 'tools_called' || !isRejectionBecause(because) ? {} : { because }; +} + function ownTypeRejectionOf(kind: KindWithType, detail: string, because: string | undefined): Settlement { - return kind === 'tools_called' - ? { status: 'rejected', reason: 'conflict', detail, kind } - : { - status: 'rejected', - reason: 'unavailable', - detail, - kind, - ...(isUnavailableBecause(because) ? { because } : {}), - }; + const known = becauseOf(kind, because); + return isConflictKind(kind) + ? { status: 'rejected', reason: 'conflict', detail, kind, ...known } + : { status: 'rejected', reason: 'unavailable', detail, kind, ...known }; } function settledRejectionOf(error: DslError): Settlement { diff --git a/scripts/docs-interaction.test.ts b/scripts/docs-interaction.test.ts index bc3143bce..33e715898 100644 --- a/scripts/docs-interaction.test.ts +++ b/scripts/docs-interaction.test.ts @@ -30,7 +30,11 @@ await test('interaction functions are available in a self-hosted runtime, with t assert.ok(routes.indexOf('/reference/reasoning-format') < routes.indexOf('/reference/interaction-format')); assert.ok(routes.indexOf('/reference/interaction-format') < routes.indexOf('/reference/computation-format')); assert.ok(urlsInCode(format).some((href) => href === unansweredType.href)); - assert.match(format, /^## Sending through a tool\n[\s\S]*^### Attempts\n[\s\S]*^## Fields$/mu); + assert.match( + format, + /^## Asking a system\n[\s\S]*^## Sending through a tool\n[\s\S]*^### Attempts\n[\s\S]*^## Fields$/mu, + ); + assert.match(format, /^\| `call` +\| The tool the function asks: /mu); assert.match(format, /^\| `deliver` +\| The tool the request is sent through: /mu); assert.match(format, /^### Answering by reply$/mu); }); @@ -42,7 +46,11 @@ await test('the words of interaction are on the terminology page, and the first for (const term of ['Request', 'Inbox', 'Delivery', 'Answer', 'Notification']) { assert.match(terminology, new RegExp(`^\\| ${term} +\\| `, 'mu'), `The terminology page needs ${term}`); } - assert.ok(terminology.includes('An interaction function asks a person or a system and takes the answer later')); + assert.ok( + terminology.includes( + 'An interaction function asks a system and answers at once, or asks a person and takes the answer later', + ), + ); assert.ok(!tutorial.includes('deliver:')); assert.ok(tutorial.includes('answer_interaction')); assert.ok(workflows.includes('An approval is an interaction function'));