diff --git a/.changeset/fix-eve-update-vendored-types-for-modern-eve.md b/.changeset/fix-eve-update-vendored-types-for-modern-eve.md new file mode 100644 index 000000000..3a4c0c92b --- /dev/null +++ b/.changeset/fix-eve-update-vendored-types-for-modern-eve.md @@ -0,0 +1,5 @@ +--- +"braintrust": patch +--- + +fix(eve): update vendored types for modern Eve diff --git a/js/src/instrumentation/plugins/eve-plugin.ts b/js/src/instrumentation/plugins/eve-plugin.ts index 407d92d93..a740929b8 100644 --- a/js/src/instrumentation/plugins/eve-plugin.ts +++ b/js/src/instrumentation/plugins/eve-plugin.ts @@ -29,7 +29,7 @@ import type { EveRuntimeActionResult, EveRuntimeToolCallActionRequest, EveRuntimeToolResultActionResult, - EveSystemModelMessage, + EveStreamEvent, } from "../../vendor-sdk-types/eve"; type SpanState = { @@ -132,7 +132,9 @@ export function braintrustEveHook(options: { const bridge = new EveBridge(state); return { events: { - "*": async (event: EveHandleMessageStreamEvent, ctx: EveHookContext) => { + // Eve delivers every accepted stream event here, including ones newer + // than the union we model; `bridge.handle` narrows and ignores the rest. + "*": async (event: EveStreamEvent, ctx: EveHookContext) => { await bridge.handle(event, ctx, options.metadata); }, }, @@ -153,6 +155,9 @@ export function createLegacyEveInstrumentation(options: { } catch (error) { debugLogger.warn("Error in Eve LLM input capture:", error); } + // We capture input for our own spans and contribute no runtime context + // to Eve's AI SDK telemetry spans. + return undefined; }, }, recordInputs: false, @@ -167,6 +172,29 @@ function isEveHandleMessageStreamEvent( return isObject(event) && typeof event["type"] === "string"; } +/** + * Eve types model messages as the AI SDK's `ModelMessage`, whose shape moves + * between `ai` majors, so the boundary is untyped and asserted here instead. + * This checks only what the serializer relies on: a known role and a content + * value it can walk. + */ +function isEveModelMessage(message: unknown): message is EveModelMessage { + if (!isObject(message)) { + return false; + } + const role = message["role"]; + if ( + role !== "system" && + role !== "user" && + role !== "assistant" && + role !== "tool" + ) { + return false; + } + const content = message["content"]; + return typeof content === "string" || Array.isArray(content); +} + class ResumedEveSpan implements EveSpan { private endTime: number | undefined; @@ -1965,11 +1993,15 @@ export function capturedModelInput( if (typeof instructions === "string") { value.push({ content: instructions, role: "system" }); } else if (Array.isArray(instructions)) { - value.push(...instructions.map(capturedEveModelMessage)); - } else if (instructions) { - value.push(capturedEveModelMessage(instructions as EveSystemModelMessage)); + value.push( + ...instructions.filter(isEveModelMessage).map(capturedEveModelMessage), + ); + } else if (isEveModelMessage(instructions)) { + value.push(capturedEveModelMessage(instructions)); } - value.push(...messages.map(capturedEveModelMessage)); + value.push( + ...messages.filter(isEveModelMessage).map(capturedEveModelMessage), + ); try { const cloned: unknown = JSON.parse(JSON.stringify(value)); @@ -2110,6 +2142,13 @@ function capturedEveModelContentPart( providerReference: part.providerReference, type: part.type, }; + default: + // Eve can deliver content parts newer than the ones we model. Record the + // discriminator so the message still round-trips instead of becoming a + // hole in the captured input. + return isObject(part) && typeof part["type"] === "string" + ? { type: part["type"] } + : {}; } } diff --git a/js/src/vendor-sdk-types/eve.ts b/js/src/vendor-sdk-types/eve.ts index b8d535a88..8b08a136b 100644 --- a/js/src/vendor-sdk-types/eve.ts +++ b/js/src/vendor-sdk-types/eve.ts @@ -291,18 +291,29 @@ export type EveHandleMessageStreamEvent = readonly type: "session.completed"; }; +/** + * Loose shape of any runtime stream event delivered to the wildcard hook. + * + * Eve's `HookEventMap` is a closed map that grows over time (eve@0.48 exposes + * 33 events to authored hooks). Our handler must accept every one of them, so + * the boundary is typed loosely here and narrowed to + * {@link EveHandleMessageStreamEvent} inside the plugin. Keeping the two apart + * is what lets the event union above stay a discriminated union: folding a + * catch-all member into it would widen `data` to `unknown` after every + * `type` narrowing. + */ +export interface EveStreamEvent { + readonly data?: unknown; + readonly meta?: unknown; + readonly type: string; +} + export interface EveHookDefinition { readonly events?: { readonly "*"?: ( - event: EveHandleMessageStreamEvent, + event: EveStreamEvent, ctx: EveHookContext, ) => void | Promise; - readonly [eventType: string]: - | (( - event: EveHandleMessageStreamEvent, - ctx: EveHookContext, - ) => void | Promise) - | undefined; }; } @@ -475,12 +486,19 @@ export type EveModelMessage = readonly role: "tool"; }; +/** + * Model input snapshot handed to `step.started`. + * + * Eve types `messages` as the AI SDK's `ModelMessage` and `instructions` as + * `SystemModelMessage`, both of which change shape between `ai` majors. We only + * read these to serialize them, so the boundary stays untyped and the parts we + * capture are narrowed defensively by `capturedModelInput`. The capture + * contract itself lives in {@link EveModelMessage} and + * {@link EveModelMessageContentPart}. + */ export interface EveInstrumentationModelInput { - readonly instructions?: - | string - | EveSystemModelMessage - | readonly EveSystemModelMessage[]; - readonly messages: readonly EveModelMessage[]; + readonly instructions?: unknown; + readonly messages: readonly unknown[]; } export interface EveInstrumentationStepStartedEventInput { @@ -499,14 +517,14 @@ export interface EveInstrumentationStepStartedEventInput { export interface EveInstrumentationDefinition { readonly events?: { + /** + * Eve requires this to return the runtime context to merge into the AI SDK + * telemetry span, or `undefined` to contribute none. `void` is not an + * accepted return type, so handlers must return explicitly. + */ readonly "step.started"?: ( input: EveInstrumentationStepStartedEventInput, - ) => void | { readonly runtimeContext?: EveJsonObject }; - readonly [eventType: string]: - | (( - input: EveInstrumentationStepStartedEventInput, - ) => void | { readonly runtimeContext?: EveJsonObject }) - | undefined; + ) => { readonly runtimeContext: EveJsonObject } | undefined; }; readonly recordInputs?: boolean; readonly recordOutputs?: boolean; @@ -526,7 +544,12 @@ export interface EveProviderSetupContext { } export interface EveProviderState { - get(): EveJsonValue | undefined; + /** + * Read as `unknown` rather than `EveJsonValue`: eve declares its JSON value + * as an interface-based union that is not structurally assignable to ours, + * and we narrow the stored value defensively at every read anyway. + */ + get(): unknown; set(value: EveJsonValue | undefined): void; }