diff --git a/src/adapter.test.ts b/src/adapter.test.ts index 379bcf3..fe47010 100644 --- a/src/adapter.test.ts +++ b/src/adapter.test.ts @@ -94,4 +94,18 @@ describe("adapter", () => { } expect(missingValue).toBeInstanceOf(ProtocolMismatchError); }); + + test("usage source reports the responding model, not the last request", () => { + const adapter = createSystemOneAdapter(); + adapter.buildRequest(turns(), "model-a", {}); + adapter.buildRequest(turns(), "model-b", {}); + const answers = { response: { type: "noul", noul: 0.5 } }; + const usageModel = (body: object) => { + const last = adapter.parseJSONResponse(JSON.stringify(body)).at(-1); + if (last?.type !== "inference.usage") throw new Error("expected usage"); + return last.data.source.model; + }; + expect(usageModel({ answers, model: "model-a-v1" })).toBe("model-a-v1"); + expect(usageModel({ answers })).toBe("jev-latest"); + }); }); diff --git a/src/adapter.ts b/src/adapter.ts index 431ca77..14e1e51 100644 --- a/src/adapter.ts +++ b/src/adapter.ts @@ -109,7 +109,11 @@ function toTokenUsage(usage: WireUsage | undefined): TokenUsage { function parseDecisions( body: string, provider: string, -): { decisions: Decision[]; usage: WireUsage | undefined } { +): { + decisions: Decision[]; + model: string | undefined; + usage: WireUsage | undefined; +} { let parsed: unknown; try { parsed = JSON.parse(body); @@ -143,7 +147,7 @@ function parseDecisions( ); } }); - return { decisions, usage: envelope.usage }; + return { decisions, model: envelope.model, usage: envelope.usage }; } /** @@ -157,12 +161,6 @@ export function createSystemOneAdapter( config?: EvaluateConfig, ): ProviderAdapter { const endpoint = resolveEndpoint(config?.endpoint); - const source = { - sourceId: SYSTEM_ONE_PROVIDER, - provider: SYSTEM_ONE_PROVIDER, - model: endpoint.model, - }; - const buildRequest = ( messages: ConversationTurn[], model: string, @@ -183,7 +181,6 @@ export function createSystemOneAdapter( if (overrides.questions !== undefined) questions = overrides.questions; } const requestModel = model || endpoint.model; - source.model = requestModel; const headers: Record = { "content-type": "application/json", accept: "application/json", @@ -201,7 +198,7 @@ export function createSystemOneAdapter( }; const parseJSONResponse = (responseBody: string): InferenceEvent[] => { - const { decisions, usage } = parseDecisions( + const { decisions, model, usage } = parseDecisions( responseBody, SYSTEM_ONE_PROVIDER, ); @@ -222,9 +219,9 @@ export function createSystemOneAdapter( data: { usage: toTokenUsage(usage), source: { - sourceId: source.sourceId, - provider: source.provider, - model: source.model, + sourceId: SYSTEM_ONE_PROVIDER, + provider: SYSTEM_ONE_PROVIDER, + model: model ?? endpoint.model, }, }, }); diff --git a/src/evaluate.test.ts b/src/evaluate.test.ts index f176ddc..e640e99 100644 --- a/src/evaluate.test.ts +++ b/src/evaluate.test.ts @@ -66,6 +66,7 @@ describe("checkAnswers", () => { ["kind mismatch", (a) => (a["gate"] = { type: "choice", choice: "x" })], ["noul without value", (a) => (a["gate"] = { type: "noul" })], ["illegal option", (a) => (must(a["route"]).choice = "maybe")], + ["inherited option", (a) => (must(a["route"]).choice = "constructor")], ["score out of range", (a) => (must(a["risk"]).score = 9)], ["choice without confidence", (a) => delete must(a["route"]).confidence], ["missing legend", (a) => delete must(a["risk"]).legend], diff --git a/src/evaluate.ts b/src/evaluate.ts index db4348b..5fcff32 100644 --- a/src/evaluate.ts +++ b/src/evaluate.ts @@ -125,7 +125,7 @@ function checkAnswer( } if (question.type === "choice" && answer.type === "choice") { if (answer.choice === undefined) return `answer "${id}" is missing choice`; - if (!(answer.choice in question.criteria)) { + if (!Object.hasOwn(question.criteria, answer.choice)) { return `answer "${id}" illegal option ${JSON.stringify(answer.choice)}`; } return checkProbabilities(