fix: per-call usage model and own-property choice check

src/adapter.test.ts

@@ -94,4 +94,18 @@ describe("adapter", () => {
     }
     expect(missingValue).toBeInstanceOf(ProtocolMismatchError);
   });
+
+  test("usage source reports the responding model, not the last request", () => {
+    const adapter = createSystemOneAdapter();
+    adapter.buildRequest(turns(), "model-a", {});
+    adapter.buildRequest(turns(), "model-b", {});
+    const answers = { response: { type: "noul", noul: 0.5 } };
+    const usageModel = (body: object) => {
+      const last = adapter.parseJSONResponse(JSON.stringify(body)).at(-1);
+      if (last?.type !== "inference.usage") throw new Error("expected usage");
+      return last.data.source.model;
+    };
+    expect(usageModel({ answers, model: "model-a-v1" })).toBe("model-a-v1");
+    expect(usageModel({ answers })).toBe("jev-latest");
+  });
 });

src/adapter.ts

@@ -109,7 +109,11 @@ function toTokenUsage(usage: WireUsage | undefined): TokenUsage {
 function parseDecisions(
   body: string,
   provider: string,
-): { decisions: Decision[]; usage: WireUsage | undefined } {
+): {
+  decisions: Decision[];
+  model: string | undefined;
+  usage: WireUsage | undefined;
+} {
   let parsed: unknown;
   try {
     parsed = JSON.parse(body);
@@ -143,7 +147,7 @@ function parseDecisions(
       );
     }
   });
-  return { decisions, usage: envelope.usage };
+  return { decisions, model: envelope.model, usage: envelope.usage };
 }
 
 /**
@@ -157,12 +161,6 @@ export function createSystemOneAdapter(
   config?: EvaluateConfig,
 ): ProviderAdapter {
   const endpoint = resolveEndpoint(config?.endpoint);
-  const source = {
-    sourceId: SYSTEM_ONE_PROVIDER,
-    provider: SYSTEM_ONE_PROVIDER,
-    model: endpoint.model,
-  };
-
   const buildRequest = (
     messages: ConversationTurn[],
     model: string,
@@ -183,7 +181,6 @@ export function createSystemOneAdapter(
       if (overrides.questions !== undefined) questions = overrides.questions;
     }
     const requestModel = model || endpoint.model;
-    source.model = requestModel;
     const headers: Record<string, string> = {
       "content-type": "application/json",
       accept: "application/json",
@@ -201,7 +198,7 @@ export function createSystemOneAdapter(
   };
 
   const parseJSONResponse = (responseBody: string): InferenceEvent[] => {
-    const { decisions, usage } = parseDecisions(
+    const { decisions, model, usage } = parseDecisions(
       responseBody,
       SYSTEM_ONE_PROVIDER,
     );
@@ -222,9 +219,9 @@ export function createSystemOneAdapter(
       data: {
         usage: toTokenUsage(usage),
         source: {
-          sourceId: source.sourceId,
-          provider: source.provider,
-          model: source.model,
+          sourceId: SYSTEM_ONE_PROVIDER,
+          provider: SYSTEM_ONE_PROVIDER,
+          model: model ?? endpoint.model,
         },
       },
     });

src/evaluate.test.ts

@@ -66,6 +66,7 @@ describe("checkAnswers", () => {
     ["kind mismatch", (a) => (a["gate"] = { type: "choice", choice: "x" })],
     ["noul without value", (a) => (a["gate"] = { type: "noul" })],
     ["illegal option", (a) => (must(a["route"]).choice = "maybe")],
+    ["inherited option", (a) => (must(a["route"]).choice = "constructor")],
     ["score out of range", (a) => (must(a["risk"]).score = 9)],
     ["choice without confidence", (a) => delete must(a["route"]).confidence],
     ["missing legend", (a) => delete must(a["risk"]).legend],

src/evaluate.ts

@@ -125,7 +125,7 @@ function checkAnswer(
   }
   if (question.type === "choice" && answer.type === "choice") {
     if (answer.choice === undefined) return `answer "${id}" is missing choice`;
-    if (!(answer.choice in question.criteria)) {
+    if (!Object.hasOwn(question.criteria, answer.choice)) {
       return `answer "${id}" illegal option ${JSON.stringify(answer.choice)}`;
     }
     return checkProbabilities(