Strip review assignment metadata from eval rows

js/src/framework.test.ts

@@ -212,6 +212,7 @@ describe("runEvaluator", () => {
     datum: {
       input: number;
       id: string;
+      metadata?: Record<string, unknown>;
       _xact_id?: string;
       created?: string;
       origin?: {
@@ -309,6 +310,41 @@ describe("runEvaluator", () => {
     });
   });
 
+  test("strips review assignment metadata from dataset-backed eval rows", async () => {
+    const data = makeDatasetData("00000000-0000-0000-0000-000000000001", {
+      input: 1,
+      id: "dataset-row-1",
+      metadata: {
+        keep: "yes",
+        "~__bt_assignments": ["user-id"],
+        "~__bt_review_lists": {
+          __bt_default_review_list: { status: "PENDING" },
+        },
+      },
+    });
+    let taskMetadata: Record<string, unknown> | null = null;
+
+    const out = await runEvaluator(
+      null,
+      {
+        projectName: "proj",
+        evalName: "eval",
+        data,
+        task: async (input: number, { metadata }) => {
+          taskMetadata = { ...metadata };
+          return input * 2;
+        },
+        scores: [],
+      },
+      new NoopProgressReporter(),
+      [],
+      undefined,
+    );
+
+    expect(taskMetadata).toEqual({ keep: "yes" });
+    expect(out.results[0].metadata).toEqual({ keep: "yes" });
+  });
+
   test("falls back to source origin when dataset row origin is incomplete", async () => {
     const datasetId = "00000000-0000-0000-0000-000000000001";
     const sourceOrigin = {

js/src/framework.ts

@@ -500,7 +500,7 @@
 
 export async function _internalInitEvaluatorExperiment(
   projectName: string,
   evaluator: Evaluator<any, any, any, any, any>,
   data: EvalData<any, any, any>,
   options: {
     disabled?: boolean;
@@ -975,6 +975,24 @@
   return classifier.name || `classifier_${classifier_idx}`;
 }
 
+const REVIEW_ASSIGNMENT_METADATA_KEYS = new Set([
+  "~__bt_assignments",
+  "~__bt_review_lists",
+]);
+
+function stripReviewAssignmentMetadata(
+  metadata: Record<string, unknown> | undefined,
+): Record<string, unknown> {
+  if (metadata === undefined) {
+    return {};
+  }
+  return Object.fromEntries(
+    Object.entries(metadata).filter(
+      ([key]) => !REVIEW_ASSIGNMENT_METADATA_KEYS.has(key),
+    ),
+  );
+}
+
 export async function _internalRunEvaluatorTask(
   task: EvalTask<any, any, any, any, any>,
   datum: EvalCase<any, any, any>,
@@ -987,9 +1005,9 @@
   metadata: Record<string, unknown>;
   tags: string[];
 }> {
-  const metadata: Record<string, unknown> = {
-    ...("metadata" in datum ? datum.metadata : {}),
-  };
+  const metadata = stripReviewAssignmentMetadata(
+    "metadata" in datum ? datum.metadata : undefined,
+  );
   const hooks: EvalHooks<unknown, Record<string, unknown>, EvalParameters> = {
     meta(value) {
       Object.assign(metadata, value);