Harden shape memory: contracts, expiry, freshness, key classifiers

packages/core/sdk/src/executor.ts

@@ -4072,7 +4072,7 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea
         // The schema's description marks it as observed.
         const observed =
           outputSchema === undefined
-            ? yield* shapeMemory.recall(String(address), parsed.owner)
+            ? yield* shapeMemory.recall(String(address), parsed.owner, "direct")
             : null;
         const effectiveOutputSchema =
           outputSchema !== undefined
@@ -4803,7 +4803,7 @@ export const createExecutor = <const TPlugins extends readonly AnyPlugin[] = rea
           if (!parsed) return Effect.void;
           const data = isToolResult(result) ? (result.ok ? result.data : undefined) : result;
           if (data === undefined) return Effect.void;
-          return shapeMemory.observe(String(address), parsed.owner, data);
+          return shapeMemory.observe(String(address), parsed.owner, "direct", data);
         }),
         Effect.withSpan("executor.tool.execute", {
           attributes: {

packages/core/sdk/src/shape-inference.test.ts

@@ -47,6 +47,50 @@ describe("inferShape", () => {
     });
   });
 
+  it("collapses narrow objects with data-bearing keys to a map", () => {
+    // Two entries is far below the width threshold — the keys themselves are
+    // the tell. None of these may persist as schema "field names".
+    const cases: Record<string, unknown>[] = [
+      { "alice@example.com": { active: true }, "bob@example.com": { active: false } },
+      { "3f2b8c1e-79aa-4f10-8d5c-0a1b2c3d4e5f": 1 },
+      { "2026-08-27T01:00:00Z": "event" },
+      { "12345": { qty: 2 }, "67890": { qty: 1 } },
+      { "https://example.com/page": 3 },
+      { deadbeefdeadbeef00: true },
+      { U012ABCDEF: { presence: "active" } },
+      { cus_9s6XKzkNRiz8i3: { plan: "pro" } },
+      { "10.0.0.7": "reachable" },
+      { sk4bcD3fGh1jKlMnOpQr: true },
+    ];
+    for (const value of cases) {
+      const shape = inferShape(value);
+      expect(shape.properties, JSON.stringify(value)).toBeUndefined();
+      expect(shape.additionalProperties, JSON.stringify(value)).toBeDefined();
+    }
+  });
+
+  it("keeps ordinary API field names as properties", () => {
+    const shape = inferShape({
+      id: 1,
+      created_at: "2026-01-01",
+      pageUrl: "https://x",
+      email2fa: true,
+      "@odata.context": "ctx",
+      organizationMembershipSettings: {},
+      sha256Fingerprint: "…",
+    });
+    expect(shape.properties).toBeDefined();
+    expect(Object.keys(shape.properties ?? {}).sort()).toEqual([
+      "@odata.context",
+      "created_at",
+      "email2fa",
+      "id",
+      "organizationMembershipSettings",
+      "pageUrl",
+      "sha256Fingerprint",
+    ]);
+  });
+
   it("degrades to unknown past the depth bound", () => {
     let value: unknown = "leaf";
     for (let i = 0; i < 10; i++) value = { child: value };

packages/core/sdk/src/shape-inference.ts

@@ -45,6 +45,43 @@ const MAX_ANYOF = 4;
 const isUnknown = (shape: InferredShape): boolean =>
   shape.type === undefined && shape.anyOf === undefined;
 
+/**
+ * Keys that look like DATA rather than API surface: emails, UUIDs,
+ * timestamps, URLs, bare numbers, long random tokens. Struct field names are
+ * schema; map keys are values, and values must never persist. Width alone
+ * (`MAX_OBJECT_KEYS`) misses a two-entry object keyed by email addresses, so
+ * any single data-looking key collapses the whole object to a map. No
+ * classifier is a proof — only declared schemas are — so this errs toward
+ * collapsing.
+ */
+const DATA_KEY_PATTERNS: readonly RegExp[] = [
+  // Email addresses (full-string — `@odata.context`-style annotation keys are
+  // legitimate API surface and must NOT collapse).
+  /^[^\s@]+@[^\s@]+\.[^\s@]+$/,
+  // UUIDs.
+  /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i,
+  // Timestamps / dates.
+  /^\d{4}-\d{2}-\d{2}/,
+  // Bare numbers and IPv4 addresses.
+  /^\d+$/,
+  /^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$/,
+  // URLs.
+  /^https?:\/\//,
+  // Long hex tokens (hashes, ids).
+  /^[0-9a-f]{16,}$/i,
+  // Platform opaque ids: Slack-style ALL-CAPS ids (U012ABCDEF), and
+  // prefix_body ids (cus_..., price_..., asst_...). Real field names in
+  // snake_case are lowercase words, not lowercase prefix + mixed-case body.
+  /^[A-Z][A-Z0-9]{8,}$/,
+  /^[a-z]{1,6}_(?=.*[A-Z0-9])[A-Za-z0-9]{10,}$/,
+  // Generic digit-bearing opaque tokens (API keys, base62/base64 ids). Long
+  // camelCase field names rarely contain digits at this length.
+  /^(?=.*\d)[A-Za-z0-9+/=_-]{20,}$/,
+];
+
+const looksLikeDataKey = (key: string): boolean =>
+  DATA_KEY_PATTERNS.some((pattern) => pattern.test(key));
+
 /** Infer the shape of one observed value. Reads structure only, never values. */
 export const inferShape = (value: unknown, depth = 0): InferredShape => {
   if (value === null || value === undefined) return { type: "null" };
@@ -64,7 +101,9 @@ export const inferShape = (value: unknown, depth = 0): InferredShape => {
 
   if (typeof value === "object") {
     const entries = Object.entries(value as Record<string, unknown>);
-    if (entries.length > MAX_OBJECT_KEYS) {
+    // Both branches guarantee at least one entry, so the seedless reduce is
+    // safe (an UNKNOWN seed would absorb every merge).
+    if (entries.length > MAX_OBJECT_KEYS || entries.some(([key]) => looksLikeDataKey(key))) {
       const merged = entries
         .slice(0, MAX_ARRAY_SAMPLE)
         .map(([, item]) => inferShape(item, depth + 1))

packages/core/sdk/src/shape-memory.test.ts

@@ -0,0 +1,187 @@
+import { describe, expect, it } from "@effect/vitest";
+import { Effect } from "effect";
+import * as TestClock from "effect/testing/TestClock";
+
+import type { Owner } from "./ids";
+import type { PluginStorageEntry, PluginStorageFacade } from "./plugin-storage";
+import { makeShapeMemory } from "./shape-memory";
+
+const OWNER: Owner = "org";
+const ADDRESS = "tools.demo.org.main.run";
+
+/** Map-backed stand-in for plugin_storage, with a write counter. */
+const makeStubStorage = () => {
+  const rows = new Map<string, unknown>();
+  let writes = 0;
+  const entryFor = <T>(key: string): PluginStorageEntry<T> | null => {
+    const data = rows.get(key);
+    if (data === undefined) return null;
+    return {
+      id: key,
+      owner: OWNER,
+      pluginId: "executor.shape-memory",
+      collection: "observed-output-shapes",
+      key,
+      data: data as T,
+      createdAt: new Date(0),
+      updatedAt: new Date(0),
+    };
+  };
+  let failNextWrites = 0;
+  const unsupported = (member: string) => () =>
+    Effect.die(`stub storage does not implement ${member}`);
+  const storage: PluginStorageFacade = {
+    collection: () => ({
+      get: unsupported("collection.get"),
+      getForOwner: unsupported("collection.getForOwner"),
+      list: unsupported("collection.list"),
+      put: unsupported("collection.put"),
+      query: unsupported("collection.query"),
+      count: unsupported("collection.count"),
+      remove: unsupported("collection.remove"),
+    }),
+    get: (input) => Effect.sync(() => entryFor(input.key)),
+    getForOwner: (input) => Effect.sync(() => entryFor(input.key)),
+    list: unsupported("list"),
+    put: (input) =>
+      Effect.suspend(() => {
+        if (failNextWrites > 0) {
+          failNextWrites -= 1;
+          return Effect.fail({ _tag: "StorageError" as const }) as never;
+        }
+        writes += 1;
+        rows.set(input.key, input.data);
+        return Effect.sync(() => entryFor(input.key) as never);
+      }),
+    putMany: unsupported("putMany"),
+    remove: unsupported("remove"),
+    removeMany: unsupported("removeMany"),
+  };
+  return {
+    storage,
+    rows,
+    writeCount: () => writes,
+    failWrites: (count: number) => {
+      failNextWrites = count;
+    },
+  };
+};
+
+const HOUR = 60 * 60 * 1000;
+const DAY = 24 * HOUR;
+
+describe("makeShapeMemory", () => {
+  it.effect("recalls what it observed, and writes only on change or freshness", () =>
+    Effect.gen(function* () {
+      const stub = makeStubStorage();
+      const memory = makeShapeMemory(stub.storage);
+
+      yield* memory.observe(ADDRESS, OWNER, "direct", { id: 1 });
+      expect(stub.writeCount(), "first observation persists").toBe(1);
+
+      // Identical shape shortly after: no write.
+      yield* TestClock.adjust("1 minute");
+      yield* memory.observe(ADDRESS, OWNER, "direct", { id: 2 });
+      expect(stub.writeCount(), "stable shape does not write").toBe(1);
+
+      // Shape change: writes.
+      yield* memory.observe(ADDRESS, OWNER, "direct", { id: 3, extra: true });
+      expect(stub.writeCount(), "changed shape writes").toBe(2);
+
+      const recalled = yield* memory.recall(ADDRESS, OWNER, "direct");
+      expect(recalled?.observations).toBe(3);
+      expect(recalled?.schema.required).toEqual(["id"]);
+    }),
+  );
+
+  it.effect("persists freshness on the interval even when the shape is stable", () =>
+    Effect.gen(function* () {
+      const stub = makeStubStorage();
+      const memory = makeShapeMemory(stub.storage);
+
+      yield* memory.observe(ADDRESS, OWNER, "direct", { id: 1 });
+      yield* memory.observe(ADDRESS, OWNER, "direct", { id: 2 });
+      expect(stub.writeCount()).toBe(1);
+
+      yield* TestClock.adjust("7 hours");
+      yield* memory.observe(ADDRESS, OWNER, "direct", { id: 3 });
+      expect(stub.writeCount(), "staleness alone forces a freshness write").toBe(2);
+      const stored = stub.rows.get(ADDRESS) as { observations: number; updatedAt: number };
+      expect(stored.observations, "persisted counters are current").toBe(3);
+      expect(stored.updatedAt).toBe(7 * HOUR + 0);
+    }),
+  );
+
+  it.effect("ignores a record observed under a different contract and restarts on observe", () =>
+    Effect.gen(function* () {
+      const stub = makeStubStorage();
+      const memory = makeShapeMemory(stub.storage);
+
+      yield* memory.observe(ADDRESS, OWNER, "direct", { content: [{ type: "text" }] });
+      expect(yield* memory.recall(ADDRESS, OWNER, "direct")).not.toBeNull();
+      expect(
+        yield* memory.recall(ADDRESS, OWNER, "mcp-call-tool-result-v2"),
+        "other contract sees nothing",
+      ).toBeNull();
+
+      // Observing under the new contract replaces rather than merges.
+      yield* memory.observe(ADDRESS, OWNER, "mcp-call-tool-result-v2", { issues: [] });
+      const fresh = yield* memory.recall(ADDRESS, OWNER, "mcp-call-tool-result-v2");
+      expect(fresh?.observations).toBe(1);
+      expect(Object.keys(fresh?.schema.properties ?? {})).toEqual(["issues"]);
+    }),
+  );
+
+  it.effect("retries after a failed write instead of pretending it persisted", () =>
+    Effect.gen(function* () {
+      const stub = makeStubStorage();
+      const memory = makeShapeMemory(stub.storage);
+
+      stub.failWrites(1);
+      yield* memory.observe(ADDRESS, OWNER, "direct", { id: 1 });
+      expect(stub.rows.has(ADDRESS), "the failed write stored nothing").toBe(false);
+
+      // The very next observation retries — no waiting out the freshness
+      // interval on bookkeeping that lied about persisting.
+      yield* memory.observe(ADDRESS, OWNER, "direct", { id: 2 });
+      expect(stub.rows.has(ADDRESS), "the retry persisted").toBe(true);
+      const stored = stub.rows.get(ADDRESS) as { observations: number };
+      expect(stored.observations).toBe(2);
+    }),
+  );
+
+  it.effect("treats legacy records without a contract field as direct", () =>
+    Effect.gen(function* () {
+      const stub = makeStubStorage();
+      stub.rows.set(ADDRESS, {
+        schema: { type: "object", properties: { ran: { type: "string" } }, required: ["ran"] },
+        observations: 4,
+        updatedAt: 0,
+      });
+      const memory = makeShapeMemory(stub.storage);
+      const recalled = yield* memory.recall(ADDRESS, OWNER, "direct");
+      expect(recalled?.observations).toBe(4);
+    }),
+  );
+
+  it.effect("expires shapes that have not been reinforced", () =>
+    Effect.gen(function* () {
+      const stub = makeStubStorage();
+      const memory = makeShapeMemory(stub.storage);
+
+      yield* memory.observe(ADDRESS, OWNER, "direct", { id: 1 });
+      yield* TestClock.adjust("29 days");
+      expect(yield* memory.recall(ADDRESS, OWNER, "direct"), "fresh enough").not.toBeNull();
+
+      yield* TestClock.adjust("2 days");
+      expect(yield* memory.recall(ADDRESS, OWNER, "direct"), "expired").toBeNull();
+
+      // The next observation restarts instead of merging into the fossil.
+      yield* memory.observe(ADDRESS, OWNER, "direct", { fresh: true });
+      const restarted = yield* memory.recall(ADDRESS, OWNER, "direct");
+      expect(restarted?.observations).toBe(1);
+      expect(Object.keys(restarted?.schema.properties ?? {})).toEqual(["fresh"]);
+      void DAY;
+    }),
+  );
+});