Drop platform resets reported by the Durable Object instrumentation

apps/cloud/src/observability/index.ts

@@ -16,6 +16,7 @@ import { Cause, Effect, Layer, Predicate } from "effect";
 import type * as Tracer from "effect/Tracer";
 
 import { ErrorCapture } from "@executor-js/api";
+import { classifyDurableObjectError } from "@executor-js/cloudflare/mcp/durable-object-errors";
 import { withStableGroupingFingerprint } from "@executor-js/sdk/sentry-grouping";
 
 // Drizzle/postgres-js include the failing SQL (params + bound values) in
@@ -129,11 +130,54 @@ const isClaimedDurableObjectEcho = (event: ErrorEvent): boolean => {
   return typeof mechanism === "string" && mechanism.startsWith("auto.");
 };
 
+/**
+ * The mechanism `instrumentDurableObjectWithSentry` stamps on a rejection it
+ * catches escaping a Durable Object entry point.
+ */
+const DO_FAAS_MECHANISM = "auto.faas.cloudflare.durable_object";
+
+/**
+ * True when this event is the Durable Object instrumentation reporting the
+ * platform tearing its own object down.
+ *
+ * The capture-owner scheme above covers every reset a worker seam observed: the
+ * seam classifies the failure, answers the client with a retryable envelope and
+ * claims the cause, so the instrumentation's echo is dropped. A deploy has no
+ * such seam. The runtime resets every live object as the new script rolls out,
+ * and the instrumentation captures that from INSIDE the dying object, where
+ * there is no request to own it — so a handful of these file on every deploy
+ * with nothing behind them. The client-facing side is already handled: the
+ * in-flight request fails at the worker seam and is rendered as a 503 there.
+ *
+ * Both conditions are required, and the classifier is the one the seams use so
+ * this can never recognize a different set of messages than they do:
+ *
+ * - the mechanism must be the DO instrumentation, so a reset message arriving
+ *   through any other path (an ordinary worker capture, the DO's own
+ *   `captureCause` seam) is untouched;
+ * - the message must classify as a TRANSIENT platform failure, so a real defect
+ *   thrown inside a DO — an alarm crash, a broken handler — still reports. The
+ *   classifier returns null for everything it does not recognize, and this
+ *   drops nothing on a null.
+ *
+ * `session_dead` is deliberately not dropped: that disposition means our own
+ * code called `ctx.abort`, which is an act of the application rather than the
+ * platform, and how often it happens is worth seeing.
+ */
+const isDurableObjectPlatformResetNoise = (event: ErrorEvent): boolean => {
+  const exception = event.exception?.values?.[0];
+  if (exception?.mechanism?.type !== DO_FAAS_MECHANISM) return false;
+  // The exception's `value` is the workerd message verbatim, and the classifier
+  // already accepts a bare string as its input.
+  return classifyDurableObjectError(exception.value)?.disposition === "transient";
+};
+
 export const beforeSendWithOtelCorrelation = (
   event: ErrorEvent,
   options?: { readonly logPayload?: boolean },
 ): ErrorEvent | null => {
   if (isClaimedDurableObjectEcho(event)) return null;
+  if (isDurableObjectPlatformResetNoise(event)) return null;
   if (options?.logPayload) {
     console.info(
       JSON.stringify({
@@ -156,9 +200,10 @@ export const beforeSendWithOtelCorrelation = (
  * `withStableGroupingFingerprint` pins a key with the hash normalized out;
  * events with no hashed grouping input are left on the default algorithm.
  *
- * The two stages are independent and compose in this order: the capture-owner
- * pass decides WHETHER the event is reported at all (a cause the Durable
- * Object already claimed is dropped, and a dropped event is never
+ * The two stages are independent and compose in this order: the reporting pass
+ * decides WHETHER the event is reported at all (a cause the Durable Object
+ * already claimed, or a platform reset the DO instrumentation caught from
+ * inside a dying object, is dropped, and a dropped event is never
  * fingerprinted), and the grouping pass then decides HOW whatever survives is
  * grouped.
  */

apps/cloud/src/observability/observability.test.ts

@@ -221,7 +221,18 @@ describe("Durable Object capture ownership", () => {
   // the auto-instrumentation is the ONLY thing that reports it. Dropping those
   // would trade duplicate noise for silence.
   it("keeps an unclaimed Durable Object failure", () => {
-    const unclaimed = doEcho({ tags: {} });
+    const unclaimed = doEcho({
+      tags: {},
+      exception: {
+        values: [
+          {
+            type: "TypeError",
+            value: "this.scheduleNextSweep is not a function",
+            mechanism: { type: "auto.faas.cloudflare.durable_object", handled: false },
+          },
+        ],
+      },
+    });
     expect(beforeSendWithOtelCorrelation(unclaimed)).not.toBeNull();
   });
 
@@ -300,3 +311,92 @@ describe("Durable Object capture ownership", () => {
     });
   });
 });
+
+// The residue the ownership scheme above cannot reach. A deploy resets every
+// live Durable Object, and `instrumentDurableObjectWithSentry` captures that
+// from inside the object being torn down — there is no worker seam in that
+// isolate to classify the failure and claim the cause, so the event arrives
+// unclaimed and files an issue with nothing behind it. The client side is
+// already handled: the in-flight request fails at the worker seam and is
+// answered there with a retryable 503.
+//
+// Not reachable from a black-box test. The dropped event and the events kept
+// beside it declare the same exception type with the same shape; only the
+// mechanism the Sentry SDK stamps and the classifier's verdict on the message
+// separate them, and neither is observable from outside the SDK.
+describe("Durable Object platform reset noise", () => {
+  const doInstrumentationEvent = (
+    value: string,
+    mechanismType = "auto.faas.cloudflare.durable_object",
+  ): ErrorEvent => ({
+    type: undefined,
+    // Unclaimed on purpose: the whole point is that no worker seam existed to
+    // claim this one.
+    tags: {},
+    exception: {
+      values: [
+        {
+          type: "Error",
+          value,
+          mechanism: { type: mechanismType, handled: false },
+        },
+      ],
+    },
+  });
+
+  it("drops a deploy-time code-update reset reported by the DO instrumentation", () => {
+    const event = doInstrumentationEvent("Durable Object reset because its code was updated.");
+    expect(beforeSendWithOtelCorrelation(event)).toBeNull();
+  });
+
+  // The classifier is shared with the worker seams, so every transient kind it
+  // knows is covered here rather than just the deploy message.
+  it("drops the other transient platform resets the shared classifier recognizes", () => {
+    for (const value of [
+      "Durable Object storage operation exceeded timeout which caused object to be reset.",
+      "Durable Object exceeded its CPU time limit and was reset.",
+      "internal error; reference = 0123abcd4567",
+    ]) {
+      expect(beforeSendWithOtelCorrelation(doInstrumentationEvent(value))).toBeNull();
+    }
+  });
+
+  // Same message, different path: a reset seen anywhere other than the DO
+  // instrumentation is somebody's deliberate report and keeps reporting.
+  it("keeps a reset message that did not come from the DO instrumentation", () => {
+    const viaWorker = doInstrumentationEvent(
+      "Durable Object reset because its code was updated.",
+      "auto.http.cloudflare",
+    );
+    expect(beforeSendWithOtelCorrelation(viaWorker)).not.toBeNull();
+
+    const viaCaptureSeam = doInstrumentationEvent(
+      "Durable Object reset because its code was updated.",
+      "generic",
+    );
+    expect(beforeSendWithOtelCorrelation(viaCaptureSeam)).not.toBeNull();
+  });
+
+  // The load-bearing limit: a defect thrown inside a DO is reported by the
+  // instrumentation and by nothing else. The classifier does not recognize it,
+  // so it must survive.
+  it("keeps a Durable Object failure the classifier does not recognize", () => {
+    const defect = doInstrumentationEvent("Cannot read properties of undefined (reading 'meta')");
+    expect(beforeSendWithOtelCorrelation(defect)).not.toBeNull();
+  });
+
+  // The memory-limit reset is deliberately absent from the classifier: the
+  // runtime blames the application for it, so it is a defect, not noise.
+  it("keeps the memory-limit reset the classifier deliberately excludes", () => {
+    const memory = doInstrumentationEvent(
+      "Durable Object's isolate exceeded its memory limit due to overflowing the storage cache. All objects in the isolate were reset.",
+    );
+    expect(beforeSendWithOtelCorrelation(memory)).not.toBeNull();
+  });
+
+  it("the hook the worker and DOs install drops the deploy reset", () => {
+    const options = cloudSentryOptions({ SENTRY_DSN: "https://public@example.invalid/1" } as Env);
+    const event = doInstrumentationEvent("Durable Object reset because its code was updated.");
+    expect(options.beforeSend(event)).toBeNull();
+  });
+});