diff --git a/apps/cloud/src/observability/index.ts b/apps/cloud/src/observability/index.ts index 2ad2755a9..88b88419e 100644 --- a/apps/cloud/src/observability/index.ts +++ b/apps/cloud/src/observability/index.ts @@ -16,6 +16,7 @@ import { Cause, Effect, Layer, Predicate } from "effect"; import type * as Tracer from "effect/Tracer"; import { ErrorCapture } from "@executor-js/api"; +import { classifyDurableObjectError } from "@executor-js/cloudflare/mcp/durable-object-errors"; import { withStableGroupingFingerprint } from "@executor-js/sdk/sentry-grouping"; // Drizzle/postgres-js include the failing SQL (params + bound values) in @@ -129,11 +130,54 @@ const isClaimedDurableObjectEcho = (event: ErrorEvent): boolean => { return typeof mechanism === "string" && mechanism.startsWith("auto."); }; +/** + * The mechanism `instrumentDurableObjectWithSentry` stamps on a rejection it + * catches escaping a Durable Object entry point. + */ +const DO_FAAS_MECHANISM = "auto.faas.cloudflare.durable_object"; + +/** + * True when this event is the Durable Object instrumentation reporting the + * platform tearing its own object down. + * + * The capture-owner scheme above covers every reset a worker seam observed: the + * seam classifies the failure, answers the client with a retryable envelope and + * claims the cause, so the instrumentation's echo is dropped. A deploy has no + * such seam. The runtime resets every live object as the new script rolls out, + * and the instrumentation captures that from INSIDE the dying object, where + * there is no request to own it — so a handful of these file on every deploy + * with nothing behind them. The client-facing side is already handled: the + * in-flight request fails at the worker seam and is rendered as a 503 there. + * + * Both conditions are required, and the classifier is the one the seams use so + * this can never recognize a different set of messages than they do: + * + * - the mechanism must be the DO instrumentation, so a reset message arriving + * through any other path (an ordinary worker capture, the DO's own + * `captureCause` seam) is untouched; + * - the message must classify as a TRANSIENT platform failure, so a real defect + * thrown inside a DO — an alarm crash, a broken handler — still reports. The + * classifier returns null for everything it does not recognize, and this + * drops nothing on a null. + * + * `session_dead` is deliberately not dropped: that disposition means our own + * code called `ctx.abort`, which is an act of the application rather than the + * platform, and how often it happens is worth seeing. + */ +const isDurableObjectPlatformResetNoise = (event: ErrorEvent): boolean => { + const exception = event.exception?.values?.[0]; + if (exception?.mechanism?.type !== DO_FAAS_MECHANISM) return false; + // The exception's `value` is the workerd message verbatim, and the classifier + // already accepts a bare string as its input. + return classifyDurableObjectError(exception.value)?.disposition === "transient"; +}; + export const beforeSendWithOtelCorrelation = ( event: ErrorEvent, options?: { readonly logPayload?: boolean }, ): ErrorEvent | null => { if (isClaimedDurableObjectEcho(event)) return null; + if (isDurableObjectPlatformResetNoise(event)) return null; if (options?.logPayload) { console.info( JSON.stringify({ @@ -156,9 +200,10 @@ export const beforeSendWithOtelCorrelation = ( * `withStableGroupingFingerprint` pins a key with the hash normalized out; * events with no hashed grouping input are left on the default algorithm. * - * The two stages are independent and compose in this order: the capture-owner - * pass decides WHETHER the event is reported at all (a cause the Durable - * Object already claimed is dropped, and a dropped event is never + * The two stages are independent and compose in this order: the reporting pass + * decides WHETHER the event is reported at all (a cause the Durable Object + * already claimed, or a platform reset the DO instrumentation caught from + * inside a dying object, is dropped, and a dropped event is never * fingerprinted), and the grouping pass then decides HOW whatever survives is * grouped. */ diff --git a/apps/cloud/src/observability/observability.test.ts b/apps/cloud/src/observability/observability.test.ts index a75a8e617..b95ea61d7 100644 --- a/apps/cloud/src/observability/observability.test.ts +++ b/apps/cloud/src/observability/observability.test.ts @@ -221,7 +221,18 @@ describe("Durable Object capture ownership", () => { // the auto-instrumentation is the ONLY thing that reports it. Dropping those // would trade duplicate noise for silence. it("keeps an unclaimed Durable Object failure", () => { - const unclaimed = doEcho({ tags: {} }); + const unclaimed = doEcho({ + tags: {}, + exception: { + values: [ + { + type: "TypeError", + value: "this.scheduleNextSweep is not a function", + mechanism: { type: "auto.faas.cloudflare.durable_object", handled: false }, + }, + ], + }, + }); expect(beforeSendWithOtelCorrelation(unclaimed)).not.toBeNull(); }); @@ -300,3 +311,92 @@ describe("Durable Object capture ownership", () => { }); }); }); + +// The residue the ownership scheme above cannot reach. A deploy resets every +// live Durable Object, and `instrumentDurableObjectWithSentry` captures that +// from inside the object being torn down — there is no worker seam in that +// isolate to classify the failure and claim the cause, so the event arrives +// unclaimed and files an issue with nothing behind it. The client side is +// already handled: the in-flight request fails at the worker seam and is +// answered there with a retryable 503. +// +// Not reachable from a black-box test. The dropped event and the events kept +// beside it declare the same exception type with the same shape; only the +// mechanism the Sentry SDK stamps and the classifier's verdict on the message +// separate them, and neither is observable from outside the SDK. +describe("Durable Object platform reset noise", () => { + const doInstrumentationEvent = ( + value: string, + mechanismType = "auto.faas.cloudflare.durable_object", + ): ErrorEvent => ({ + type: undefined, + // Unclaimed on purpose: the whole point is that no worker seam existed to + // claim this one. + tags: {}, + exception: { + values: [ + { + type: "Error", + value, + mechanism: { type: mechanismType, handled: false }, + }, + ], + }, + }); + + it("drops a deploy-time code-update reset reported by the DO instrumentation", () => { + const event = doInstrumentationEvent("Durable Object reset because its code was updated."); + expect(beforeSendWithOtelCorrelation(event)).toBeNull(); + }); + + // The classifier is shared with the worker seams, so every transient kind it + // knows is covered here rather than just the deploy message. + it("drops the other transient platform resets the shared classifier recognizes", () => { + for (const value of [ + "Durable Object storage operation exceeded timeout which caused object to be reset.", + "Durable Object exceeded its CPU time limit and was reset.", + "internal error; reference = 0123abcd4567", + ]) { + expect(beforeSendWithOtelCorrelation(doInstrumentationEvent(value))).toBeNull(); + } + }); + + // Same message, different path: a reset seen anywhere other than the DO + // instrumentation is somebody's deliberate report and keeps reporting. + it("keeps a reset message that did not come from the DO instrumentation", () => { + const viaWorker = doInstrumentationEvent( + "Durable Object reset because its code was updated.", + "auto.http.cloudflare", + ); + expect(beforeSendWithOtelCorrelation(viaWorker)).not.toBeNull(); + + const viaCaptureSeam = doInstrumentationEvent( + "Durable Object reset because its code was updated.", + "generic", + ); + expect(beforeSendWithOtelCorrelation(viaCaptureSeam)).not.toBeNull(); + }); + + // The load-bearing limit: a defect thrown inside a DO is reported by the + // instrumentation and by nothing else. The classifier does not recognize it, + // so it must survive. + it("keeps a Durable Object failure the classifier does not recognize", () => { + const defect = doInstrumentationEvent("Cannot read properties of undefined (reading 'meta')"); + expect(beforeSendWithOtelCorrelation(defect)).not.toBeNull(); + }); + + // The memory-limit reset is deliberately absent from the classifier: the + // runtime blames the application for it, so it is a defect, not noise. + it("keeps the memory-limit reset the classifier deliberately excludes", () => { + const memory = doInstrumentationEvent( + "Durable Object's isolate exceeded its memory limit due to overflowing the storage cache. All objects in the isolate were reset.", + ); + expect(beforeSendWithOtelCorrelation(memory)).not.toBeNull(); + }); + + it("the hook the worker and DOs install drops the deploy reset", () => { + const options = cloudSentryOptions({ SENTRY_DSN: "https://public@example.invalid/1" } as Env); + const event = doInstrumentationEvent("Durable Object reset because its code was updated."); + expect(options.beforeSend(event)).toBeNull(); + }); +});