Classify Durable Object platform failures as retryable protocol errors

apps/cloud/src/mcp/agent-handler.ts

@@ -17,6 +17,11 @@ import {
   withVerifiedIdentityHeaders,
 } from "@executor-js/cloudflare/mcp/do-headers";
 import type { McpSessionProps } from "@executor-js/cloudflare/mcp/agent-durable-object";
+import {
+  classifyDurableObjectError,
+  durableObjectFailureResponse,
+  type DurableObjectFailure,
+} from "@executor-js/cloudflare/mcp/durable-object-errors";
 import { mcpSessionStub } from "@executor-js/cloudflare/mcp/session-stub";
 
 import { wrapMcpSseResponse } from "../observability/memory-metrics";
@@ -80,6 +85,41 @@ const renderAuthError = (
   });
 };
 
+/**
+ * A Cloudflare *platform* Durable Object failure happened at one of this
+ * handler's stub touchpoints. Record what kind it was — on an exported span and
+ * in a structured log — so the production volume stays countable per cause
+ * (deploy reset vs storage timeout vs destroyed session) now that it is no
+ * longer a pile of 500s.
+ *
+ * Talking to a session DO means talking to a process the platform can reset out
+ * from under us: a deploy, a storage timeout, a backend blip, the session's own
+ * `ctx.abort("destroyed")`. None of those are application defects. An
+ * unrecognized failure never reaches here and keeps escaping as before.
+ */
+const recordDurableObjectFailure = (
+  failure: DurableObjectFailure,
+  operation: string,
+): Effect.Effect<void> =>
+  Effect.sync(() => {
+    console.warn(
+      JSON.stringify({
+        event: "mcp_durable_object_platform_failure",
+        operation,
+        resetKind: failure.kind,
+        disposition: failure.disposition,
+      }),
+    );
+  }).pipe(
+    Effect.withSpan("mcp.do.platform_failure", {
+      attributes: {
+        "mcp.do.reset_kind": failure.kind,
+        "mcp.do.reset_disposition": failure.disposition,
+        "mcp.do.reset_operation": operation,
+      },
+    }),
+  );
+
 const authenticate = (request: Request) =>
   Effect.gen(function* () {
     const auth = yield* McpAuthProvider;
@@ -201,10 +241,27 @@ export const makeCloudMcpAgentHandler = () => {
     }
 
     if (sessionId) {
-      const owner = await mcpSessionStub(env.MCP_SESSION, sessionId).validateMcpSessionOwner({
-        accountId: outcome.principal.accountId,
-        organizationId: outcome.principal.organizationId,
-      });
+      let owner: "ok" | "not_found" | "forbidden" | "terminated";
+      // oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary: a Durable Object stub RPC rejects with a plain platform Error, never a typed failure
+      try {
+        owner = await mcpSessionStub(env.MCP_SESSION, sessionId).validateMcpSessionOwner({
+          accountId: outcome.principal.accountId,
+          organizationId: outcome.principal.organizationId,
+        });
+      } catch (error) {
+        // The sibling stub touchpoints in this handler are both guarded — the
+        // `_cf_scheduleDestroy` call above with `Effect.ignore`, the
+        // `target.fetch` below with a catch — and this one was not, so a session
+        // whose DO had been destroyed or reset by the platform 500ed here before
+        // any of that handling could run.
+        const failure = classifyDurableObjectError(error);
+        if (!failure) {
+          // oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary: an unrecognized failure is a real defect and must reach the runtime unchanged
+          throw error;
+        }
+        await runTraced(request, recordDurableObjectFailure(failure, "validate_session_owner"));
+        return durableObjectFailureResponse(failure);
+      }
       if (owner === "not_found") {
         return jsonRpcResponse(404, -32001, "Session not found");
       }
@@ -244,12 +301,18 @@ export const makeCloudMcpAgentHandler = () => {
       // DO ever getting to answer. Map it to the old envelope's reconnect
       // error for a dead session (e2e/cloud/mcp-protocol.test.ts expects the
       // client to be told to reconnect, matching a timed-out session).
-      // oxlint-disable-next-line executor/no-unknown-error-message -- adapter boundary: the abort reason is a plain runtime Error whose message IS the signal
-      if (Predicate.isError(error) && error.message === "destroyed") {
-        return jsonRpcResponse(404, -32001, "Session timed out, please reconnect");
+      //
+      // The same catch now also covers the rest of the platform's reset
+      // vocabulary — a deploy, a storage timeout, a cancelled
+      // blockConcurrencyWhile — which reaches here through the agents SDK's own
+      // `getServerByName` retry and used to 500 identically.
+      const failure = classifyDurableObjectError(error);
+      if (!failure) {
+        // oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary: rethrow anything that isn't a recognized platform failure to the Workers runtime unchanged
+        throw error;
       }
-      // oxlint-disable-next-line executor/no-try-catch-or-throw -- adapter boundary: rethrow anything that isn't the condemned-DO abort to the Workers runtime unchanged
-      throw error;
+      await runTraced(request, recordDurableObjectFailure(failure, "session_fetch"));
+      return durableObjectFailureResponse(failure);
     }
     // The agents SDK answers a bare DELETE with 204; the old envelope's
     // contract (see above) was 200 — rewrite for consistency.

apps/cloud/src/mcp/session-durable-object.ts

@@ -71,6 +71,7 @@ import { DoTelemetryLive, flushTracerProvider } from "../observability/telemetry
 import {
   captureCause as reportCause,
   captureCauseEffect as reportCauseEffect,
+  claimCauseHandledByDurableObject,
   tagCurrentSentryScopeWithCurrentOtelSpan,
 } from "../observability";
 import { parseTraceparent } from "./traceparent";
@@ -368,6 +369,12 @@ export class McpSessionDOSqlite extends McpAgentSessionDOBase<Env, CloudSessionD
     return Effect.asVoid(tagCurrentSentryScopeWithCurrentOtelSpan);
   }
 
+  // The DO owns this cause; `instrumentDurableObjectWithSentry` (server.ts) must
+  // not report it a second time when the rejection escapes the method.
+  protected override claimCauseHandled(_cause: Cause.Cause<unknown>): Effect.Effect<void> {
+    return claimCauseHandledByDurableObject;
+  }
+
   // Best-effort export the DO isolate's buffered spans after the RPC settles,
   // so a dying init/handleRequest can ship its own spans (and the exception +
   // stack recorded on them) — not just the worker-side `mcp.do.*` span. Keep it

apps/cloud/src/observability/index.ts

@@ -30,6 +30,21 @@ export const OTEL_TRACE_ID_TAG = "otel_trace_id";
 export const OTEL_SPAN_ID_TAG = "otel_span_id";
 export const SENTRY_EVENT_ID_ATTRIBUTE = "sentry.event_id";
 
+/**
+ * Set by the MCP session Durable Object when it has finished deciding what to
+ * do about a cause — reported it, or classified it as an expected Cloudflare
+ * platform reset and deliberately not reported it.
+ *
+ * `instrumentDurableObjectWithSentry` wraps the DO's entry points and captures
+ * the same rejection again as it escapes, which is why one platform reset
+ * opened two issues for the same event. The DO is the better owner — it has
+ * the session, the org, the OTEL correlation and the classification — so its
+ * claim wins and the auto-instrumentation's echo is dropped in `beforeSend`.
+ * Nothing the DO does not claim is affected.
+ */
+export const DO_CAUSE_OWNER_TAG = "mcp.do.cause_owner";
+export const DO_CAUSE_OWNER_VALUE = "durable_object";
+
 export type OtelCorrelationContext = {
   readonly traceId: string;
   readonly spanId: string;
@@ -99,10 +114,25 @@ export const tagCurrentSentryScopeWithCurrentOtelSpan: Effect.Effect<OtelCorrela
     return context;
   });
 
+/**
+ * True when this event is the auto-instrumentation's copy of a cause the
+ * Durable Object already claimed.
+ *
+ * Both conditions matter. The tag alone would drop the DO's own report; the
+ * mechanism alone would drop genuinely unhandled DO failures (an alarm crash,
+ * say) that nothing else reports. Together they identify exactly the echo.
+ */
+const isClaimedDurableObjectEcho = (event: ErrorEvent): boolean => {
+  if (event.tags?.[DO_CAUSE_OWNER_TAG] !== DO_CAUSE_OWNER_VALUE) return false;
+  const mechanism = event.exception?.values?.[0]?.mechanism?.type;
+  return typeof mechanism === "string" && mechanism.startsWith("auto.");
+};
+
 export const beforeSendWithOtelCorrelation = (
   event: ErrorEvent,
   options?: { readonly logPayload?: boolean },
-): ErrorEvent => {
+): ErrorEvent | null => {
+  if (isClaimedDurableObjectEcho(event)) return null;
   if (options?.logPayload) {
     console.info(
       JSON.stringify({
@@ -167,6 +197,15 @@ export const captureCauseEffect = (input: unknown): Effect.Effect<string | undef
     return eventId;
   });
 
+/**
+ * Mark the current Sentry scope as "the Durable Object has already dealt with
+ * this cause". Call it AFTER any `captureCauseEffect`, so the DO's own event —
+ * captured against the scope as it was — is not itself mistaken for the echo.
+ */
+export const claimCauseHandledByDurableObject: Effect.Effect<void> = Effect.sync(() => {
+  Sentry.getCurrentScope().setTag(DO_CAUSE_OWNER_TAG, DO_CAUSE_OWNER_VALUE);
+});
+
 export const ErrorCaptureLive: Layer.Layer<ErrorCapture> = Layer.succeed(
   ErrorCapture,
   ErrorCapture.of({

apps/cloud/src/observability/observability.test.ts

@@ -2,8 +2,13 @@ import { describe, expect, it } from "@effect/vitest";
 import { Cause, Effect } from "effect";
 import type * as Tracer from "effect/Tracer";
 
+import type { ErrorEvent } from "@sentry/cloudflare";
+
 import {
   addCurrentOtelCorrelationTags,
+  beforeSendWithOtelCorrelation,
+  DO_CAUSE_OWNER_TAG,
+  DO_CAUSE_OWNER_VALUE,
   OTEL_SPAN_ID_TAG,
   OTEL_TRACE_ID_TAG,
   sentryPayloadForCause,
@@ -91,3 +96,70 @@ describe("Sentry OTel correlation", () => {
     }).pipe(Effect.withSpan("test.sentry_capture"), Effect.withTracer(makeFixedTracer())),
   );
 });
+
+// One Durable Object failure used to open two Sentry issues: the DO's own
+// `captureCause` seam reported it (mechanism `generic`), and then
+// `instrumentDurableObjectWithSentry` reported the very same rejection again as
+// it escaped the method (mechanism `auto.faas.cloudflare.durable_object`). The
+// DO is the owner — it has the session, the classification and the OTel
+// correlation — so its claim suppresses the echo and nothing else.
+describe("Durable Object capture ownership", () => {
+  const doEcho = (overrides: Partial<ErrorEvent> = {}): ErrorEvent => ({
+    type: undefined,
+    tags: { [DO_CAUSE_OWNER_TAG]: DO_CAUSE_OWNER_VALUE },
+    exception: {
+      values: [
+        {
+          type: "Error",
+          value: "Durable Object reset because its code was updated.",
+          mechanism: { type: "auto.faas.cloudflare.durable_object", handled: false },
+        },
+      ],
+    },
+    ...overrides,
+  });
+
+  it("drops the auto-instrumentation's copy of a cause the DO already claimed", () => {
+    expect(beforeSendWithOtelCorrelation(doEcho())).toBeNull();
+  });
+
+  it("keeps the DO's own report, which carries no auto mechanism", () => {
+    const own = doEcho({
+      exception: {
+        values: [
+          {
+            type: "Error",
+            value: "Durable Object reset because its code was updated.",
+            mechanism: { type: "generic", handled: true },
+          },
+        ],
+      },
+    });
+    expect(beforeSendWithOtelCorrelation(own)).not.toBeNull();
+  });
+
+  // An alarm crash or a transport fault is never claimed by the DO seam, and
+  // the auto-instrumentation is the ONLY thing that reports it. Dropping those
+  // would trade duplicate noise for silence.
+  it("keeps an unclaimed Durable Object failure", () => {
+    const unclaimed = doEcho({ tags: {} });
+    expect(beforeSendWithOtelCorrelation(unclaimed)).not.toBeNull();
+  });
+
+  it("keeps ordinary worker events untouched", () => {
+    const workerEvent: ErrorEvent = {
+      type: undefined,
+      tags: { [OTEL_TRACE_ID_TAG]: traceId },
+      exception: {
+        values: [
+          {
+            type: "TypeError",
+            value: "x is not a function",
+            mechanism: { type: "auto.http.cloudflare", handled: false },
+          },
+        ],
+      },
+    };
+    expect(beforeSendWithOtelCorrelation(workerEvent)).not.toBeNull();
+  });
+});