Stable Sentry error grouping across deploys

apps/cloud/src/observability/index.ts

@@ -16,6 +16,7 @@ import { Cause, Effect, Layer } from "effect";
 import type * as Tracer from "effect/Tracer";
 
 import { ErrorCapture } from "@executor-js/api";
+import { withStableGroupingFingerprint } from "@executor-js/sdk/sentry-grouping";
 
 // Drizzle/postgres-js include the failing SQL (params + bound values) in
 // their error message. For OpenAPI source inserts that's 1MB+ of spec
@@ -146,6 +147,53 @@ export const beforeSendWithOtelCorrelation = (
   return event;
 };
 
+/**
+ * The single `beforeSend` the worker and its Durable Objects install.
+ *
+ * The worker ships as content-hashed chunks and its frames are not resolved
+ * back to source, so Sentry's default grouping keys on names like
+ * `execution-rate-limit-<hash>` and re-opens every issue on the next deploy.
+ * `withStableGroupingFingerprint` pins a key with the hash normalized out;
+ * events with no hashed grouping input are left on the default algorithm.
+ *
+ * The two stages are independent and compose in this order: the capture-owner
+ * pass decides WHETHER the event is reported at all (a cause the Durable
+ * Object already claimed is dropped, and a dropped event is never
+ * fingerprinted), and the grouping pass then decides HOW whatever survives is
+ * grouped.
+ */
+export const beforeSendCloudEvent = (
+  event: ErrorEvent,
+  options?: { readonly logPayload?: boolean },
+): ErrorEvent | null => {
+  const reported = beforeSendWithOtelCorrelation(event, options);
+  return reported === null ? null : withStableGroupingFingerprint(reported);
+};
+
+/**
+ * The Sentry options the worker and every Durable Object install. It lives
+ * beside the `beforeSend` it wires so the composition is covered by
+ * observability.test.ts; `server.ts` only passes this through.
+ *
+ * NOTE: do NOT enable `instrumentPrototypeMethods`. It walks the DO prototype
+ * and reads every property — including accessors — to find methods to wrap,
+ * which invokes the `sessionId` getter with `this` bound to the prototype
+ * (where `ctx` is undefined) and throws during construction, 500ing every
+ * session create / cold restore. The DO captures its own errors via the
+ * `captureCause` seam (→ Sentry) instead.
+ */
+export const cloudSentryOptions = (env: Env) => ({
+  dsn: env.SENTRY_DSN,
+  tracesSampleRate: 0,
+  enableLogs: true,
+  sendDefaultPii: true,
+  skipOpenTelemetrySetup: true,
+  beforeSend: (event: ErrorEvent) =>
+    beforeSendCloudEvent(event, {
+      logPayload: !env.SENTRY_DSN || env.SENTRY_OTEL_LOG_PAYLOAD === "true",
+    }),
+});
+
 export const addCurrentOtelCorrelationTags = <
   T extends { readonly tags?: Record<string, unknown> },
 >(

apps/cloud/src/observability/observability.test.ts

@@ -6,7 +6,9 @@ import type { ErrorEvent } from "@sentry/cloudflare";
 
 import {
   addCurrentOtelCorrelationTags,
+  beforeSendCloudEvent,
   beforeSendWithOtelCorrelation,
+  cloudSentryOptions,
   DO_CAUSE_OWNER_TAG,
   DO_CAUSE_OWNER_VALUE,
   OTEL_SPAN_ID_TAG,
@@ -84,6 +86,83 @@ describe("sentryPayloadForCause", () => {
   });
 });
 
+// Grouping keys are decided inside the Sentry SDK and never appear on any
+// product surface, so the e2e harness cannot observe them. The split is:
+// e2e/cloud/sentry-otel-correlation.test.ts proves the worker really installs
+// `cloudSentryOptions.beforeSend` (its correlation payload only exists if that
+// hook ran), and the tests here prove the hook it installs fingerprints.
+describe("Sentry grouping", () => {
+  // The worker bundle ships as content-hashed chunks, so the only module name
+  // Sentry ever sees for a given frame changes on every deploy.
+  const workerEvent = (chunkHash: string): ErrorEvent => ({
+    type: undefined,
+    exception: {
+      values: [
+        {
+          type: "GateCheckTimeoutError",
+          value: "balance check timed out",
+          stacktrace: {
+            frames: [
+              {
+                filename: `/assets/execution-rate-limit-${chunkHash}.js`,
+                module: `execution-rate-limit-${chunkHash}`,
+                function: "timeoutOrElse",
+                in_app: true,
+              },
+            ],
+          },
+        },
+      ],
+    },
+  });
+
+  it("pins one fingerprint across two deploys of the same chunk", () => {
+    const before = beforeSendCloudEvent(workerEvent("BAuwphPA"), {});
+    const after = beforeSendCloudEvent(workerEvent("DkcPBbWe"), {});
+
+    expect(before?.fingerprint).toBeDefined();
+    expect(before?.fingerprint).toEqual(after?.fingerprint);
+  });
+
+  it("leaves unhashed events on Sentry's default grouping", () => {
+    const event: ErrorEvent = {
+      type: undefined,
+      exception: {
+        values: [
+          {
+            type: "AutumnError",
+            stacktrace: {
+              frames: [
+                { filename: "/src/engine/execution-gate.ts", function: "checkExecutionBalance" },
+              ],
+            },
+          },
+        ],
+      },
+    };
+
+    const sent = beforeSendCloudEvent(event, {});
+
+    expect(sent).not.toBeNull();
+    expect(sent?.fingerprint).toBeUndefined();
+  });
+
+  // The wiring check: this is the exact object handed to `Sentry.withSentry`
+  // and `instrumentDurableObjectWithSentry` in server.ts. If the normalizer is
+  // ever dropped from the hook the worker installs, this fails.
+  it("the options the worker and DOs install carry the fingerprinting hook", () => {
+    const options = cloudSentryOptions({ SENTRY_DSN: "https://public@example.invalid/1" } as Env);
+    const sent = options.beforeSend(workerEvent("BAuwphPA"));
+
+    expect(sent?.fingerprint).toEqual([
+      "GateCheckTimeoutError",
+      "timeoutOrElse@execution-rate-limit",
+    ]);
+    // Same source, next deploy, new chunk hash — one issue, not two.
+    expect(options.beforeSend(workerEvent("DkcPBbWe"))?.fingerprint).toEqual(sent?.fingerprint);
+  });
+});
+
 describe("Sentry OTel correlation", () => {
   it.effect("adds tags from the active Effect span", () =>
     Effect.gen(function* () {
@@ -162,4 +241,62 @@ describe("Durable Object capture ownership", () => {
     };
     expect(beforeSendWithOtelCorrelation(workerEvent)).not.toBeNull();
   });
+
+  // The two stages of the installed `beforeSend` answer different questions and
+  // must both keep working: capture ownership decides WHETHER an event is
+  // reported, stable grouping decides HOW a reported one is grouped. A dropped
+  // event is never fingerprinted, and a surviving one still is.
+  describe("composed with stable grouping", () => {
+    const hashedFrames = (chunkHash: string) => ({
+      stacktrace: {
+        frames: [
+          {
+            filename: `/assets/session-durable-object-${chunkHash}.js`,
+            module: `session-durable-object-${chunkHash}`,
+            function: "handleSessionRequest",
+            in_app: true,
+          },
+        ],
+      },
+    });
+
+    it("drops a claimed echo rather than fingerprinting it", () => {
+      const echo = doEcho({
+        exception: {
+          values: [
+            {
+              type: "Error",
+              value: "Durable Object reset because its code was updated.",
+              mechanism: { type: "auto.faas.cloudflare.durable_object", handled: false },
+              ...hashedFrames("BAuwphPA"),
+            },
+          ],
+        },
+      });
+
+      expect(beforeSendCloudEvent(echo, {})).toBeNull();
+    });
+
+    it("pins a stable fingerprint on the report the DO itself owns", () => {
+      const ownReport = (chunkHash: string): ErrorEvent =>
+        doEcho({
+          exception: {
+            values: [
+              {
+                type: "Error",
+                value: "Durable Object reset because its code was updated.",
+                mechanism: { type: "generic", handled: true },
+                ...hashedFrames(chunkHash),
+              },
+            ],
+          },
+        });
+
+      const before = beforeSendCloudEvent(ownReport("BAuwphPA"), {});
+      const after = beforeSendCloudEvent(ownReport("DkcPBbWe"), {});
+
+      expect(before?.fingerprint).toBeDefined();
+      expect(before?.fingerprint).toEqual(after?.fingerprint);
+    });
+  });
 });

apps/cloud/src/server.ts

@@ -1,6 +1,5 @@
 import { DurableObject } from "cloudflare:workers";
 import { SpanKind, SpanStatusCode, context, trace, type SpanContext } from "@opentelemetry/api";
-import type { ErrorEvent } from "@sentry/cloudflare";
 import {
   ATTR_HTTP_REQUEST_METHOD,
   ATTR_HTTP_RESPONSE_STATUS_CODE,
@@ -19,7 +18,7 @@ import { classifyMcpPath, prepareMcpOrgScope } from "./mcp/mount";
 import { parseTraceparent } from "./mcp/traceparent";
 import { McpSessionDOSqlite as McpSessionDOBase } from "./mcp/session-durable-object";
 import {
-  beforeSendWithOtelCorrelation,
+  cloudSentryOptions,
   captureCause,
   otelCorrelationContextFromOpenTelemetrySpan,
   SENTRY_EVENT_ID_ATTRIBUTE,
@@ -28,28 +27,6 @@ import {
 import { browserTracesResponse } from "./observability/browser-traces";
 import { flushTracerProvider, installTracerProvider } from "./observability/telemetry";
 
-// ---------------------------------------------------------------------------
-// Sentry config
-// ---------------------------------------------------------------------------
-
-const sentryOptions = (env: Env) => ({
-  dsn: env.SENTRY_DSN,
-  tracesSampleRate: 0,
-  enableLogs: true,
-  sendDefaultPii: true,
-  skipOpenTelemetrySetup: true,
-  beforeSend: (event: ErrorEvent) =>
-    beforeSendWithOtelCorrelation(event, {
-      logPayload: !env.SENTRY_DSN || env.SENTRY_OTEL_LOG_PAYLOAD === "true",
-    }),
-  // NOTE: do NOT enable `instrumentPrototypeMethods`. It walks the DO prototype
-  // and reads every property — including accessors — to find methods to wrap,
-  // which invokes the `sessionId` getter with `this` bound to the prototype
-  // (where `ctx` is undefined) and throws during construction, 500ing every
-  // session create / cold restore. The DO captures its own errors via the
-  // `captureCause` seam (→ Sentry) instead.
-});
-
 // ---------------------------------------------------------------------------
 // Durable Object — wrapped with Sentry so DO errors land in Sentry (inits the
 // client inside the DO isolate, which plain `Sentry.captureException` cannot
@@ -58,7 +35,7 @@ const sentryOptions = (env: Env) => ({
 // ---------------------------------------------------------------------------
 
 export const McpSessionDOSqlite = Sentry.instrumentDurableObjectWithSentry(
-  sentryOptions,
+  cloudSentryOptions,
   McpSessionDOBase,
 );
 
@@ -458,4 +435,4 @@ const cloudflareHandler: ExportedHandler<Env> = {
   },
 };
 
-export default Sentry.withSentry(sentryOptions, cloudflareHandler);
+export default Sentry.withSentry(cloudSentryOptions, cloudflareHandler);