Keep Sentry issue grouping stable across deploys

Normalize build content hashes out of the grouping key in the cloud, desktop main and desktop renderer Sentry inits, and merge address-keyed Chromium soft-assert minidumps into one fingerprint.

Author: RhysSullivan

apps/cloud/src/observability/index.ts

@@ -16,6 +16,7 @@ import { Cause, Effect, Layer } from "effect";
 import type * as Tracer from "effect/Tracer";
 
 import { ErrorCapture } from "@executor-js/api";
+import { stableGroupingFingerprint, type GroupingEvent } from "@executor-js/sdk/sentry-grouping";
 
 // Drizzle/postgres-js include the failing SQL (params + bound values) in
 // their error message. For OpenAPI source inserts that's 1MB+ of spec
@@ -116,6 +117,24 @@ export const beforeSendWithOtelCorrelation = (
   return event;
 };
 
+/**
+ * The worker ships as content-hashed chunks and its frames are not resolved
+ * back to source, so Sentry's default grouping keys on names like
+ * `execution-rate-limit-<hash>` and re-opens every issue on the next deploy.
+ * Pin a fingerprint with the hash normalized out; events with no hashed
+ * grouping input are left on the default algorithm.
+ */
+export const withStableGroupingFingerprint = <T extends GroupingEvent>(event: T): T => {
+  const fingerprint = stableGroupingFingerprint(event);
+  return fingerprint ? { ...event, fingerprint: [...fingerprint] } : event;
+};
+
+/** The single `beforeSend` the worker and its Durable Objects install. */
+export const beforeSendCloudEvent = (
+  event: ErrorEvent,
+  options?: { readonly logPayload?: boolean },
+): ErrorEvent => withStableGroupingFingerprint(beforeSendWithOtelCorrelation(event, options));
+
 export const addCurrentOtelCorrelationTags = <
   T extends { readonly tags?: Record<string, unknown> },
 >(

apps/cloud/src/observability/observability.test.ts

@@ -2,8 +2,11 @@ import { describe, expect, it } from "@effect/vitest";
 import { Cause, Effect } from "effect";
 import type * as Tracer from "effect/Tracer";
 
+import type { ErrorEvent } from "@sentry/cloudflare";
+
 import {
   addCurrentOtelCorrelationTags,
+  beforeSendCloudEvent,
   OTEL_SPAN_ID_TAG,
   OTEL_TRACE_ID_TAG,
   sentryPayloadForCause,
@@ -79,6 +82,71 @@ describe("sentryPayloadForCause", () => {
   });
 });
 
+describe("Sentry grouping", () => {
+  // The worker bundle ships as content-hashed chunks, so the only module name
+  // Sentry ever sees for a given frame changes on every deploy.
+  const workerEvent = (chunkHash: string): ErrorEvent => ({
+    type: undefined,
+    exception: {
+      values: [
+        {
+          type: "GateCheckTimeoutError",
+          value: "balance check timed out",
+          stacktrace: {
+            frames: [
+              {
+                filename: `/assets/execution-rate-limit-${chunkHash}.js`,
+                module: `execution-rate-limit-${chunkHash}`,
+                function: "timeoutOrElse",
+                in_app: true,
+              },
+            ],
+          },
+        },
+      ],
+    },
+  });
+
+  it("pins one fingerprint across two deploys of the same chunk", () => {
+    const before = beforeSendCloudEvent(workerEvent("BAuwphPA"), {});
+    const after = beforeSendCloudEvent(workerEvent("DkcPBbWe"), {});
+
+    expect(before.fingerprint).toBeDefined();
+    expect(before.fingerprint).toEqual(after.fingerprint);
+  });
+
+  it("leaves unhashed events on Sentry's default grouping", () => {
+    const event: ErrorEvent = {
+      type: undefined,
+      exception: {
+        values: [
+          {
+            type: "AutumnError",
+            stacktrace: {
+              frames: [
+                { filename: "/src/engine/execution-gate.ts", function: "checkExecutionBalance" },
+              ],
+            },
+          },
+        ],
+      },
+    };
+
+    expect(beforeSendCloudEvent(event, {}).fingerprint).toBeUndefined();
+  });
+
+  it("keeps the OTel correlation tags it is composed with", () => {
+    const event: ErrorEvent = {
+      type: undefined,
+      tags: { [OTEL_TRACE_ID_TAG]: traceId, [OTEL_SPAN_ID_TAG]: spanId },
+    };
+
+    const result = beforeSendCloudEvent(event, {});
+    expect(result.tags?.[OTEL_TRACE_ID_TAG]).toBe(traceId);
+    expect(result.tags?.[OTEL_SPAN_ID_TAG]).toBe(spanId);
+  });
+});
+
 describe("Sentry OTel correlation", () => {
   it.effect("adds tags from the active Effect span", () =>
     Effect.gen(function* () {

apps/cloud/src/server.ts

@@ -19,7 +19,7 @@ import { classifyMcpPath, prepareMcpOrgScope } from "./mcp/mount";
 import { parseTraceparent } from "./mcp/traceparent";
 import { McpSessionDOSqlite as McpSessionDOBase } from "./mcp/session-durable-object";
 import {
-  beforeSendWithOtelCorrelation,
+  beforeSendCloudEvent,
   captureCause,
   otelCorrelationContextFromOpenTelemetrySpan,
   SENTRY_EVENT_ID_ATTRIBUTE,
@@ -39,7 +39,7 @@ const sentryOptions = (env: Env) => ({
   sendDefaultPii: true,
   skipOpenTelemetrySetup: true,
   beforeSend: (event: ErrorEvent) =>
-    beforeSendWithOtelCorrelation(event, {
+    beforeSendCloudEvent(event, {
       logPayload: !env.SENTRY_DSN || env.SENTRY_OTEL_LOG_PAYLOAD === "true",
     }),
   // NOTE: do NOT enable `instrumentPrototypeMethods`. It walks the DO prototype

apps/desktop/src/main/crash-fingerprint.test.ts

@@ -0,0 +1,105 @@
+import { expect, test } from "@effect/vitest";
+
+import { crashReportFingerprint, type CrashEvent } from "./crash-fingerprint";
+
+// Chromium's soft-assert path (`NOTREACHED()`/`DCHECK`) dumps and keeps
+// running. Sentry titles each one with the faulting load address, so one
+// condition arrives as a new issue every time the address moves.
+const softAssertEvent = (address: string): CrashEvent => ({
+  exception: {
+    values: [
+      {
+        type: "Fatal Error",
+        value: `Simulated Exception / ${address}`,
+        mechanism: { type: "minidump" },
+        stacktrace: {
+          frames: [
+            { function: "logging::NotReachedLogMessage::~NotReachedLogMessage" },
+            { function: "logging::HandleCheckErrorLogMessage" },
+            { function: "base::debug::DumpWithoutCrashing" },
+            { function: "crash_reporter::DumpWithoutCrashing" },
+          ],
+        },
+      },
+    ],
+  },
+});
+
+test("address-keyed Chromium soft asserts collapse to one fingerprint", () => {
+  const first = crashReportFingerprint(softAssertEvent("0x00000001a2b3c4d5"));
+  const second = crashReportFingerprint(softAssertEvent("0x00000007e8f9a0b1"));
+
+  expect(first).toEqual(["chromium-dump-without-crashing"]);
+  expect(first).toEqual(second);
+});
+
+test("a real native abort keeps Sentry's own grouping", () => {
+  const abortEvent: CrashEvent = {
+    exception: {
+      values: [
+        {
+          type: "EXC_CRASH",
+          value: "SIGABRT",
+          mechanism: { type: "minidump" },
+          stacktrace: {
+            frames: [
+              { function: "abort" },
+              { function: "pthread_kill" },
+              { function: "__pthread_kill" },
+            ],
+          },
+        },
+      ],
+    },
+  };
+
+  expect(crashReportFingerprint(abortEvent)).toBeUndefined();
+});
+
+test("renderer chunk hashes are normalized out of the fingerprint", () => {
+  const rendererEvent = (chunkHash: string): CrashEvent => ({
+    culprit: `loadConnections(assets/atoms-${chunkHash})`,
+    exception: {
+      values: [
+        {
+          type: "TypeError",
+          value: "cannot read properties of undefined",
+          stacktrace: {
+            frames: [
+              {
+                filename: `http://127.0.0.1:4789/assets/atoms-${chunkHash}.js`,
+                module: `atoms-${chunkHash}`,
+                function: "loadConnections",
+                in_app: true,
+              },
+            ],
+          },
+        },
+      ],
+    },
+  });
+
+  const release1 = crashReportFingerprint(rendererEvent("Yemn7yhP"));
+  const release2 = crashReportFingerprint(rendererEvent("CeCENfWa"));
+
+  expect(release1).toEqual(["TypeError", "loadConnections@atoms"]);
+  expect(release1).toEqual(release2);
+});
+
+test("events with no volatile grouping input are left alone", () => {
+  expect(crashReportFingerprint({})).toBeUndefined();
+  expect(
+    crashReportFingerprint({
+      exception: {
+        values: [
+          {
+            type: "Error",
+            stacktrace: {
+              frames: [{ filename: "/src/main/sidecar.ts", function: "startSidecar" }],
+            },
+          },
+        ],
+      },
+    }),
+  ).toBeUndefined();
+});

apps/desktop/src/main/crash-fingerprint.ts

@@ -0,0 +1,39 @@
+/**
+ * Grouping keys for desktop crash reports.
+ *
+ * Two things split one desktop problem across many Sentry issues:
+ *
+ *  - Chromium's soft-assert path (`NOTREACHED()`/`DCHECK`) calls
+ *    `DumpWithoutCrashing`, which files a minidump and lets the process carry
+ *    on. Sentry titles each one with the faulting load address, so the same
+ *    condition arrives as a new issue every time the address moves.
+ *  - Renderer and main bundles ship as content-hashed chunks, so unresolved
+ *    frames name `atoms-<hash>` and re-group on every release.
+ *
+ * Both are grouping problems only: nothing here drops, downgrades or edits an
+ * event, and the frames keep their hashes so sourcemap resolution still works.
+ */
+
+import {
+  stableGroupingFingerprint,
+  type GroupingEvent,
+  type GroupingFrame,
+} from "@executor-js/sdk/sentry-grouping";
+
+export type CrashEvent = GroupingEvent;
+
+export const CHROMIUM_SOFT_ASSERT_FINGERPRINT = "chromium-dump-without-crashing";
+
+const isSoftAssertFrame = (frame: GroupingFrame): boolean =>
+  (frame.function ?? "").includes("DumpWithoutCrashing");
+
+/**
+ * The fingerprint to attach to a crash event, or `undefined` to keep Sentry's
+ * default grouping (which is right for a real native abort — one issue per
+ * distinct stack is what we want there).
+ */
+export const crashReportFingerprint = (event: CrashEvent): readonly string[] | undefined => {
+  const frames = (event.exception?.values ?? []).flatMap((value) => value.stacktrace?.frames ?? []);
+  if (frames.some(isSoftAssertFrame)) return [CHROMIUM_SOFT_ASSERT_FINGERPRINT];
+  return stableGroupingFingerprint(event);
+};

apps/desktop/src/main/diagnostics.ts

@@ -23,6 +23,7 @@ import { dirname, join } from "node:path";
 import { app, crashReporter, dialog, shell } from "electron";
 import log from "electron-log/main.js";
 import * as Sentry from "@sentry/electron/main";
+import { crashReportFingerprint } from "./crash-fingerprint";
 import { getServerSettings } from "./settings";
 
 const sentryDsn = __EXECUTOR_SENTRY_DSN__;
@@ -89,6 +90,11 @@ export const initErrorReporting = () => {
           runId,
         },
       },
+      // Grouping only — the event is forwarded untouched otherwise.
+      beforeSend: (event) => {
+        const fingerprint = crashReportFingerprint(event);
+        return fingerprint ? { ...event, fingerprint: [...fingerprint] } : event;
+      },
     });
   } else {
     // No DSN baked in — keep native crash dumps local so a user-reported

packages/app/src/crash-reporting.ts

@@ -12,6 +12,8 @@
  * once initialized — no reporter rewiring needed.
  */
 
+import { stableGroupingFingerprint } from "@executor-js/sdk/sentry-grouping";
+
 interface CrashReportingConfig {
   readonly dsn: string;
   readonly release: string;
@@ -45,6 +47,14 @@ export const initDesktopCrashReporting = (): void => {
             runId: config.runId,
           },
         },
+        // Route chunks are content-hashed, so an unresolved frame names
+        // `atoms-<hash>` and one bug re-groups on every release. Pin a
+        // fingerprint with the hash normalized out; the event itself keeps its
+        // hashed filenames so sourcemap resolution is unaffected.
+        beforeSend: (event) => {
+          const fingerprint = stableGroupingFingerprint(event);
+          return fingerprint ? { ...event, fingerprint: [...fingerprint] } : event;
+        },
       });
     } catch {
       // Reporting failures stay silent — there is nowhere left to report them.

packages/core/sdk/package.json

@@ -25,7 +25,8 @@
     "./testing": "./src/testing.ts",
     "./migration": "./src/migration-spec.ts",
     "./http-auth": "./src/http-auth/index.ts",
-    "./public-origin": "./src/public-origin.ts"
+    "./public-origin": "./src/public-origin.ts",
+    "./sentry-grouping": "./src/sentry-grouping.ts"
   },
   "publishConfig": {
     "access": "public",
@@ -83,6 +84,12 @@
           "types": "./dist/public-origin.d.ts",
           "default": "./dist/public-origin.js"
         }
+      },
+      "./sentry-grouping": {
+        "import": {
+          "types": "./dist/sentry-grouping.d.ts",
+          "default": "./dist/sentry-grouping.js"
+        }
       }
     }
   },