From 4c3f2225487fbe6110a8e86d74b898a4476360f7 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Thu, 27 Aug 2026 13:08:32 -0700 Subject: [PATCH 1/3] Keep Sentry issue grouping stable across deploys Normalize build content hashes out of the grouping key in the cloud, desktop main and desktop renderer Sentry inits, and merge address-keyed Chromium soft-assert minidumps into one fingerprint. --- apps/cloud/src/observability/index.ts | 30 +++ .../src/observability/observability.test.ts | 119 +++++++++++ apps/cloud/src/server.ts | 4 +- .../src/main/crash-fingerprint.test.ts | 105 +++++++++ apps/desktop/src/main/crash-fingerprint.ts | 39 ++++ apps/desktop/src/main/diagnostics.ts | 6 + packages/app/src/crash-reporting.ts | 10 + packages/core/sdk/package.json | 9 +- packages/core/sdk/src/sentry-grouping.test.ts | 201 ++++++++++++++++++ packages/core/sdk/src/sentry-grouping.ts | 157 ++++++++++++++ packages/core/sdk/tsup.config.ts | 1 + 11 files changed, 678 insertions(+), 3 deletions(-) create mode 100644 apps/desktop/src/main/crash-fingerprint.test.ts create mode 100644 apps/desktop/src/main/crash-fingerprint.ts create mode 100644 packages/core/sdk/src/sentry-grouping.test.ts create mode 100644 packages/core/sdk/src/sentry-grouping.ts diff --git a/apps/cloud/src/observability/index.ts b/apps/cloud/src/observability/index.ts index a56118a6c..bc2ca5fd9 100644 --- a/apps/cloud/src/observability/index.ts +++ b/apps/cloud/src/observability/index.ts @@ -16,6 +16,7 @@ import { Cause, Effect, Layer } from "effect"; import type * as Tracer from "effect/Tracer"; import { ErrorCapture } from "@executor-js/api"; +import { stableGroupingFingerprint, type GroupingEvent } from "@executor-js/sdk/sentry-grouping"; // Drizzle/postgres-js include the failing SQL (params + bound values) in // their error message. For OpenAPI source inserts that's 1MB+ of spec @@ -146,6 +147,35 @@ export const beforeSendWithOtelCorrelation = ( return event; }; +/** + * The worker ships as content-hashed chunks and its frames are not resolved + * back to source, so Sentry's default grouping keys on names like + * `execution-rate-limit-` and re-opens every issue on the next deploy. + * Pin a fingerprint with the hash normalized out; events with no hashed + * grouping input are left on the default algorithm. + */ +export const withStableGroupingFingerprint = (event: T): T => { + const fingerprint = stableGroupingFingerprint(event); + return fingerprint ? { ...event, fingerprint: [...fingerprint] } : event; +}; + +/** + * The single `beforeSend` the worker and its Durable Objects install. + * + * The two stages are independent and compose in this order: the capture-owner + * pass decides WHETHER the event is reported at all (a cause the Durable + * Object already claimed is dropped, and a dropped event is never + * fingerprinted), and the grouping pass then decides HOW whatever survives is + * grouped. + */ +export const beforeSendCloudEvent = ( + event: ErrorEvent, + options?: { readonly logPayload?: boolean }, +): ErrorEvent | null => { + const reported = beforeSendWithOtelCorrelation(event, options); + return reported === null ? null : withStableGroupingFingerprint(reported); +}; + export const addCurrentOtelCorrelationTags = < T extends { readonly tags?: Record }, >( diff --git a/apps/cloud/src/observability/observability.test.ts b/apps/cloud/src/observability/observability.test.ts index 9185cbec8..69906078f 100644 --- a/apps/cloud/src/observability/observability.test.ts +++ b/apps/cloud/src/observability/observability.test.ts @@ -6,6 +6,7 @@ import type { ErrorEvent } from "@sentry/cloudflare"; import { addCurrentOtelCorrelationTags, + beforeSendCloudEvent, beforeSendWithOtelCorrelation, DO_CAUSE_OWNER_TAG, DO_CAUSE_OWNER_VALUE, @@ -84,6 +85,66 @@ describe("sentryPayloadForCause", () => { }); }); +// Grouping keys are decided inside the Sentry SDK and never appear on any +// product surface, so the e2e harness cannot observe them; the running +// beforeSend itself is covered by e2e/cloud/sentry-otel-correlation.test.ts. +describe("Sentry grouping", () => { + // The worker bundle ships as content-hashed chunks, so the only module name + // Sentry ever sees for a given frame changes on every deploy. + const workerEvent = (chunkHash: string): ErrorEvent => ({ + type: undefined, + exception: { + values: [ + { + type: "GateCheckTimeoutError", + value: "balance check timed out", + stacktrace: { + frames: [ + { + filename: `/assets/execution-rate-limit-${chunkHash}.js`, + module: `execution-rate-limit-${chunkHash}`, + function: "timeoutOrElse", + in_app: true, + }, + ], + }, + }, + ], + }, + }); + + it("pins one fingerprint across two deploys of the same chunk", () => { + const before = beforeSendCloudEvent(workerEvent("BAuwphPA"), {}); + const after = beforeSendCloudEvent(workerEvent("DkcPBbWe"), {}); + + expect(before?.fingerprint).toBeDefined(); + expect(before?.fingerprint).toEqual(after?.fingerprint); + }); + + it("leaves unhashed events on Sentry's default grouping", () => { + const event: ErrorEvent = { + type: undefined, + exception: { + values: [ + { + type: "AutumnError", + stacktrace: { + frames: [ + { filename: "/src/engine/execution-gate.ts", function: "checkExecutionBalance" }, + ], + }, + }, + ], + }, + }; + + const sent = beforeSendCloudEvent(event, {}); + + expect(sent).not.toBeNull(); + expect(sent?.fingerprint).toBeUndefined(); + }); +}); + describe("Sentry OTel correlation", () => { it.effect("adds tags from the active Effect span", () => Effect.gen(function* () { @@ -162,4 +223,62 @@ describe("Durable Object capture ownership", () => { }; expect(beforeSendWithOtelCorrelation(workerEvent)).not.toBeNull(); }); + + // The two stages of the installed `beforeSend` answer different questions and + // must both keep working: capture ownership decides WHETHER an event is + // reported, stable grouping decides HOW a reported one is grouped. A dropped + // event is never fingerprinted, and a surviving one still is. + describe("composed with stable grouping", () => { + const hashedFrames = (chunkHash: string) => ({ + stacktrace: { + frames: [ + { + filename: `/assets/session-durable-object-${chunkHash}.js`, + module: `session-durable-object-${chunkHash}`, + function: "handleSessionRequest", + in_app: true, + }, + ], + }, + }); + + it("drops a claimed echo rather than fingerprinting it", () => { + const echo = doEcho({ + exception: { + values: [ + { + type: "Error", + value: "Durable Object reset because its code was updated.", + mechanism: { type: "auto.faas.cloudflare.durable_object", handled: false }, + ...hashedFrames("BAuwphPA"), + }, + ], + }, + }); + + expect(beforeSendCloudEvent(echo, {})).toBeNull(); + }); + + it("pins a stable fingerprint on the report the DO itself owns", () => { + const ownReport = (chunkHash: string): ErrorEvent => + doEcho({ + exception: { + values: [ + { + type: "Error", + value: "Durable Object reset because its code was updated.", + mechanism: { type: "generic", handled: true }, + ...hashedFrames(chunkHash), + }, + ], + }, + }); + + const before = beforeSendCloudEvent(ownReport("BAuwphPA"), {}); + const after = beforeSendCloudEvent(ownReport("DkcPBbWe"), {}); + + expect(before?.fingerprint).toBeDefined(); + expect(before?.fingerprint).toEqual(after?.fingerprint); + }); + }); }); diff --git a/apps/cloud/src/server.ts b/apps/cloud/src/server.ts index ec4771e96..942127cbd 100644 --- a/apps/cloud/src/server.ts +++ b/apps/cloud/src/server.ts @@ -19,7 +19,7 @@ import { classifyMcpPath, prepareMcpOrgScope } from "./mcp/mount"; import { parseTraceparent } from "./mcp/traceparent"; import { McpSessionDOSqlite as McpSessionDOBase } from "./mcp/session-durable-object"; import { - beforeSendWithOtelCorrelation, + beforeSendCloudEvent, captureCause, otelCorrelationContextFromOpenTelemetrySpan, SENTRY_EVENT_ID_ATTRIBUTE, @@ -39,7 +39,7 @@ const sentryOptions = (env: Env) => ({ sendDefaultPii: true, skipOpenTelemetrySetup: true, beforeSend: (event: ErrorEvent) => - beforeSendWithOtelCorrelation(event, { + beforeSendCloudEvent(event, { logPayload: !env.SENTRY_DSN || env.SENTRY_OTEL_LOG_PAYLOAD === "true", }), // NOTE: do NOT enable `instrumentPrototypeMethods`. It walks the DO prototype diff --git a/apps/desktop/src/main/crash-fingerprint.test.ts b/apps/desktop/src/main/crash-fingerprint.test.ts new file mode 100644 index 000000000..55448eb1e --- /dev/null +++ b/apps/desktop/src/main/crash-fingerprint.test.ts @@ -0,0 +1,105 @@ +import { expect, test } from "@effect/vitest"; + +import { crashReportFingerprint, type CrashEvent } from "./crash-fingerprint"; + +// Chromium's soft-assert path (`NOTREACHED()`/`DCHECK`) dumps and keeps +// running. Sentry titles each one with the faulting load address, so one +// condition arrives as a new issue every time the address moves. +const softAssertEvent = (address: string): CrashEvent => ({ + exception: { + values: [ + { + type: "Fatal Error", + value: `Simulated Exception / ${address}`, + mechanism: { type: "minidump" }, + stacktrace: { + frames: [ + { function: "logging::NotReachedLogMessage::~NotReachedLogMessage" }, + { function: "logging::HandleCheckErrorLogMessage" }, + { function: "base::debug::DumpWithoutCrashing" }, + { function: "crash_reporter::DumpWithoutCrashing" }, + ], + }, + }, + ], + }, +}); + +test("address-keyed Chromium soft asserts collapse to one fingerprint", () => { + const first = crashReportFingerprint(softAssertEvent("0x00000001a2b3c4d5")); + const second = crashReportFingerprint(softAssertEvent("0x00000007e8f9a0b1")); + + expect(first).toEqual(["chromium-dump-without-crashing"]); + expect(first).toEqual(second); +}); + +test("a real native abort keeps Sentry's own grouping", () => { + const abortEvent: CrashEvent = { + exception: { + values: [ + { + type: "EXC_CRASH", + value: "SIGABRT", + mechanism: { type: "minidump" }, + stacktrace: { + frames: [ + { function: "abort" }, + { function: "pthread_kill" }, + { function: "__pthread_kill" }, + ], + }, + }, + ], + }, + }; + + expect(crashReportFingerprint(abortEvent)).toBeUndefined(); +}); + +test("renderer chunk hashes are normalized out of the fingerprint", () => { + const rendererEvent = (chunkHash: string): CrashEvent => ({ + culprit: `loadConnections(assets/atoms-${chunkHash})`, + exception: { + values: [ + { + type: "TypeError", + value: "cannot read properties of undefined", + stacktrace: { + frames: [ + { + filename: `http://127.0.0.1:4789/assets/atoms-${chunkHash}.js`, + module: `atoms-${chunkHash}`, + function: "loadConnections", + in_app: true, + }, + ], + }, + }, + ], + }, + }); + + const release1 = crashReportFingerprint(rendererEvent("Yemn7yhP")); + const release2 = crashReportFingerprint(rendererEvent("CeCENfWa")); + + expect(release1).toEqual(["TypeError", "loadConnections@atoms"]); + expect(release1).toEqual(release2); +}); + +test("events with no volatile grouping input are left alone", () => { + expect(crashReportFingerprint({})).toBeUndefined(); + expect( + crashReportFingerprint({ + exception: { + values: [ + { + type: "Error", + stacktrace: { + frames: [{ filename: "/src/main/sidecar.ts", function: "startSidecar" }], + }, + }, + ], + }, + }), + ).toBeUndefined(); +}); diff --git a/apps/desktop/src/main/crash-fingerprint.ts b/apps/desktop/src/main/crash-fingerprint.ts new file mode 100644 index 000000000..b0718b753 --- /dev/null +++ b/apps/desktop/src/main/crash-fingerprint.ts @@ -0,0 +1,39 @@ +/** + * Grouping keys for desktop crash reports. + * + * Two things split one desktop problem across many Sentry issues: + * + * - Chromium's soft-assert path (`NOTREACHED()`/`DCHECK`) calls + * `DumpWithoutCrashing`, which files a minidump and lets the process carry + * on. Sentry titles each one with the faulting load address, so the same + * condition arrives as a new issue every time the address moves. + * - Renderer and main bundles ship as content-hashed chunks, so unresolved + * frames name `atoms-` and re-group on every release. + * + * Both are grouping problems only: nothing here drops, downgrades or edits an + * event, and the frames keep their hashes so sourcemap resolution still works. + */ + +import { + stableGroupingFingerprint, + type GroupingEvent, + type GroupingFrame, +} from "@executor-js/sdk/sentry-grouping"; + +export type CrashEvent = GroupingEvent; + +export const CHROMIUM_SOFT_ASSERT_FINGERPRINT = "chromium-dump-without-crashing"; + +const isSoftAssertFrame = (frame: GroupingFrame): boolean => + (frame.function ?? "").includes("DumpWithoutCrashing"); + +/** + * The fingerprint to attach to a crash event, or `undefined` to keep Sentry's + * default grouping (which is right for a real native abort — one issue per + * distinct stack is what we want there). + */ +export const crashReportFingerprint = (event: CrashEvent): readonly string[] | undefined => { + const frames = (event.exception?.values ?? []).flatMap((value) => value.stacktrace?.frames ?? []); + if (frames.some(isSoftAssertFrame)) return [CHROMIUM_SOFT_ASSERT_FINGERPRINT]; + return stableGroupingFingerprint(event); +}; diff --git a/apps/desktop/src/main/diagnostics.ts b/apps/desktop/src/main/diagnostics.ts index 3d04b1cc2..e724f6dea 100644 --- a/apps/desktop/src/main/diagnostics.ts +++ b/apps/desktop/src/main/diagnostics.ts @@ -23,6 +23,7 @@ import { dirname, join } from "node:path"; import { app, crashReporter, dialog, shell } from "electron"; import log from "electron-log/main.js"; import * as Sentry from "@sentry/electron/main"; +import { crashReportFingerprint } from "./crash-fingerprint"; import { getServerSettings } from "./settings"; const sentryDsn = __EXECUTOR_SENTRY_DSN__; @@ -89,6 +90,11 @@ export const initErrorReporting = () => { runId, }, }, + // Grouping only — the event is forwarded untouched otherwise. + beforeSend: (event) => { + const fingerprint = crashReportFingerprint(event); + return fingerprint ? { ...event, fingerprint: [...fingerprint] } : event; + }, }); } else { // No DSN baked in — keep native crash dumps local so a user-reported diff --git a/packages/app/src/crash-reporting.ts b/packages/app/src/crash-reporting.ts index e06146aa7..41225f024 100644 --- a/packages/app/src/crash-reporting.ts +++ b/packages/app/src/crash-reporting.ts @@ -12,6 +12,8 @@ * once initialized — no reporter rewiring needed. */ +import { stableGroupingFingerprint } from "@executor-js/sdk/sentry-grouping"; + interface CrashReportingConfig { readonly dsn: string; readonly release: string; @@ -45,6 +47,14 @@ export const initDesktopCrashReporting = (): void => { runId: config.runId, }, }, + // Route chunks are content-hashed, so an unresolved frame names + // `atoms-` and one bug re-groups on every release. Pin a + // fingerprint with the hash normalized out; the event itself keeps its + // hashed filenames so sourcemap resolution is unaffected. + beforeSend: (event) => { + const fingerprint = stableGroupingFingerprint(event); + return fingerprint ? { ...event, fingerprint: [...fingerprint] } : event; + }, }); } catch { // Reporting failures stay silent — there is nowhere left to report them. diff --git a/packages/core/sdk/package.json b/packages/core/sdk/package.json index 17874f8d7..289b54834 100644 --- a/packages/core/sdk/package.json +++ b/packages/core/sdk/package.json @@ -25,7 +25,8 @@ "./testing": "./src/testing.ts", "./migration": "./src/migration-spec.ts", "./http-auth": "./src/http-auth/index.ts", - "./public-origin": "./src/public-origin.ts" + "./public-origin": "./src/public-origin.ts", + "./sentry-grouping": "./src/sentry-grouping.ts" }, "publishConfig": { "access": "public", @@ -83,6 +84,12 @@ "types": "./dist/public-origin.d.ts", "default": "./dist/public-origin.js" } + }, + "./sentry-grouping": { + "import": { + "types": "./dist/sentry-grouping.d.ts", + "default": "./dist/sentry-grouping.js" + } } } }, diff --git a/packages/core/sdk/src/sentry-grouping.test.ts b/packages/core/sdk/src/sentry-grouping.test.ts new file mode 100644 index 000000000..ffefb1017 --- /dev/null +++ b/packages/core/sdk/src/sentry-grouping.test.ts @@ -0,0 +1,201 @@ +import { expect, test } from "@effect/vitest"; + +import { + containsContentHash, + stableGroupingFingerprint, + stripContentHashes, + type GroupingEvent, +} from "./sentry-grouping"; + +// Two builds of the same source: only the Vite content hash differs. +const workerFrame = (hash: string) => ({ + filename: `/assets/execution-rate-limit-${hash}.js`, + module: `execution-rate-limit-${hash}`, + function: "timeoutOrElse", + in_app: true, +}); + +const workerEvent = (hash: string): GroupingEvent => ({ + culprit: `timeoutOrElse(execution-rate-limit-${hash})`, + exception: { + values: [ + { + type: "GateCheckTimeoutError", + value: "balance check timed out", + stacktrace: { frames: [workerFrame(hash)] }, + }, + ], + }, +}); + +test("stripContentHashes rewrites chunk hashes and leaves everything else alone", () => { + expect(stripContentHashes("/assets/execution-rate-limit-BAuwphPA.js")).toBe( + "/assets/execution-rate-limit.js", + ); + expect(stripContentHashes("/assets/execution-rate-limit-DkcPBbWe.js")).toBe( + "/assets/execution-rate-limit.js", + ); + // Rollup hashes may themselves contain a dash — the whole 8-char tail goes. + expect(stripContentHashes("http://127.0.0.1:4789/assets/AddGraphqlIntegration-BCr-oWx4.js")).toBe( + "http://127.0.0.1:4789/assets/AddGraphqlIntegration.js", + ); + // Culprit strings wrap the chunk name in parentheses. + expect(stripContentHashes("U(assets/atoms-Yemn7yhP)")).toBe("U(assets/atoms)"); + expect(stripContentHashes("W(assets/atoms-CeCENfWa)")).toBe("W(assets/atoms)"); +}); + +test("stripContentHashes does not eat real name segments", () => { + // 8 lowercase letters is a word, not a hash. + expect(stripContentHashes("/src/auth/oauth-callback.ts")).toBe("/src/auth/oauth-callback.ts"); + expect(stripContentHashes("apps/cloud/src/engine/execution-gate.ts")).toBe( + "apps/cloud/src/engine/execution-gate.ts", + ); + expect(stripContentHashes("/assets/execution-rate-limit.js")).toBe( + "/assets/execution-rate-limit.js", + ); + expect(stripContentHashes("")).toBe(""); +}); + +test("stripContentHashes keeps genuinely different modules distinct", () => { + expect(stripContentHashes("/assets/atoms-Yemn7yhP.js")).not.toBe( + stripContentHashes("/assets/router-CeCENfWa.js"), + ); + expect(stripContentHashes("/assets/atoms-Yemn7yhP.js")).not.toBe( + stripContentHashes("/assets/atoms-shell-CeCENfWa.js"), + ); +}); + +test("containsContentHash detects only hashed paths", () => { + expect(containsContentHash("/assets/atoms-Yemn7yhP.js")).toBe(true); + expect(containsContentHash("apps/cloud/src/engine/execution-gate.ts")).toBe(false); +}); + +test("the same logical frame fingerprints identically across two deploys", () => { + const first = stableGroupingFingerprint(workerEvent("BAuwphPA")); + const second = stableGroupingFingerprint(workerEvent("DkcPBbWe")); + expect(first).toBeDefined(); + expect(first).toEqual(second); + expect(first).toEqual(["GateCheckTimeoutError", "timeoutOrElse@execution-rate-limit"]); +}); + +test("different modules keep different fingerprints", () => { + const other: GroupingEvent = { + exception: { + values: [ + { + type: "GateCheckTimeoutError", + stacktrace: { + frames: [ + { + filename: "/assets/atoms-Yemn7yhP.js", + module: "atoms-Yemn7yhP", + function: "loadConnections", + in_app: true, + }, + ], + }, + }, + ], + }, + }; + expect(stableGroupingFingerprint(other)).not.toEqual( + stableGroupingFingerprint(workerEvent("BAuwphPA")), + ); +}); + +test("minified single-letter frame functions are left out of the fingerprint", () => { + // Minified identifiers rotate with every build exactly like chunk hashes, so + // keeping them would re-split the issue on the next deploy. + const atoms = (fn: string, hash: string): GroupingEvent => ({ + exception: { + values: [ + { + type: "TypeError", + stacktrace: { + frames: [ + { filename: `/assets/atoms-${hash}.js`, module: `atoms-${hash}`, function: fn }, + ], + }, + }, + ], + }, + }); + expect(stableGroupingFingerprint(atoms("U", "Yemn7yhP"))).toEqual( + stableGroupingFingerprint(atoms("W", "CeCENfWa")), + ); + expect(stableGroupingFingerprint(atoms("U", "Yemn7yhP"))).toEqual(["TypeError", "atoms"]); +}); + +test("two vendor chunks sharing a name stay apart on their callers", () => { + // A single build emits many `dist-.js` chunks from unrelated packages, + // so the top frame alone is not a safe key. + const vendorEvent = (caller: string, hash: string): GroupingEvent => ({ + exception: { + values: [ + { + type: "TypeError", + stacktrace: { + frames: [ + { module: `${caller}-${hash}`, function: caller, in_app: true }, + { module: `dist-${hash}`, function: "x", in_app: false }, + ], + }, + }, + ], + }, + }); + + expect(stableGroupingFingerprint(vendorEvent("uploadArtifact", "BQZkXWT2"))).not.toEqual( + stableGroupingFingerprint(vendorEvent("renderMarkdown", "BQZkXWT2")), + ); +}); + +test("events without a chunk hash keep Sentry's default grouping", () => { + const unhashed: GroupingEvent = { + culprit: "checkExecutionBalance(execution-gate.ts)", + exception: { + values: [ + { + type: "AutumnError", + stacktrace: { + frames: [ + { + filename: "/apps/cloud/src/engine/execution-gate.ts", + function: "checkExecutionBalance", + in_app: true, + }, + ], + }, + }, + ], + }, + }; + expect(stableGroupingFingerprint(unhashed)).toBeUndefined(); + expect(stableGroupingFingerprint({})).toBeUndefined(); + expect(stableGroupingFingerprint({ exception: { values: [] } })).toBeUndefined(); +}); + +test("a hashed culprit alone is enough to pin the fingerprint", () => { + const culpritOnly: GroupingEvent = { + culprit: "orElse(execution-rate-limit-BAuwphPA)", + exception: { values: [{ type: "TypeError", value: "destroyed" }] }, + }; + expect(stableGroupingFingerprint(culpritOnly)).toEqual([ + "TypeError", + "orElse(execution-rate-limit)", + ]); +}); + +test("the outermost exception drives the fingerprint", () => { + // Sentry orders `values` innermost-first; the last entry is the one whose + // type the issue is titled with. + const chained: GroupingEvent = { + exception: { + values: [ + { type: "InnerError", stacktrace: { frames: [workerFrame("BAuwphPA")] } }, + { type: "OuterError", stacktrace: { frames: [workerFrame("BAuwphPA")] } }, + ], + }, + }; + expect(stableGroupingFingerprint(chained)?.[0]).toBe("OuterError"); +}); diff --git a/packages/core/sdk/src/sentry-grouping.ts b/packages/core/sdk/src/sentry-grouping.ts new file mode 100644 index 000000000..abf836eb6 --- /dev/null +++ b/packages/core/sdk/src/sentry-grouping.ts @@ -0,0 +1,157 @@ +// --------------------------------------------------------------------------- +// Stable Sentry grouping across deploys. +// +// Every bundle we ship names its chunks `-.js`, and the +// hash rotates on every build. When a stack frame is not resolved back to +// source (no uploaded sourcemap for that release), Sentry groups on those +// minified names, so the SAME error opens a brand-new issue after each +// deploy — one bug spread over N issues, no counts, no regression history. +// +// The fix is a `beforeSend` that computes an explicit `fingerprint` from the +// event's grouping inputs with the volatile hash segment removed. The event +// itself is left untouched: filenames still carry their hashes so server-side +// sourcemap resolution keeps working — only the grouping key is normalized. +// +// Deliberately narrow: a fingerprint is returned ONLY for events that actually +// carry a content hash. Everything else keeps Sentry's default grouping. +// --------------------------------------------------------------------------- + +/** The shape of a Sentry stack frame this module reads. Structural on purpose: + * the same helper serves @sentry/cloudflare, /electron and /browser events. */ +export type GroupingFrame = { + readonly filename?: string | undefined; + readonly abs_path?: string | undefined; + readonly module?: string | undefined; + readonly function?: string | undefined; + readonly in_app?: boolean | undefined; +}; + +export type GroupingExceptionValue = { + readonly type?: string | undefined; + readonly value?: string | undefined; + readonly stacktrace?: { readonly frames?: readonly GroupingFrame[] | undefined } | undefined; + readonly mechanism?: { readonly type?: string | undefined } | undefined; +}; + +export type GroupingEvent = { + readonly culprit?: string | undefined; + readonly exception?: { readonly values?: readonly GroupingExceptionValue[] | undefined }; +}; + +/** + * A `-XXXXXXXX` tail that ends a path segment — i.e. is followed by nothing, + * by file extensions, or by a delimiter such as the `)` in a Sentry culprit + * (`orElse(execution-rate-limit-BAuwphPA)`). Eight characters is the Vite and + * Rollup default; the hash alphabet includes `-` and `_`, so a hash can itself + * contain a dash (`BCr-oWx4`) and the whole 8-char tail must go at once. + */ +const HASH_TAIL = /-([A-Za-z0-9_-]{8})(?=(?:\.[A-Za-z0-9]+)*(?:$|[)\]}'"\s,:;?#]))/g; + +/** + * Whether an 8-character segment reads like a content hash rather than a word. + * Build hashes are random base64url, so they mix cases and digits; real name + * segments (`callback`, `grouping`, `provider`) are all lowercase. Requiring + * that mix is what keeps `oauth-callback` from collapsing to `oauth`. The cost + * is that the occasional word-shaped hash (one capital, no digits) is left + * alone and keeps re-grouping — 1 of 147 chunks in a sample desktop build. A + * missed normalization is recoverable; a wrongly merged issue is not. + */ +const looksLikeContentHash = (segment: string): boolean => { + let upper = 0; + let digits = 0; + for (const char of segment) { + if (char >= "A" && char <= "Z") upper += 1; + else if (char >= "0" && char <= "9") digits += 1; + } + return upper >= 2 || (upper >= 1 && digits >= 1) || digits >= 2; +}; + +/** + * Remove build content hashes from a path, module name or culprit string, + * leaving every other character in place. + */ +export const stripContentHashes = (value: string): string => + value.replace(HASH_TAIL, (match, segment: string) => + looksLikeContentHash(segment) ? "" : match, + ); + +/** True when the value carries at least one build content hash. */ +export const containsContentHash = (value: string): boolean => stripContentHashes(value) !== value; + +/** + * Minified identifiers rotate with every build exactly like chunk hashes, so a + * fingerprint containing one is no more stable than the hash it replaced. Names + * of three characters or fewer are treated as minified and dropped; the frame + * still contributes its module. + */ +const MINIFIED_FUNCTION = /^[A-Za-z_$][A-Za-z0-9_$]{0,2}$/; + +const usableFunction = (name: string | undefined): string | undefined => { + if (!name || name === "" || name === "?") return undefined; + return MINIFIED_FUNCTION.test(name) ? undefined : name; +}; + +/** Last path segment, without query or fragment. Chunk paths are served from + * host- and port-specific origins (`http://127.0.0.1:4789/assets/…`), which are + * volatile in their own right. */ +const basename = (path: string): string => { + const clean = path.split(/[?#]/)[0] ?? path; + const segments = clean.split(/[/\\]/); + return segments[segments.length - 1] || clean; +}; + +const frameLocation = (frame: GroupingFrame): string | undefined => { + if (frame.module) return stripContentHashes(frame.module); + const path = frame.filename ?? frame.abs_path; + return path ? stripContentHashes(basename(path)) : undefined; +}; + +const frameKey = (frame: GroupingFrame): string | undefined => { + const location = frameLocation(frame); + if (!location) return undefined; + const fn = usableFunction(frame.function); + return fn ? `${fn}@${location}` : location; +}; + +const frameHasContentHash = (frame: GroupingFrame): boolean => + containsContentHash(frame.module ?? "") || + containsContentHash(frame.filename ?? "") || + containsContentHash(frame.abs_path ?? ""); + +/** + * How many frames from the top of the stack enter the fingerprint. Bundlers + * emit several unrelated chunks under the same name (nine different + * `dist-.js` in one desktop build), so the top frame alone would merge + * unrelated vendor code; the caller chain is what keeps them apart. The + * resulting key is still strictly coarser than Sentry's default input — same + * frames, minus the volatile hash and minified identifiers — so this can only + * merge issues, never split one further. + */ +const FINGERPRINT_FRAMES = 8; + +/** + * A deploy-stable fingerprint for an event whose grouping input carries a build + * content hash, or `undefined` to leave Sentry's default grouping in place. + */ +export const stableGroupingFingerprint = (event: GroupingEvent): readonly string[] | undefined => { + const values = event.exception?.values ?? []; + // Sentry orders chained exceptions innermost-first; the last entry is the one + // the issue is titled with. + const primary = values[values.length - 1]; + if (!primary) return undefined; + + const frames = primary.stacktrace?.frames ?? []; + const culprit = event.culprit ?? ""; + if (!frames.some(frameHasContentHash) && !containsContentHash(culprit)) return undefined; + + const inApp = frames.filter((frame) => frame.in_app !== false); + const considered = (inApp.length > 0 ? inApp : frames).slice(-FINGERPRINT_FRAMES).reverse(); + const keys = considered.flatMap((frame) => { + const key = frameKey(frame); + return key ? [key] : []; + }); + + const type = primary.type ?? "Error"; + if (keys.length > 0) return [type, ...keys]; + return culprit ? [type, stripContentHashes(culprit)] : undefined; +}; diff --git a/packages/core/sdk/tsup.config.ts b/packages/core/sdk/tsup.config.ts index e18e7a71b..338335e48 100644 --- a/packages/core/sdk/tsup.config.ts +++ b/packages/core/sdk/tsup.config.ts @@ -10,6 +10,7 @@ export default defineConfig({ "migration-spec": "src/migration-spec.ts", "http-auth": "src/http-auth/index.ts", testing: "src/testing.ts", + "sentry-grouping": "src/sentry-grouping.ts", }, format: ["esm"], dts: false, From f8f033f20e00e511eb395fec84b802905972b9ca Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:39:19 -0700 Subject: [PATCH 2/3] Cover the fingerprint wiring and pin the hash threshold --- apps/cloud/src/observability/index.ts | 40 +++++-- .../src/observability/observability.test.ts | 22 +++- apps/cloud/src/server.ts | 29 +---- .../src/main/crash-fingerprint.test.ts | 27 ++++- apps/desktop/src/main/crash-fingerprint.ts | 13 +++ apps/desktop/src/main/diagnostics.ts | 7 +- packages/app/src/crash-reporting.ts | 7 +- packages/core/sdk/src/sentry-grouping.test.ts | 106 ++++++++++++++---- packages/core/sdk/src/sentry-grouping.ts | 12 ++ 9 files changed, 193 insertions(+), 70 deletions(-) diff --git a/apps/cloud/src/observability/index.ts b/apps/cloud/src/observability/index.ts index bc2ca5fd9..e539baf5c 100644 --- a/apps/cloud/src/observability/index.ts +++ b/apps/cloud/src/observability/index.ts @@ -16,7 +16,7 @@ import { Cause, Effect, Layer } from "effect"; import type * as Tracer from "effect/Tracer"; import { ErrorCapture } from "@executor-js/api"; -import { stableGroupingFingerprint, type GroupingEvent } from "@executor-js/sdk/sentry-grouping"; +import { withStableGroupingFingerprint } from "@executor-js/sdk/sentry-grouping"; // Drizzle/postgres-js include the failing SQL (params + bound values) in // their error message. For OpenAPI source inserts that's 1MB+ of spec @@ -148,19 +148,13 @@ export const beforeSendWithOtelCorrelation = ( }; /** + * The single `beforeSend` the worker and its Durable Objects install. + * * The worker ships as content-hashed chunks and its frames are not resolved * back to source, so Sentry's default grouping keys on names like * `execution-rate-limit-` and re-opens every issue on the next deploy. - * Pin a fingerprint with the hash normalized out; events with no hashed - * grouping input are left on the default algorithm. - */ -export const withStableGroupingFingerprint = (event: T): T => { - const fingerprint = stableGroupingFingerprint(event); - return fingerprint ? { ...event, fingerprint: [...fingerprint] } : event; -}; - -/** - * The single `beforeSend` the worker and its Durable Objects install. + * `withStableGroupingFingerprint` pins a key with the hash normalized out; + * events with no hashed grouping input are left on the default algorithm. * * The two stages are independent and compose in this order: the capture-owner * pass decides WHETHER the event is reported at all (a cause the Durable @@ -176,6 +170,30 @@ export const beforeSendCloudEvent = ( return reported === null ? null : withStableGroupingFingerprint(reported); }; +/** + * The Sentry options the worker and every Durable Object install. It lives + * beside the `beforeSend` it wires so the composition is covered by + * observability.test.ts; `server.ts` only passes this through. + * + * NOTE: do NOT enable `instrumentPrototypeMethods`. It walks the DO prototype + * and reads every property — including accessors — to find methods to wrap, + * which invokes the `sessionId` getter with `this` bound to the prototype + * (where `ctx` is undefined) and throws during construction, 500ing every + * session create / cold restore. The DO captures its own errors via the + * `captureCause` seam (→ Sentry) instead. + */ +export const cloudSentryOptions = (env: Env) => ({ + dsn: env.SENTRY_DSN, + tracesSampleRate: 0, + enableLogs: true, + sendDefaultPii: true, + skipOpenTelemetrySetup: true, + beforeSend: (event: ErrorEvent) => + beforeSendCloudEvent(event, { + logPayload: !env.SENTRY_DSN || env.SENTRY_OTEL_LOG_PAYLOAD === "true", + }), +}); + export const addCurrentOtelCorrelationTags = < T extends { readonly tags?: Record }, >( diff --git a/apps/cloud/src/observability/observability.test.ts b/apps/cloud/src/observability/observability.test.ts index 69906078f..a75a8e617 100644 --- a/apps/cloud/src/observability/observability.test.ts +++ b/apps/cloud/src/observability/observability.test.ts @@ -8,6 +8,7 @@ import { addCurrentOtelCorrelationTags, beforeSendCloudEvent, beforeSendWithOtelCorrelation, + cloudSentryOptions, DO_CAUSE_OWNER_TAG, DO_CAUSE_OWNER_VALUE, OTEL_SPAN_ID_TAG, @@ -86,8 +87,10 @@ describe("sentryPayloadForCause", () => { }); // Grouping keys are decided inside the Sentry SDK and never appear on any -// product surface, so the e2e harness cannot observe them; the running -// beforeSend itself is covered by e2e/cloud/sentry-otel-correlation.test.ts. +// product surface, so the e2e harness cannot observe them. The split is: +// e2e/cloud/sentry-otel-correlation.test.ts proves the worker really installs +// `cloudSentryOptions.beforeSend` (its correlation payload only exists if that +// hook ran), and the tests here prove the hook it installs fingerprints. describe("Sentry grouping", () => { // The worker bundle ships as content-hashed chunks, so the only module name // Sentry ever sees for a given frame changes on every deploy. @@ -143,6 +146,21 @@ describe("Sentry grouping", () => { expect(sent).not.toBeNull(); expect(sent?.fingerprint).toBeUndefined(); }); + + // The wiring check: this is the exact object handed to `Sentry.withSentry` + // and `instrumentDurableObjectWithSentry` in server.ts. If the normalizer is + // ever dropped from the hook the worker installs, this fails. + it("the options the worker and DOs install carry the fingerprinting hook", () => { + const options = cloudSentryOptions({ SENTRY_DSN: "https://public@example.invalid/1" } as Env); + const sent = options.beforeSend(workerEvent("BAuwphPA")); + + expect(sent?.fingerprint).toEqual([ + "GateCheckTimeoutError", + "timeoutOrElse@execution-rate-limit", + ]); + // Same source, next deploy, new chunk hash — one issue, not two. + expect(options.beforeSend(workerEvent("DkcPBbWe"))?.fingerprint).toEqual(sent?.fingerprint); + }); }); describe("Sentry OTel correlation", () => { diff --git a/apps/cloud/src/server.ts b/apps/cloud/src/server.ts index 942127cbd..4f5bf7628 100644 --- a/apps/cloud/src/server.ts +++ b/apps/cloud/src/server.ts @@ -1,6 +1,5 @@ import { DurableObject } from "cloudflare:workers"; import { SpanKind, SpanStatusCode, context, trace, type SpanContext } from "@opentelemetry/api"; -import type { ErrorEvent } from "@sentry/cloudflare"; import { ATTR_HTTP_REQUEST_METHOD, ATTR_HTTP_RESPONSE_STATUS_CODE, @@ -19,7 +18,7 @@ import { classifyMcpPath, prepareMcpOrgScope } from "./mcp/mount"; import { parseTraceparent } from "./mcp/traceparent"; import { McpSessionDOSqlite as McpSessionDOBase } from "./mcp/session-durable-object"; import { - beforeSendCloudEvent, + cloudSentryOptions, captureCause, otelCorrelationContextFromOpenTelemetrySpan, SENTRY_EVENT_ID_ATTRIBUTE, @@ -28,28 +27,6 @@ import { import { browserTracesResponse } from "./observability/browser-traces"; import { flushTracerProvider, installTracerProvider } from "./observability/telemetry"; -// --------------------------------------------------------------------------- -// Sentry config -// --------------------------------------------------------------------------- - -const sentryOptions = (env: Env) => ({ - dsn: env.SENTRY_DSN, - tracesSampleRate: 0, - enableLogs: true, - sendDefaultPii: true, - skipOpenTelemetrySetup: true, - beforeSend: (event: ErrorEvent) => - beforeSendCloudEvent(event, { - logPayload: !env.SENTRY_DSN || env.SENTRY_OTEL_LOG_PAYLOAD === "true", - }), - // NOTE: do NOT enable `instrumentPrototypeMethods`. It walks the DO prototype - // and reads every property — including accessors — to find methods to wrap, - // which invokes the `sessionId` getter with `this` bound to the prototype - // (where `ctx` is undefined) and throws during construction, 500ing every - // session create / cold restore. The DO captures its own errors via the - // `captureCause` seam (→ Sentry) instead. -}); - // --------------------------------------------------------------------------- // Durable Object — wrapped with Sentry so DO errors land in Sentry (inits the // client inside the DO isolate, which plain `Sentry.captureException` cannot @@ -58,7 +35,7 @@ const sentryOptions = (env: Env) => ({ // --------------------------------------------------------------------------- export const McpSessionDOSqlite = Sentry.instrumentDurableObjectWithSentry( - sentryOptions, + cloudSentryOptions, McpSessionDOBase, ); @@ -458,4 +435,4 @@ const cloudflareHandler: ExportedHandler = { }, }; -export default Sentry.withSentry(sentryOptions, cloudflareHandler); +export default Sentry.withSentry(cloudSentryOptions, cloudflareHandler); diff --git a/apps/desktop/src/main/crash-fingerprint.test.ts b/apps/desktop/src/main/crash-fingerprint.test.ts index 55448eb1e..e7385ac19 100644 --- a/apps/desktop/src/main/crash-fingerprint.test.ts +++ b/apps/desktop/src/main/crash-fingerprint.test.ts @@ -1,11 +1,19 @@ import { expect, test } from "@effect/vitest"; -import { crashReportFingerprint, type CrashEvent } from "./crash-fingerprint"; +import { + crashReportFingerprint, + withCrashReportFingerprint, + type CrashEvent, +} from "./crash-fingerprint"; + +/** A crash event as Sentry hands it to `beforeSend` — with the grouping key + * slot the hook is allowed to fill. */ +type SentCrashEvent = CrashEvent & { readonly fingerprint?: readonly string[] | undefined }; // Chromium's soft-assert path (`NOTREACHED()`/`DCHECK`) dumps and keeps // running. Sentry titles each one with the faulting load address, so one // condition arrives as a new issue every time the address moves. -const softAssertEvent = (address: string): CrashEvent => ({ +const softAssertEvent = (address: string): SentCrashEvent => ({ exception: { values: [ { @@ -86,6 +94,21 @@ test("renderer chunk hashes are normalized out of the fingerprint", () => { expect(release1).toEqual(release2); }); +// `withCrashReportFingerprint` is the function object diagnostics.ts installs +// as its `beforeSend`, so these assertions cover the main-process wiring and +// not just the classifier behind it. +test("the main-process beforeSend pins the key and forwards the event", () => { + const collapsed = withCrashReportFingerprint(softAssertEvent("0x00000001a2b3c4d5")); + expect(collapsed.fingerprint).toEqual(["chromium-dump-without-crashing"]); + // Nothing else about the event is touched — this is grouping only. + expect(collapsed.exception?.values?.[0]?.value).toBe("Simulated Exception / 0x00000001a2b3c4d5"); + + const untouched: SentCrashEvent = { + exception: { values: [{ type: "EXC_CRASH", stacktrace: { frames: [{ function: "abort" }] } }] }, + }; + expect(withCrashReportFingerprint(untouched)).toBe(untouched); +}); + test("events with no volatile grouping input are left alone", () => { expect(crashReportFingerprint({})).toBeUndefined(); expect( diff --git a/apps/desktop/src/main/crash-fingerprint.ts b/apps/desktop/src/main/crash-fingerprint.ts index b0718b753..a9853d2f2 100644 --- a/apps/desktop/src/main/crash-fingerprint.ts +++ b/apps/desktop/src/main/crash-fingerprint.ts @@ -16,6 +16,7 @@ import { stableGroupingFingerprint, + withStableGroupingFingerprint, type GroupingEvent, type GroupingFrame, } from "@executor-js/sdk/sentry-grouping"; @@ -37,3 +38,15 @@ export const crashReportFingerprint = (event: CrashEvent): readonly string[] | u if (frames.some(isSoftAssertFrame)) return [CHROMIUM_SOFT_ASSERT_FINGERPRINT]; return stableGroupingFingerprint(event); }; + +/** + * The `beforeSend` the Electron main process installs. Grouping only — an + * event with no volatile grouping input is forwarded unchanged. + */ +export const withCrashReportFingerprint = (event: T): T => { + const frames = (event.exception?.values ?? []).flatMap((value) => value.stacktrace?.frames ?? []); + if (frames.some(isSoftAssertFrame)) { + return { ...event, fingerprint: [CHROMIUM_SOFT_ASSERT_FINGERPRINT] }; + } + return withStableGroupingFingerprint(event); +}; diff --git a/apps/desktop/src/main/diagnostics.ts b/apps/desktop/src/main/diagnostics.ts index e724f6dea..ad1041eb8 100644 --- a/apps/desktop/src/main/diagnostics.ts +++ b/apps/desktop/src/main/diagnostics.ts @@ -23,7 +23,7 @@ import { dirname, join } from "node:path"; import { app, crashReporter, dialog, shell } from "electron"; import log from "electron-log/main.js"; import * as Sentry from "@sentry/electron/main"; -import { crashReportFingerprint } from "./crash-fingerprint"; +import { withCrashReportFingerprint } from "./crash-fingerprint"; import { getServerSettings } from "./settings"; const sentryDsn = __EXECUTOR_SENTRY_DSN__; @@ -91,10 +91,7 @@ export const initErrorReporting = () => { }, }, // Grouping only — the event is forwarded untouched otherwise. - beforeSend: (event) => { - const fingerprint = crashReportFingerprint(event); - return fingerprint ? { ...event, fingerprint: [...fingerprint] } : event; - }, + beforeSend: withCrashReportFingerprint, }); } else { // No DSN baked in — keep native crash dumps local so a user-reported diff --git a/packages/app/src/crash-reporting.ts b/packages/app/src/crash-reporting.ts index 41225f024..01c078922 100644 --- a/packages/app/src/crash-reporting.ts +++ b/packages/app/src/crash-reporting.ts @@ -12,7 +12,7 @@ * once initialized — no reporter rewiring needed. */ -import { stableGroupingFingerprint } from "@executor-js/sdk/sentry-grouping"; +import { withStableGroupingFingerprint } from "@executor-js/sdk/sentry-grouping"; interface CrashReportingConfig { readonly dsn: string; @@ -51,10 +51,7 @@ export const initDesktopCrashReporting = (): void => { // `atoms-` and one bug re-groups on every release. Pin a // fingerprint with the hash normalized out; the event itself keeps its // hashed filenames so sourcemap resolution is unaffected. - beforeSend: (event) => { - const fingerprint = stableGroupingFingerprint(event); - return fingerprint ? { ...event, fingerprint: [...fingerprint] } : event; - }, + beforeSend: withStableGroupingFingerprint, }); } catch { // Reporting failures stay silent — there is nowhere left to report them. diff --git a/packages/core/sdk/src/sentry-grouping.test.ts b/packages/core/sdk/src/sentry-grouping.test.ts index ffefb1017..f0f8c6fb6 100644 --- a/packages/core/sdk/src/sentry-grouping.test.ts +++ b/packages/core/sdk/src/sentry-grouping.test.ts @@ -4,6 +4,7 @@ import { containsContentHash, stableGroupingFingerprint, stripContentHashes, + withStableGroupingFingerprint, type GroupingEvent, } from "./sentry-grouping"; @@ -65,6 +66,30 @@ test("stripContentHashes keeps genuinely different modules distinct", () => { ); }); +// Merging two unrelated bugs into one issue is unrecoverable, so the "is this +// a hash?" threshold is pinned from both sides: a segment needs two of the +// three hash signals (uppercase, uppercase+digit, digits) before it is eaten. +// Loosening it to one signal — the tempting fix for the word-shaped hashes +// this deliberately misses — silently collapses real chunk names. +test("a name segment one signal short of a hash is left in place", () => { + // One capital, no digits. + expect(stripContentHashes("/assets/connections-Settings.js")).toBe( + "/assets/connections-Settings.js", + ); + // One digit, no capitals. + expect(stripContentHashes("/assets/storage-s3client.js")).toBe("/assets/storage-s3client.js"); + // ...and neither may be mistaken for the chunk it would collapse onto. + expect(stripContentHashes("/assets/connections-Settings.js")).not.toBe( + stripContentHashes("/assets/connections-BAuwphPA.js"), + ); +}); + +test("a segment with two hash signals is eaten", () => { + expect(stripContentHashes("/assets/chunk-AbcdefgH.js")).toBe("/assets/chunk.js"); + expect(stripContentHashes("/assets/chunk-Abcdefg1.js")).toBe("/assets/chunk.js"); + expect(stripContentHashes("/assets/chunk-abcdef12.js")).toBe("/assets/chunk.js"); +}); + test("containsContentHash detects only hashed paths", () => { expect(containsContentHash("/assets/atoms-Yemn7yhP.js")).toBe(true); expect(containsContentHash("apps/cloud/src/engine/execution-gate.ts")).toBe(false); @@ -126,27 +151,41 @@ test("minified single-letter frame functions are left out of the fingerprint", ( expect(stableGroupingFingerprint(atoms("U", "Yemn7yhP"))).toEqual(["TypeError", "atoms"]); }); -test("two vendor chunks sharing a name stay apart on their callers", () => { - // A single build emits many `dist-.js` chunks from unrelated packages, - // so the top frame alone is not a safe key. - const vendorEvent = (caller: string, hash: string): GroupingEvent => ({ - exception: { - values: [ - { - type: "TypeError", - stacktrace: { - frames: [ - { module: `${caller}-${hash}`, function: caller, in_app: true }, - { module: `dist-${hash}`, function: "x", in_app: false }, - ], - }, +// A single build emits many `dist-.js` chunks from unrelated packages, +// so the crashing frame alone is not a safe key — two different bugs would +// land in one issue. Frames are oldest-first, so the shared chunk is last. +const vendorEvent = (caller: string, hash: string): GroupingEvent => ({ + exception: { + values: [ + { + type: "TypeError", + stacktrace: { + frames: [ + { module: `${caller}-${hash}`, function: caller, in_app: true }, + { module: `dist-${hash}`, function: "throwHelper", in_app: true }, + ], }, - ], - }, - }); + }, + ], + }, +}); + +test("two vendor chunks sharing a name stay apart on their callers", () => { + const upload = stableGroupingFingerprint(vendorEvent("uploadArtifact", "BQZkXWT2")); + const render = stableGroupingFingerprint(vendorEvent("renderMarkdown", "BQZkXWT2")); - expect(stableGroupingFingerprint(vendorEvent("uploadArtifact", "BQZkXWT2"))).not.toEqual( - stableGroupingFingerprint(vendorEvent("renderMarkdown", "BQZkXWT2")), + // The crashing frame is identical in both — only the caller chain separates + // them, so a fingerprint built from the top frame alone would over-merge. + expect(upload?.slice(0, 2)).toEqual(["TypeError", "throwHelper@dist"]); + expect(render?.slice(0, 2)).toEqual(upload?.slice(0, 2)); + expect(upload).not.toEqual(render); + expect(upload).toEqual(["TypeError", "throwHelper@dist", "uploadArtifact@uploadArtifact"]); +}); + +test("the caller chain is itself deploy-stable", () => { + // Every frame in the chain must lose its hash, not just the crashing one. + expect(stableGroupingFingerprint(vendorEvent("uploadArtifact", "BQZkXWT2"))).toEqual( + stableGroupingFingerprint(vendorEvent("uploadArtifact", "Yemn7yhP")), ); }); @@ -186,6 +225,35 @@ test("a hashed culprit alone is enough to pin the fingerprint", () => { ]); }); +// The wrapper every process installs as its `beforeSend`. `fingerprint` and +// `tags` stand in for the fields a real Sentry event carries around it. +type SentEvent = GroupingEvent & { + readonly fingerprint?: readonly string[] | undefined; + readonly tags?: Record | undefined; +}; + +test("withStableGroupingFingerprint pins the key and changes nothing else", () => { + const input: SentEvent = { ...workerEvent("BAuwphPA"), tags: { a: "b" } }; + const hashed = withStableGroupingFingerprint(input); + expect(hashed.fingerprint).toEqual([ + "GateCheckTimeoutError", + "timeoutOrElse@execution-rate-limit", + ]); + // The event still carries its hashed filename, or server-side sourcemap + // resolution would stop finding the release's artifacts. + expect(hashed.exception?.values?.[0]?.stacktrace?.frames?.[0]?.filename).toBe( + "/assets/execution-rate-limit-BAuwphPA.js", + ); + expect(hashed.tags).toEqual({ a: "b" }); +}); + +test("withStableGroupingFingerprint forwards an unhashed event untouched", () => { + const plain: SentEvent = { culprit: "checkExecutionBalance(execution-gate.ts)" }; + const out = withStableGroupingFingerprint(plain); + expect(out).toBe(plain); + expect("fingerprint" in out).toBe(false); +}); + test("the outermost exception drives the fingerprint", () => { // Sentry orders `values` innermost-first; the last entry is the one whose // type the issue is titled with. diff --git a/packages/core/sdk/src/sentry-grouping.ts b/packages/core/sdk/src/sentry-grouping.ts index abf836eb6..6f07385ee 100644 --- a/packages/core/sdk/src/sentry-grouping.ts +++ b/packages/core/sdk/src/sentry-grouping.ts @@ -155,3 +155,15 @@ export const stableGroupingFingerprint = (event: GroupingEvent): readonly string if (keys.length > 0) return [type, ...keys]; return culprit ? [type, stripContentHashes(culprit)] : undefined; }; + +/** + * The whole `beforeSend` contract: pin the deploy-stable fingerprint when the + * event has a volatile grouping input, and forward the event untouched when it + * does not. Every call site (cloud worker/DO, desktop main, desktop renderer) + * installs this one symbol rather than its own copy, so the wiring is covered + * by the tests below instead of being retyped per process. + */ +export const withStableGroupingFingerprint = (event: T): T => { + const fingerprint = stableGroupingFingerprint(event); + return fingerprint ? { ...event, fingerprint: [...fingerprint] } : event; +}; From ccc859fceb3a1c95f590bb11d95939e287347a86 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Thu, 27 Aug 2026 14:42:33 -0700 Subject: [PATCH 3/3] Assemble desktop main Sentry options where they are tested --- .../src/main/crash-fingerprint.test.ts | 15 ++++++++++ apps/desktop/src/main/crash-fingerprint.ts | 28 +++++++++++++++++++ apps/desktop/src/main/diagnostics.ts | 24 ++++++---------- 3 files changed, 52 insertions(+), 15 deletions(-) diff --git a/apps/desktop/src/main/crash-fingerprint.test.ts b/apps/desktop/src/main/crash-fingerprint.test.ts index e7385ac19..057498f9c 100644 --- a/apps/desktop/src/main/crash-fingerprint.test.ts +++ b/apps/desktop/src/main/crash-fingerprint.test.ts @@ -2,6 +2,7 @@ import { expect, test } from "@effect/vitest"; import { crashReportFingerprint, + mainCrashReportingOptions, withCrashReportFingerprint, type CrashEvent, } from "./crash-fingerprint"; @@ -109,6 +110,20 @@ test("the main-process beforeSend pins the key and forwards the event", () => { expect(withCrashReportFingerprint(untouched)).toBe(untouched); }); +// The wiring check: this is the exact object handed to `Sentry.init` in +// diagnostics.ts. If the hook is ever dropped from the main process, this +// fails. +test("the options the main process installs carry the fingerprinting hook", () => { + const options = mainCrashReportingOptions({ + dsn: "https://public@example.invalid/1", + release: "executor-desktop@0.0.0", + environment: "production", + runId: "abcdef123456", + }); + const sent = options.beforeSend(softAssertEvent("0x00000001a2b3c4d5")); + expect(sent.fingerprint).toEqual(["chromium-dump-without-crashing"]); +}); + test("events with no volatile grouping input are left alone", () => { expect(crashReportFingerprint({})).toBeUndefined(); expect( diff --git a/apps/desktop/src/main/crash-fingerprint.ts b/apps/desktop/src/main/crash-fingerprint.ts index a9853d2f2..105f60a10 100644 --- a/apps/desktop/src/main/crash-fingerprint.ts +++ b/apps/desktop/src/main/crash-fingerprint.ts @@ -50,3 +50,31 @@ export const withCrashReportFingerprint = (event: T): T => } return withStableGroupingFingerprint(event); }; + +/** + * The Sentry options the Electron main process installs, assembled here rather + * than inline at the `Sentry.init` call so the `beforeSend` wiring is covered + * by crash-fingerprint.test.ts. `diagnostics.ts` only passes this through. + * + * Typed structurally on purpose: this module stays importable without pulling + * in electron, which is what keeps it unit-testable at all. + */ +export const mainCrashReportingOptions = (config: { + readonly dsn: string; + readonly release: string; + readonly environment: string; + readonly runId: string; +}) => ({ + dsn: config.dsn, + release: config.release, + environment: config.environment, + initialScope: { + tags: { + platform: process.platform, + arch: process.arch, + runId: config.runId, + }, + }, + // Grouping only — the event is forwarded untouched otherwise. + beforeSend: withCrashReportFingerprint, +}); diff --git a/apps/desktop/src/main/diagnostics.ts b/apps/desktop/src/main/diagnostics.ts index ad1041eb8..c10127a47 100644 --- a/apps/desktop/src/main/diagnostics.ts +++ b/apps/desktop/src/main/diagnostics.ts @@ -23,7 +23,7 @@ import { dirname, join } from "node:path"; import { app, crashReporter, dialog, shell } from "electron"; import log from "electron-log/main.js"; import * as Sentry from "@sentry/electron/main"; -import { withCrashReportFingerprint } from "./crash-fingerprint"; +import { mainCrashReportingOptions } from "./crash-fingerprint"; import { getServerSettings } from "./settings"; const sentryDsn = __EXECUTOR_SENTRY_DSN__; @@ -79,20 +79,14 @@ export const sidecarCrashReportingEnv = (): Record => */ export const initErrorReporting = () => { if (errorReportingEnabled) { - Sentry.init({ - dsn: sentryDsn, - release: releaseTag(), - environment: environmentTag(), - initialScope: { - tags: { - platform: process.platform, - arch: process.arch, - runId, - }, - }, - // Grouping only — the event is forwarded untouched otherwise. - beforeSend: withCrashReportFingerprint, - }); + Sentry.init( + mainCrashReportingOptions({ + dsn: sentryDsn, + release: releaseTag(), + environment: environmentTag(), + runId, + }), + ); } else { // No DSN baked in — keep native crash dumps local so a user-reported // crash still leaves minidumps for the diagnostics zip to collect.