From 26499bc82865da7a81b18749e38a74020a3ebf8c Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Sun, 27 Sep 2026 22:18:39 -0700 Subject: [PATCH 1/2] feat(telemetry): emit session_end for exec and surface on cli_start --- src/exec/runner.ts | 42 +- src/index.ts | 4 +- src/telemetry/first-run.ts | 3 +- src/telemetry/index.ts | 2 +- tests/unit/telemetry-session-end-exec.test.ts | 411 ++++++++++++++++++ 5 files changed, 458 insertions(+), 4 deletions(-) create mode 100644 tests/unit/telemetry-session-end-exec.test.ts diff --git a/src/exec/runner.ts b/src/exec/runner.ts index ecbd598fe..f00271d01 100644 --- a/src/exec/runner.ts +++ b/src/exec/runner.ts @@ -371,6 +371,33 @@ export interface ExecResult { model?: string; } +/** + * Builds the `session_end` payload for exec runs, mirroring the TUI exit + * reporter's status/exit_reason contract. Falls back to the live turn count + * and wall clock when the run never produced a result. + */ +export function execSessionEndProperties( + result: ExecResult | undefined, + startedAt: number, + sinkTurns: number, +): { + status: "done" | "failed" | "cancelled"; + turn_count: number; + duration_ms: number; + session_mode: "exec"; + exit_reason: "done" | "error" | "cancelled"; +} { + const status = result?.status ?? "failed"; + return { + status, + turn_count: result?.turnsUsed ?? sinkTurns, + duration_ms: result?.durationMs ?? Date.now() - startedAt, + session_mode: "exec", + exit_reason: + status === "done" ? "done" : status === "failed" ? "error" : "cancelled", + }; +} + export function createExecToolCallGate( isAdvertised: (name: string) => boolean, ): (name: string) => boolean { @@ -406,7 +433,7 @@ export async function runExec(config: Config): Promise { const task = config.task.trim(); if (task.length === 0) { stderr.write('Usage: corbits exec ""\n'); - return { + const result: ExecResult = { exitCode: 2, sessionId: config.sessionId, text: "", @@ -425,6 +452,11 @@ export async function runExec(config: Config): Promise { provider: config.providerName, model: config.model, }; + liveTelemetry.capture( + "session_end", + execSessionEndProperties(result, Date.now(), 0), + ); + return result; } const sessionId = @@ -1307,6 +1339,14 @@ export async function runExec(config: Config): Promise { result.error = `runtime dispose failed: ${message}`; } } + liveTelemetry.capture( + "session_end", + execSessionEndProperties( + result, + startedAt, + runSink?.getTurnCount() ?? turnsUsed, + ), + ); clearActiveDisposeHost(); } } diff --git a/src/index.ts b/src/index.ts index eb5329ad8..e3d953725 100644 --- a/src/index.ts +++ b/src/index.ts @@ -79,7 +79,9 @@ export async function mainWithRunners( if (settings?.telemetry?.noticeShown === true) { const telemetry = createTelemetry({ settings }); setTelemetry(telemetry); - telemetry.capture("cli_start"); + telemetry.capture("cli_start", { + surface: config.command === "exec" ? "exec" : "tui", + }); } } diff --git a/src/telemetry/first-run.ts b/src/telemetry/first-run.ts index cf6edc62f..a83ac3b86 100644 --- a/src/telemetry/first-run.ts +++ b/src/telemetry/first-run.ts @@ -60,6 +60,7 @@ export async function activateHeldTelemetry( globalSettingsPath: string, confirmIntent: () => boolean = () => true, deps: FirstRunDeps = defaultDeps, + surface: "tui" | "exec" = "tui", ): Promise { try { await deps.markTelemetryNoticeShown(globalSettingsPath); @@ -89,5 +90,5 @@ export async function activateHeldTelemetry( if (!confirmIntent()) return; const telemetry = deps.createTelemetry({ settings }); deps.setTelemetry(telemetry); - telemetry.capture("cli_start"); + telemetry.capture("cli_start", { surface }); } diff --git a/src/telemetry/index.ts b/src/telemetry/index.ts index 8e6688a1d..90cbb2eb6 100644 --- a/src/telemetry/index.ts +++ b/src/telemetry/index.ts @@ -106,7 +106,7 @@ export function getSessionId(): string { // capture() appends ($app_version, service_version, os_type, os_arch, // schema_version, session_id), this bounds everything telemetry can ever contain. const EVENT_PROPERTY_ALLOWLIST: Record = { - cli_start: [], + cli_start: ["surface"], session_end: [ "status", "turn_count", diff --git a/tests/unit/telemetry-session-end-exec.test.ts b/tests/unit/telemetry-session-end-exec.test.ts new file mode 100644 index 000000000..3335614aa --- /dev/null +++ b/tests/unit/telemetry-session-end-exec.test.ts @@ -0,0 +1,411 @@ +import { afterEach, expect, mock, test } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { Config } from "../../src/config/index.js"; +import { + resetPricingMetadataRefreshForTests, + schedulePricingMetadataRefresh, +} from "../../src/cost/pricing-metadata.js"; +import { execSessionEndProperties, runExec } from "../../src/exec/runner.js"; +import { + clearActiveRun, + getActiveRun, + setActiveRun, +} from "../../src/session/active-run.js"; +import { + activateHeldTelemetry, + type FirstRunDeps, +} from "../../src/telemetry/first-run.js"; +import { + createTelemetry, + NOOP_TELEMETRY, + type Telemetry, + type TelemetryEvent, +} from "../../src/telemetry/index.js"; +import { setTelemetry } from "../../src/telemetry/singleton.js"; +import { defined } from "../helpers/defined.js"; +import { + withMockedHomedir, + withMockedModuleDuring, +} from "../helpers/mock-module.js"; +import { createTempDirs } from "../helpers/temporary-dirs.js"; + +afterEach(() => { + setTelemetry(NOOP_TELEMETRY); +}); + +interface BatchBody { + api_key: string; + batch: { + event: string; + timestamp: string; + properties: Record; + }[]; +} + +function recordingFetch() { + const bodies: BatchBody[] = []; + const fetchFn = ((_url: string, init: RequestInit) => { + bodies.push(JSON.parse(init.body as string) as BatchBody); + return Promise.resolve(new Response("1", { status: 200 })); + }) as unknown as typeof fetch; + return { + fetchFn, + events: () => bodies.flatMap((body) => body.batch), + }; +} + +function settingsWithId() { + return { providers: {}, telemetry: { installationId: "id" } }; +} + +test("cli_start allowlist passes surface through and strips everything else", async () => { + const { fetchFn, events } = recordingFetch(); + const telemetry = createTelemetry({ + settings: settingsWithId(), + env: {}, + fetchFn, + apiKey: "test-key", + }); + telemetry.capture("cli_start", { surface: "exec", prompt: "must-not-leave" }); + await telemetry.flush(); + expect(events()).toHaveLength(1); + const body = defined(events()[0], "telemetry event"); + expect(body.event).toBe("cli_start"); + expect(body.properties.surface).toBe("exec"); + expect(body.properties.prompt).toBeUndefined(); +}); + +test("session_end allowlist passes the exec-shaped payload and strips extras", async () => { + const { fetchFn, events } = recordingFetch(); + const telemetry = createTelemetry({ + settings: settingsWithId(), + env: {}, + fetchFn, + apiKey: "test-key", + }); + telemetry.capture("session_end", { + status: "done", + turn_count: 2, + duration_ms: 150, + session_mode: "exec", + exit_reason: "done", + task: "must-not-leave", + }); + await telemetry.flush(); + expect(events()).toHaveLength(1); + const body = defined(events()[0], "telemetry event"); + expect(body.properties.status).toBe("done"); + expect(body.properties.turn_count).toBe(2); + expect(body.properties.duration_ms).toBe(150); + expect(body.properties.session_mode).toBe("exec"); + expect(body.properties.exit_reason).toBe("done"); + expect(body.properties.task).toBeUndefined(); +}); + +function recordingFirstRunDeps() { + const { fetchFn, events } = recordingFetch(); + let instance: Telemetry | undefined; + const deps: FirstRunDeps = { + loadSettings: async () => settingsWithId(), + markTelemetryNoticeShown: async () => undefined, + createTelemetry: (opts) => + createTelemetry({ ...opts, env: {}, apiKey: "test-key", fetchFn }), + setTelemetry: (telemetry) => { + instance = telemetry; + }, + }; + return { deps, getInstance: () => instance, events }; +} + +test("activateHeldTelemetry fires the held cli_start with tui surface by default", async () => { + const { deps, getInstance, events } = recordingFirstRunDeps(); + await activateHeldTelemetry("/fake/path", () => true, deps); + await defined(getInstance(), "telemetry instance").flush(); + const cliStarts = events().filter((event) => event.event === "cli_start"); + expect(cliStarts).toHaveLength(1); + expect(defined(cliStarts[0], "cli_start").properties.surface).toBe("tui"); +}); + +test("activateHeldTelemetry forwards an explicit exec surface on cli_start", async () => { + const { deps, getInstance, events } = recordingFirstRunDeps(); + await activateHeldTelemetry("/fake/path", () => true, deps, "exec"); + await defined(getInstance(), "telemetry instance").flush(); + const cliStarts = events().filter((event) => event.event === "cli_start"); + expect(cliStarts).toHaveLength(1); + expect(defined(cliStarts[0], "cli_start").properties.surface).toBe("exec"); +}); + +test("execSessionEndProperties mirrors the TUI exit_reason contract", () => { + expect( + execSessionEndProperties( + { + exitCode: 0, + sessionId: "s", + text: "ok", + status: "done", + durationMs: 150, + turnsUsed: 3, + }, + 0, + 0, + ), + ).toEqual({ + status: "done", + turn_count: 3, + duration_ms: 150, + session_mode: "exec", + exit_reason: "done", + }); + const failed = execSessionEndProperties(undefined, Date.now() - 40, 1); + expect(failed.status).toBe("failed"); + expect(failed.turn_count).toBe(1); + expect(failed.session_mode).toBe("exec"); + expect(failed.exit_reason).toBe("error"); + expect(typeof failed.duration_ms).toBe("number"); + const cancelled = execSessionEndProperties( + { + exitCode: 1, + sessionId: "s", + text: "", + status: "cancelled", + durationMs: 90, + turnsUsed: 2, + }, + 0, + 0, + ); + expect(cancelled.status).toBe("cancelled"); + expect(cancelled.exit_reason).toBe("cancelled"); +}); + +function silenceStderr(): () => void { + const original = process.stderr.write; + process.stderr.write = (() => true) as typeof process.stderr.write; + return () => { + process.stderr.write = original; + }; +} + +function bareConfig(task: string): Config { + return { + command: "exec", + task, + cwd: process.cwd(), + configured: true, + providerName: "test", + model: "test", + providers: {}, + dangerouslySkipPermissions: true, + autoMode: false, + sessionId: "test-session", + } as unknown as Config; +} + +test("runExec with a missing prompt emits exactly one failed exec session_end", async () => { + const { fetchFn, events } = recordingFetch(); + const telemetry = createTelemetry({ + settings: settingsWithId(), + env: {}, + fetchFn, + apiKey: "test-key", + }); + setTelemetry(telemetry); + const restoreStderr = silenceStderr(); + try { + const result = await runExec(bareConfig(" ")); + expect(result.status).toBe("failed"); + } finally { + restoreStderr(); + } + await telemetry.flush(); + const ends = events().filter((event) => event.event === "session_end"); + expect(ends).toHaveLength(1); + const body = defined(ends[0], "session_end"); + expect(body.properties.status).toBe("failed"); + expect(body.properties.turn_count).toBe(0); + expect(body.properties.duration_ms).toBe(0); + expect(body.properties.session_mode).toBe("exec"); + expect(body.properties.exit_reason).toBe("error"); +}); + +test("runExec bootstrap failure emits exactly one failed exec session_end", async () => { + const previous = getActiveRun(); + clearActiveRun(); + const { cwd, home, cleanup } = createTempDirs( + "corbits-session-end-cwd-", + "corbits-session-end-home-", + ); + const { fetchFn, events } = recordingFetch(); + const telemetry = createTelemetry({ + settings: settingsWithId(), + env: {}, + fetchFn, + apiKey: "test-key", + }); + setTelemetry(telemetry); + const restoreStderr = silenceStderr(); + try { + await withMockedHomedir(home, async () => { + await withMockedModuleDuring( + import.meta.resolve("../../src/session/assemble-runtime.js"), + (real: typeof import("../../src/session/assemble-runtime.js")) => ({ + ...real, + assembleInferenceBase: () => + Promise.reject(new Error("bootstrap failed")), + }), + async () => { + const { runExec: runExecUnderMock } = + await import("../../src/exec/runner.js"); + const result = await runExecUnderMock({ + ...bareConfig("do the thing"), + cwd, + sessionId: "exec-session-end-fail", + }); + expect(result.exitCode).toBe(1); + expect(result.status).toBe("failed"); + }, + ); + }); + } finally { + restoreStderr(); + if (previous !== null) setActiveRun(previous); + else clearActiveRun(); + cleanup(); + } + await telemetry.flush(); + const ends = events().filter((event) => event.event === "session_end"); + expect(ends).toHaveLength(1); + const body = defined(ends[0], "session_end"); + expect(body.properties.status).toBe("failed"); + expect(body.properties.session_mode).toBe("exec"); + expect(body.properties.exit_reason).toBe("error"); +}); + +function writeSandboxSettings(root: string): void { + const settingsDir = join(root, "home", ".corbits"); + mkdirSync(settingsDir, { recursive: true }); + writeFileSync( + join(settingsDir, "settings.json"), + JSON.stringify({ + providers: { + "test-provider": { + baseURL: "http://localhost:1234", + apiKey: "test-key", + models: ["test-model"], + defaultModel: "test-model", + }, + }, + defaultProvider: "test-provider", + }), + ); + mkdirSync(join(root, "project"), { recursive: true }); +} + +async function withoutTelemetryKills(fn: () => Promise): Promise { + const saved: Record = {}; + for (const key of ["CORBITS_TELEMETRY", "DO_NOT_TRACK"]) { + saved[key] = process.env[key]; + Reflect.deleteProperty(process.env, key); + } + try { + await fn(); + } finally { + for (const [key, value] of Object.entries(saved)) { + if (value === undefined) Reflect.deleteProperty(process.env, key); + else process.env[key] = value; + } + } +} + +async function cliStartSurfaces( + subcommand: readonly string[], + rest: readonly string[] = [], +): Promise { + const sandbox = mkdtempSync(join(tmpdir(), "corbits-session-end-cli-")); + writeSandboxSettings(sandbox); + resetPricingMetadataRefreshForTests(); + schedulePricingMetadataRefresh({ + cachePath: join( + sandbox, + "home", + ".corbits", + "cache", + "models-pricing.json", + ), + fetchImpl: () => Promise.reject(new Error("network disabled in tests")), + }); + const captured: { + event: TelemetryEvent; + properties: Record | undefined; + }[] = []; + const stub: Telemetry = { + enabled: true, + installationId: "test-installation", + capture: (event, properties) => { + captured.push({ event, properties }); + }, + captureIntentional: () => false, + flush: async () => undefined, + discard: () => undefined, + }; + try { + await withoutTelemetryKills(async () => { + await withMockedModuleDuring( + import.meta.resolve("../../src/telemetry/index.js"), + (real: typeof import("../../src/telemetry/index.js")) => ({ + ...real, + telemetryDisabledByEnv: () => false, + createTelemetry: () => stub, + }), + async () => { + await withMockedModuleDuring( + import.meta.resolve("../../src/config/settings.js"), + (real: typeof import("../../src/config/settings.js")) => ({ + ...real, + ensureTelemetrySettings: async () => ({ + providers: {}, + telemetry: { + installationId: "test-installation", + noticeShown: true, + }, + }), + globalSettingsPath: () => join(sandbox, "global.json"), + }), + async () => { + const { mainWithRunners } = await import("../../src/index.js"); + const runTUI = mock((_config: Config) => Promise.resolve(0)); + const runExec = mock((_config: Config) => Promise.resolve(0)); + const runOnboarding = mock(() => Promise.resolve(0)); + await mainWithRunners( + [ + ...subcommand, + "--cwd", + join(sandbox, "project"), + "--config", + join(sandbox, "home", ".corbits", "settings.json"), + ...rest, + ], + { runTUI, runExec, runOnboarding }, + ); + }, + ); + }, + ); + }); + } finally { + rmSync(sandbox, { recursive: true, force: true }); + } + return captured + .filter((entry) => entry.event === "cli_start") + .map((entry) => entry.properties?.surface); +} + +test("startup cli_start carries surface exec on the exec path", async () => { + expect(await cliStartSurfaces(["exec"], ["say hello"])).toEqual(["exec"]); +}); + +test("startup cli_start carries surface tui on the interactive path", async () => { + expect(await cliStartSurfaces([])).toEqual(["tui"]); +}); From fe2e87f62a3ed3275e30723a7a8b04f7f2ed8d3f Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Mon, 28 Sep 2026 08:15:13 -0700 Subject: [PATCH 2/2] fix(telemetry): close exec funnel gaps without counting usage errors Pre-try setup failures and unconfigured exec now emit a failed session_end so every cli_start(surface exec) is paired. Missing-prompt usage errors stay emission-free: exit 2 with no run is not a session. --- src/exec/runner.ts | 55 +++- src/index.ts | 8 + tests/unit/telemetry-session-end-exec.test.ts | 269 ++++++++++++++++-- 3 files changed, 298 insertions(+), 34 deletions(-) diff --git a/src/exec/runner.ts b/src/exec/runner.ts index f00271d01..d4cfd4e13 100644 --- a/src/exec/runner.ts +++ b/src/exec/runner.ts @@ -103,6 +103,7 @@ import { loadState, saveState, type ConnectedMcpServer, + type RunState, } from "../session/state.js"; import { resolveExecRunStatus, type RunSink } from "../session/run-sink.js"; import { createRunSummary } from "../session/hooks.js"; @@ -452,10 +453,10 @@ export async function runExec(config: Config): Promise { provider: config.providerName, model: config.model, }; - liveTelemetry.capture( - "session_end", - execSessionEndProperties(result, Date.now(), 0), - ); + // Deliberate funnel gap: a missing prompt is a usage error (exit 2) and + // no run ever started, so there is no session to close. Emitting a + // failed session_end here would pollute failed counts with invocations + // that never ran. return result; } @@ -463,12 +464,46 @@ export async function runExec(config: Config): Promise { config.sessionId.length > 0 ? config.sessionId : generateSessionId(); const startedAt = Date.now(); const workdir = sessionContextDir(config.cwd, sessionId); - await initSessionDir(config.cwd, sessionId); - const prior = - config.sessionId.length > 0 - ? await loadState(config.cwd, config.sessionId) - : undefined; - const priorState = prior?.kind === "ok" ? prior.state : undefined; + // Setup runs before the main try below: initSessionDir/loadState hit disk + // before any session_end coverage exists, so an EACCES/EROFS here would + // reject with zero session_end and orphan the cli_start funnel. Emit a + // minimal failed session_end on this window instead of letting it throw. + let priorState: RunState | undefined; + try { + await initSessionDir(config.cwd, sessionId); + const prior = + config.sessionId.length > 0 + ? await loadState(config.cwd, config.sessionId) + : undefined; + priorState = prior?.kind === "ok" ? prior.state : undefined; + } catch (err) { + const message = formatCaughtError(err); + logger.error("exec setup failed: {error}", { error: message }); + stderr.write(`Error: ${message}\n`); + liveTelemetry.capture( + "session_end", + execSessionEndProperties(undefined, startedAt, 0), + ); + return { + exitCode: 1, + sessionId, + text: "", + error: message, + status: "failed", + durationMs: Date.now() - startedAt, + turnsUsed: 0, + toolCallCount: 0, + tokenUsage: { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + thinking: 0, + }, + provider: config.providerName, + model: config.model, + }; + } let connectedMcp: ConnectedMcpServer[] = []; let agent: Agent | null = null; diff --git a/src/index.ts b/src/index.ts index e3d953725..1e0b8230e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -94,6 +94,14 @@ export async function mainWithRunners( "No provider configured. Run `corbits` (interactive) once to complete setup, " + "or pass --provider / --model with credentials.\n", ); + // cli_start (surface exec) already emitted above while runExec never + // runs on this branch — emit a minimal failed session_end so the + // funnel stays paired instead of orphaning the start. + const { execSessionEndProperties } = await import("./exec/runner.js"); + getTelemetry().capture( + "session_end", + execSessionEndProperties(undefined, Date.now(), 0), + ); exitCode = 2; } else { exitCode = await runners.runOnboarding(config); diff --git a/tests/unit/telemetry-session-end-exec.test.ts b/tests/unit/telemetry-session-end-exec.test.ts index 3335614aa..aef020ee1 100644 --- a/tests/unit/telemetry-session-end-exec.test.ts +++ b/tests/unit/telemetry-session-end-exec.test.ts @@ -203,7 +203,7 @@ function bareConfig(task: string): Config { } as unknown as Config; } -test("runExec with a missing prompt emits exactly one failed exec session_end", async () => { +test("runExec with a missing prompt emits no session_end: usage is not a session", async () => { const { fetchFn, events } = recordingFetch(); const telemetry = createTelemetry({ settings: settingsWithId(), @@ -221,13 +221,7 @@ test("runExec with a missing prompt emits exactly one failed exec session_end", } await telemetry.flush(); const ends = events().filter((event) => event.event === "session_end"); - expect(ends).toHaveLength(1); - const body = defined(ends[0], "session_end"); - expect(body.properties.status).toBe("failed"); - expect(body.properties.turn_count).toBe(0); - expect(body.properties.duration_ms).toBe(0); - expect(body.properties.session_mode).toBe("exec"); - expect(body.properties.exit_reason).toBe("error"); + expect(ends).toHaveLength(0); }); test("runExec bootstrap failure emits exactly one failed exec session_end", async () => { @@ -283,22 +277,206 @@ test("runExec bootstrap failure emits exactly one failed exec session_end", asyn expect(body.properties.exit_reason).toBe("error"); }); -function writeSandboxSettings(root: string): void { +test("runExec pre-try setup failure returns failed and still closes the funnel", async () => { + const previous = getActiveRun(); + clearActiveRun(); + const { cwd, home, cleanup } = createTempDirs( + "corbits-session-end-setup-", + "corbits-session-end-setup-home-", + ); + const { fetchFn, events } = recordingFetch(); + const telemetry = createTelemetry({ + settings: settingsWithId(), + env: {}, + fetchFn, + apiKey: "test-key", + }); + setTelemetry(telemetry); + const restoreStderr = silenceStderr(); + try { + await withMockedHomedir(home, async () => { + await withMockedModuleDuring( + import.meta.resolve("../../src/session/index.js"), + (real: typeof import("../../src/session/index.js")) => ({ + ...real, + initSessionDir: () => + Promise.reject(new Error("EACCES: permission denied")), + }), + async () => { + const { runExec: runExecUnderMock } = + await import("../../src/exec/runner.js"); + const result = await runExecUnderMock({ + ...bareConfig("do the thing"), + cwd, + sessionId: "exec-session-end-setup", + }); + expect(result.exitCode).toBe(1); + expect(result.status).toBe("failed"); + }, + ); + }); + } finally { + restoreStderr(); + if (previous !== null) setActiveRun(previous); + else clearActiveRun(); + cleanup(); + } + await telemetry.flush(); + const ends = events().filter((event) => event.event === "session_end"); + expect(ends).toHaveLength(1); + const body = defined(ends[0], "session_end"); + expect(body.properties.status).toBe("failed"); + expect(body.properties.turn_count).toBe(0); + expect(body.properties.session_mode).toBe("exec"); + expect(body.properties.exit_reason).toBe("error"); +}); + +test("runExec success emits exactly one done exec session_end", async () => { + const previous = getActiveRun(); + clearActiveRun(); + const { cwd, home, cleanup } = createTempDirs( + "corbits-session-end-done-", + "corbits-session-end-done-home-", + ); + const { fetchFn, events } = recordingFetch(); + const telemetry = createTelemetry({ + settings: settingsWithId(), + env: {}, + fetchFn, + apiKey: "test-key", + }); + setTelemetry(telemetry); + const restoreStderr = silenceStderr(); + async function* emptyEvents(): AsyncGenerator { + // No agent events: the run completes without turns or tool calls. + } + const fakeAgent = { + send: async () => ({ type: "completed" }), + stream: () => emptyEvents(), + close: async () => undefined, + deliver: () => undefined, + }; + const fakeToolset = { + dynamicRunner: { + setCallGate: () => undefined, + currentDefinitions: () => [], + }, + skills: [], + setToolPromoter: () => undefined, + dispose: async () => undefined, + }; + const fakeSink = { + getTurnCollector: () => null, + getTurnCount: () => 0, + getTokenUsage: () => ({ + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + thinking: 0, + }), + getToolCallCount: () => 0, + getStatus: () => "cancelled", + getRunError: () => undefined, + }; + try { + await withMockedHomedir(home, async () => { + await withMockedModuleDuring( + import.meta.resolve("../../src/session/assemble-runtime.js"), + (real: typeof import("../../src/session/assemble-runtime.js")) => ({ + ...real, + assembleInferenceBase: async () => ({}), + resolveLiveSessionSources: () => ({ + sources: [], + defaultSource: [], + selected: { id: "test-source", credentialId: "test-cred" }, + }), + assembleSessionTrust: async () => ({ + projectTrust: {}, + pluginModules: [], + diagnostics: { warnings: [] }, + }), + assembleSessionGate: async () => ({ + gate: { clearDenials: () => undefined }, + }), + assembleSessionLifecycle: async () => ({ + hookManager: { dispatchPostRun: async () => undefined }, + runSink: fakeSink, + cycleRecorder: { + handleEvent: () => undefined, + dispose: async () => undefined, + }, + }), + assembleChatAgent: (opts: { + onBuilt: (agent: unknown, storage: unknown) => void; + }) => ({ + directorHolder: { instance: undefined }, + buildAgent: async () => { + opts.onBuilt(fakeAgent, {}); + return fakeAgent; + }, + }), + loadSessionLocalSettings: async () => undefined, + }), + async () => { + await withMockedModuleDuring( + import.meta.resolve("../../src/agent/tools.js"), + (real: typeof import("../../src/agent/tools.js")) => ({ + ...real, + createAgentToolset: async () => fakeToolset, + }), + async () => { + const { runExec: runExecUnderMock } = + await import("../../src/exec/runner.js"); + const result = await runExecUnderMock({ + ...bareConfig("do the thing"), + cwd, + sessionId: "exec-session-end-done", + globalSettingsPath: join(home, "global.json"), + providers: [], + }); + expect(result.exitCode).toBe(0); + expect(result.status).toBe("done"); + }, + ); + }, + ); + }); + } finally { + restoreStderr(); + if (previous !== null) setActiveRun(previous); + else clearActiveRun(); + cleanup(); + } + await telemetry.flush(); + const ends = events().filter((event) => event.event === "session_end"); + expect(ends).toHaveLength(1); + const body = defined(ends[0], "session_end"); + expect(body.properties.status).toBe("done"); + expect(body.properties.session_mode).toBe("exec"); + expect(body.properties.exit_reason).toBe("done"); +}); + +function writeSandboxSettings(root: string, configured = true): void { const settingsDir = join(root, "home", ".corbits"); mkdirSync(settingsDir, { recursive: true }); writeFileSync( join(settingsDir, "settings.json"), - JSON.stringify({ - providers: { - "test-provider": { - baseURL: "http://localhost:1234", - apiKey: "test-key", - models: ["test-model"], - defaultModel: "test-model", - }, - }, - defaultProvider: "test-provider", - }), + JSON.stringify( + configured + ? { + providers: { + "test-provider": { + baseURL: "http://localhost:1234", + apiKey: "test-key", + models: ["test-model"], + defaultModel: "test-model", + }, + }, + defaultProvider: "test-provider", + } + : { providers: {} }, + ), ); mkdirSync(join(root, "project"), { recursive: true }); } @@ -319,12 +497,13 @@ async function withoutTelemetryKills(fn: () => Promise): Promise { } } -async function cliStartSurfaces( +async function runMainWithStub( subcommand: readonly string[], rest: readonly string[] = [], -): Promise { + configured = true, +) { const sandbox = mkdtempSync(join(tmpdir(), "corbits-session-end-cli-")); - writeSandboxSettings(sandbox); + writeSandboxSettings(sandbox, configured); resetPricingMetadataRefreshForTests(); schedulePricingMetadataRefresh({ cachePath: join( @@ -350,6 +529,14 @@ async function cliStartSurfaces( flush: async () => undefined, discard: () => undefined, }; + let outcome: + | { + code: number; + runTUI: unknown; + runExec: unknown; + runOnboarding: unknown; + } + | undefined; try { await withoutTelemetryKills(async () => { await withMockedModuleDuring( @@ -378,7 +565,7 @@ async function cliStartSurfaces( const runTUI = mock((_config: Config) => Promise.resolve(0)); const runExec = mock((_config: Config) => Promise.resolve(0)); const runOnboarding = mock(() => Promise.resolve(0)); - await mainWithRunners( + const code = await mainWithRunners( [ ...subcommand, "--cwd", @@ -389,6 +576,7 @@ async function cliStartSurfaces( ], { runTUI, runExec, runOnboarding }, ); + outcome = { code, runTUI, runExec, runOnboarding }; }, ); }, @@ -397,6 +585,14 @@ async function cliStartSurfaces( } finally { rmSync(sandbox, { recursive: true, force: true }); } + return { ...defined(outcome, "main outcome"), captured }; +} + +async function cliStartSurfaces( + subcommand: readonly string[], + rest: readonly string[] = [], +): Promise { + const { captured } = await runMainWithStub(subcommand, rest); return captured .filter((entry) => entry.event === "cli_start") .map((entry) => entry.properties?.surface); @@ -409,3 +605,28 @@ test("startup cli_start carries surface exec on the exec path", async () => { test("startup cli_start carries surface tui on the interactive path", async () => { expect(await cliStartSurfaces([])).toEqual(["tui"]); }); + +test("unconfigured exec closes the funnel with one failed session_end", async () => { + const restoreStderr = silenceStderr(); + try { + const { code, captured, runExec } = await runMainWithStub( + ["exec"], + ["say hello"], + false, + ); + expect(code).toBe(2); + expect(runExec).not.toHaveBeenCalled(); + const surfaces = captured + .filter((entry) => entry.event === "cli_start") + .map((entry) => entry.properties?.surface); + expect(surfaces).toEqual(["exec"]); + const ends = captured.filter((entry) => entry.event === "session_end"); + expect(ends).toHaveLength(1); + const body = defined(ends[0], "session_end"); + expect(body.properties?.status).toBe("failed"); + expect(body.properties?.session_mode).toBe("exec"); + expect(body.properties?.exit_reason).toBe("error"); + } finally { + restoreStderr(); + } +});