From e9e2fd960a175cc93527468725ea382e5542de89 Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Fri, 30 Oct 2026 17:00:22 +0530 Subject: [PATCH 01/22] test(control-plane): characterize failure, replay and restart paths Pin today's behaviour before moving integrations behind a common interface: failure notices per round kind for task failures and missing workspaces, replayed harness events, webhook redeliveries, restarts and Slack gating. Tests for behaviour the refactor changes on purpose are marked, so they are flipped knowingly. Co-Authored-By: Claude Opus 5.5 --- .../githubWebhook.controller.test.ts | 34 ++++ .../__tests__/inbox.repository.test.ts | 60 ++++++ .../__tests__/task.repository.test.ts | 117 +++++++++++ .../__tests__/agentSession.service.test.ts | 9 + .../__tests__/apiTask.service.test.ts | 58 ++++++ .../__tests__/eventDispatcher.test.ts | 187 +++++++++++++++++- .../services/__tests__/inbox.service.test.ts | 150 ++++++++++++++ .../__tests__/sandboxManager.service.test.ts | 23 +++ .../__tests__/slackNotifier.service.test.ts | 6 + .../__tests__/taskProcessor.service.test.ts | 171 +++++++++++++++- 10 files changed, 812 insertions(+), 3 deletions(-) diff --git a/control-plane/src/controllers/__tests__/githubWebhook.controller.test.ts b/control-plane/src/controllers/__tests__/githubWebhook.controller.test.ts index 36c831b..da566ad 100644 --- a/control-plane/src/controllers/__tests__/githubWebhook.controller.test.ts +++ b/control-plane/src/controllers/__tests__/githubWebhook.controller.test.ts @@ -462,6 +462,17 @@ describe("GitHubWebhookController.handleWebhook", () => { expect(addIssueCommentReaction).toHaveBeenCalledWith("o", "r", 555, "eyes"); }); + // No delivery-id dedupe here: the inbox's ON CONFLICT drops the duplicate row. + it("a redelivered comment enqueues and reacts again", async () => { + const body = commentBody("alice"); + expect((await post(body, "issue_comment", sign(body))).status).toBe(200); + expect((await post(body, "issue_comment", sign(body))).status).toBe(200); + await flushMicrotasks(); + expect(enqueueComment).toHaveBeenCalledTimes(2); + expect(enqueueComment.mock.calls[1]).toEqual(enqueueComment.mock.calls[0]); + expect(addIssueCommentReaction).toHaveBeenCalledTimes(2); + }); + it("does not react to a shortened alias of the bot login", async () => { const body = commentBody("alice", "acme-bot[bot]", "@bot fix"); const res = await post(body, "issue_comment", sign(body)); @@ -689,6 +700,29 @@ describe("GitHubWebhookController.handleWebhook", () => { expect(addReviewCommentReaction).toHaveBeenCalledWith("o", "r", 12, "eyes"); }); + it("a redelivered review refetches its inline comments, enqueues and reacts again", async () => { + getReviewComments.mockResolvedValue([ + { id: 11, path: "a.ts", line: 3, body: "nit", user: { login: "reviewer" } }, + ]); + const body = JSON.stringify({ + action: "submitted", + repository: { name: "r", owner: { login: "o" } }, + pull_request: { number: 7, user: { login: "acme-bot[bot]" } }, + review: { + id: 88, + body: "@acme-bot please handle this review", + user: { login: "a", id: 2 }, + submitted_at: "2026-04-23T12:00:00Z", + }, + }); + expect((await post(body, "pull_request_review", sign(body))).status).toBe(200); + expect((await post(body, "pull_request_review", sign(body))).status).toBe(200); + await flushMicrotasks(); + expect(getReviewComments).toHaveBeenCalledTimes(2); + expect(enqueueReview).toHaveBeenCalledTimes(2); + expect(addReviewCommentReaction).toHaveBeenCalledTimes(2); + }); + it("enqueues a review on an API task's PR without parsing its URL", async () => { findByPrUrl.mockResolvedValue(apiTask); const body = JSON.stringify({ diff --git a/control-plane/src/repositories/__tests__/inbox.repository.test.ts b/control-plane/src/repositories/__tests__/inbox.repository.test.ts index 941a99b..fe0e137 100644 --- a/control-plane/src/repositories/__tests__/inbox.repository.test.ts +++ b/control-plane/src/repositories/__tests__/inbox.repository.test.ts @@ -116,6 +116,66 @@ describe("InboxRepository", () => { const rows = await db.execute<{ n: string }>(sql`SELECT count(*) AS n FROM inbox`); expect(Number(rows[0]?.n)).toBe(1); }); + + const ghComment = (externalId: string) => ({ + source: "github" as const, + kind: "comment" as const, + external_id: externalId, + payload: { + target_id: externalId, + body: "@bot fix", + task_id: "task-1", + task_key: "T-1", + pr: { repo: "r", pr_url: "https://github.com/o/r/pull/7" }, + created_at: "", + }, + }); + + it("a redelivered comment adds nothing, even after its round finished", async () => { + const task = await insertTask(); + const first = await repo.appendInbox({ + taskId: task.id, + message: ghComment("555"), + replyTargets: [], + }); + await db.execute(sql`UPDATE inbox SET status = 'responded'`); + + expect( + await repo.appendInbox({ taskId: task.id, message: ghComment("555"), replyTargets: [] }), + ).toBeNull(); + const rows = await db.execute<{ id: string; status: string }>( + sql`SELECT id, status FROM inbox`, + ); + expect([...rows]).toEqual([{ id: first?.id as string, status: "responded" }]); + }); + + it("dedupes across kinds: a review whose id equals a stored comment id is dropped", async () => { + // characterization: changes in phase 2 (spec I9: unique on (source, kind, external_id)) + const task = await insertTask(); + await repo.appendInbox({ taskId: task.id, message: ghComment("555"), replyTargets: [] }); + + const review = await repo.appendInbox({ + taskId: task.id, + message: { + source: "github", + kind: "review", + external_id: "555", + payload: { + review_body: { target_id: "555", body: "@bot" }, + inline_comments: [], + task_id: "task-1", + task_key: "T-1", + pr: { repo: "r", pr_url: "https://github.com/o/r/pull/7" }, + created_at: "", + }, + }, + replyTargets: [], + }); + + expect(review).toBeNull(); + const kinds = await db.execute<{ kind: string }>(sql`SELECT kind FROM inbox`); + expect([...kinds]).toEqual([{ kind: "comment" }]); + }); }); describe("listPendingByTask", () => { diff --git a/control-plane/src/repositories/__tests__/task.repository.test.ts b/control-plane/src/repositories/__tests__/task.repository.test.ts index 2a1f8ed..c78bf02 100644 --- a/control-plane/src/repositories/__tests__/task.repository.test.ts +++ b/control-plane/src/repositories/__tests__/task.repository.test.ts @@ -200,9 +200,111 @@ describe("TaskRepository", () => { expect(rows).toHaveLength(1); expect(rows[0]?.reply_targets[0]?.id).toBe("sess-1"); }); + + // The Linear `created` handler runs this upsert first and restarts only when + // the stored session id differs, so a same-session redelivery ends here. + it("a duplicate created (same session) refreshes title, URL and flags but leaves the run alone", async () => { + // characterization: changes in phase 2 (spec §13: a duplicate created no longer refreshes title and URL) + // Only the title / URL / flags assertions flip; the run stays untouched. + const { taskId } = await repo.upsertFromAgentSessionWithInbox({ + taskKey: "ISS-1", + taskExternalId: "issue-1", + taskTitle: "Fix auth", + taskExternalUrl: "https://linear.app/x/issue/ISS-1", + inboxPayload: assignmentContent, + inboxReplyTargets: [linearTarget("sess-1")], + metadata: { experimentalFlags: ["OLD"] }, + }); + await db.execute(sql` + UPDATE tasks + SET owner_id = 'pod-a', heartbeat_at = now(), sandbox_id = 'sb-1', pvc_name = 'pvc-1', + last_observed_event_seq = 7, + pr_urls = '[{"repo":"r","url":"https://github.com/o/r/pull/1","branch":"b","commit_sha":"c"}]'::jsonb + WHERE id = ${taskId}; + `); + await db.execute(sql`UPDATE inbox SET status = 'delivered'`); + const [before] = await db.execute<{ id: string }>(sql`SELECT id FROM inbox`); + + await repo.upsertFromAgentSessionWithInbox({ + taskKey: "ISS-1", + taskExternalId: "issue-1", + taskTitle: "Fix auth (edited)", + taskExternalUrl: "https://linear.app/x/issue/ISS-1/fix-auth-edited", + inboxPayload: { ...assignmentContent, title: "Fix auth (edited)" }, + inboxReplyTargets: [linearTarget("sess-1")], + metadata: { experimentalFlags: ["NEW"] }, + }); + + const task = await repo.getById(taskId); + expect({ title: task?.title, url: task?.externalUrl, metadata: task?.metadata }).toEqual({ + title: "Fix auth (edited)", + url: "https://linear.app/x/issue/ISS-1/fix-auth-edited", + metadata: { experimentalFlags: ["NEW"] }, + }); + expect({ + ownerId: task?.ownerId, + sandboxId: task?.sandboxId, + pvcName: task?.pvcName, + seq: Number(task?.lastObservedEventSeq), + prs: task?.prUrls, + }).toEqual({ + ownerId: "pod-a", + sandboxId: "sb-1", + pvcName: "pvc-1", + seq: 7, + prs: [{ repo: "r", url: "https://github.com/o/r/pull/1", branch: "b", commit_sha: "c" }], + }); + const rows = await db.execute<{ id: string; status: string; payload: { title: string } }>( + sql`SELECT id, status, payload FROM inbox`, + ); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + id: (before as { id: string }).id, + status: "delivered", + payload: { title: "Fix auth" }, + }); + }); }); describe("restartTaskWithInbox", () => { + it("re-delegation: the conflict upsert refreshes title, URL and flags; the restart drops retryCount", async () => { + const { taskId } = await repo.upsertFromAgentSessionWithInbox({ + taskKey: "ISS-1", + taskExternalId: "issue-1", + taskTitle: "Fix auth", + taskExternalUrl: "https://linear.app/x/issue/ISS-1", + inboxPayload: assignmentContent, + inboxReplyTargets: [linearTarget("sess-1")], + metadata: { experimentalFlags: ["OLD"] }, + }); + await db.execute(sql` + UPDATE tasks SET metadata = metadata || '{"retryCount":2}'::jsonb WHERE id = ${taskId}; + `); + const next = { + taskKey: "ISS-1", + taskExternalId: "issue-1", + taskTitle: "Fix auth v2", + taskExternalUrl: "https://linear.app/x/issue/ISS-1/v2", + inboxPayload: { ...assignmentContent, title: "Fix auth v2", target_id: "sess-2" }, + inboxReplyTargets: [linearTarget("sess-2")], + metadata: { experimentalFlags: ["NEW"] }, + }; + + await repo.upsertFromAgentSessionWithInbox(next); + await repo.restartTaskWithInbox({ + taskId, + inboxPayload: next.inboxPayload, + inboxReplyTargets: next.inboxReplyTargets, + }); + + const task = await repo.getById(taskId); + expect({ title: task?.title, url: task?.externalUrl, metadata: task?.metadata }).toEqual({ + title: "Fix auth v2", + url: "https://linear.app/x/issue/ISS-1/v2", + metadata: { experimentalFlags: ["NEW"] }, + }); + }); + it("stamps task_id/task_key on the assignment payload from the task row", async () => { const { taskId } = await repo.upsertFromAgentSessionWithInbox({ taskExternalId: "issue-1", @@ -874,6 +976,21 @@ describe("TaskRepository", () => { expect(urls).toEqual(["https://github.com/o/r/pull/1", "https://github.com/o/r/pull/2"]); }); + it("re-appending a known url still returns the row (no first-time signal)", async () => { + // characterization: changes in phase 1b (spec §15 bug 4, I11 firstTime) + const t = await insertTask({ ownerId: "pod-a" }); + const pr = { + taskId: t.id, + ownerId: "pod-a", + prUrl: "https://github.com/o/r/pull/1", + repo: "r", + branch: "b", + commitSha: "abc", + }; + await repo.appendPrUrl(pr); + expect(await repo.appendPrUrl(pr)).toEqual([{ id: t.id }]); + }); + it("owner-guarded: returns 0 rows on stale-lease takeover", async () => { const t = await insertTask({ ownerId: "pod-a" }); const wrong = await repo.appendPrUrl({ diff --git a/control-plane/src/services/__tests__/agentSession.service.test.ts b/control-plane/src/services/__tests__/agentSession.service.test.ts index bdd7ff2..27358e2 100644 --- a/control-plane/src/services/__tests__/agentSession.service.test.ts +++ b/control-plane/src/services/__tests__/agentSession.service.test.ts @@ -285,6 +285,15 @@ describe("AgentSessionService", () => { expect(emitAction).toHaveBeenCalledWith(expect.anything(), "sess-1", "Evaluating", ""); }); + it("a redelivered prompted re-acks and enqueues again (the inbox drops the duplicate row)", async () => { + await service.handleEvent(promptedEvent()); + await service.handleEvent(promptedEvent()); + await flushMicrotasks(); + expect(enqueueLinearComment).toHaveBeenCalledTimes(2); + expect(enqueueLinearComment.mock.calls[1]).toEqual(enqueueLinearComment.mock.calls[0]); + expect(emitAction).toHaveBeenCalledTimes(2); + }); + it("fires the Evaluating ack even when the task is not yet known (session_not_found)", async () => { findByLinearIssueId.mockResolvedValue(null); diff --git a/control-plane/src/services/__tests__/apiTask.service.test.ts b/control-plane/src/services/__tests__/apiTask.service.test.ts index bb27bef..bdadb1d 100644 --- a/control-plane/src/services/__tests__/apiTask.service.test.ts +++ b/control-plane/src/services/__tests__/apiTask.service.test.ts @@ -208,6 +208,64 @@ describe("ApiTaskService", () => { expect(killSandbox).toHaveBeenCalledWith("sbx-old"); }); + it("restart resets the run, re-pends a failed assignment and only kills the old sandbox", async () => { + const { id } = await service.create(principal, { + title: "T", + description: null, + experimentalFlags: ["FOO"], + }); + await db.execute(sql` + UPDATE tasks + SET owner_id = 'pod-a', heartbeat_at = now(), sandbox_id = 'sbx-old', pvc_name = 'pvc-old', + last_observed_event_seq = 9, metadata = metadata || '{"retryCount":2}'::jsonb, + pr_urls = '[{"repo":"r","url":"https://github.com/o/r/pull/1","branch":"b","commit_sha":"c"}]'::jsonb + WHERE id = ${id}`); + await db.execute(sql` + UPDATE inbox SET status = 'failed', failure_reason = 'boom', outcome = 'failed' + WHERE task_id = ${id}`); + const [before] = await db.execute<{ id: string }>(sql`SELECT id FROM inbox`); + + await service.restart("DBX-API-1", { title: "T2", description: null, experimentalFlags: [] }); + + const [task] = await db.execute(sql` + SELECT key, external_url, created_by, owner_id, heartbeat_at, sandbox_id, pvc_name, + last_observed_event_seq::int AS seq, pr_urls, metadata + FROM tasks WHERE id = ${id}`); + expect(task).toEqual({ + key: "DBX-API-1", + external_url: "https://api.example/api/tasks/DBX-API-1", + created_by: principal.tokenId, + owner_id: null, + heartbeat_at: null, + sandbox_id: null, + pvc_name: null, + seq: 0, + pr_urls: [], + metadata: { experimentalFlags: [] }, + }); + const rows = await db.execute<{ id: string }>( + sql`SELECT id, status, failure_reason, outcome FROM inbox`, + ); + expect([...rows]).toEqual([ + { id: expect.any(String), status: "pending", failure_reason: null, outcome: null }, + ]); + expect(rows[0]?.id).not.toBe((before as { id: string }).id); + // The old workspace volume is not deleted, only unlinked (spec §15 bug 6, out of scope). + expect(killSandbox).toHaveBeenCalledTimes(1); + expect(killSandbox).toHaveBeenCalledWith("sbx-old"); + }); + + it("sendMessage is not idempotent: the same body twice queues two rounds", async () => { + await service.create(null, { title: "T", description: null, experimentalFlags: [] }); + const a = await service.sendMessage("DBX-API-1", "again"); + const b = await service.sendMessage("DBX-API-1", "again"); + expect(a.round_id).not.toBe(b.round_id); + const queued = await db.execute( + sql`SELECT type FROM task_events WHERE type = 'message_queued'`, + ); + expect(queued).toHaveLength(2); + }); + it("restart with no previous sandbox kills nothing", async () => { await service.create(null, { title: "T", description: null, experimentalFlags: [] }); await service.restart("DBX-API-1", { title: "T2", description: null, experimentalFlags: [] }); diff --git a/control-plane/src/services/__tests__/eventDispatcher.test.ts b/control-plane/src/services/__tests__/eventDispatcher.test.ts index 38ad1eb..53ae24a 100644 --- a/control-plane/src/services/__tests__/eventDispatcher.test.ts +++ b/control-plane/src/services/__tests__/eventDispatcher.test.ts @@ -150,6 +150,7 @@ describe("EventDispatcher", () => { let getByIdForTask: ReturnType; let markResponded: ReturnType; let notifyTerminal: ReturnType; + let extractArchive: ReturnType; let slackNotifier: SlackNotifierService; let dispatcher: InstanceType; @@ -161,6 +162,7 @@ describe("EventDispatcher", () => { emitError, setIssueStarted, attachUrlToIssue, + uploadIssueAttachment, ]) { m.mockReset(); m.mockResolvedValue(undefined); @@ -188,7 +190,8 @@ describe("EventDispatcher", () => { markResponded, } as unknown as InboxRepository; - sandboxManager = {} as unknown as SandboxManager; + extractArchive = mock(() => Promise.resolve(null as Uint8Array | null)); + sandboxManager = { extractArchive } as unknown as SandboxManager; linearTokenProvider = { getAuthorizedLinearClient: mock(() => Promise.resolve({} as unknown as LinearClient)), } as unknown as Pick; @@ -435,6 +438,27 @@ describe("EventDispatcher", () => { expect(emitThought).not.toHaveBeenCalled(); }); + it("Linear task: a token fetch failure rejects handle() before any side effect (cursor held)", async () => { + (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( + new Error("token refresh failed"), + ); + await expect( + dispatcher.handle( + { + seq: 1, + ts: "t", + type: "agent_thought", + inbox_id: "inbox-1", + target_id: "sess-1", + body: "x", + }, + makeTask(), + ), + ).rejects.toThrow("token refresh failed"); + expect(getByIdForTask).not.toHaveBeenCalled(); + expect(emitThought).not.toHaveBeenCalled(); + }); + it("foreign / unknown inbox_id → cross-task leak: aborts, no emit", async () => { // A foreign inbox_id resolving to null means this task's stream is // carrying an event it doesn't own (cross-task leak). The dispatcher @@ -459,6 +483,55 @@ describe("EventDispatcher", () => { }); }); + describe("agent_error routing", () => { + // The harness's round-crash format (_emit_round_failure), aimed at the + // round's primary target. + const errorEvent = (targetId: string): Extract => ({ + seq: 1, + ts: "t", + type: "agent_error", + inbox_id: "inbox-1", + target_id: targetId, + body: "⚠ Something went wrong — RuntimeError: boom", + }); + const ghRow = (kind: string, target: InboxRow["reply_targets"][number]): InboxRow => + ({ ...linearRow(), source: "github", kind, reply_targets: [target] }) as InboxRow; + + it("a GitHub comment or review round failure is posted nowhere", async () => { + // characterization: changes in phase 1a (spec §13, §15 bug 1) + getByIdForTask.mockResolvedValueOnce( + ghRow("comment", { + id: "555", + spec: { kind: "issue_comment", owner: "o", repo: "r", prNumber: 1, commentId: 555 }, + }), + ); + expect(await dispatcher.handle(errorEvent("555"), makeTask())).toBe("ok"); + getByIdForTask.mockResolvedValueOnce( + ghRow("review", { + id: "88", + spec: { kind: "review_body", owner: "o", repo: "r", prNumber: 1, reviewId: 88 }, + }), + ); + expect(await dispatcher.handle(errorEvent("88"), makeTask())).toBe("ok"); + + expect(postIssueComment).not.toHaveBeenCalled(); + expect(postReviewCommentReply).not.toHaveBeenCalled(); + expect(emitError).not.toHaveBeenCalled(); + }); + + it("a CI round failure is posted nowhere", async () => { + getByIdForTask.mockResolvedValue( + ghRow("ci_result", { + id: "9001", + spec: { kind: "issue_comment", owner: "o", repo: "r", prNumber: 1, commentId: 0 }, + }), + ); + expect(await dispatcher.handle(errorEvent("9001"), makeTask())).toBe("ok"); + expect(postIssueComment).not.toHaveBeenCalled(); + expect(emitError).not.toHaveBeenCalled(); + }); + }); + describe("pr_opened", () => { it("appends the PR url and attaches it to the issue", async () => { const result = await dispatcher.handle(prEvent(), makeTask()); @@ -720,6 +793,39 @@ describe("EventDispatcher", () => { expect(emitResponse).toHaveBeenCalledWith(expect.anything(), "sess-1", "good catch, fixing"); }); + it("a failed GitHub reply is logged and dropped: handle() resolves ok, so the cursor advances", async () => { + // characterization: changes in phase 1b (spec §13, §4 I14: pre-send failures hold) + getByIdForTask.mockResolvedValue(ghCommentRow()); + postIssueComment.mockRejectedValue(new Error("GitHub 429")); + expect(await dispatcher.handle(responseEvent("555"), makeTask())).toBe("ok"); + expect(postIssueComment).toHaveBeenCalledTimes(1); + }); + + it("still posts GitHub replies when GitHub webhooks are off", async () => { + // characterization: changes in phase 2 (spec §13: a disabled integration's replies are skipped) + const webhooks = mockedConfig.github.webhooks; + mockedConfig.github.webhooks = undefined; + try { + getByIdForTask.mockResolvedValue(ghCommentRow()); + await dispatcher.handle(responseEvent("555"), makeTask()); + expect(postIssueComment).toHaveBeenCalledWith("o", "r", 1, "good catch, fixing"); + } finally { + mockedConfig.github.webhooks = webhooks; + } + }); + + it("a Linear token failure also holds a GitHub round's reply on a Linear task", async () => { + // characterization: changes in phase 1a (spec §4 routes by round.source; not listed in §13) + (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( + new Error("token refresh failed"), + ); + getByIdForTask.mockResolvedValue(ghCommentRow()); + await expect(dispatcher.handle(responseEvent("555"), makeTask())).rejects.toThrow( + "token refresh failed", + ); + expect(postIssueComment).not.toHaveBeenCalled(); + }); + it("drops a target_id absent from reply_targets (no throw, no post)", async () => { getByIdForTask.mockResolvedValue(ghCommentRow()); const result = await dispatcher.handle(responseEvent("not-a-target"), makeTask()); @@ -831,6 +937,22 @@ describe("EventDispatcher", () => { expect(markResponded).not.toHaveBeenCalled(); }); + it("uploads the run archive to the Linear issue for an assignment round", async () => { + extractArchive.mockResolvedValue(new Uint8Array([1, 2])); + await dispatcher.handle(completedEvent(), makeTask()); + await flushMicrotasks(); + expect(extractArchive).toHaveBeenCalledWith("sbx-1"); + expect(uploadIssueAttachment).toHaveBeenCalledWith(expect.anything(), { + issueId: "issue-1", + title: "Harness run archive", + subtitle: "Task task-1", + filename: "harness-run-task-1.tar.gz", + contentType: "application/gzip", + bytes: new Uint8Array([1, 2]), + metadata: { taskId: "task-1" }, + }); + }); + it("posts a Slack completion update on the first transition (assignment round)", async () => { await dispatcher.handle(completedEvent(), makeTask()); expect(notifyTerminal).toHaveBeenCalledTimes(1); @@ -1011,4 +1133,67 @@ describe("EventDispatcher", () => { expect(markResponded).toHaveBeenCalled(); }); }); + + // reconcileOne advances the cursor only after handle() returns, so a lost + // lease or a crash in between dispatches the same events again. + describe("replay (same event dispatched again)", () => { + it("completed: the run archive upload re-fires on the already-responded row", async () => { + // characterization: changes in phase 1b (spec §15 bug 3, I11 firstTransition) + extractArchive.mockResolvedValue(new Uint8Array([1])); + const done: HarnessEvent = { + seq: 5, + ts: "t", + type: "completed", + inbox_id: "inbox-1", + status: "completed", + }; + await dispatcher.handle(done, makeTask()); + getByIdForTask.mockResolvedValue(linearRow({ status: "responded" })); + await dispatcher.handle(done, makeTask()); + await flushMicrotasks(); + + expect(markResponded).toHaveBeenCalledTimes(1); + expect(uploadIssueAttachment).toHaveBeenCalledTimes(2); + }); + + it("pr_opened: the Linear attach re-runs (appendPrUrl returns the row for a known URL)", async () => { + await dispatcher.handle(prEvent(), makeTask()); + await dispatcher.handle(prEvent(), makeTask()); + expect(appendPrUrl).toHaveBeenCalledTimes(2); + expect(attachUrlToIssue).toHaveBeenCalledTimes(2); + }); + + it("pr_opened: the onboarding comment and the PR assignee mirror re-run", async () => { + // characterization: changes in phase 1b (spec §13, §15 bug 4, I11 firstTime) + getIssueHumanAssignee.mockResolvedValue({ + id: "usr-1", + name: "Ada Lovelace", + displayName: "ada", + email: "ada@acme.com", + gitHubUserId: "octocat", + }); + addPrAssignees.mockResolvedValue({ assigned: ["octocat"] }); + await dispatcher.handle(prEvent(), makeTask()); + await dispatcher.handle(prEvent(), makeTask()); + await flushMicrotasks(); + + expect(postIssueComment).toHaveBeenCalledTimes(2); + expect(addPrAssignees).toHaveBeenCalledTimes(2); + }); + + it("agent_response and agent_error post again on every dispatch", async () => { + const reply: HarnessEvent = { + seq: 2, + ts: "t", + type: "agent_response", + inbox_id: "inbox-1", + target_id: "sess-1", + body: "done", + }; + const error: HarnessEvent = { ...reply, seq: 3, type: "agent_error", body: "bad" }; + for (const e of [reply, reply, error, error]) await dispatcher.handle(e, makeTask()); + expect(emitResponse).toHaveBeenCalledTimes(2); + expect(emitError).toHaveBeenCalledTimes(2); + }); + }); }); diff --git a/control-plane/src/services/__tests__/inbox.service.test.ts b/control-plane/src/services/__tests__/inbox.service.test.ts index 0cee8a7..319fb3f 100644 --- a/control-plane/src/services/__tests__/inbox.service.test.ts +++ b/control-plane/src/services/__tests__/inbox.service.test.ts @@ -298,6 +298,156 @@ describe("InboxService", () => { expect(emitError).not.toHaveBeenCalled(); expect(markFailed).not.toHaveBeenCalled(); }); + + // One open row per kind on a Linear task. getById echoes each row so the + // per-row status re-check passes. + const stageRows = (rows: InboxRow[]) => { + listNonTerminalByTask.mockResolvedValue(rows); + getById.mockImplementation((id: string) => + Promise.resolve(rows.find((r) => r.id === id) ?? null), + ); + }; + const linearRow = (id: string, sessionId: string, kind = "comment"): InboxRow => + ({ + ...ghCommentRow(id), + source: "linear", + kind, + reply_targets: [{ id: sessionId, spec: { kind: "linear_session" } }], + }) as InboxRow; + + it("notifies each row by kind: Linear session error, review-comment reply + 😕, review body as a PR comment, CI silent", async () => { + stageRows([ + linearRow("lin", "sess-2"), + { + ...ghCommentRow("rc"), + reply_targets: [ + { + id: "556", + spec: { + kind: "review_comment", + owner: "o", + repo: "r", + prNumber: 7, + commentId: 556, + inReplyTo: 556, + path: "a.ts", + line: 3, + }, + }, + ], + }, + { + ...ghCommentRow("rev"), + kind: "review", + reply_targets: [ + { + id: "11", + spec: { + kind: "review_comment", + owner: "o", + repo: "r", + prNumber: 8, + commentId: 11, + inReplyTo: 11, + path: "b.ts", + line: 9, + }, + }, + { + id: "88", + spec: { kind: "review_body", owner: "o", repo: "r", prNumber: 8, reviewId: 88 }, + }, + ], + } as InboxRow, + { + ...ghCommentRow("ci"), + kind: "ci_result", + reply_targets: [ + { + id: "9001", + spec: { kind: "issue_comment", owner: "o", repo: "r", prNumber: 9, commentId: 0 }, + }, + ], + } as InboxRow, + ]); + + await service.deadLetterWorkspaceMissing(makeTask({ sandboxId: null })); + + expect(emitError).toHaveBeenCalledTimes(1); + expect(emitError).toHaveBeenCalledWith( + expect.anything(), + "sess-2", + WORKSPACE_MISSING_MESSAGE, + ); + expect(postReviewCommentReply).toHaveBeenCalledTimes(1); + expect(postReviewCommentReply).toHaveBeenCalledWith( + "o", + "r", + 7, + 556, + WORKSPACE_MISSING_MESSAGE, + ); + expect(addReviewCommentReaction).toHaveBeenCalledTimes(1); + expect(addReviewCommentReaction).toHaveBeenCalledWith("o", "r", 556, "confused"); + // Only the review body posts; its inline comments and the CI row get nothing. + expect(postIssueComment).toHaveBeenCalledTimes(1); + expect(postIssueComment).toHaveBeenCalledWith("o", "r", 8, WORKSPACE_MISSING_MESSAGE); + expect(addIssueCommentReaction).not.toHaveBeenCalled(); + expect(markFailed.mock.calls.map((c) => (c[0] as { inboxId: string }).inboxId)).toEqual([ + "lin", + "rc", + "rev", + "ci", + ]); + }); + + it("posts per row, not per target: two Linear rows on one session get two errors", async () => { + // failTask dedupes by target id; this path does not. + stageRows([linearRow("a", "sess-1", "assignment"), linearRow("b", "sess-1")]); + + await service.deadLetterWorkspaceMissing(makeTask({ sandboxId: null })); + + expect(emitError).toHaveBeenCalledTimes(2); + expect(markFailed).toHaveBeenCalledTimes(2); + }); + + it("posts before marking the row, and posts even when markFailed transitions nothing", async () => { + // characterization: changes in phase 1a (spec §4 I15 / §15 bug 7) + const order: string[] = []; + postIssueComment.mockImplementation(() => { + order.push("post"); + return Promise.resolve({ id: 1, url: "u" }); + }); + addIssueCommentReaction.mockImplementation(() => { + order.push("react"); + return Promise.resolve(); + }); + markFailed.mockImplementation(() => { + order.push("mark"); + return Promise.resolve([]); + }); + listNonTerminalByTask.mockResolvedValue([ghCommentRow("a")]); + + await service.deadLetterWorkspaceMissing(makeTask({ sandboxId: null })); + + expect(order).toEqual(["post", "react", "mark"]); + }); + + it("a Linear token failure escapes: that row is still marked failed, later rows are untouched", async () => { + // characterization: changes in phase 1a (spec §15 bug 2) + stageRows([linearRow("a", "sess-1"), ghCommentRow("b")]); + (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( + new Error("token refresh failed"), + ); + + await expect( + service.deadLetterWorkspaceMissing(makeTask({ sandboxId: null })), + ).rejects.toThrow("token refresh failed"); + + expect(markFailed).toHaveBeenCalledTimes(1); + expect(markFailed).toHaveBeenCalledWith(expect.objectContaining({ inboxId: "a" })); + expect(postIssueComment).not.toHaveBeenCalled(); + }); }); describe("enqueueComment", () => { diff --git a/control-plane/src/services/__tests__/sandboxManager.service.test.ts b/control-plane/src/services/__tests__/sandboxManager.service.test.ts index caa1e81..64f9aff 100644 --- a/control-plane/src/services/__tests__/sandboxManager.service.test.ts +++ b/control-plane/src/services/__tests__/sandboxManager.service.test.ts @@ -833,6 +833,29 @@ describe("SandboxManager", () => { expect(err.body.length).toBeLessThanOrEqual(2001); }); + it("drops the harness's retryable_assignment_failure flag from an agent_error", async () => { + // characterization: changes in phase 1a (spec §4: validateHarnessEvent copies it when === true) + const raw = JSON.stringify({ + seq: 1, + ts: "t", + type: "agent_error", + inbox_id: "i", + target_id: "tgt", + body: "RuntimeError: boom", + retryable_assignment_failure: true, + }); + expect(await mgrWith(raw).readEvents("sbx", 0)).toEqual([ + { + seq: 1, + ts: "t", + type: "agent_error", + inbox_id: "i", + target_id: "tgt", + body: "RuntimeError: boom", + }, + ]); + }); + it("agent_thought/action/response use the wider 60k cap (plan markdown + completion summaries can run long)", async () => { const huge = "y".repeat(5000); const raw = [ diff --git a/control-plane/src/services/__tests__/slackNotifier.service.test.ts b/control-plane/src/services/__tests__/slackNotifier.service.test.ts index 99975df..21ebf1d 100644 --- a/control-plane/src/services/__tests__/slackNotifier.service.test.ts +++ b/control-plane/src/services/__tests__/slackNotifier.service.test.ts @@ -127,6 +127,12 @@ describe("SlackNotifierService", () => { expect(text).toContain("sandbox ran out of memory"); }); + it("labels the ticket link 'Linear ticket' when the task has no title", async () => { + getById.mockResolvedValue(makeTask({ title: "", prUrls: [PR] })); + await make().notifyTerminal(client, makeTask({ title: "" }), "completed"); + expect(postedText()).toContain(""); + }); + it("re-reads the task for the freshest pr_urls (stale in-memory task had none)", async () => { // The task handed in predates the pr_opened write; getById returns the PR. await make().notifyTerminal(client, makeTask({ prUrls: [] }), "completed"); diff --git a/control-plane/src/services/__tests__/taskProcessor.service.test.ts b/control-plane/src/services/__tests__/taskProcessor.service.test.ts index 95e632a..e2fda71 100644 --- a/control-plane/src/services/__tests__/taskProcessor.service.test.ts +++ b/control-plane/src/services/__tests__/taskProcessor.service.test.ts @@ -73,11 +73,13 @@ mock.module("../../clients/linear", () => ({ // only). Mock the barrel before importing, like the other consumers' tests. // The mock leaks to later files too, so it stubs every export they import. // See control-plane/README.md (Conventions). +const postIssueComment = mock(() => Promise.resolve({ id: 1, url: "u" })); +const postReviewCommentReply = mock(() => Promise.resolve({ id: 1, url: "u" })); mock.module("../../clients/github", () => ({ addIssueCommentReaction: mock(() => Promise.resolve()), addReviewCommentReaction: mock(() => Promise.resolve()), - postIssueComment: mock(() => Promise.resolve({ id: 1, url: "u" })), - postReviewCommentReply: mock(() => Promise.resolve({ id: 1, url: "u" })), + postIssueComment, + postReviewCommentReply, getReviewComments: mock(() => Promise.resolve([])), fetchOrgMembers: mock(() => Promise.resolve(new Set())), addPrAssignees: mock(() => Promise.resolve({ assigned: [] })), @@ -183,6 +185,10 @@ describe("TaskProcessorService", () => { emitThought.mockResolvedValue(undefined); emitError.mockReset(); emitError.mockResolvedValue(undefined); + postIssueComment.mockReset(); + postIssueComment.mockResolvedValue({ id: 1, url: "u" }); + postReviewCommentReply.mockReset(); + postReviewCommentReply.mockResolvedValue({ id: 1, url: "u" }); heartbeatOwnedBy = mock(() => Promise.resolve()); claimNext = mock(() => Promise.resolve(null)); @@ -347,6 +353,12 @@ describe("TaskProcessorService", () => { }); expect(setIssueStarted).toHaveBeenCalledWith(expect.anything(), "issue-1"); expect(emitAction).toHaveBeenCalledWith(expect.anything(), "sess-1", "Provisioning", "infra"); + // characterization: changes in phase 1b (spec §13: the "Sandbox provisioned" thought is removed) + expect(emitThought).toHaveBeenCalledWith( + expect.anything(), + "sess-1", + "Sandbox provisioned.\n- sandboxId: `sbx-1`\n- pvcName: `devbox-pvc-task-1-aaaaaa`", + ); }); it("logs and still provisions when the team has no started state", async () => { @@ -625,6 +637,137 @@ describe("TaskProcessorService", () => { }); }); + describe("handleTaskError — pruned workspace", () => { + it("a throwing workspace-missing dead-letter (Linear token) escapes before the lease is released", async () => { + // characterization: changes in phase 1a (spec §15 bug 2) + deadLetterWorkspaceMissing.mockRejectedValue(new Error("token refresh failed")); + const task = makeTask({ sandboxId: null, pvcName: "pvc-1" }); + + await expect( + ( + service as unknown as { handleTaskError(t: Task, e: unknown): Promise } + ).handleTaskError(task, new PvcNotFoundError("pvc-1")), + ).rejects.toThrow("token refresh failed"); + + expect(releaseLease).not.toHaveBeenCalled(); + }); + }); + + describe("failTask — failure broadcast", () => { + const MSG = "Task failed — check control-plane logs for details."; + const failTask = (task: Task): Promise => + (service as unknown as { failTask(t: Task, m: string): Promise }).failTask(task, MSG); + // biome-ignore lint/suspicious/noExplicitAny: test stub shape, cast at use site + const row = (id: string, kind: string, source: string, replyTargets: unknown[]): any => ({ + ...survivingNonTerminalRow(), + id, + kind, + source, + reply_targets: replyTargets, + }); + const reviewComment = (id: number, prNumber: number) => ({ + id: String(id), + spec: { + kind: "review_comment", + owner: "o", + repo: "r", + prNumber, + commentId: id, + inReplyTo: id, + path: "a.ts", + line: 1, + }, + }); + const issueComment = (id: string, prNumber: number, commentId: number) => ({ + id, + spec: { kind: "issue_comment", owner: "o", repo: "r", prNumber, commentId }, + }); + + it("posts once per distinct target id across every open row, by target kind", async () => { + (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValue([ + row("a", "assignment", "linear", [{ id: "sess-1", spec: { kind: "linear_session" } }]), + // Same session as the assignment → deduped. + row("lc", "comment", "linear", [{ id: "sess-1", spec: { kind: "linear_session" } }]), + row("gc", "comment", "github", [issueComment("555", 7, 555)]), + row("rv", "review", "github", [ + reviewComment(11, 8), + reviewComment(12, 8), + { + id: "88", + spec: { kind: "review_body", owner: "o", repo: "r", prNumber: 8, reviewId: 88 }, + }, + ]), + // CI target: commentId 0 = "post a new PR comment". + row("ci", "ci_result", "github", [issueComment("9001", 9, 0)]), + ]); + + await failTask(makeTask()); + + expect(emitError).toHaveBeenCalledTimes(1); + expect(emitError).toHaveBeenCalledWith(expect.anything(), "sess-1", MSG); + // PR comments for the comment, the review body and the CI row, in row order. + expect(postIssueComment.mock.calls as unknown[]).toEqual([ + ["o", "r", 7, MSG], + ["o", "r", 8, MSG], + ["o", "r", 9, MSG], + ]); + expect(postReviewCommentReply.mock.calls as unknown[]).toEqual([ + ["o", "r", 8, 11, MSG], + ["o", "r", 8, 12, MSG], + ]); + expect(failTaskInbox).toHaveBeenCalledWith({ + taskId: "task-1", + ownerId: "pod-test", + reason: MSG, + }); + }); + + it("posts before dead-lettering, and posts even when failTaskInbox transitions no row", async () => { + // characterization: changes in phase 1a (spec §4 I15 / §15 bug 7) + const order: string[] = []; + emitError.mockImplementation(() => { + order.push("post"); + return Promise.resolve(); + }); + getByIdOwnedBy.mockResolvedValue(makeTask({ sandboxId: "sbx-live" })); + killSandbox.mockImplementation(() => { + order.push("kill"); + return Promise.resolve(); + }); + failTaskInbox.mockImplementation(() => { + order.push("mark"); + return Promise.resolve([]); + }); + releaseLease.mockImplementation(() => { + order.push("release"); + return Promise.resolve([{ id: "task-1" }]); + }); + + await failTask(makeTask()); + + expect(order).toEqual(["post", "kill", "mark", "release"]); + }); + + it("a Linear token failure skips only the Linear post; GitHub posts, dead-letter and release go on", async () => { + getAuthorizedLinearClient.mockRejectedValue(new Error("token refresh failed")); + (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValue([ + row("a", "assignment", "linear", [{ id: "sess-1", spec: { kind: "linear_session" } }]), + row("gc", "comment", "github", [issueComment("555", 7, 555)]), + ]); + + await failTask(makeTask()); + + expect(emitError).not.toHaveBeenCalled(); + expect(postIssueComment).toHaveBeenCalledWith("o", "r", 7, MSG); + expect(failTaskInbox).toHaveBeenCalled(); + expect(releaseLease).toHaveBeenCalledWith({ + taskId: "task-1", + ownerId: "pod-test", + alsoClearSandbox: true, + }); + }); + }); + describe("runTask — adoption (sandbox_id already set)", () => { it("attaches to the existing sandbox without provisioning a new one", async () => { const task = makeTask({ sandboxId: "sbx-alive" }); @@ -923,6 +1066,30 @@ describe("TaskProcessorService", () => { expect(dispatcherHandle.mock.calls.length).toBe(1); }); + it("a dispatcher throw (e.g. a Linear token outage) holds the cursor and fails nothing", async () => { + const task = makeTask({ sandboxId: "sbx-1" }); + listInFlightOwnedBy.mockResolvedValueOnce([task]).mockResolvedValue([]); + readEvents + .mockResolvedValueOnce([ + { seq: 1, ts: "t", type: "agent_thought", inbox_id: "i", target_id: "s", body: "x" }, + { seq: 2, ts: "t", type: "agent_thought", inbox_id: "i", target_id: "s", body: "y" }, + ] as HarnessEvent[]) + .mockResolvedValue([]); + dispatcherHandle.mockRejectedValueOnce(new Error("token refresh failed")); + + service.start(); + await waitFor(() => dispatcherHandle.mock.calls.length >= 1); + await new Promise((r) => setTimeout(r, 30)); + await service.stop(); + + // The throw ends this task's tick: seq 2 is not dispatched and seq 1 is + // re-read next tick from the unchanged cursor. + expect(dispatcherHandle).toHaveBeenCalledTimes(1); + expect(recordEventCursor).not.toHaveBeenCalled(); + expect(failTaskInbox).not.toHaveBeenCalled(); + expect(requeueForRetry).not.toHaveBeenCalled(); + }); + it("calls sweepUndelivered before reading events", async () => { const task = makeTask({ sandboxId: "sbx-1" }); listInFlightOwnedBy.mockResolvedValueOnce([task]).mockResolvedValue([]); From 127956410dbd84ec8604cbdabcd9b21b45d0233c Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Fri, 30 Oct 2026 17:12:50 +0530 Subject: [PATCH 02/22] test(control-plane): tighten characterization tests Stub the rows failTaskInbox marks, so failure tests don't depend on posting before marking. Cover agent_error on Linear comment rounds and Slack during a Linear outage, use the GitHub client's real 429 error, and wait on observable calls instead of fixed sleeps. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/eventDispatcher.test.ts | 25 ++++++++++++++++--- .../__tests__/taskProcessor.service.test.ts | 11 ++++++-- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/control-plane/src/services/__tests__/eventDispatcher.test.ts b/control-plane/src/services/__tests__/eventDispatcher.test.ts index 53ae24a..d92efa2 100644 --- a/control-plane/src/services/__tests__/eventDispatcher.test.ts +++ b/control-plane/src/services/__tests__/eventDispatcher.test.ts @@ -438,7 +438,8 @@ describe("EventDispatcher", () => { expect(emitThought).not.toHaveBeenCalled(); }); - it("Linear task: a token fetch failure rejects handle() before any side effect (cursor held)", async () => { + it("Linear task: a token fetch failure posts nothing and does not return ok (cursor held)", async () => { + // characterization: changes in phase 1b (handle returns "hold" instead of rejecting) (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( new Error("token refresh failed"), ); @@ -455,8 +456,9 @@ describe("EventDispatcher", () => { makeTask(), ), ).rejects.toThrow("token refresh failed"); - expect(getByIdForTask).not.toHaveBeenCalled(); - expect(emitThought).not.toHaveBeenCalled(); + for (const m of [emitThought, emitAction, emitResponse, emitError, postIssueComment]) { + expect(m).not.toHaveBeenCalled(); + } }); it("foreign / unknown inbox_id → cross-task leak: aborts, no emit", async () => { @@ -497,6 +499,16 @@ describe("EventDispatcher", () => { const ghRow = (kind: string, target: InboxRow["reply_targets"][number]): InboxRow => ({ ...linearRow(), source: "github", kind, reply_targets: [target] }) as InboxRow; + it("a Linear comment round failure → emitError on the row's session", async () => { + getByIdForTask.mockResolvedValue({ ...linearRow(), kind: "comment" } as InboxRow); + expect(await dispatcher.handle(errorEvent("sess-1"), makeTask())).toBe("ok"); + expect(emitError).toHaveBeenCalledWith( + expect.anything(), + "sess-1", + "⚠ Something went wrong — RuntimeError: boom", + ); + }); + it("a GitHub comment or review round failure is posted nowhere", async () => { // characterization: changes in phase 1a (spec §13, §15 bug 1) getByIdForTask.mockResolvedValueOnce( @@ -796,7 +808,12 @@ describe("EventDispatcher", () => { it("a failed GitHub reply is logged and dropped: handle() resolves ok, so the cursor advances", async () => { // characterization: changes in phase 1b (spec §13, §4 I14: pre-send failures hold) getByIdForTask.mockResolvedValue(ghCommentRow()); - postIssueComment.mockRejectedValue(new Error("GitHub 429")); + // githubFetch's error for a 429: a plain Error, the status only in the message. + postIssueComment.mockRejectedValue( + new Error( + 'GitHub POST /repos/o/r/issues/1/comments failed: HTTP 429 {"message":"API rate limit exceeded"}', + ), + ); expect(await dispatcher.handle(responseEvent("555"), makeTask())).toBe("ok"); expect(postIssueComment).toHaveBeenCalledTimes(1); }); diff --git a/control-plane/src/services/__tests__/taskProcessor.service.test.ts b/control-plane/src/services/__tests__/taskProcessor.service.test.ts index e2fda71..9dbe90b 100644 --- a/control-plane/src/services/__tests__/taskProcessor.service.test.ts +++ b/control-plane/src/services/__tests__/taskProcessor.service.test.ts @@ -485,6 +485,7 @@ describe("TaskProcessorService", () => { updated_at: new Date(), }, ]); + failTaskInbox.mockResolvedValue([{ id: "inbox-1" }]); service.start(); await waitFor(() => releaseLease.mock.calls.length > 0); @@ -700,6 +701,7 @@ describe("TaskProcessorService", () => { // CI target: commentId 0 = "post a new PR comment". row("ci", "ci_result", "github", [issueComment("9001", 9, 0)]), ]); + failTaskInbox.mockResolvedValue(["a", "lc", "gc", "rv", "ci"].map((id) => ({ id }))); await failTask(makeTask()); @@ -754,6 +756,7 @@ describe("TaskProcessorService", () => { row("a", "assignment", "linear", [{ id: "sess-1", spec: { kind: "linear_session" } }]), row("gc", "comment", "github", [issueComment("555", 7, 555)]), ]); + failTaskInbox.mockResolvedValue([{ id: "a" }, { id: "gc" }]); await failTask(makeTask()); @@ -765,6 +768,8 @@ describe("TaskProcessorService", () => { ownerId: "pod-test", alsoClearSandbox: true, }); + // characterization: changes in phase 2 (Slack notifier sends without the assignee mention; spec §13) + expect(notifyTerminal).not.toHaveBeenCalled(); }); }); @@ -1078,8 +1083,8 @@ describe("TaskProcessorService", () => { dispatcherHandle.mockRejectedValueOnce(new Error("token refresh failed")); service.start(); - await waitFor(() => dispatcherHandle.mock.calls.length >= 1); - await new Promise((r) => setTimeout(r, 30)); + // assessReconcileError reads the pod (unknown here → log only). + await waitFor(() => getSandboxPodStatus.mock.calls.length > 0); await service.stop(); // The throw ends this task's tick: seq 2 is not dispatched and seq 1 is @@ -1176,6 +1181,7 @@ describe("TaskProcessorService", () => { getSandboxPodStatus.mockResolvedValue({ exists: true, phase: "Running", oomKilled: false }); getSandboxInfo.mockResolvedValue(makeSandboxInfo({ metadata: { task_id: "other-task" } })); stubLinearTarget(task); + failTaskInbox.mockResolvedValue([{ id: "inbox-1" }]); service.start(); await waitFor(() => releaseLease.mock.calls.length > 0); @@ -1313,6 +1319,7 @@ describe("TaskProcessorService", () => { it("retries exhausted → dead-letters + fails with the give-up message", async () => { const task = makeTask({ sandboxId: "sbx-oom" }); stubLinearTarget(task); + failTaskInbox.mockResolvedValue([{ id: "inbox-1" }]); getSandboxPodStatus.mockResolvedValue({ exists: true, phase: "Failed", oomKilled: true }); // The count is durable in tasks.metadata; requeueForRetry persists it. Mirror // that on the shared task object so the next assess() reads the bumped count From 8283fe099b3dc2206ae806e06a36ac6530d7e920 Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Fri, 30 Oct 2026 17:28:50 +0530 Subject: [PATCH 03/22] refactor(control-plane): route outbound messages through integration deliver Linear, GitHub and the API each get an integration module with one deliver(ctx, target, message). Core only labels harness events as a reply, progress or failure and routes them by the round's source; the module decides where, or whether, each one shows up. Failures now mark rows first and notify only the rows actually marked, so a late completion or a restart can't get a contradictory "task failed". A crashed GitHub comment or review round now gets a short retry hint on the PR instead of nothing, and a Linear token error no longer leaves the task's other rows open and its lease held. Co-Authored-By: Claude Opus 5.5 --- control-plane/src/app.ts | 15 +- control-plane/src/integrations/api/index.ts | 10 + .../src/integrations/github/index.ts | 117 ++++++++++++ .../src/integrations/linear/index.ts | 60 ++++++ control-plane/src/integrations/outbound.ts | 29 +++ control-plane/src/integrations/types.ts | 13 ++ .../src/repositories/inbox.repository.ts | 10 +- .../__tests__/eventDispatcher.test.ts | 118 +++++++++++- .../services/__tests__/inbox.service.test.ts | 39 ++-- .../__tests__/sandboxManager.service.test.ts | 31 ++-- .../__tests__/taskProcessor.service.test.ts | 59 ++++-- control-plane/src/services/eventDispatcher.ts | 175 +++++++----------- control-plane/src/services/inbox.service.ts | 120 ++++-------- .../src/services/sandboxManager.service.ts | 4 + .../src/services/taskProcessor.service.ts | 170 +++++++---------- control-plane/src/types/harnessEvent.ts | 14 +- 16 files changed, 621 insertions(+), 363 deletions(-) create mode 100644 control-plane/src/integrations/api/index.ts create mode 100644 control-plane/src/integrations/github/index.ts create mode 100644 control-plane/src/integrations/linear/index.ts create mode 100644 control-plane/src/integrations/outbound.ts create mode 100644 control-plane/src/integrations/types.ts diff --git a/control-plane/src/app.ts b/control-plane/src/app.ts index d8bda75..21f1bf2 100644 --- a/control-plane/src/app.ts +++ b/control-plane/src/app.ts @@ -7,6 +7,10 @@ import { GitHubWebhookController } from "./controllers/githubWebhook.controller" import { HealthController } from "./controllers/health.controller"; import { LinearWebhookController } from "./controllers/linearWebhook.controller"; import { db } from "./db"; +import { apiIntegration } from "./integrations/api"; +import { githubIntegration } from "./integrations/github"; +import { LinearIntegration } from "./integrations/linear"; +import type { Integration, IntegrationName } from "./integrations/types"; import { logger } from "./logger"; import { apiTaskRepository } from "./repositories/apiTask.repository"; import { apiTokenRepository } from "./repositories/apiToken.repository"; @@ -73,7 +77,14 @@ export function createApp(): AppBundle { trustedUsers: joinIfAny(config.github.trustedUsers), }); - const inboxService = new InboxService(inboxRepository, sandboxManager, linearTokenProvider); + // Every outbound message goes through integrations[round.source].deliver. + const integrations = { + linear: new LinearIntegration(linearTokenProvider), + api: apiIntegration, + github: githubIntegration, + } satisfies Record; + + const inboxService = new InboxService(inboxRepository, sandboxManager, integrations); const agentSessionService = new AgentSessionService( taskRepository, @@ -95,6 +106,7 @@ export function createApp(): AppBundle { sandboxManager, linearTokenProvider, slackNotifier, + integrations, ); const taskProcessor = new TaskProcessorService( @@ -105,6 +117,7 @@ export function createApp(): AppBundle { eventDispatcher, inboxService, slackNotifier, + integrations, ); const linearStatus = new LinearStatusService( diff --git a/control-plane/src/integrations/api/index.ts b/control-plane/src/integrations/api/index.ts new file mode 100644 index 0000000..25d7f17 --- /dev/null +++ b/control-plane/src/integrations/api/index.ts @@ -0,0 +1,10 @@ +import type { Integration } from "../types"; + +// API tasks have no outbound channel: the task_events log carries replies and +// failures (failTaskInbox / markFailed write round_failed there). +export const apiIntegration: Integration = { + name: "api", + async deliver() { + // Nothing to post. + }, +}; diff --git a/control-plane/src/integrations/github/index.ts b/control-plane/src/integrations/github/index.ts new file mode 100644 index 0000000..58d4fa4 --- /dev/null +++ b/control-plane/src/integrations/github/index.ts @@ -0,0 +1,117 @@ +import { + addIssueCommentReaction, + addReviewCommentReaction, + postIssueComment, + postReviewCommentReply, +} from "../../clients/github"; +import { config } from "../../config"; +import type { InboxRow, ReplyTargetSpec } from "../../repositories/inbox.repository"; +import { WORKSPACE_MISSING_MESSAGE } from "../../services/inbox.service"; +import type { Outbound, SendCtx } from "../outbound"; +import type { Integration } from "../types"; + +type GithubSpec = Extract< + ReplyTargetSpec, + { kind: "issue_comment" | "review_comment" | "review_body" } +>; + +// GitHub rounds (comments, reviews, CI results) post on the PR. A PR shows no +// live progress, and it is public: a failure's raw detail never goes there. +export const githubIntegration: Integration = { + name: "github", + async deliver(ctx, target, message) { + const spec = target.spec; + if (spec.kind === "linear_session" || spec.kind === "api_session") { + throw new Error(`github cannot deliver to a ${spec.kind} target`); + } + switch (message.type) { + case "reply": + await post(spec, message.body); + return; + case "progress": + return; + case "failure": + await deliverFailure(ctx, spec, message); + return; + } + }, +}; + +async function deliverFailure( + ctx: SendCtx, + spec: GithubSpec, + message: Extract, +): Promise { + const { round } = message; + switch (message.cause) { + case "task_failed": + // failTask's message is control-plane-generated and scrubbed: public-safe. + if (message.summary) await post(spec, message.summary); + return; + case "round_failed": { + // A failed CI-fix round stays silent. + if (round.kind === "ci_result") return; + // The retry hint is an @-mention, which only works with webhooks on. + const botLogin = config.github.webhooks?.botLogin; + if (!botLogin) return; + await post( + spec, + `⚠️ Something went wrong while working on this. Mention @${botLogin} in a new comment to retry.`, + ); + return; + } + case "workspace_missing": + await workspaceMissing(ctx, spec, round); + return; + } +} + +// One notice per round. A comment gets a reply plus 😕. A review gets only a +// top-level PR comment for its summary, whatever target core passed (the +// review body has no reply / reactions API). A CI result is informational and +// dead-lettered silently. +async function workspaceMissing(ctx: SendCtx, spec: GithubSpec, round: InboxRow): Promise { + if (round.kind === "review") { + const body = round.reply_targets.find((t) => t.spec.kind === "review_body"); + if (body?.spec.kind === "review_body") await post(body.spec, WORKSPACE_MISSING_MESSAGE); + return; + } + if (round.kind !== "comment") return; + try { + await post(spec, WORKSPACE_MISSING_MESSAGE); + } finally { + // The reaction is independent of the reply: try it even if the reply failed. + await confusedReaction(spec).catch((err: unknown) => { + const e = err instanceof Error ? err : new Error(String(err)); + ctx.log.error("workspace_missing reaction failed", { message: e.message, stack: e.stack }); + }); + } +} + +async function confusedReaction(spec: GithubSpec): Promise { + if (spec.kind === "issue_comment") { + await addIssueCommentReaction(spec.owner, spec.repo, spec.commentId, "confused"); + } else if (spec.kind === "review_comment") { + await addReviewCommentReaction(spec.owner, spec.repo, spec.commentId, "confused"); + } +} + +// Posts `body` verbatim to one PR thread, by target kind. CP never formats it. +async function post(spec: GithubSpec, body: string): Promise { + switch (spec.kind) { + case "issue_comment": + await postIssueComment(spec.owner, spec.repo, spec.prNumber, body); + return; + case "review_comment": + await postReviewCommentReply(spec.owner, spec.repo, spec.prNumber, spec.inReplyTo, body); + return; + case "review_body": + // Top-level PR comment — GH has no reply-to-review-body API. + await postIssueComment(spec.owner, spec.repo, spec.prNumber, body); + return; + default: { + const _exhaustive: never = spec; + void _exhaustive; + } + } +} diff --git a/control-plane/src/integrations/linear/index.ts b/control-plane/src/integrations/linear/index.ts new file mode 100644 index 0000000..5af8243 --- /dev/null +++ b/control-plane/src/integrations/linear/index.ts @@ -0,0 +1,60 @@ +import type { LinearClient } from "@linear/sdk"; +import { emitAction, emitError, emitResponse, emitThought } from "../../clients/linear"; +import type { ReplyTarget } from "../../repositories/inbox.repository"; +import { WORKSPACE_MISSING_MESSAGE } from "../../services/inbox.service"; +import type { LinearTokenProvider } from "../../services/linearTokenProvider"; +import { type Outbound, RetryableError, type SendCtx } from "../outbound"; +import type { Integration } from "../types"; + +// Linear rounds post to the agent session. target.id IS the agent session id +// by construction (the enqueue paths stamp the live session id). The session +// is private to the workspace, so a failure shows its raw detail. +export class LinearIntegration implements Integration { + readonly name = "linear"; + + constructor( + private linearTokenProvider: Pick, + ) {} + + async deliver(_ctx: SendCtx, target: ReplyTarget, message: Outbound): Promise { + if (target.spec.kind !== "linear_session") { + throw new Error(`linear cannot deliver to a ${target.spec.kind} target`); + } + const client = await this.client(); + switch (message.type) { + case "reply": + await emitResponse(client, target.id, message.body); + return; + case "progress": + if (message.kind === "thought") await emitThought(client, target.id, message.body); + // The daemon's body is the full action label (e.g. "Planning", + // "Addressing comment"). Linear renders ` ` + // concatenated, so a non-empty parameter appends a noisy suffix + // ("Addressing comment task"). Pass empty parameter — the body + // already carries everything the user should see. + else if (message.kind === "action") await emitAction(client, target.id, message.body, ""); + else await emitError(client, target.id, message.body); + return; + case "failure": { + // task_failed and a failed assignment carry the summary; a crashed + // comment round carries only the detail. + const text = + message.cause === "workspace_missing" + ? WORKSPACE_MISSING_MESSAGE + : (message.summary ?? message.detail); + if (text) await emitError(client, target.id, text); + return; + } + } + } + + // Nothing has been sent when the token fetch fails, so the caller may retry. + private async client(): Promise { + try { + return await this.linearTokenProvider.getAuthorizedLinearClient(); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + throw new RetryableError(`Linear token fetch failed: ${msg}`); + } + } +} diff --git a/control-plane/src/integrations/outbound.ts b/control-plane/src/integrations/outbound.ts new file mode 100644 index 0000000..47e9d6a --- /dev/null +++ b/control-plane/src/integrations/outbound.ts @@ -0,0 +1,29 @@ +import type { logger } from "../logger"; +import type { InboxRow } from "../repositories/inbox.repository"; + +type Logger = typeof logger; + +// What core hands to deliver(). Core says WHAT a message is; the integration +// decides WHERE (or whether) it shows up. +export type Outbound = + | { type: "reply"; body: string } // the agent's answer + | { type: "progress"; kind: "thought" | "action" | "error"; body: string } + | { + type: "failure"; + cause: "round_failed" | "task_failed" | "workspace_missing"; + round: InboxRow; + summary?: string; // user-safe text (the harness's "what failed" summary) + detail?: string; // raw error text (exception message): never post it on a public channel + }; +export type SendCtx = { signal: AbortSignal; log: Logger }; + +// Throw only before the first request is sent (token fetch failed, 429, +// connection refused, DNS, TLS handshake). +export class RetryableError extends Error { + constructor( + msg: string, + readonly retryAfterMs?: number, + ) { + super(msg); + } +} diff --git a/control-plane/src/integrations/types.ts b/control-plane/src/integrations/types.ts new file mode 100644 index 0000000..2a37133 --- /dev/null +++ b/control-plane/src/integrations/types.ts @@ -0,0 +1,13 @@ +import type { ReplyTarget } from "../repositories/inbox.repository"; +import type { Outbound, SendCtx } from "./outbound"; + +// One per task/round source (today's InboxSource). +export type IntegrationName = "linear" | "api" | "github"; + +export interface Integration { + name: IntegrationName; + + // Outgoing. Core labels each message; the integration decides where (or + // whether) it shows up. Throws RetryableError only before anything was sent. + deliver(ctx: SendCtx, target: ReplyTarget, message: Outbound): Promise; +} diff --git a/control-plane/src/repositories/inbox.repository.ts b/control-plane/src/repositories/inbox.repository.ts index b35c88a..aa21715 100644 --- a/control-plane/src/repositories/inbox.repository.ts +++ b/control-plane/src/repositories/inbox.repository.ts @@ -187,9 +187,9 @@ export class InboxRepository { return asRows(result).map(mapRow); } - // Every non-terminal row of a task (pending or delivered). failTask walks - // these to broadcast the failure message to each reply target before - // dead-lettering. + // Every non-terminal row of a task (pending or delivered). failTask reads + // these before dead-lettering, then sends the failure message to the reply + // targets of the rows failTaskInbox transitioned. async listNonTerminalByTask(taskId: string): Promise { const result = await db.execute(sql` SELECT * FROM inbox @@ -258,8 +258,8 @@ export class InboxRepository { // task-failure paths (provision failure, sandbox crash/timeout) call this so // a failed task cannot leave a row stuck pre-terminal — a non-terminal row // pins the idle reaper, and on a no-sandbox provision failure a 'pending' - // row would otherwise stay claimable forever. Returns the affected row ids - // for the caller to log. + // row would otherwise stay claimable forever. Returns the affected row ids: + // the caller logs them and sends the failure message only to those rows. async failTaskInbox(params: { taskId: string; ownerId: string; diff --git a/control-plane/src/services/__tests__/eventDispatcher.test.ts b/control-plane/src/services/__tests__/eventDispatcher.test.ts index d92efa2..4ca61ba 100644 --- a/control-plane/src/services/__tests__/eventDispatcher.test.ts +++ b/control-plane/src/services/__tests__/eventDispatcher.test.ts @@ -78,6 +78,9 @@ mock.module("../../clients/github", () => ({ })); const { EventDispatcher } = await import("../eventDispatcher"); +const { LinearIntegration } = await import("../../integrations/linear"); +const { githubIntegration } = await import("../../integrations/github"); +const { apiIntegration } = await import("../../integrations/api"); const { config: mockedConfig } = (await import("../../config")) as unknown as { config: { github: { webhooks: { botLogin: string } | undefined }; @@ -205,6 +208,11 @@ describe("EventDispatcher", () => { sandboxManager, linearTokenProvider, slackNotifier, + { + linear: new LinearIntegration(linearTokenProvider), + api: apiIntegration, + github: githubIntegration, + }, ); }); @@ -509,8 +517,7 @@ describe("EventDispatcher", () => { ); }); - it("a GitHub comment or review round failure is posted nowhere", async () => { - // characterization: changes in phase 1a (spec §13, §15 bug 1) + it("a GitHub comment or review round failure posts the generic retry message, not the error", async () => { getByIdForTask.mockResolvedValueOnce( ghRow("comment", { id: "555", @@ -526,7 +533,13 @@ describe("EventDispatcher", () => { ); expect(await dispatcher.handle(errorEvent("88"), makeTask())).toBe("ok"); - expect(postIssueComment).not.toHaveBeenCalled(); + const retry = + "⚠️ Something went wrong while working on this. Mention @harness in a new comment to retry."; + // The comment's thread and the review summary (a top-level PR comment). + expect(postIssueComment.mock.calls as unknown[]).toEqual([ + ["o", "r", 1, retry], + ["o", "r", 1, retry], + ]); expect(postReviewCommentReply).not.toHaveBeenCalled(); expect(emitError).not.toHaveBeenCalled(); }); @@ -544,6 +557,84 @@ describe("EventDispatcher", () => { }); }); + describe("labelling", () => { + it("labels each agent emit and hands it to the round's integration with the event's target", async () => { + const delivered: unknown[] = []; + const record = (name: "linear" | "api" | "github") => ({ + name, + deliver: async (_ctx: unknown, target: unknown, message: unknown) => { + delivered.push([name, target, message]); + }, + }); + const d = new EventDispatcher( + taskRepo, + inboxRepo, + sandboxManager, + linearTokenProvider, + slackNotifier, + { linear: record("linear"), api: record("api"), github: record("github") }, + ); + const assignment = linearRow(); + const ghComment = { + ...linearRow(), + source: "github", + kind: "comment", + reply_targets: [ + { + id: "555", + spec: { kind: "issue_comment", owner: "o", repo: "r", prNumber: 1, commentId: 555 }, + }, + ], + } as InboxRow; + const emit = (type: string, targetId: string, extra = {}) => + ({ + seq: 1, + ts: "t", + type, + inbox_id: "inbox-1", + target_id: targetId, + body: "b", + ...extra, + }) as HarnessEvent; + const cases: Array<[InboxRow, HarnessEvent, unknown]> = [ + [assignment, emit("agent_response", "sess-1"), { type: "reply", body: "b" }], + [ + assignment, + emit("agent_thought", "sess-1"), + { type: "progress", kind: "thought", body: "b" }, + ], + [ + assignment, + emit("agent_action", "sess-1"), + { type: "progress", kind: "action", body: "b" }, + ], + [ + assignment, + emit("agent_error", "sess-1", { retryable_assignment_failure: true }), + { type: "progress", kind: "error", body: "b" }, + ], + [ + assignment, + emit("agent_error", "sess-1"), + { type: "failure", cause: "round_failed", round: assignment, summary: "b" }, + ], + [ + ghComment, + emit("agent_error", "555"), + { type: "failure", cause: "round_failed", round: ghComment, detail: "b" }, + ], + ]; + for (const [row, event] of cases) { + getByIdForTask.mockResolvedValueOnce(row); + expect(await d.handle(event, makeTask())).toBe("ok"); + } + // Routed by round.source (the task is a Linear task throughout). + expect(delivered).toEqual( + cases.map(([row, , message]) => [row.source, row.reply_targets[0], message]), + ); + }); + }); + describe("pr_opened", () => { it("appends the PR url and attaches it to the issue", async () => { const result = await dispatcher.handle(prEvent(), makeTask()); @@ -831,16 +922,25 @@ describe("EventDispatcher", () => { } }); - it("a Linear token failure also holds a GitHub round's reply on a Linear task", async () => { - // characterization: changes in phase 1a (spec §4 routes by round.source; not listed in §13) + it("a Linear token failure does not hold a GitHub round's reply or completion on a Linear task", async () => { (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( new Error("token refresh failed"), ); getByIdForTask.mockResolvedValue(ghCommentRow()); - await expect(dispatcher.handle(responseEvent("555"), makeTask())).rejects.toThrow( - "token refresh failed", - ); - expect(postIssueComment).not.toHaveBeenCalled(); + expect(await dispatcher.handle(responseEvent("555"), makeTask())).toBe("ok"); + expect( + await dispatcher.handle( + { seq: 2, ts: "t", type: "completed", inbox_id: "inbox-2" }, + makeTask(), + ), + ).toBe("ok"); + expect(postIssueComment).toHaveBeenCalledWith("o", "r", 1, "good catch, fixing"); + expect(markResponded).toHaveBeenCalledWith({ + inboxId: "inbox-2", + ownerId: "pod-test", + outcome: null, + }); + expect(linearTokenProvider.getAuthorizedLinearClient).not.toHaveBeenCalled(); }); it("drops a target_id absent from reply_targets (no throw, no post)", async () => { diff --git a/control-plane/src/services/__tests__/inbox.service.test.ts b/control-plane/src/services/__tests__/inbox.service.test.ts index 319fb3f..edcb436 100644 --- a/control-plane/src/services/__tests__/inbox.service.test.ts +++ b/control-plane/src/services/__tests__/inbox.service.test.ts @@ -68,6 +68,9 @@ mock.module("../../clients/github", () => ({ })); const { InboxService, WORKSPACE_MISSING_MESSAGE } = await import("../inbox.service"); +const { LinearIntegration } = await import("../../integrations/linear"); +const { githubIntegration } = await import("../../integrations/github"); +const { apiIntegration } = await import("../../integrations/api"); const makeTask = (overrides: Partial = {}): Task => ({ id: "task-1", @@ -176,7 +179,11 @@ describe("InboxService", () => { getAuthorizedLinearClient: mock(() => Promise.resolve({})), } as unknown as Pick; - service = new InboxService(inboxRepo, sandboxManager, linearTokenProvider); + service = new InboxService(inboxRepo, sandboxManager, { + linear: new LinearIntegration(linearTokenProvider), + api: apiIntegration, + github: githubIntegration, + }); }); describe("sweepUndelivered", () => { @@ -411,8 +418,7 @@ describe("InboxService", () => { expect(markFailed).toHaveBeenCalledTimes(2); }); - it("posts before marking the row, and posts even when markFailed transitions nothing", async () => { - // characterization: changes in phase 1a (spec §4 I15 / §15 bug 7) + it("marks the row before posting, and posts nothing when markFailed transitions nothing", async () => { const order: string[] = []; postIssueComment.mockImplementation(() => { order.push("post"); @@ -422,31 +428,32 @@ describe("InboxService", () => { order.push("react"); return Promise.resolve(); }); - markFailed.mockImplementation(() => { - order.push("mark"); - return Promise.resolve([]); + // Row b went terminal under us (a racing restart or a late `completed`). + markFailed.mockImplementation(({ inboxId }: { inboxId: string }) => { + order.push(`mark ${inboxId}`); + return Promise.resolve(inboxId === "a" ? [{ id: "a" }] : []); }); - listNonTerminalByTask.mockResolvedValue([ghCommentRow("a")]); + stageRows([ghCommentRow("a"), ghCommentRow("b")]); await service.deadLetterWorkspaceMissing(makeTask({ sandboxId: null })); - expect(order).toEqual(["post", "react", "mark"]); + expect(order).toEqual(["mark a", "post", "react", "mark b"]); }); - it("a Linear token failure escapes: that row is still marked failed, later rows are untouched", async () => { - // characterization: changes in phase 1a (spec §15 bug 2) + it("a Linear token failure drops only that row's notice: every row is marked failed, later rows still post", async () => { stageRows([linearRow("a", "sess-1"), ghCommentRow("b")]); (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( new Error("token refresh failed"), ); - await expect( - service.deadLetterWorkspaceMissing(makeTask({ sandboxId: null })), - ).rejects.toThrow("token refresh failed"); + await service.deadLetterWorkspaceMissing(makeTask({ sandboxId: null })); - expect(markFailed).toHaveBeenCalledTimes(1); - expect(markFailed).toHaveBeenCalledWith(expect.objectContaining({ inboxId: "a" })); - expect(postIssueComment).not.toHaveBeenCalled(); + expect(emitError).not.toHaveBeenCalled(); + expect(markFailed.mock.calls.map((c) => (c[0] as { inboxId: string }).inboxId)).toEqual([ + "a", + "b", + ]); + expect(postIssueComment).toHaveBeenCalledWith("o", "r", 7, WORKSPACE_MISSING_MESSAGE); }); }); diff --git a/control-plane/src/services/__tests__/sandboxManager.service.test.ts b/control-plane/src/services/__tests__/sandboxManager.service.test.ts index 64f9aff..ca68ebe 100644 --- a/control-plane/src/services/__tests__/sandboxManager.service.test.ts +++ b/control-plane/src/services/__tests__/sandboxManager.service.test.ts @@ -833,9 +833,23 @@ describe("SandboxManager", () => { expect(err.body.length).toBeLessThanOrEqual(2001); }); - it("drops the harness's retryable_assignment_failure flag from an agent_error", async () => { - // characterization: changes in phase 1a (spec §4: validateHarnessEvent copies it when === true) - const raw = JSON.stringify({ + it("keeps the harness's retryable_assignment_failure flag on an agent_error only when it is true", async () => { + const line = (seq: number, flag: unknown) => + JSON.stringify({ + seq, + ts: "t", + type: "agent_error", + inbox_id: "i", + target_id: "tgt", + body: "RuntimeError: boom", + retryable_assignment_failure: flag, + }); + const events = await mgrWith([line(1, true), line(2, "true")].join("\n")).readEvents( + "sbx", + 0, + ); + expect(events).toHaveLength(2); + expect(events[0]).toEqual({ seq: 1, ts: "t", type: "agent_error", @@ -844,16 +858,7 @@ describe("SandboxManager", () => { body: "RuntimeError: boom", retryable_assignment_failure: true, }); - expect(await mgrWith(raw).readEvents("sbx", 0)).toEqual([ - { - seq: 1, - ts: "t", - type: "agent_error", - inbox_id: "i", - target_id: "tgt", - body: "RuntimeError: boom", - }, - ]); + expect("retryable_assignment_failure" in (events[1] ?? {})).toBe(false); }); it("agent_thought/action/response use the wider 60k cap (plan markdown + completion summaries can run long)", async () => { diff --git a/control-plane/src/services/__tests__/taskProcessor.service.test.ts b/control-plane/src/services/__tests__/taskProcessor.service.test.ts index 9dbe90b..76bc2de 100644 --- a/control-plane/src/services/__tests__/taskProcessor.service.test.ts +++ b/control-plane/src/services/__tests__/taskProcessor.service.test.ts @@ -90,6 +90,9 @@ mock.module("../../clients/github", () => ({ })); const { TaskProcessorService } = await import("../taskProcessor.service"); +const { LinearIntegration } = await import("../../integrations/linear"); +const { githubIntegration } = await import("../../integrations/github"); +const { apiIntegration } = await import("../../integrations/api"); const { logger } = await import("../../logger"); const makeTask = (overrides: Partial = {}): Task => ({ @@ -276,6 +279,11 @@ describe("TaskProcessorService", () => { eventDispatcher, inboxService, slackNotifier, + { + linear: new LinearIntegration(linearTokenProvider), + api: apiIntegration, + github: githubIntegration, + }, ); }); @@ -639,18 +647,38 @@ describe("TaskProcessorService", () => { }); describe("handleTaskError — pruned workspace", () => { - it("a throwing workspace-missing dead-letter (Linear token) escapes before the lease is released", async () => { - // characterization: changes in phase 1a (spec §15 bug 2) - deadLetterWorkspaceMissing.mockRejectedValue(new Error("token refresh failed")); + it("a throwing workspace-missing dead-letter is logged and the lease is still released", async () => { + deadLetterWorkspaceMissing.mockRejectedValue(new Error("db connection reset")); const task = makeTask({ sandboxId: null, pvcName: "pvc-1" }); - await expect( - ( - service as unknown as { handleTaskError(t: Task, e: unknown): Promise } - ).handleTaskError(task, new PvcNotFoundError("pvc-1")), - ).rejects.toThrow("token refresh failed"); + await ( + service as unknown as { handleTaskError(t: Task, e: unknown): Promise } + ).handleTaskError(task, new PvcNotFoundError("pvc-1")); - expect(releaseLease).not.toHaveBeenCalled(); + expect(releaseLease).toHaveBeenCalledWith({ + taskId: "task-1", + ownerId: "pod-test", + alsoClearSandbox: true, + }); + }); + }); + + describe("dispatch", () => { + it("a throw from handleTaskError itself is logged, not left unhandled", async () => { + // Transient provision failure → releaseLeaseIfNoSandbox, whose release throws. + createSandboxWithPvc.mockRejectedValue(new SandboxApiException({ statusCode: 503 })); + releaseLease.mockRejectedValue(new Error("db connection reset")); + const error = spyOn(logger, "error").mockImplementation(() => logger); + claimNext.mockResolvedValueOnce(makeTask()).mockResolvedValue(null); + try { + service.start(); + await waitFor(() => + error.mock.calls.some(([msg]) => msg === "handleTaskError failed for task task-1"), + ); + await service.stop(); + } finally { + error.mockRestore(); + } }); }); @@ -724,8 +752,12 @@ describe("TaskProcessorService", () => { }); }); - it("posts before dead-lettering, and posts even when failTaskInbox transitions no row", async () => { - // characterization: changes in phase 1a (spec §4 I15 / §15 bug 7) + it("dead-letters before posting, and posts only to the rows it dead-lettered", async () => { + (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValue([ + row("a", "assignment", "linear", [{ id: "sess-1", spec: { kind: "linear_session" } }]), + // Responded by a late `completed` before failTaskInbox ran: not transitioned. + row("gc", "comment", "github", [issueComment("555", 7, 555)]), + ]); const order: string[] = []; emitError.mockImplementation(() => { order.push("post"); @@ -738,7 +770,7 @@ describe("TaskProcessorService", () => { }); failTaskInbox.mockImplementation(() => { order.push("mark"); - return Promise.resolve([]); + return Promise.resolve([{ id: "a" }]); }); releaseLease.mockImplementation(() => { order.push("release"); @@ -747,7 +779,8 @@ describe("TaskProcessorService", () => { await failTask(makeTask()); - expect(order).toEqual(["post", "kill", "mark", "release"]); + expect(order).toEqual(["mark", "post", "kill", "release"]); + expect(postIssueComment).not.toHaveBeenCalled(); }); it("a Linear token failure skips only the Linear post; GitHub posts, dead-letter and release go on", async () => { diff --git a/control-plane/src/services/eventDispatcher.ts b/control-plane/src/services/eventDispatcher.ts index e607349..db2342d 100644 --- a/control-plane/src/services/eventDispatcher.ts +++ b/control-plane/src/services/eventDispatcher.ts @@ -4,24 +4,17 @@ import { getUserById, parsePrUrl, postIssueComment, - postReviewCommentReply, requestReviewers, } from "../clients/github"; -import { - attachUrlToIssue, - emitAction, - emitError, - emitResponse, - emitThought, - getIssueHumanAssignee, - uploadIssueAttachment, -} from "../clients/linear"; +import { attachUrlToIssue, getIssueHumanAssignee, uploadIssueAttachment } from "../clients/linear"; import { config } from "../config"; import type { Task } from "../db/schema"; +import { type Outbound, RetryableError } from "../integrations/outbound"; +import type { Integration, IntegrationName } from "../integrations/types"; import { logger } from "../logger"; import type { InboxRepository, InboxRow } from "../repositories/inbox.repository"; import type { TaskRepository } from "../repositories/task.repository"; -import type { HarnessEvent } from "../types/harnessEvent"; +import type { AgentEmit, HarnessEvent } from "../types/harnessEvent"; import type { LinearTokenProvider } from "./linearTokenProvider"; import type { SandboxManager } from "./sandboxManager.service"; import type { SlackNotifierService } from "./slackNotifier.service"; @@ -65,12 +58,10 @@ export class EventDispatcher { private sandboxManager: SandboxManager, private linearTokenProvider: Pick, private slackNotifier: SlackNotifierService, + private integrations: Record, ) {} async handle(event: HarnessEvent, task: Task): Promise { - const linearClient = - task.source === "linear" ? await this.linearTokenProvider.getAuthorizedLinearClient() : null; - // event.inbox_id is sandbox-written; resolve it task-scoped. A null row // means this task's event stream is carrying an event whose inbox row it // does NOT own — a cross-task LEAK (a sibling's event in our stream, our own @@ -103,23 +94,22 @@ export class EventDispatcher { } } switch (event.type) { - // Streaming progress / errors → forwarded to the round's Linear session. - // The session id is pulled from inboxRow.reply_targets (the linear_session - // target), so a github-sourced round (whose reply_targets carry GH specs - // only) implicitly skips the Linear forward — no progress noise on the - // Linear UI when the user is driving from GitHub. + // Agent emits → labelled, then delivered by the round's integration + // (routed by round.source, not task.source) to the target named by + // event.target_id. A Linear outage therefore holds only Linear rounds' + // messages, never a GitHub round's. case "agent_thought": case "agent_action": - case "agent_error": - await this.forwardAgentEmit(event, inboxRow, linearClient); - break; - - // Per-item reply → routed by target_id through inboxRow.reply_targets[]. case "agent_response": - await this.postTargetedReply(event, inboxRow, task, linearClient); + case "agent_error": + await this.deliver(event, inboxRow, task); break; case "pr_opened": { + // The Linear attach and assignee mirror below need a client. Fetched + // first, so a Linear token failure rejects before any side effect and + // the cursor holds. + const linearClient = await this.linearClientFor(task); // appendPrUrl is owner-guarded; a 0-row return means the lease was // taken over (stale-lease takeover). Self-evict BEFORE attachUrlToIssue // and before the cursor advance — a sibling pod now owns this task. @@ -160,6 +150,11 @@ export class EventDispatcher { } case "completed": { + // The Slack notify below needs a Linear client for a Linear task's + // assignment round. Fetched before marking, so a Linear token failure + // rejects with the row still open and the cursor held. + const linearClient = + inboxRow?.kind === "assignment" ? await this.linearClientFor(task) : null; // Bare terminal marker — the user-facing message already went out via // the preceding agent_response / agent_error emit. Mark the inbox row // responded so the reaper guard clears and the sandbox can be freed. @@ -204,57 +199,19 @@ export class EventDispatcher { return "ok"; } - // Streaming forwarder for thought / action / error → Linear agent session. - // target_id on the event is informational; routing is by the inbox row's - // linear_session reply target — target.id IS the agent session id by - // construction (set by enqueue paths). A row whose reply_targets carry no - // linear_session (i.e. a GitHub-sourced comment/review round) yields a null - // session id → drop. A foreign / unknown inbox_id resolves inboxRow to null - // and also drops: a confused emit is safer dropped than routed to "the - // task's session" somewhere unrelated. Final user-facing replies route - // through postTargetedReply, which honors reply_targets and lands on GitHub - // for github-sourced rows. - private async forwardAgentEmit( - event: Extract, - inboxRow: InboxRow | null, - linearClient: LinearClient | null, - ): Promise { - const agentSessionId = - inboxRow?.reply_targets.find((t) => t.spec.kind === "linear_session")?.id ?? null; - if (!linearClient || !agentSessionId) return; - try { - switch (event.type) { - case "agent_thought": - await emitThought(linearClient, agentSessionId, event.body); - break; - case "agent_action": - // The daemon's body is the full action label (e.g. "Planning", - // "Addressing comment"). Linear renders ` ` - // concatenated, so a non-empty parameter appends a noisy suffix - // ("Addressing comment task"). Pass empty parameter — the body - // already carries everything the user should see. - await emitAction(linearClient, agentSessionId, event.body, ""); - break; - case "agent_error": - await emitError(linearClient, agentSessionId, event.body); - break; - } - } catch (err) { - logErr(`forward ${event.type} failed`, err); - } + private async linearClientFor(task: Task): Promise { + return task.source === "linear" ? this.linearTokenProvider.getAuthorizedLinearClient() : null; } - // Posts one agent reply to one thread — looks up the target on the inbox - // row's reply_targets[] by target_id, then posts event.body verbatim, routing - // by spec.kind. CP never formats the text. - private async postTargetedReply( - event: Extract, - inboxRow: InboxRow | null, - task: Task, - linearClient: LinearClient | null, - ): Promise { + // Delivers one agent emit to one thread: looks up the target on the inbox + // row's reply_targets[] by target_id, labels the emit, and hands both to the + // round's integration, which decides where (or whether) it shows up. Core + // never checks what an integration can show. A RetryableError (nothing was + // sent) propagates, so handle() rejects and the cursor holds; any other + // failure is logged and the event dropped. + private async deliver(event: AgentEmit, inboxRow: InboxRow | null, task: Task): Promise { if (!inboxRow) { - logger.warn("agent_response references missing inbox row", { + logger.warn(`${event.type} references missing inbox row`, { taskId: task.id, inboxId: event.inbox_id, }); @@ -262,7 +219,7 @@ export class EventDispatcher { } const target = inboxRow.reply_targets.find((t) => t.id === event.target_id); if (!target) { - logger.warn("agent_response target_id not in reply_targets", { + logger.warn(`${event.type} target_id not in reply_targets`, { taskId: task.id, inboxId: inboxRow.id, targetId: event.target_id, @@ -271,46 +228,14 @@ export class EventDispatcher { } try { - switch (target.spec.kind) { - case "issue_comment": - await postIssueComment( - target.spec.owner, - target.spec.repo, - target.spec.prNumber, - event.body, - ); - break; - case "review_comment": - await postReviewCommentReply( - target.spec.owner, - target.spec.repo, - target.spec.prNumber, - target.spec.inReplyTo, - event.body, - ); - break; - case "review_body": - // Top-level PR comment — GH has no reply-to-review-body API. - await postIssueComment( - target.spec.owner, - target.spec.repo, - target.spec.prNumber, - event.body, - ); - break; - case "linear_session": - // target.id IS the agent session id by construction (enqueue paths - // stamp the live session id, refreshed on every `prompted`). - if (linearClient) { - await emitResponse(linearClient, target.id, event.body); - } - break; - case "api_session": - break; // API tasks: the event log carries the reply - } + await this.integrations[inboxRow.source].deliver( + { signal: new AbortController().signal, log: logger }, + target, + labelAgentEmit(event, inboxRow), + ); } catch (err) { - logErr(`post targeted reply failed for target ${target.id}`, err); - // At-least-once: cursor-replay re-fires this single agent_response retries. + if (err instanceof RetryableError) throw err; + logErr(`deliver ${event.type} failed for target ${target.id}`, err); } } @@ -530,6 +455,30 @@ export class EventDispatcher { } } +// What an agent emit is, as the integration sees it. A non-retryable +// agent_error relies on the harness contract: on an assignment its body is the +// user-safe summary (_format_completion_body); on a comment, review or CI round +// it carries exception text (_emit_round_failure). Neither marks the row failed +// nor writes a task_event: the following `completed` marks it responded. +function labelAgentEmit(event: AgentEmit, round: InboxRow): Outbound { + switch (event.type) { + case "agent_response": + return { type: "reply", body: event.body }; + case "agent_thought": + return { type: "progress", kind: "thought", body: event.body }; + case "agent_action": + return { type: "progress", kind: "action", body: event.body }; + case "agent_error": + // A retried assignment attempt; the final failure arrives through failTask. + if (event.retryable_assignment_failure) { + return { type: "progress", kind: "error", body: event.body }; + } + return round.kind === "assignment" + ? { type: "failure", cause: "round_failed", round, summary: event.body } + : { type: "failure", cause: "round_failed", round, detail: event.body }; + } +} + function reviewPrCoordinates( inboxRow: Extract, ): { owner: string; repo: string; prNumber: number } | null { diff --git a/control-plane/src/services/inbox.service.ts b/control-plane/src/services/inbox.service.ts index e9787f7..fe973f6 100644 --- a/control-plane/src/services/inbox.service.ts +++ b/control-plane/src/services/inbox.service.ts @@ -1,12 +1,6 @@ -import { - addIssueCommentReaction, - addReviewCommentReaction, - postIssueComment, - postReviewCommentReply, -} from "../clients/github"; -import { emitError } from "../clients/linear"; import { config } from "../config"; import type { Task } from "../db/schema"; +import type { Integration, IntegrationName } from "../integrations/types"; import { logger } from "../logger"; import type { InboxRepository, @@ -17,7 +11,6 @@ import type { } from "../repositories/inbox.repository"; import { taskIdentity } from "../repositories/inbox.repository"; import type { InboxMessage } from "../types/inboxMessage"; -import type { LinearTokenProvider } from "./linearTokenProvider"; import type { SandboxManager } from "./sandboxManager.service"; // InboxService owns inbox enqueue (comment/review/Linear-comment), the sweep @@ -31,7 +24,7 @@ export class InboxService { constructor( private inboxRepo: InboxRepository, private sandboxManager: SandboxManager, - private linearTokenProvider: Pick, + private integrations: Record, ) {} // ──── Enqueue paths ──── @@ -303,8 +296,8 @@ export class InboxService { } // A task whose PVC no longer exists cannot be resumed. Dead-letters every - // still-open ('pending'/'delivered') row of the task (per-row user notice + mark - // failed) so it stops pinning the reaper. Called by runTask's recreate-from-PVC + // still-open ('pending'/'delivered') row of the task (mark failed + per-row + // user notice) so it stops pinning the reaper. Called by runTask's recreate-from-PVC // path when createSandboxFromExistingPvc reports the PVC gone. // Re-checks each row's status from DB so a row a reassignment superseded mid-tick // is left untouched. @@ -317,79 +310,35 @@ export class InboxService { } } - // PVC-missing per-row dead-letter. Once-per-row: emit a Linear error / GH - // reply + confused reaction, then mark the row failed so the sweep skips it. + // PVC-missing per-row dead-letter. Marks the row failed first, so the sweep + // skips it, then hands a workspace_missing failure to the round's integration + // only if this call transitioned the row (a racing restart or a late + // `completed` gets no contradictory notice). One notice per row, no target-id + // dedupe (unlike failTask); reply_targets[0] is the round's thread, and the + // integration words the notice by round kind. At most once: there is no + // cursor to hold, so a delivery error (RetryableError included) is logged and + // dropped, and the task's remaining rows still dead-letter. private async handlePvcMissing(entry: InboxRow): Promise { - const userMessage = WORKSPACE_MISSING_MESSAGE; - - try { - if (entry.source === "api") { - // No outbound channel: the round_failed event markFailed writes below - // is how the user sees it. - } else if (entry.source === "linear") { - // reply_targets[0].id IS the agent session id by construction - // (assignment and prompted enqueue paths both stamp it). - const sessionTarget = entry.reply_targets.find((t) => t.spec.kind === "linear_session"); - if (sessionTarget) { - const linearClient = await this.linearTokenProvider.getAuthorizedLinearClient(); - await emitError(linearClient, sessionTarget.id, userMessage).catch(logErrSwallow); - } - } else if (entry.source === "github" && entry.kind === "comment") { - const target = entry.reply_targets[0]; - if (target?.spec.kind === "issue_comment") { - await postIssueComment( - target.spec.owner, - target.spec.repo, - target.spec.prNumber, - userMessage, - ).catch(logErrSwallow); - await addIssueCommentReaction( - target.spec.owner, - target.spec.repo, - target.spec.commentId, - "confused", - ).catch(logErrSwallow); - } else if (target?.spec.kind === "review_comment") { - await postReviewCommentReply( - target.spec.owner, - target.spec.repo, - target.spec.prNumber, - target.spec.inReplyTo, - userMessage, - ).catch(logErrSwallow); - await addReviewCommentReaction( - target.spec.owner, - target.spec.repo, - target.spec.commentId, - "confused", - ).catch(logErrSwallow); - } - } else if (entry.source === "github" && entry.kind === "review") { - // Top-level PR comment — the review body has no reply / reactions API. - const reviewBodyTarget = entry.reply_targets.find((t) => t.spec.kind === "review_body"); - if (reviewBodyTarget?.spec.kind === "review_body") { - await postIssueComment( - reviewBodyTarget.spec.owner, - reviewBodyTarget.spec.repo, - reviewBodyTarget.spec.prNumber, - userMessage, - ).catch(logErrSwallow); - } - } else if (entry.source === "github" && entry.kind === "ci_result") { - // CI result rows are informational — silently dead-letter on pruned - // workspace. No user-facing error is posted; the finally block below - // calls markFailed to clear the row from the sweep. - } - } finally { - await this.inboxRepo - .markFailed({ - inboxId: entry.id, - ownerId: config.podId, - reason: "workspace_missing", - userMessage, - }) - .catch((err) => logErr(`markFailed failed for inbox ${entry.id}`, err)); - } + const marked = await this.inboxRepo + .markFailed({ + inboxId: entry.id, + ownerId: config.podId, + reason: "workspace_missing", + userMessage: WORKSPACE_MISSING_MESSAGE, + }) + .catch((err) => { + logErr(`markFailed failed for inbox ${entry.id}`, err); + return []; + }); + const target = entry.reply_targets[0]; + if (marked.length === 0 || !target) return; + await this.integrations[entry.source] + .deliver({ signal: new AbortController().signal, log: logger }, target, { + type: "failure", + cause: "workspace_missing", + round: entry, + }) + .catch((err) => logErr(`workspace_missing notice failed for inbox ${entry.id}`, err)); } } @@ -403,11 +352,6 @@ function logDebugErr(msg: string, err: unknown): void { logger.debug(msg, { message: e.message, stack: e.stack }); } -// 1-arg variant for `.catch(logErrSwallow)` on fire-and-forget calls. -function logErrSwallow(err: unknown): void { - logErr("fire-and-forget call failed", err); -} - // Strip CP-internal persistence metadata from a row, keeping only the wire // contract fields the daemon expects to read from `inbox/.json`. function toInboxMessage(row: InboxRow): InboxMessage { diff --git a/control-plane/src/services/sandboxManager.service.ts b/control-plane/src/services/sandboxManager.service.ts index 380fb24..ef687a9 100644 --- a/control-plane/src/services/sandboxManager.service.ts +++ b/control-plane/src/services/sandboxManager.service.ts @@ -864,6 +864,10 @@ export function validateHarnessEvent(parsed: unknown): HarnessEvent | null { inbox_id: parsed.inbox_id, target_id: parsed.target_id, body: capString(parsed.body, maxLen) ?? "", + // Kept only when exactly `true`; the dispatcher labels on it. + ...(parsed.type === "agent_error" && parsed.retryable_assignment_failure === true + ? { retryable_assignment_failure: true } + : {}), }; } diff --git a/control-plane/src/services/taskProcessor.service.ts b/control-plane/src/services/taskProcessor.service.ts index 70a60dc..e71c223 100644 --- a/control-plane/src/services/taskProcessor.service.ts +++ b/control-plane/src/services/taskProcessor.service.ts @@ -1,12 +1,11 @@ import type { SandboxInfo } from "@alibaba-group/opensandbox"; -import type { LinearClient } from "@linear/sdk"; -import { postIssueComment, postReviewCommentReply } from "../clients/github"; -import { emitAction, emitError, emitThought, setIssueStarted } from "../clients/linear"; +import { emitAction, emitThought, setIssueStarted } from "../clients/linear"; import { config } from "../config"; import type { Task } from "../db/schema"; import { PvcNotFoundError, SandboxCreateError } from "../errors"; +import type { Integration, IntegrationName } from "../integrations/types"; import { logger } from "../logger"; -import type { InboxRepository, ReplyTarget } from "../repositories/inbox.repository"; +import type { InboxRepository } from "../repositories/inbox.repository"; import type { TaskRepository } from "../repositories/task.repository"; import type { EventDispatcher } from "./eventDispatcher"; import { extractExperimentalFlags } from "./experimentalFlags"; @@ -42,6 +41,7 @@ export class TaskProcessorService { private readonly eventDispatcher: EventDispatcher; private readonly inboxService: InboxService; private readonly slackNotifier: SlackNotifierService; + private readonly integrations: Record; constructor( taskRepo: TaskRepository, @@ -51,6 +51,7 @@ export class TaskProcessorService { eventDispatcher: EventDispatcher, inboxService: InboxService, slackNotifier: SlackNotifierService, + integrations: Record, ) { this.taskRepo = taskRepo; this.inboxRepo = inboxRepo; @@ -59,6 +60,7 @@ export class TaskProcessorService { this.eventDispatcher = eventDispatcher; this.inboxService = inboxService; this.slackNotifier = slackNotifier; + this.integrations = integrations; } start(): void { @@ -166,6 +168,9 @@ export class TaskProcessorService { private dispatch(task: Task): void { const p = this.runTask(task) .catch((err) => this.handleTaskError(task, err)) + // Nothing awaits p, so a throw from handleTaskError itself (e.g. a DB + // error releasing the lease) would be an unhandled rejection. Log it. + .catch((err) => logErr(`handleTaskError failed for task ${task.id}`, err)) .finally(() => this.inflight.delete(p)); this.inflight.add(p); } @@ -713,18 +718,17 @@ export class TaskProcessorService { } // Hard task failure (provision failure, sandbox timeout/crash, sandbox - // unreachable). Broadcast the failure message to every reply target on the - // task's non-terminal inbox rows (so each pending @harness comment + the - // Linear session sees why their round didn't complete), kill the sandbox, - // dead-letter the rows, release the lease. The dead-letter is what keeps - // the task from wedging: a 'pending'/'delivered' row left behind is - // un-claimable to no end (claimNext needs 'pending' but the task has no - // sandbox) and pins the idle reaper. + // unreachable). Dead-letter the task's open inbox rows, tell every reply + // target on the rows this call dead-lettered why their round didn't complete + // (each @harness comment + the Linear session), kill the sandbox, release the + // lease. The dead-letter is what keeps the task from wedging: a + // 'pending'/'delivered' row left behind is un-claimable to no end (claimNext + // needs 'pending' but the task has no sandbox) and pins the idle reaper. // killSandbox=false is for a crossed binding: the sandbox isn't ours to kill (a // sibling task owns it). We still dead-letter our rows + release the lease // (which clears our sandbox_id), so the task fails cleanly. The structured // failure log lives at each call site (with its pod/forensic detail); failTask - // only does the broadcast + teardown. + // only does the dead-letter, broadcast + teardown. private async failTask( task: Task, message: string, @@ -737,28 +741,43 @@ export class TaskProcessorService { logErr(`failTask: listNonTerminalByTask failed for ${task.id}`, e); return []; }); + // Dead-letter the inbox rows FIRST, and BEFORE releaseLease — failTaskInbox + // is owner-guarded and needs owner_id still set. Only the rows it + // transitions get a failure message, so a racing restart or a late + // `completed` can't draw a "Task failed" on a live or finished round. + const failed = await this.inboxRepo + .failTaskInbox({ taskId: task.id, ownerId: config.podId, reason: message }) + .catch((e) => { + logErr(`failTaskInbox failed for task ${task.id}`, e); + return [] as { id: string }[]; + }); + const failedIds = new Set(failed.map((r) => r.id)); + const failedRows = nonTerminal.filter((r) => failedIds.has(r.id)); + if (failed.length > 0) { + logger.warn("inbox rows dead-lettered on task failure", { + taskId: task.id, + inboxIds: failed.map((r) => r.id), + reason: message, + }); + } + // Dedupe by target id — a Linear-source task has many rows sharing the // same linear_session target id; posting once per id keeps the user from - // seeing the same error N times. + // seeing the same error N times. At most once: there is no cursor to hold, + // so a delivery error (RetryableError included) is logged and dropped. const seen = new Set(); - let cachedLinearClient: LinearClient | null = null; - const getLinearClient = async (): Promise => { - if (cachedLinearClient) return cachedLinearClient; - try { - cachedLinearClient = await this.linearTokenProvider.getAuthorizedLinearClient(); - return cachedLinearClient; - } catch (e) { - logErr("failTask: getAuthorizedLinearClient failed", e); - return null; - } - }; - for (const row of nonTerminal) { + for (const row of failedRows) { for (const t of row.reply_targets) { if (seen.has(t.id)) continue; seen.add(t.id); - await this.postFailureToTarget(t, message, getLinearClient).catch((e) => - logErr(`failTask: post to ${t.spec.kind} target ${t.id} failed`, e), - ); + await this.integrations[row.source] + .deliver({ signal: new AbortController().signal, log: logger }, t, { + type: "failure", + cause: "task_failed", + round: row, + summary: message, + }) + .catch((e) => logErr(`failTask: post to ${t.spec.kind} target ${t.id} failed`, e)); } } @@ -771,42 +790,25 @@ export class TaskProcessorService { const current = await this.taskRepo.getByIdOwnedBy(task.id, config.podId); if (current?.sandboxId) await this.killSandboxQuietly(current.sandboxId); } - // Dead-letter the inbox rows BEFORE releaseLease — failTaskInbox is - // owner-guarded and needs owner_id still set. - const failed = await this.inboxRepo - .failTaskInbox({ taskId: task.id, ownerId: config.podId, reason: message }) - .catch((e) => { - logErr(`failTaskInbox failed for task ${task.id}`, e); - return [] as { id: string }[]; - }); - if (failed.length > 0) { - logger.warn("inbox rows dead-lettered on task failure", { - taskId: task.id, - inboxIds: failed.map((r) => r.id), - reason: message, + // Slack failure update to the configured channel. Gated on the ASSIGNMENT + // (top-level) round being among the rows just dead-lettered, mirroring the + // completion path's kind === "assignment" gate in eventDispatcher. Without + // this, a failing follow-up comment/review/ci_result round (a new inbox row + // on the SAME Linear task, appended after the assignment already completed + // and posted "✅ devbox completed") would post a contradictory "❌ devbox + // failed" into that same thread — only the top-level delegated task should + // ever notify Slack. Tying it to this failTaskInbox transition also keeps it + // at-most-once. Fire-and-forget: a Slack outage must not affect teardown; + // the notifier no-ops when Slack is disabled. + if (failedRows.some((r) => r.kind === "assignment") && task.source === "linear") { + const client = await this.linearTokenProvider.getAuthorizedLinearClient().catch((e) => { + logErr("failTask: getAuthorizedLinearClient failed", e); + return null; }); - // Slack failure update to the configured channel. Gated on - // the ASSIGNMENT (top-level) round being among the rows just dead-lettered, - // mirroring the completion path's kind === "assignment" gate in - // eventDispatcher. Without this, a failing follow-up comment/review/ - // ci_result round (a new inbox row on the SAME Linear task, appended after - // the assignment already completed and posted "✅ devbox completed") would - // post a contradictory "❌ devbox failed" into that same thread — only the - // top-level delegated task should ever notify Slack. Tying it to this - // failTaskInbox transition also keeps it at-most-once. Fire-and-forget: a - // Slack outage must not affect teardown; the notifier no-ops when Slack is - // disabled. - const failedIds = new Set(failed.map((r) => r.id)); - const assignmentFailed = nonTerminal.some( - (r) => r.kind === "assignment" && failedIds.has(r.id), - ); - if (assignmentFailed && task.source === "linear") { - const client = await getLinearClient(); - if (client) { - void this.slackNotifier - .notifyTerminal(client, task, "failed", message) - .catch((e) => logErr(`slack failure notify failed for task ${task.id}`, e)); - } + if (client) { + void this.slackNotifier + .notifyTerminal(client, task, "failed", message) + .catch((e) => logErr(`slack failure notify failed for task ${task.id}`, e)); } } await this.taskRepo.releaseLease({ @@ -816,42 +818,6 @@ export class TaskProcessorService { }); } - // Posts `message` to one reply target, switching on spec.kind. Used by - // failTask to broadcast a failure message to every pending thread before - // dead-lettering. Mirrors handlePvcMissing in inbox.service.ts — same per- - // spec branching; if a third caller appears, extract into a shared helper. - private async postFailureToTarget( - target: ReplyTarget, - message: string, - getLinearClient: () => Promise, - ): Promise { - switch (target.spec.kind) { - case "linear_session": { - const client = await getLinearClient(); - if (client) await emitError(client, target.id, message); - return; - } - case "issue_comment": - await postIssueComment(target.spec.owner, target.spec.repo, target.spec.prNumber, message); - return; - case "review_comment": - await postReviewCommentReply( - target.spec.owner, - target.spec.repo, - target.spec.prNumber, - target.spec.inReplyTo, - message, - ); - return; - case "review_body": - // No reply-to-review-body API — top-level PR comment instead. - await postIssueComment(target.spec.owner, target.spec.repo, target.spec.prNumber, message); - return; - case "api_session": - return; // API tasks: round_failed in the event log carries it - } - } - // THE single classifier for everything runTask raises (it's the dispatch // `.catch`). runTask is pure work; this is the only place that decides what a // provisioning failure means: @@ -863,7 +829,11 @@ export class TaskProcessorService { // stays in server logs only; the user sees a generic message. private async handleTaskError(task: Task, err: unknown): Promise { if (err instanceof PvcNotFoundError) { - await this.inboxService.deadLetterWorkspaceMissing(task); + // A throw here (a DB error) must not skip the release: an unreleased lease + // with no sandbox stays heartbeated by this pod and is never re-claimed. + await this.inboxService + .deadLetterWorkspaceMissing(task) + .catch((e) => logErr(`deadLetterWorkspaceMissing failed for task ${task.id}`, e)); await this.taskRepo .releaseLease({ taskId: task.id, ownerId: config.podId, alsoClearSandbox: true }) .catch((e) => logErr(`releaseLease after workspace-missing failed for task ${task.id}`, e)); diff --git a/control-plane/src/types/harnessEvent.ts b/control-plane/src/types/harnessEvent.ts index 5b006c2..30567ac 100644 --- a/control-plane/src/types/harnessEvent.ts +++ b/control-plane/src/types/harnessEvent.ts @@ -3,12 +3,13 @@ // // Two categories: // - Generic agent emits (agent_thought / agent_action / agent_response / -// agent_error): the dispatcher forwards each to the source-specific emit. +// agent_error): the dispatcher labels each and hands it to the round's +// integration. // - Structural events (pr_opened, completed): source-independent CP-side // state changes. -// Generic agent emits — dispatcher forwards to source-aware emit. All four -// share one shape, parametrized over `type` so `Extract` still narrows correctly at call sites. +// Generic agent emits. All four share one shape, parametrized over `type` so +// `Extract` still narrows correctly at call +// sites. // // `target_id` is always present and matches an inbox row's reply_targets[] // entry: @@ -31,7 +32,10 @@ export type AgentEmit = | AgentEmitOf<"agent_thought"> | AgentEmitOf<"agent_action"> | AgentEmitOf<"agent_response"> - | AgentEmitOf<"agent_error">; + // retryable_assignment_failure: one failed assignment attempt. The daemon + // exits and the control plane retries the sandbox from the preserved inbox; + // the final failure, if any, arrives through failTask. + | (AgentEmitOf<"agent_error"> & { retryable_assignment_failure?: boolean }); export type HarnessEvent = | AgentEmit From 877c016e1b981f7f03f77386bdda9c102289d791 Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Fri, 30 Oct 2026 17:43:19 +0530 Subject: [PATCH 04/22] test(control-plane): cover Linear-outage holds and webhook-off failures Pin that a Linear token failure on pr_opened and on an assignment's completion waits instead of advancing, and that no retry hint is posted when GitHub webhooks are off. Rows whose source has no integration are skipped instead of throwing. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/eventDispatcher.test.ts | 38 +++++++++++++++++++ .../__tests__/taskProcessor.service.test.ts | 17 +++++++++ control-plane/src/services/inbox.service.ts | 12 +++++- .../src/services/taskProcessor.service.ts | 13 ++++++- 4 files changed, 78 insertions(+), 2 deletions(-) diff --git a/control-plane/src/services/__tests__/eventDispatcher.test.ts b/control-plane/src/services/__tests__/eventDispatcher.test.ts index 4ca61ba..d740179 100644 --- a/control-plane/src/services/__tests__/eventDispatcher.test.ts +++ b/control-plane/src/services/__tests__/eventDispatcher.test.ts @@ -555,6 +555,24 @@ describe("EventDispatcher", () => { expect(postIssueComment).not.toHaveBeenCalled(); expect(emitError).not.toHaveBeenCalled(); }); + + it("a GitHub comment round failure posts nothing when GitHub webhooks are off (no bot login)", async () => { + const webhooks = mockedConfig.github.webhooks; + mockedConfig.github.webhooks = undefined; + try { + getByIdForTask.mockResolvedValue( + ghRow("comment", { + id: "555", + spec: { kind: "issue_comment", owner: "o", repo: "r", prNumber: 1, commentId: 555 }, + }), + ); + expect(await dispatcher.handle(errorEvent("555"), makeTask())).toBe("ok"); + expect(postIssueComment).not.toHaveBeenCalled(); + expect(postReviewCommentReply).not.toHaveBeenCalled(); + } finally { + mockedConfig.github.webhooks = webhooks; + } + }); }); describe("labelling", () => { @@ -655,6 +673,16 @@ describe("EventDispatcher", () => { ); }); + it("Linear task: a token fetch failure rejects before appendPrUrl (cursor held)", async () => { + (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( + new Error("token refresh failed"), + ); + await expect(dispatcher.handle(prEvent(), makeTask())).rejects.toThrow( + "token refresh failed", + ); + expect(appendPrUrl).not.toHaveBeenCalled(); + }); + it("self-evicts when appendPrUrl returns 0 rows (lease lost); skips attachUrlToIssue", async () => { appendPrUrl.mockResolvedValue([]); const result = await dispatcher.handle(prEvent(), makeTask()); @@ -1240,6 +1268,16 @@ describe("EventDispatcher", () => { expect(markResponded).toHaveBeenCalled(); }); + it("Linear assignment: a token fetch failure rejects before markResponded (cursor held)", async () => { + (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( + new Error("token refresh failed"), + ); + await expect(dispatcher.handle(completedEvent(), makeTask())).rejects.toThrow( + "token refresh failed", + ); + expect(markResponded).not.toHaveBeenCalled(); + }); + it("still marks responded when linearClient is null", async () => { // The marker frees the sandbox regardless of Linear availability — it // never touches linearClient. diff --git a/control-plane/src/services/__tests__/taskProcessor.service.test.ts b/control-plane/src/services/__tests__/taskProcessor.service.test.ts index 76bc2de..14d23c0 100644 --- a/control-plane/src/services/__tests__/taskProcessor.service.test.ts +++ b/control-plane/src/services/__tests__/taskProcessor.service.test.ts @@ -783,6 +783,23 @@ describe("TaskProcessorService", () => { expect(postIssueComment).not.toHaveBeenCalled(); }); + it("a row whose source has no integration is skipped; other rows post and the lease is released", async () => { + (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValue([ + row("x", "comment", "unknown-source", [issueComment("777", 7, 777)]), + row("gc", "comment", "github", [issueComment("555", 7, 555)]), + ]); + failTaskInbox.mockResolvedValue([{ id: "x" }, { id: "gc" }]); + + await failTask(makeTask()); + + expect(postIssueComment.mock.calls as unknown[]).toEqual([["o", "r", 7, MSG]]); + expect(releaseLease).toHaveBeenCalledWith({ + taskId: "task-1", + ownerId: "pod-test", + alsoClearSandbox: true, + }); + }); + it("a Linear token failure skips only the Linear post; GitHub posts, dead-letter and release go on", async () => { getAuthorizedLinearClient.mockRejectedValue(new Error("token refresh failed")); (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValue([ diff --git a/control-plane/src/services/inbox.service.ts b/control-plane/src/services/inbox.service.ts index fe973f6..4cebd67 100644 --- a/control-plane/src/services/inbox.service.ts +++ b/control-plane/src/services/inbox.service.ts @@ -332,7 +332,17 @@ export class InboxService { }); const target = entry.reply_targets[0]; if (marked.length === 0 || !target) return; - await this.integrations[entry.source] + // A source this build has no integration for (say, rows written by a newer + // build before a rollback) gets no notice; its row is still failed. + const integration: Integration | undefined = this.integrations[entry.source]; + if (!integration) { + logger.warn("workspace_missing: no integration for the row's source; notice skipped", { + inboxId: entry.id, + source: entry.source, + }); + return; + } + await integration .deliver({ signal: new AbortController().signal, log: logger }, target, { type: "failure", cause: "workspace_missing", diff --git a/control-plane/src/services/taskProcessor.service.ts b/control-plane/src/services/taskProcessor.service.ts index e71c223..570d50f 100644 --- a/control-plane/src/services/taskProcessor.service.ts +++ b/control-plane/src/services/taskProcessor.service.ts @@ -767,10 +767,21 @@ export class TaskProcessorService { // so a delivery error (RetryableError included) is logged and dropped. const seen = new Set(); for (const row of failedRows) { + // A source this build has no integration for (say, rows written by a + // newer build before a rollback) gets no notice; its row is still failed. + const integration: Integration | undefined = this.integrations[row.source]; + if (!integration) { + logger.warn("failTask: no integration for the row's source; notice skipped", { + taskId: task.id, + inboxId: row.id, + source: row.source, + }); + continue; + } for (const t of row.reply_targets) { if (seen.has(t.id)) continue; seen.add(t.id); - await this.integrations[row.source] + await integration .deliver({ signal: new AbortController().signal, log: logger }, t, { type: "failure", cause: "task_failed", From d86504635b0fe8c17e3ba8a93e3dbffbebb43ab0 Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Fri, 30 Oct 2026 18:06:27 +0530 Subject: [PATCH 05/22] feat(control-plane): hold retryable sends and bound delivery time A send that fails before anything went out (rate limit, connection refused, DNS, TLS, token fetch) now holds the event and retries it, honouring Retry-After, instead of dropping the reply. Every send runs under a 20 s timeout; a timed-out or failed send is dropped and the task yields the tick, so one slow platform can't stall the others. Leases are renewed before each task's reconcile, and pods only claim tasks whose source they know, which keeps rolling deploys and rollbacks safe. Co-Authored-By: Claude Opus 5.5 --- control-plane/src/app.ts | 12 + .../src/clients/__tests__/github.test.ts | 39 +++ control-plane/src/clients/githubApi.ts | 29 +- control-plane/src/clients/linearApi.ts | 3 + control-plane/src/clients/linearOauth.ts | 7 +- control-plane/src/clients/slackApi.ts | 8 +- .../src/integrations/__tests__/http.test.ts | 267 ++++++++++++++++++ .../src/integrations/github/index.ts | 62 ++-- control-plane/src/integrations/http.ts | 87 ++++++ .../src/integrations/linear/index.ts | 76 +++-- .../__tests__/task.repository.test.ts | 70 ++++- .../src/repositories/task.repository.ts | 32 ++- .../__tests__/eventDispatcher.test.ts | 109 +++++-- .../services/__tests__/inbox.service.test.ts | 33 ++- .../src/services/__tests__/replyHolds.test.ts | 85 ++++++ .../__tests__/taskProcessor.service.test.ts | 138 ++++++++- control-plane/src/services/eventDispatcher.ts | 78 +++-- control-plane/src/services/inbox.service.ts | 9 +- .../src/services/linearTokenProvider.ts | 6 +- control-plane/src/services/replyHolds.ts | 82 ++++++ .../src/services/taskProcessor.service.ts | 115 +++++--- 21 files changed, 1185 insertions(+), 162 deletions(-) create mode 100644 control-plane/src/integrations/__tests__/http.test.ts create mode 100644 control-plane/src/integrations/http.ts create mode 100644 control-plane/src/services/__tests__/replyHolds.test.ts create mode 100644 control-plane/src/services/replyHolds.ts diff --git a/control-plane/src/app.ts b/control-plane/src/app.ts index 21f1bf2..6347050 100644 --- a/control-plane/src/app.ts +++ b/control-plane/src/app.ts @@ -7,6 +7,7 @@ import { GitHubWebhookController } from "./controllers/githubWebhook.controller" import { HealthController } from "./controllers/health.controller"; import { LinearWebhookController } from "./controllers/linearWebhook.controller"; import { db } from "./db"; +import { taskSource } from "./db/schema"; import { apiIntegration } from "./integrations/api"; import { githubIntegration } from "./integrations/github"; import { LinearIntegration } from "./integrations/linear"; @@ -28,6 +29,7 @@ import { GithubStatusService } from "./services/githubStatus.service"; import { InboxService } from "./services/inbox.service"; import { LinearStatusService } from "./services/linearStatus.service"; import { LinearTokenProvider } from "./services/linearTokenProvider"; +import { ReplyHolds } from "./services/replyHolds"; import { SandboxManager } from "./services/sandboxManager.service"; import { SlackNotifierService } from "./services/slackNotifier.service"; import { TaskProcessorService } from "./services/taskProcessor.service"; @@ -83,6 +85,13 @@ export function createApp(): AppBundle { api: apiIntegration, github: githubIntegration, } satisfies Record; + // The registered integrations that create tasks (github creates none). Only + // these sources are claimed and reconciled, so an older image never works a + // task whose source it doesn't know (§10). + const taskSources = taskSource.enumValues.filter((s) => s in integrations); + // Sends held because they failed before anything went out; the status + // routes will report them per integration (phase 2). + const replyHolds = new ReplyHolds(); const inboxService = new InboxService(inboxRepository, sandboxManager, integrations); @@ -107,6 +116,7 @@ export function createApp(): AppBundle { linearTokenProvider, slackNotifier, integrations, + replyHolds, ); const taskProcessor = new TaskProcessorService( @@ -118,6 +128,8 @@ export function createApp(): AppBundle { inboxService, slackNotifier, integrations, + replyHolds, + taskSources, ); const linearStatus = new LinearStatusService( diff --git a/control-plane/src/clients/__tests__/github.test.ts b/control-plane/src/clients/__tests__/github.test.ts index 385caa7..c32c031 100644 --- a/control-plane/src/clients/__tests__/github.test.ts +++ b/control-plane/src/clients/__tests__/github.test.ts @@ -25,8 +25,10 @@ const { getUserById, getUserIdByLogin, parsePrUrl, + postIssueComment, requestReviewers, } = (await import("../githubApi.ts" + "?github-impl-test")) as typeof import("../githubApi"); +const { HttpError } = await import("../../integrations/http"); // Build a fetch stub that returns each page in `pages` as a 200 JSON response, // recording the requested URLs. githubFetch does resp.text() → JSON.parse, so @@ -352,6 +354,43 @@ describe("requestReviewers", () => { }); }); +describe("githubFetch errors and signal", () => { + const originalFetch = globalThis.fetch; + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + it("a non-2xx is an HttpError with status and Retry-After; the message is unchanged", async () => { + globalThis.fetch = (() => + Promise.resolve( + new Response('{"message":"API rate limit exceeded"}', { + status: 429, + headers: { "retry-after": "30" }, + }), + )) as unknown as typeof fetch; + + const err = await postIssueComment("o", "r", 1, "hi").catch((e: unknown) => e); + expect(err).toBeInstanceOf(HttpError); + expect((err as InstanceType).status).toBe(429); + expect((err as InstanceType).retryAfterMs).toBe(30_000); + expect((err as Error).message).toBe( + 'GitHub POST /repos/o/r/issues/1/comments failed: HTTP 429 {"message":"API rate limit exceeded"}', + ); + }); + + it("passes the caller's signal to fetch", async () => { + let seen: AbortSignal | null | undefined; + globalThis.fetch = ((_url: string, init?: RequestInit) => { + seen = init?.signal; + return Promise.resolve(Response.json({ id: 1, html_url: "u" })); + }) as unknown as typeof fetch; + + const ac = new AbortController(); + await postIssueComment("o", "r", 1, "hi", ac.signal); + expect(seen).toBe(ac.signal); + }); +}); + describe("addPrAssignees", () => { const originalFetch = globalThis.fetch; afterEach(() => { diff --git a/control-plane/src/clients/githubApi.ts b/control-plane/src/clients/githubApi.ts index 1eb2e0b..3d08cfd 100644 --- a/control-plane/src/clients/githubApi.ts +++ b/control-plane/src/clients/githubApi.ts @@ -13,6 +13,7 @@ import { readFileSync } from "node:fs"; import { config } from "../config"; import { githubApiBaseUrl } from "../githubToken/mint"; +import { HttpError, parseRetryAfter, TokenFetchError } from "../integrations/http"; import { logger } from "../logger"; import { type GithubTokenProvider, @@ -42,8 +43,13 @@ function tokenProvider(): GithubTokenProvider { return githubToken; } +// No token means nothing was sent, so a send may be retried (TokenFetchError). function token(): string { - return tokenProvider()(); + try { + return tokenProvider()(); + } catch (err) { + throw new TokenFetchError(err instanceof Error ? err.message : String(err)); + } } // The token provider's state, for GET /api/integrations/github. @@ -60,22 +66,29 @@ function githubHeaders(): Record { }; } -// One request. Throws on non-2xx with status + a truncated body snippet — the -// token is never included in the message. +// One request. Throws an HttpError on non-2xx with status + a truncated body +// snippet (and Retry-After) — the token is never included in the message. async function githubFetch( method: string, path: string, body?: unknown, + signal?: AbortSignal, ): Promise<{ status: number; json: unknown }> { const resp = await fetch(`${API_BASE}${path}`, { method, headers: githubHeaders(), body: body === undefined ? undefined : JSON.stringify(body), + signal, }); const text = await resp.text(); if (!resp.ok) { const snippet = text.slice(0, 500); - throw new Error(`GitHub ${method} ${path} failed: HTTP ${resp.status} ${snippet}`); + throw new HttpError( + `GitHub ${method} ${path} failed: HTTP ${resp.status} ${snippet}`, + resp.status, + // `?.`: the client tests stub fetch with header-less partial responses. + parseRetryAfter(resp.headers?.get("retry-after") ?? null), + ); } let json: unknown = null; if (text.length > 0) { @@ -99,11 +112,13 @@ export async function postIssueComment( repo: string, prNumber: number, body: string, + signal?: AbortSignal, ): Promise<{ id: number; url: string }> { const { json } = await githubFetch( "POST", `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/issues/${prNumber}/comments`, { body }, + signal, ); const r = asRecord(json); return { id: Number(r.id), url: String(r.html_url ?? "") }; @@ -133,11 +148,13 @@ export async function postReviewCommentReply( prNumber: number, inReplyTo: number, body: string, + signal?: AbortSignal, ): Promise<{ id: number; url: string }> { const { json } = await githubFetch( "POST", `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/pulls/${prNumber}/comments`, { body, in_reply_to: inReplyTo }, + signal, ); const r = asRecord(json); return { id: Number(r.id), url: String(r.html_url ?? "") }; @@ -150,11 +167,13 @@ export async function addIssueCommentReaction( repo: string, commentId: number, content: ReactionContent, + signal?: AbortSignal, ): Promise { await githubFetch( "POST", `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/issues/comments/${commentId}/reactions`, { content }, + signal, ); } @@ -165,11 +184,13 @@ export async function addReviewCommentReaction( repo: string, commentId: number, content: ReactionContent, + signal?: AbortSignal, ): Promise { await githubFetch( "POST", `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/pulls/comments/${commentId}/reactions`, { content }, + signal, ); } diff --git a/control-plane/src/clients/linearApi.ts b/control-plane/src/clients/linearApi.ts index eb8bcfa..b6ccbab 100644 --- a/control-plane/src/clients/linearApi.ts +++ b/control-plane/src/clients/linearApi.ts @@ -72,6 +72,8 @@ type UploadIssueAttachmentParams = { contentType: string; subtitle?: string; metadata?: Record; + // The SDK calls take the client's own signal; this one bounds the raw upload. + signal?: AbortSignal; }; // Uploads bytes to Linear-managed object storage, then attaches the uploaded @@ -104,6 +106,7 @@ export async function uploadIssueAttachment( method: "PUT", headers: uploadHeaders, body: params.bytes, + signal: params.signal, }); if (!uploadResp.ok) { const responseBody = (await uploadResp.text().catch(() => "")).slice(0, 500); diff --git a/control-plane/src/clients/linearOauth.ts b/control-plane/src/clients/linearOauth.ts index fb459ca..802c821 100644 --- a/control-plane/src/clients/linearOauth.ts +++ b/control-plane/src/clients/linearOauth.ts @@ -1,3 +1,5 @@ +import { TokenFetchError } from "../integrations/http"; + // ── Client-credentials token ───────────────────────────────────────────────── // The exports in this section are pure: they never read `config`. @@ -12,8 +14,9 @@ export type LinearCredentials = { clientId: string; clientSecret: string }; export type ClientCredentialsToken = { accessToken: string; expiresAt: Date; scope: string }; // A failed token request. `code` and `description` are the OAuth `error` and -// `error_description` fields; the response body is never kept otherwise. -export class LinearTokenError extends Error { +// `error_description` fields; the response body is never kept otherwise. A +// TokenFetchError: nothing was sent with the missing token, so a send may retry. +export class LinearTokenError extends TokenFetchError { status?: number; code?: string; description?: string; diff --git a/control-plane/src/clients/slackApi.ts b/control-plane/src/clients/slackApi.ts index 274ccdc..aa0bd64 100644 --- a/control-plane/src/clients/slackApi.ts +++ b/control-plane/src/clients/slackApi.ts @@ -27,6 +27,7 @@ export type PostSlackMessageParams = { text: string; // Optional Block Kit blocks; `text` stays the fallback/notification string. blocks?: unknown[]; + signal?: AbortSignal; }; // POST /chat.postMessage into a channel, or into a thread when threadTs is set. IMPORTANT: Slack returns HTTP 200 even @@ -46,6 +47,7 @@ export async function postSlackMessage(params: PostSlackMessageParams): Promise< text: params.text, ...(params.blocks ? { blocks: params.blocks } : {}), }), + signal: params.signal, }); const text = await resp.text(); if (!resp.ok) { @@ -68,10 +70,14 @@ export async function postSlackMessage(params: PostSlackMessageParams): Promise< // or missing_scope when the bot lacks `users:read.email`) — the caller's mention // degrades to a plain name, never worth failing a message over. Throws only on // a transport-level error, matching postSlackMessage. -export async function lookupSlackUserIdByEmail(email: string): Promise { +export async function lookupSlackUserIdByEmail( + email: string, + signal?: AbortSignal, +): Promise { const resp = await fetch(`${API_BASE}/users.lookupByEmail?email=${encodeURIComponent(email)}`, { method: "GET", headers: { Authorization: `Bearer ${slackToken()}` }, + signal, }); const text = await resp.text(); if (!resp.ok) { diff --git a/control-plane/src/integrations/__tests__/http.test.ts b/control-plane/src/integrations/__tests__/http.test.ts new file mode 100644 index 0000000..852ca0f --- /dev/null +++ b/control-plane/src/integrations/__tests__/http.test.ts @@ -0,0 +1,267 @@ +import { afterEach, describe, expect, it, mock } from "bun:test"; +import { LinearClient } from "@linear/sdk"; +import type { ReplyTarget } from "../../repositories/inbox.repository"; + +// A token file that does not exist: the GitHub client has no token to send. +mock.module("../../config", () => ({ + config: { podId: "pod-test", github: { tokenSettings: { file: "/nonexistent/devbox/token" } } }, +})); + +const { HttpError, TokenFetchError, parseRetryAfter, retryable, withTimeout } = await import( + "../http" +); +const { RetryableError } = await import("../outbound"); +const { LinearIntegration } = await import("../linear"); +const { LinearTokenError } = await import("../../clients/linearOauth"); +// A fresh instance (see github.test.ts): never the possibly-mocked barrel. +const { postIssueComment } = (await import( + "../../clients/githubApi.ts" + "?http-test" +)) as typeof import("../../clients/githubApi"); +const { logger } = await import("../../logger"); + +// What fetch throws for `url`. +async function fetchError(url: string): Promise { + try { + await fetch(url); + } catch (err) { + return err; + } + throw new Error(`fetch ${url} succeeded`); +} + +const codeOf = (err: unknown) => (err as { code?: unknown }).code; + +// A local TCP server whose sockets do `onOpen` / `onData`. +function tcpServer(handlers: { + open?: (s: { write(d: string): void; end(): void }) => void; + data?: (s: { end(): void }) => void; +}) { + return Bun.listen({ + hostname: "127.0.0.1", + port: 0, + socket: { open: (s) => handlers.open?.(s), data: (s) => handlers.data?.(s) }, + }); +} + +// The exact shapes bun 1.4.4's fetch throws; retryable() is written from these. +describe("retryable: bun fetch failures", () => { + it("connection refused is a TypeError with code ECONNREFUSED: retryable", async () => { + const closed = tcpServer({}); + const port = closed.port; + closed.stop(true); + const err = await fetchError(`http://127.0.0.1:${port}/`); + expect(err).toBeInstanceOf(TypeError); + expect(codeOf(err)).toBe("ECONNREFUSED"); + expect(retryable(err)).toBeInstanceOf(RetryableError); + }); + + it("an unknown host is a TypeError with code ENOTFOUND (EAI_AGAIN without a resolver): retryable", async () => { + const err = await fetchError("https://devbox-http-test.invalid/"); + expect(err).toBeInstanceOf(TypeError); + expect(["ENOTFOUND", "EAI_AGAIN"]).toContain(codeOf(err) as string); + expect(retryable(err)).toBeInstanceOf(RetryableError); + }); + + it("a failed TLS handshake is a TypeError with code EPROTO: retryable", async () => { + const plain = tcpServer({ + open: (s) => { + s.write("HTTP/1.1 200 OK\r\n\r\nnot tls"); + s.end(); + }, + }); + try { + const err = await fetchError(`https://127.0.0.1:${plain.port}/`); + expect(err).toBeInstanceOf(TypeError); + expect(codeOf(err)).toBe("EPROTO"); + expect(retryable(err)).toBeInstanceOf(RetryableError); + } finally { + plain.stop(true); + } + }); + + it("a reset after the request went out (ECONNRESET) is not retryable", async () => { + const resetter = tcpServer({ data: (s) => s.end() }); + try { + const err = await fetchError(`http://127.0.0.1:${resetter.port}/`); + expect(codeOf(err)).toBe("ECONNRESET"); + expect(retryable(err)).toBeUndefined(); + } finally { + resetter.stop(true); + } + }); + + it("a timed-out (aborted) fetch is not retryable: it may have landed", async () => { + const hang = tcpServer({}); + try { + const err = await withTimeout(20, (signal) => + fetch(`http://127.0.0.1:${hang.port}/`, { signal }), + ).catch((e: unknown) => e); + expect((err as Error).message).toBe("timed out after 20 ms"); + expect(retryable(err)).toBeUndefined(); + } finally { + hang.stop(true); + } + }); +}); + +describe("retryable: typed errors", () => { + it("a 429 is retryable and keeps its Retry-After; 5xx and other 4xx are not", () => { + const limited = retryable(new HttpError("HTTP 429 slow down", 429, 7000)); + expect(limited).toBeInstanceOf(RetryableError); + expect(limited?.message).toBe("HTTP 429 slow down"); + expect(limited?.retryAfterMs).toBe(7000); + for (const status of [400, 403, 404, 422, 500, 502, 503]) { + expect(retryable(new HttpError(`HTTP ${status}`, status))).toBeUndefined(); + } + }); + + it("a token-fetch failure is retryable, Linear's included", () => { + expect(retryable(new TokenFetchError("no token"))).toBeInstanceOf(RetryableError); + expect(retryable(new LinearTokenError("Linear token request failed"))).toBeInstanceOf( + RetryableError, + ); + }); + + it("anything else is not retryable", () => { + expect(retryable(new Error("boom"))).toBeUndefined(); + expect(retryable("boom")).toBeUndefined(); + expect(retryable(null)).toBeUndefined(); + }); +}); + +describe("parseRetryAfter", () => { + it("reads delay-seconds and HTTP dates; ignores anything else", () => { + expect(parseRetryAfter("7")).toBe(7000); + expect(parseRetryAfter("0")).toBe(0); + const inAMinute = parseRetryAfter(new Date(Date.now() + 60_000).toUTCString()) ?? 0; + expect(inAMinute).toBeGreaterThan(58_000); + expect(inAMinute).toBeLessThanOrEqual(60_000); + expect(parseRetryAfter(new Date(Date.now() - 60_000).toUTCString())).toBe(0); + expect(parseRetryAfter("soon")).toBeUndefined(); + expect(parseRetryAfter(null)).toBeUndefined(); + }); +}); + +describe("withTimeout", () => { + it("returns the send's result and leaves its signal alone", async () => { + let seen: AbortSignal | undefined; + const out = await withTimeout(1000, async (signal) => { + seen = signal; + return "sent"; + }); + expect(out).toBe("sent"); + expect(seen?.aborted).toBe(false); + }); + + it("gives up after the bound even when the send ignores its signal, and aborts the signal", async () => { + let seen: AbortSignal | undefined; + const err = await withTimeout(20, (signal) => { + seen = signal; + return new Promise(() => {}); + }).catch((e: unknown) => e); + expect((err as Error).message).toBe("timed out after 20 ms"); + expect(seen?.aborted).toBe(true); + }); +}); + +describe("GitHub client", () => { + const originalFetch = globalThis.fetch; + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + it("a missing token is a TokenFetchError and sends nothing", async () => { + const fetchStub = mock(() => Promise.resolve(new Response("{}"))); + globalThis.fetch = fetchStub as unknown as typeof fetch; + const err = await postIssueComment("o", "r", 1, "hi").catch((e: unknown) => e); + expect(err).toBeInstanceOf(TokenFetchError); + expect(retryable(err)).toBeInstanceOf(RetryableError); + expect(fetchStub).not.toHaveBeenCalled(); + }); +}); + +// Linear's SDK wraps every failure in a LinearError; the integration maps the +// SDK's rate-limit error and the raw transport error underneath. +describe("Linear deliver: SDK failures", () => { + const target: ReplyTarget = { id: "sess-1", spec: { kind: "linear_session" } }; + const reply = { type: "reply", body: "hi" } as const; + const integrationFor = (apiUrl: string) => + new LinearIntegration({ + getAuthorizedLinearClient: (signal?: AbortSignal) => + Promise.resolve(new LinearClient({ accessToken: "t", apiUrl, signal })), + }); + const deliverTo = (apiUrl: string, signal = new AbortController().signal) => + integrationFor(apiUrl) + .deliver({ signal, log: logger }, target, reply) + .then(() => "sent") + .catch((e: unknown) => e); + + const server = (respond: () => Response | Promise) => + Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: respond }); + + it("an HTTP 429 is retryable with Retry-After", async () => { + const s = server( + () => new Response("Too many requests", { status: 429, headers: { "retry-after": "12" } }), + ); + try { + const err = await deliverTo(`http://127.0.0.1:${s.port}/graphql`); + expect(err).toBeInstanceOf(RetryableError); + expect((err as InstanceType).retryAfterMs).toBe(12_000); + } finally { + s.stop(true); + } + }); + + it("a ratelimited GraphQL error is retryable", async () => { + const s = server(() => + Response.json( + { + errors: [ + { + message: "Rate limit exceeded", + extensions: { type: "ratelimited", code: "RATELIMITED" }, + }, + ], + }, + { status: 400 }, + ), + ); + try { + expect(await deliverTo(`http://127.0.0.1:${s.port}/graphql`)).toBeInstanceOf(RetryableError); + } finally { + s.stop(true); + } + }); + + it("a 500 is not retryable", async () => { + const s = server(() => new Response("boom", { status: 500 })); + try { + const err = await deliverTo(`http://127.0.0.1:${s.port}/graphql`); + expect(err).toBeInstanceOf(Error); + expect(err).not.toBeInstanceOf(RetryableError); + } finally { + s.stop(true); + } + }); + + it("connection refused is retryable", async () => { + const closed = tcpServer({}); + const port = closed.port; + closed.stop(true); + expect(await deliverTo(`http://127.0.0.1:${port}/graphql`)).toBeInstanceOf(RetryableError); + }); + + it("the per-call client honours the send's signal (aborted: not retryable)", async () => { + const s = server(() => new Promise(() => {})); + try { + const ac = new AbortController(); + const pending = deliverTo(`http://127.0.0.1:${s.port}/graphql`, ac.signal); + ac.abort(new Error("timed out after 20000 ms")); + const err = await pending; + expect(err).toBeInstanceOf(Error); + expect(err).not.toBeInstanceOf(RetryableError); + } finally { + s.stop(true); + } + }); +}); diff --git a/control-plane/src/integrations/github/index.ts b/control-plane/src/integrations/github/index.ts index 58d4fa4..db0f905 100644 --- a/control-plane/src/integrations/github/index.ts +++ b/control-plane/src/integrations/github/index.ts @@ -7,6 +7,7 @@ import { import { config } from "../../config"; import type { InboxRow, ReplyTargetSpec } from "../../repositories/inbox.repository"; import { WORKSPACE_MISSING_MESSAGE } from "../../services/inbox.service"; +import { retryable } from "../http"; import type { Outbound, SendCtx } from "../outbound"; import type { Integration } from "../types"; @@ -26,7 +27,7 @@ export const githubIntegration: Integration = { } switch (message.type) { case "reply": - await post(spec, message.body); + await post(ctx, spec, message.body); return; case "progress": return; @@ -46,7 +47,7 @@ async function deliverFailure( switch (message.cause) { case "task_failed": // failTask's message is control-plane-generated and scrubbed: public-safe. - if (message.summary) await post(spec, message.summary); + if (message.summary) await post(ctx, spec, message.summary); return; case "round_failed": { // A failed CI-fix round stays silent. @@ -55,6 +56,7 @@ async function deliverFailure( const botLogin = config.github.webhooks?.botLogin; if (!botLogin) return; await post( + ctx, spec, `⚠️ Something went wrong while working on this. Mention @${botLogin} in a new comment to retry.`, ); @@ -73,45 +75,59 @@ async function deliverFailure( async function workspaceMissing(ctx: SendCtx, spec: GithubSpec, round: InboxRow): Promise { if (round.kind === "review") { const body = round.reply_targets.find((t) => t.spec.kind === "review_body"); - if (body?.spec.kind === "review_body") await post(body.spec, WORKSPACE_MISSING_MESSAGE); + if (body?.spec.kind === "review_body") await post(ctx, body.spec, WORKSPACE_MISSING_MESSAGE); return; } if (round.kind !== "comment") return; try { - await post(spec, WORKSPACE_MISSING_MESSAGE); + await post(ctx, spec, WORKSPACE_MISSING_MESSAGE); } finally { // The reaction is independent of the reply: try it even if the reply failed. - await confusedReaction(spec).catch((err: unknown) => { + await confusedReaction(ctx, spec).catch((err: unknown) => { const e = err instanceof Error ? err : new Error(String(err)); ctx.log.error("workspace_missing reaction failed", { message: e.message, stack: e.stack }); }); } } -async function confusedReaction(spec: GithubSpec): Promise { +async function confusedReaction(ctx: SendCtx, spec: GithubSpec): Promise { if (spec.kind === "issue_comment") { - await addIssueCommentReaction(spec.owner, spec.repo, spec.commentId, "confused"); + await addIssueCommentReaction(spec.owner, spec.repo, spec.commentId, "confused", ctx.signal); } else if (spec.kind === "review_comment") { - await addReviewCommentReaction(spec.owner, spec.repo, spec.commentId, "confused"); + await addReviewCommentReaction(spec.owner, spec.repo, spec.commentId, "confused", ctx.signal); } } // Posts `body` verbatim to one PR thread, by target kind. CP never formats it. -async function post(spec: GithubSpec, body: string): Promise { - switch (spec.kind) { - case "issue_comment": - await postIssueComment(spec.owner, spec.repo, spec.prNumber, body); - return; - case "review_comment": - await postReviewCommentReply(spec.owner, spec.repo, spec.prNumber, spec.inReplyTo, body); - return; - case "review_body": - // Top-level PR comment — GH has no reply-to-review-body API. - await postIssueComment(spec.owner, spec.repo, spec.prNumber, body); - return; - default: { - const _exhaustive: never = spec; - void _exhaustive; +// It is the first request of every GitHub send (the 😕 reaction comes after and +// never fails the send), so a retryable failure here means nothing was posted. +async function post(ctx: SendCtx, spec: GithubSpec, body: string): Promise { + const { signal } = ctx; + try { + switch (spec.kind) { + case "issue_comment": + await postIssueComment(spec.owner, spec.repo, spec.prNumber, body, signal); + return; + case "review_comment": + await postReviewCommentReply( + spec.owner, + spec.repo, + spec.prNumber, + spec.inReplyTo, + body, + signal, + ); + return; + case "review_body": + // Top-level PR comment — GH has no reply-to-review-body API. + await postIssueComment(spec.owner, spec.repo, spec.prNumber, body, signal); + return; + default: { + const _exhaustive: never = spec; + void _exhaustive; + } } + } catch (err) { + throw retryable(err) ?? err; } } diff --git a/control-plane/src/integrations/http.ts b/control-plane/src/integrations/http.ts new file mode 100644 index 0000000..4254f75 --- /dev/null +++ b/control-plane/src/integrations/http.ts @@ -0,0 +1,87 @@ +import { RetryableError } from "./outbound"; + +// Shared rules for integrations' outbound HTTP (I16): the typed errors clients +// throw, which failures may be retried, and how long one send may take. + +// Upper bound on one deliver() call (§4 I14). +export const SEND_TIMEOUT_MS = 20_000; + +// A non-2xx answer. The message keeps the client's own wording (tests and logs +// match on it); status and Retry-After ride along for classification. +export class HttpError extends Error { + constructor( + message: string, + readonly status: number, + readonly retryAfterMs?: number, + ) { + super(message); + } +} + +// The client had no credential to send with, so nothing was sent. +export class TokenFetchError extends Error {} + +// What bun 1.4.4's fetch throws when the request never reached the server: a +// TypeError whose `code` names the cause (shapes pinned in http.test.ts). A +// reset (ECONNRESET) can come after the request went out, so it is not here. +const NOTHING_SENT = new Set([ + "ECONNREFUSED", // connection refused + "ENOTFOUND", // DNS: no such host + "EAI_AGAIN", // DNS: resolver unavailable (bun ships Node's code; not reproducible offline) + "EPROTO", // TLS handshake failed + // TLS: the server's certificate failed verification during the handshake + "DEPTH_ZERO_SELF_SIGNED_CERT", + "SELF_SIGNED_CERT_IN_CHAIN", + "UNABLE_TO_VERIFY_LEAF_SIGNATURE", + "UNABLE_TO_GET_ISSUER_CERT_LOCALLY", + "CERT_HAS_EXPIRED", + "ERR_TLS_CERT_ALTNAME_INVALID", +]); + +// The failure as a RetryableError when it is one of the exact retryable cases +// (§4): 429, connection refused, DNS, TLS handshake, token fetch. Anything else +// (timeouts, 5xx, other 4xx) is undefined: the caller logs and drops it. +export function retryable(err: unknown): RetryableError | undefined { + if (err instanceof RetryableError) return err; + if (err instanceof TokenFetchError) return new RetryableError(err.message); + if (err instanceof HttpError) { + return err.status === 429 ? new RetryableError(err.message, err.retryAfterMs) : undefined; + } + const code = (err as { code?: unknown } | null)?.code; + if (err instanceof TypeError && typeof code === "string" && NOTHING_SENT.has(code)) { + return new RetryableError(err.message); + } + return undefined; +} + +// Retry-After as delay-seconds or an HTTP date; undefined when absent or unreadable. +export function parseRetryAfter(value: string | null): number | undefined { + if (!value) return undefined; + const seconds = Number(value); + if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1000; + const at = Date.parse(value); + return Number.isNaN(at) ? undefined : Math.max(0, at - Date.now()); +} + +// Runs `send` with an abort signal and stops waiting after `ms`, whether or not +// the client honours the signal. The send may still land after the timeout, so +// a timeout is a plain failure (dropped), never a retry. +export async function withTimeout( + ms: number, + send: (signal: AbortSignal) => Promise, +): Promise { + const ac = new AbortController(); + let timer: ReturnType | undefined; + const expired = new Promise((_, reject) => { + timer = setTimeout(() => { + const err = new Error(`timed out after ${ms} ms`); + ac.abort(err); + reject(err); + }, ms); + }); + try { + return await Promise.race([send(ac.signal), expired]); + } finally { + clearTimeout(timer); + } +} diff --git a/control-plane/src/integrations/linear/index.ts b/control-plane/src/integrations/linear/index.ts index 5af8243..3a73e4c 100644 --- a/control-plane/src/integrations/linear/index.ts +++ b/control-plane/src/integrations/linear/index.ts @@ -1,8 +1,9 @@ -import type { LinearClient } from "@linear/sdk"; +import { type LinearClient, LinearError, RatelimitedLinearError } from "@linear/sdk"; import { emitAction, emitError, emitResponse, emitThought } from "../../clients/linear"; import type { ReplyTarget } from "../../repositories/inbox.repository"; import { WORKSPACE_MISSING_MESSAGE } from "../../services/inbox.service"; import type { LinearTokenProvider } from "../../services/linearTokenProvider"; +import { retryable } from "../http"; import { type Outbound, RetryableError, type SendCtx } from "../outbound"; import type { Integration } from "../types"; @@ -16,45 +17,64 @@ export class LinearIntegration implements Integration { private linearTokenProvider: Pick, ) {} - async deliver(_ctx: SendCtx, target: ReplyTarget, message: Outbound): Promise { + async deliver(ctx: SendCtx, target: ReplyTarget, message: Outbound): Promise { if (target.spec.kind !== "linear_session") { throw new Error(`linear cannot deliver to a ${target.spec.kind} target`); } - const client = await this.client(); - switch (message.type) { - case "reply": - await emitResponse(client, target.id, message.body); - return; - case "progress": - if (message.kind === "thought") await emitThought(client, target.id, message.body); - // The daemon's body is the full action label (e.g. "Planning", - // "Addressing comment"). Linear renders ` ` - // concatenated, so a non-empty parameter appends a noisy suffix - // ("Addressing comment task"). Pass empty parameter — the body - // already carries everything the user should see. - else if (message.kind === "action") await emitAction(client, target.id, message.body, ""); - else await emitError(client, target.id, message.body); - return; - case "failure": { - // task_failed and a failed assignment carry the summary; a crashed - // comment round carries only the detail. - const text = - message.cause === "workspace_missing" - ? WORKSPACE_MISSING_MESSAGE - : (message.summary ?? message.detail); - if (text) await emitError(client, target.id, text); - return; + const client = await this.client(ctx.signal); + // Each message is one activity (one request), so a retryable failure means + // nothing was posted. + try { + switch (message.type) { + case "reply": + await emitResponse(client, target.id, message.body); + return; + case "progress": + if (message.kind === "thought") await emitThought(client, target.id, message.body); + // The daemon's body is the full action label (e.g. "Planning", + // "Addressing comment"). Linear renders ` ` + // concatenated, so a non-empty parameter appends a noisy suffix + // ("Addressing comment task"). Pass empty parameter — the body + // already carries everything the user should see. + else if (message.kind === "action") await emitAction(client, target.id, message.body, ""); + else await emitError(client, target.id, message.body); + return; + case "failure": { + // task_failed and a failed assignment carry the summary; a crashed + // comment round carries only the detail. + const text = + message.cause === "workspace_missing" + ? WORKSPACE_MISSING_MESSAGE + : (message.summary ?? message.detail); + if (text) await emitError(client, target.id, text); + return; + } } + } catch (err) { + throw linearRetryable(err) ?? err; } } // Nothing has been sent when the token fetch fails, so the caller may retry. - private async client(): Promise { + // A per-call client: the signal bounds this send's requests only. + private async client(signal: AbortSignal): Promise { try { - return await this.linearTokenProvider.getAuthorizedLinearClient(); + return await this.linearTokenProvider.getAuthorizedLinearClient(signal); } catch (err) { const msg = err instanceof Error ? err.message : String(err); throw new RetryableError(`Linear token fetch failed: ${msg}`); } } } + +// The SDK wraps every failure in a LinearError whose `raw` is what fetch (or +// the token renewal after a 401) threw. A rate limit is the SDK's +// RatelimitedLinearError: an HTTP 429 or a "ratelimited" GraphQL error, with +// Retry-After in seconds. +function linearRetryable(err: unknown): RetryableError | undefined { + if (err instanceof RatelimitedLinearError) { + const s = err.retryAfter; + return new RetryableError(err.message, s !== undefined && s >= 0 ? s * 1000 : undefined); + } + return retryable(err instanceof LinearError ? err.raw : err); +} diff --git a/control-plane/src/repositories/__tests__/task.repository.test.ts b/control-plane/src/repositories/__tests__/task.repository.test.ts index c78bf02..15df5b1 100644 --- a/control-plane/src/repositories/__tests__/task.repository.test.ts +++ b/control-plane/src/repositories/__tests__/task.repository.test.ts @@ -16,6 +16,9 @@ const { migrate } = await import("drizzle-orm/bun-sql/migrator"); const repo = new TaskRepository(); +// Every task source this build knows; claimNext and listInFlightOwnedBy filter on it. +const ALL_SOURCES: ("linear" | "api")[] = ["linear", "api"]; + const linearTarget = (targetId: string) => ({ id: targetId, spec: { kind: "linear_session" as const }, @@ -545,6 +548,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 100, + sources: ALL_SOURCES, }), ).toBeNull(); }); @@ -557,6 +561,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 100, + sources: ALL_SOURCES, }); expect(task?.id).toBe(t.id); @@ -573,6 +578,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 100, + sources: ALL_SOURCES, }), ).toBeNull(); }); @@ -589,6 +595,7 @@ describe("TaskRepository", () => { ownerId: "pod-live", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 100, + sources: ALL_SOURCES, }); expect(task?.id).toBe(t.id); expect(task?.ownerId).toBe("pod-live"); @@ -611,6 +618,7 @@ describe("TaskRepository", () => { ownerId: "pod-live", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 100, + sources: ALL_SOURCES, }); expect(task?.id).toBe(t.id); expect(task?.ownerId).toBe("pod-live"); @@ -630,6 +638,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 100, + sources: ALL_SOURCES, }), ).toBeNull(); }); @@ -648,6 +657,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 100, + sources: ALL_SOURCES, }); expect(task?.id).toBe(b.id); }); @@ -663,6 +673,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 1, + sources: ALL_SOURCES, }), ).toBeNull(); }); @@ -675,6 +686,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 1, + sources: ALL_SOURCES, }); expect(task?.id).toBe(t.id); }); @@ -700,6 +712,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 1, + sources: ALL_SOURCES, }), ).toBeNull(); }); @@ -726,6 +739,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 1, + sources: ALL_SOURCES, }), ).toBeNull(); }); @@ -752,6 +766,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 1, + sources: ALL_SOURCES, }); // A claim succeeds — the stale lease did not consume the only slot. expect(task?.id).toBe(stale.id); @@ -776,6 +791,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 2, + sources: ALL_SOURCES, }); // running.n == 1 (not 2) → the fresh task is claimable. expect(task?.id).toBe(fresh.id); @@ -1059,11 +1075,62 @@ describe("TaskRepository", () => { await insertTask({ externalId: "b", ownerId: "pod-a" }); // no sandbox await insertTask({ externalId: "c", ownerId: "pod-b", sandboxId: "sbx-c" }); // other pod - const ids = (await repo.listInFlightOwnedBy("pod-a")).map((r) => r.id); + const ids = (await repo.listInFlightOwnedBy("pod-a", ALL_SOURCES)).map((r) => r.id); expect(ids).toEqual([withSbx.id]); }); }); + describe("source filter (§10: an image never works a source it doesn't know)", () => { + it("claimNext and listInFlightOwnedBy skip a task whose source is not in the list", async () => { + const api = await insertTask({ source: "api", externalId: "api-unclaimed" }); + await insertInboxRow(api.id); + const apiOwned = await insertTask({ + source: "api", + externalId: "api-owned", + ownerId: "pod-a", + sandboxId: "sbx-api", + }); + const linear = await insertTask({ + externalId: "lin-owned", + ownerId: "pod-a", + sandboxId: "sbx-l", + }); + const opts = { + leaseTimeoutMs: 10_000, + maxConcurrentSandboxes: 100, + sources: ["linear" as const], + }; + + expect(await repo.claimNext({ ownerId: "pod-b", ...opts })).toBeNull(); + expect((await repo.listInFlightOwnedBy("pod-a", ["linear"])).map((r) => r.id)).toEqual([ + linear.id, + ]); + // The same rows with the source listed: claimed and listed. + expect((await repo.claimNext({ ownerId: "pod-b", ...opts, sources: ALL_SOURCES }))?.id).toBe( + api.id, + ); + expect( + (await repo.listInFlightOwnedBy("pod-a", ALL_SOURCES)).map((r) => r.id).sort(), + ).toEqual([apiOwned.id, linear.id].sort()); + }); + }); + + describe("heartbeatTask", () => { + it("renews one owned task and returns 0 rows once the lease is someone else's", async () => { + const mine = await insertTask({ ownerId: "pod-a", heartbeatAt: new Date("2020-01-01") }); + const other = await insertTask({ ownerId: "pod-a", heartbeatAt: new Date("2020-01-01") }); + + expect(await repo.heartbeatTask({ taskId: mine.id, ownerId: "pod-a" })).toEqual([ + { id: mine.id }, + ]); + expect(await repo.heartbeatTask({ taskId: mine.id, ownerId: "pod-b" })).toEqual([]); + + const rows = await db.select().from(schema.tasks); + expect(rows.find((r) => r.id === mine.id)?.heartbeatAt?.getFullYear()).toBeGreaterThan(2020); + expect(rows.find((r) => r.id === other.id)?.heartbeatAt?.getFullYear()).toBe(2020); + }); + }); + describe("heartbeatOwnedBy / backdateHeartbeatsOwnedBy", () => { it("heartbeat updates only rows owned by us", async () => { const mine = await insertTask({ @@ -1093,6 +1160,7 @@ describe("TaskRepository", () => { ownerId: "pod-b", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 100, + sources: ALL_SOURCES, }); expect(task?.id).toBe(t.id); expect(task?.ownerId).toBe("pod-b"); diff --git a/control-plane/src/repositories/task.repository.ts b/control-plane/src/repositories/task.repository.ts index f2d12e1..dfa487c 100644 --- a/control-plane/src/repositories/task.repository.ts +++ b/control-plane/src/repositories/task.repository.ts @@ -1,4 +1,4 @@ -import { and, eq, sql } from "drizzle-orm"; +import { and, eq, inArray, sql } from "drizzle-orm"; import { db, schema } from "../db"; import type { Task } from "../db/schema"; import { logger } from "../logger"; @@ -290,10 +290,15 @@ export class TaskRepository { // on that delivered row; throughout its recreate-from-PVC window sandbox_id // is NULL, so excluding 'delivered' here would let a retry wave over-admit // past the cap. Matches the claimable predicate's IN ('pending','delivered'). + // + // Only tasks of `sources` (the sources this build has an integration for) are + // claimed, so an older image never takes a task of a source it doesn't know: + // rollbacks and rolling overlaps stay safe without draining (§10). async claimNext(params: { ownerId: string; leaseTimeoutMs: number; maxConcurrentSandboxes: number; + sources: Task["source"][]; }): Promise { const leaseCutoff = `${params.leaseTimeoutMs} milliseconds`; const rows = await db.execute(sql` @@ -315,7 +320,8 @@ export class TaskRepository { claimable AS ( SELECT t.id FROM tasks t, running - WHERE EXISTS ( + WHERE t.source IN ${params.sources} + AND EXISTS ( SELECT 1 FROM inbox i WHERE i.task_id = t.id AND i.status IN ('pending', 'delivered') @@ -352,12 +358,18 @@ export class TaskRepository { // Tasks this pod owns with a sandbox attached — the reconcile loop iterates // these every tick. Comment-pickup tasks with sandbox_id NULL are reconciled - // via claimNext + sweep, not here. - async listInFlightOwnedBy(ownerId: string): Promise { + // via claimNext + sweep, not here. Filtered to `sources` like claimNext. + async listInFlightOwnedBy(ownerId: string, sources: Task["source"][]): Promise { return db .select() .from(schema.tasks) - .where(and(eq(schema.tasks.ownerId, ownerId), sql`${schema.tasks.sandboxId} IS NOT NULL`)); + .where( + and( + eq(schema.tasks.ownerId, ownerId), + sql`${schema.tasks.sandboxId} IS NOT NULL`, + inArray(schema.tasks.source, sources), + ), + ); } // As long as a pod owns a task it heartbeats; if it loses interest it calls @@ -369,6 +381,16 @@ export class TaskRepository { .where(eq(schema.tasks.ownerId, ownerId)); } + // One task's heartbeat, renewed right before its reconcile. Owner-guarded: + // 0 rows means the lease was taken over. + async heartbeatTask(params: { taskId: string; ownerId: string }): Promise<{ id: string }[]> { + return db + .update(schema.tasks) + .set({ heartbeatAt: sql`now()` }) + .where(this.ownedByUs(params.taskId, params.ownerId)) + .returning({ id: schema.tasks.id }); + } + // Backdate heartbeats on own tasks so survivors become immediately // reclaimable instead of waiting the full LEASE_TIMEOUT_MS. async backdateHeartbeatsOwnedBy(ownerId: string): Promise { diff --git a/control-plane/src/services/__tests__/eventDispatcher.test.ts b/control-plane/src/services/__tests__/eventDispatcher.test.ts index d740179..78079a6 100644 --- a/control-plane/src/services/__tests__/eventDispatcher.test.ts +++ b/control-plane/src/services/__tests__/eventDispatcher.test.ts @@ -81,6 +81,8 @@ const { EventDispatcher } = await import("../eventDispatcher"); const { LinearIntegration } = await import("../../integrations/linear"); const { githubIntegration } = await import("../../integrations/github"); const { apiIntegration } = await import("../../integrations/api"); +const { HttpError } = await import("../../integrations/http"); +const { ReplyHolds } = await import("../replyHolds"); const { config: mockedConfig } = (await import("../../config")) as unknown as { config: { github: { webhooks: { botLogin: string } | undefined }; @@ -155,6 +157,7 @@ describe("EventDispatcher", () => { let notifyTerminal: ReturnType; let extractArchive: ReturnType; let slackNotifier: SlackNotifierService; + let holds: InstanceType; let dispatcher: InstanceType; beforeEach(() => { @@ -202,6 +205,7 @@ describe("EventDispatcher", () => { notifyTerminal = mock(() => Promise.resolve()); slackNotifier = { notifyTerminal } as unknown as SlackNotifierService; + holds = new ReplyHolds(); dispatcher = new EventDispatcher( taskRepo, inboxRepo, @@ -213,6 +217,7 @@ describe("EventDispatcher", () => { api: apiIntegration, github: githubIntegration, }, + holds, ); }); @@ -446,13 +451,12 @@ describe("EventDispatcher", () => { expect(emitThought).not.toHaveBeenCalled(); }); - it("Linear task: a token fetch failure posts nothing and does not return ok (cursor held)", async () => { - // characterization: changes in phase 1b (handle returns "hold" instead of rejecting) + it("Linear task: a token fetch failure posts nothing and holds the event", async () => { (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( new Error("token refresh failed"), ); - await expect( - dispatcher.handle( + expect( + await dispatcher.handle( { seq: 1, ts: "t", @@ -463,7 +467,7 @@ describe("EventDispatcher", () => { }, makeTask(), ), - ).rejects.toThrow("token refresh failed"); + ).toBe("hold"); for (const m of [emitThought, emitAction, emitResponse, emitError, postIssueComment]) { expect(m).not.toHaveBeenCalled(); } @@ -537,8 +541,8 @@ describe("EventDispatcher", () => { "⚠️ Something went wrong while working on this. Mention @harness in a new comment to retry."; // The comment's thread and the review summary (a top-level PR comment). expect(postIssueComment.mock.calls as unknown[]).toEqual([ - ["o", "r", 1, retry], - ["o", "r", 1, retry], + ["o", "r", 1, retry, expect.any(AbortSignal)], + ["o", "r", 1, retry, expect.any(AbortSignal)], ]); expect(postReviewCommentReply).not.toHaveBeenCalled(); expect(emitError).not.toHaveBeenCalled(); @@ -591,6 +595,7 @@ describe("EventDispatcher", () => { linearTokenProvider, slackNotifier, { linear: record("linear"), api: record("api"), github: record("github") }, + new ReplyHolds(), ); const assignment = linearRow(); const ghComment = { @@ -673,13 +678,11 @@ describe("EventDispatcher", () => { ); }); - it("Linear task: a token fetch failure rejects before appendPrUrl (cursor held)", async () => { + it("Linear task: a token fetch failure holds the event before appendPrUrl", async () => { (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( new Error("token refresh failed"), ); - await expect(dispatcher.handle(prEvent(), makeTask())).rejects.toThrow( - "token refresh failed", - ); + expect(await dispatcher.handle(prEvent(), makeTask())).toBe("hold"); expect(appendPrUrl).not.toHaveBeenCalled(); }); @@ -888,7 +891,13 @@ describe("EventDispatcher", () => { it("routes an issue_comment target to postIssueComment", async () => { getByIdForTask.mockResolvedValue(ghCommentRow()); await dispatcher.handle(responseEvent("555"), makeTask()); - expect(postIssueComment).toHaveBeenCalledWith("o", "r", 1, "good catch, fixing"); + expect(postIssueComment).toHaveBeenCalledWith( + "o", + "r", + 1, + "good catch, fixing", + expect.any(AbortSignal), + ); expect(emitResponse).not.toHaveBeenCalled(); }); @@ -911,7 +920,14 @@ describe("EventDispatcher", () => { ]; getByIdForTask.mockResolvedValue(row); await dispatcher.handle(responseEvent("555"), makeTask()); - expect(postReviewCommentReply).toHaveBeenCalledWith("o", "r", 1, 555, "good catch, fixing"); + expect(postReviewCommentReply).toHaveBeenCalledWith( + "o", + "r", + 1, + 555, + "good catch, fixing", + expect.any(AbortSignal), + ); }); it("routes a linear_session target to emitResponse using target.id (the session id)", async () => { @@ -924,17 +940,23 @@ describe("EventDispatcher", () => { expect(emitResponse).toHaveBeenCalledWith(expect.anything(), "sess-1", "good catch, fixing"); }); - it("a failed GitHub reply is logged and dropped: handle() resolves ok, so the cursor advances", async () => { - // characterization: changes in phase 1b (spec §13, §4 I14: pre-send failures hold) + it("a GitHub reply that fails before sending (429) holds; any other failure is dropped", async () => { getByIdForTask.mockResolvedValue(ghCommentRow()); - // githubFetch's error for a 429: a plain Error, the status only in the message. - postIssueComment.mockRejectedValue( - new Error( + // githubFetch's error for a 429 carries the status and Retry-After. + postIssueComment.mockRejectedValueOnce( + new HttpError( 'GitHub POST /repos/o/r/issues/1/comments failed: HTTP 429 {"message":"API rate limit exceeded"}', + 429, + 30_000, ), ); - expect(await dispatcher.handle(responseEvent("555"), makeTask())).toBe("ok"); - expect(postIssueComment).toHaveBeenCalledTimes(1); + expect(await dispatcher.handle(responseEvent("555"), makeTask())).toBe("hold"); + // A 5xx may have landed: dropped, the cursor advances past it. + postIssueComment.mockRejectedValueOnce( + new HttpError("GitHub POST /repos/o/r/issues/1/comments failed: HTTP 502 ", 502), + ); + expect(await dispatcher.handle(responseEvent("555"), makeTask())).toBe("dropped"); + expect(postIssueComment).toHaveBeenCalledTimes(2); }); it("still posts GitHub replies when GitHub webhooks are off", async () => { @@ -944,7 +966,13 @@ describe("EventDispatcher", () => { try { getByIdForTask.mockResolvedValue(ghCommentRow()); await dispatcher.handle(responseEvent("555"), makeTask()); - expect(postIssueComment).toHaveBeenCalledWith("o", "r", 1, "good catch, fixing"); + expect(postIssueComment).toHaveBeenCalledWith( + "o", + "r", + 1, + "good catch, fixing", + expect.any(AbortSignal), + ); } finally { mockedConfig.github.webhooks = webhooks; } @@ -962,7 +990,13 @@ describe("EventDispatcher", () => { makeTask(), ), ).toBe("ok"); - expect(postIssueComment).toHaveBeenCalledWith("o", "r", 1, "good catch, fixing"); + expect(postIssueComment).toHaveBeenCalledWith( + "o", + "r", + 1, + "good catch, fixing", + expect.any(AbortSignal), + ); expect(markResponded).toHaveBeenCalledWith({ inboxId: "inbox-2", ownerId: "pod-test", @@ -977,6 +1011,31 @@ describe("EventDispatcher", () => { expect(result).toBe("ok"); expect(postIssueComment).not.toHaveBeenCalled(); }); + + it("a send holds on the round's integration; the Linear client fetch on the task's", async () => { + // A GitHub round on a Linear task: the 429 is GitHub's hold. + getByIdForTask.mockResolvedValue(ghCommentRow()); + postIssueComment.mockRejectedValueOnce(new HttpError("HTTP 429", 429)); + expect(await dispatcher.handle(responseEvent("555"), makeTask())).toBe("hold"); + expect(Object.keys(holds.stats())).toEqual(["github"]); + // pr_opened's Linear client fetch is the task's (Linear's) hold. + (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( + new Error("token refresh failed"), + ); + expect(await dispatcher.handle(prEvent(), makeTask({ id: "task-2" }))).toBe("hold"); + expect(holds.stats().linear?.heldTasks).toBe(1); + }); + + it("a Linear send gets a per-call client bound to the send's signal", async () => { + const row = ghCommentRow(); + row.source = "linear"; + row.reply_targets = [{ id: "sess-1", spec: { kind: "linear_session" } }]; + getByIdForTask.mockResolvedValue(row); + expect(await dispatcher.handle(responseEvent("sess-1"), makeTask())).toBe("ok"); + expect(linearTokenProvider.getAuthorizedLinearClient).toHaveBeenCalledWith( + expect.any(AbortSignal), + ); + }); }); describe("completed", () => { @@ -1268,13 +1327,11 @@ describe("EventDispatcher", () => { expect(markResponded).toHaveBeenCalled(); }); - it("Linear assignment: a token fetch failure rejects before markResponded (cursor held)", async () => { + it("Linear assignment: a token fetch failure holds the event before markResponded", async () => { (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( new Error("token refresh failed"), ); - await expect(dispatcher.handle(completedEvent(), makeTask())).rejects.toThrow( - "token refresh failed", - ); + expect(await dispatcher.handle(completedEvent(), makeTask())).toBe("hold"); expect(markResponded).not.toHaveBeenCalled(); }); diff --git a/control-plane/src/services/__tests__/inbox.service.test.ts b/control-plane/src/services/__tests__/inbox.service.test.ts index edcb436..ca46f8e 100644 --- a/control-plane/src/services/__tests__/inbox.service.test.ts +++ b/control-plane/src/services/__tests__/inbox.service.test.ts @@ -233,7 +233,13 @@ describe("InboxService", () => { await service.deadLetterWorkspaceMissing(makeTask({ sandboxId: null })); expect(postIssueComment).toHaveBeenCalledTimes(1); - expect(addIssueCommentReaction).toHaveBeenCalledWith("o", "r", 555, "confused"); + expect(addIssueCommentReaction).toHaveBeenCalledWith( + "o", + "r", + 555, + "confused", + expect.any(AbortSignal), + ); expect(markFailed).toHaveBeenCalledWith({ inboxId: "a", ownerId: "pod-test", @@ -393,12 +399,25 @@ describe("InboxService", () => { 7, 556, WORKSPACE_MISSING_MESSAGE, + expect.any(AbortSignal), ); expect(addReviewCommentReaction).toHaveBeenCalledTimes(1); - expect(addReviewCommentReaction).toHaveBeenCalledWith("o", "r", 556, "confused"); + expect(addReviewCommentReaction).toHaveBeenCalledWith( + "o", + "r", + 556, + "confused", + expect.any(AbortSignal), + ); // Only the review body posts; its inline comments and the CI row get nothing. expect(postIssueComment).toHaveBeenCalledTimes(1); - expect(postIssueComment).toHaveBeenCalledWith("o", "r", 8, WORKSPACE_MISSING_MESSAGE); + expect(postIssueComment).toHaveBeenCalledWith( + "o", + "r", + 8, + WORKSPACE_MISSING_MESSAGE, + expect.any(AbortSignal), + ); expect(addIssueCommentReaction).not.toHaveBeenCalled(); expect(markFailed.mock.calls.map((c) => (c[0] as { inboxId: string }).inboxId)).toEqual([ "lin", @@ -453,7 +472,13 @@ describe("InboxService", () => { "a", "b", ]); - expect(postIssueComment).toHaveBeenCalledWith("o", "r", 7, WORKSPACE_MISSING_MESSAGE); + expect(postIssueComment).toHaveBeenCalledWith( + "o", + "r", + 7, + WORKSPACE_MISSING_MESSAGE, + expect.any(AbortSignal), + ); }); }); diff --git a/control-plane/src/services/__tests__/replyHolds.test.ts b/control-plane/src/services/__tests__/replyHolds.test.ts new file mode 100644 index 0000000..6f6b99b --- /dev/null +++ b/control-plane/src/services/__tests__/replyHolds.test.ts @@ -0,0 +1,85 @@ +import { afterEach, describe, expect, it, setSystemTime, spyOn } from "bun:test"; + +const { ReplyHolds } = await import("../replyHolds"); +const { RetryableError } = await import("../../integrations/outbound"); +const { logger } = await import("../../logger"); + +const T0 = new Date("2026-10-30T12:00:00Z").getTime(); +const at = (ms: number) => setSystemTime(new Date(T0 + ms)); +const event = (seq: number, inbox_id = "inbox-1") => ({ seq, inbox_id }); + +describe("ReplyHolds", () => { + afterEach(() => { + setSystemTime(); + }); + + it("without Retry-After the next attempt is the next tick; with it, after the delay", () => { + const holds = new ReplyHolds(); + at(0); + holds.hold("t1", event(1), "linear", new RetryableError("token fetch failed")); + expect(holds.waiting("t1")).toBe(false); + + holds.hold("t2", event(1), "github", new RetryableError("HTTP 429", 30_000)); + expect(holds.waiting("t2")).toBe(true); + at(29_999); + expect(holds.waiting("t2")).toBe(true); + at(30_000); + expect(holds.waiting("t2")).toBe(false); + }); + + it("warns reply_held at most every 10 minutes per task; a new event starts a new hold", () => { + const warn = spyOn(logger, "warn").mockImplementation(() => logger); + try { + const holds = new ReplyHolds(); + const err = new RetryableError("HTTP 429"); + at(0); + holds.hold("t1", event(5), "github", err); + at(9 * 60_000); + holds.hold("t1", event(5), "github", err); + at(10 * 60_000); + holds.hold("t1", event(5), "github", err); + expect(warn.mock.calls.map((c) => c[1])).toEqual([ + { + event: "reply_held", + taskId: "t1", + integration: "github", + heldForMin: 0, + reason: "HTTP 429", + }, + { + event: "reply_held", + taskId: "t1", + integration: "github", + heldForMin: 10, + reason: "HTTP 429", + }, + ]); + // The same task held on another event is a new hold: its age restarts. + at(15 * 60_000); + holds.hold("t1", event(6), "github", err); + expect(warn).toHaveBeenCalledTimes(3); + expect(holds.stats()).toEqual({ github: { heldTasks: 1, oldestHoldMs: 0 } }); + } finally { + warn.mockRestore(); + } + }); + + it("reports held tasks and the oldest hold age per integration; release and retain drop holds", () => { + const holds = new ReplyHolds(); + const err = new RetryableError("x"); + at(0); + holds.hold("a", event(1), "linear", err); + at(60_000); + holds.hold("b", event(1), "linear", err); + holds.hold("c", event(1), "github", err); + at(120_000); + expect(holds.stats()).toEqual({ + linear: { heldTasks: 2, oldestHoldMs: 120_000 }, + github: { heldTasks: 1, oldestHoldMs: 60_000 }, + }); + + holds.release("a"); + holds.retain(new Set(["a", "b"])); + expect(holds.stats()).toEqual({ linear: { heldTasks: 1, oldestHoldMs: 60_000 } }); + }); +}); diff --git a/control-plane/src/services/__tests__/taskProcessor.service.test.ts b/control-plane/src/services/__tests__/taskProcessor.service.test.ts index 14d23c0..23d5d40 100644 --- a/control-plane/src/services/__tests__/taskProcessor.service.test.ts +++ b/control-plane/src/services/__tests__/taskProcessor.service.test.ts @@ -94,6 +94,8 @@ const { LinearIntegration } = await import("../../integrations/linear"); const { githubIntegration } = await import("../../integrations/github"); const { apiIntegration } = await import("../../integrations/api"); const { logger } = await import("../../logger"); +const { ReplyHolds } = await import("../replyHolds"); +const { RetryableError } = await import("../../integrations/outbound"); const makeTask = (overrides: Partial = {}): Task => ({ id: "task-1", @@ -145,6 +147,7 @@ const survivingNonTerminalRow = (taskId = "task-1", externalId = "issue-1"): any describe("TaskProcessorService", () => { let heartbeatOwnedBy: ReturnType; + let heartbeatTask: ReturnType; let claimNext: ReturnType; let attachSandboxResources: ReturnType; let backdateHeartbeatsOwnedBy: ReturnType; @@ -177,6 +180,7 @@ describe("TaskProcessorService", () => { let inboxService: InboxService; let notifyTerminal: ReturnType; let slackNotifier: SlackNotifierService; + let holds: InstanceType; let service: InstanceType; beforeEach(() => { @@ -194,6 +198,7 @@ describe("TaskProcessorService", () => { postReviewCommentReply.mockResolvedValue({ id: 1, url: "u" }); heartbeatOwnedBy = mock(() => Promise.resolve()); + heartbeatTask = mock(() => Promise.resolve([{ id: "task-1" }])); claimNext = mock(() => Promise.resolve(null)); attachSandboxResources = mock(() => Promise.resolve([{ id: "task-1" }])); backdateHeartbeatsOwnedBy = mock(() => Promise.resolve()); @@ -207,6 +212,7 @@ describe("TaskProcessorService", () => { getByIdOwnedBy = mock((id: string) => Promise.resolve(makeTask({ id }))); taskRepo = { heartbeatOwnedBy, + heartbeatTask, claimNext, attachSandboxResources, backdateHeartbeatsOwnedBy, @@ -271,6 +277,7 @@ describe("TaskProcessorService", () => { notifyTerminal = mock(() => Promise.resolve()); slackNotifier = { notifyTerminal } as unknown as SlackNotifierService; + holds = new ReplyHolds(); service = new TaskProcessorService( taskRepo, inboxRepo, @@ -284,6 +291,8 @@ describe("TaskProcessorService", () => { api: apiIntegration, github: githubIntegration, }, + holds, + ["linear", "api"], ); }); @@ -736,14 +745,15 @@ describe("TaskProcessorService", () => { expect(emitError).toHaveBeenCalledTimes(1); expect(emitError).toHaveBeenCalledWith(expect.anything(), "sess-1", MSG); // PR comments for the comment, the review body and the CI row, in row order. + const signal = expect.any(AbortSignal); expect(postIssueComment.mock.calls as unknown[]).toEqual([ - ["o", "r", 7, MSG], - ["o", "r", 8, MSG], - ["o", "r", 9, MSG], + ["o", "r", 7, MSG, signal], + ["o", "r", 8, MSG, signal], + ["o", "r", 9, MSG, signal], ]); expect(postReviewCommentReply.mock.calls as unknown[]).toEqual([ - ["o", "r", 8, 11, MSG], - ["o", "r", 8, 12, MSG], + ["o", "r", 8, 11, MSG, signal], + ["o", "r", 8, 12, MSG, signal], ]); expect(failTaskInbox).toHaveBeenCalledWith({ taskId: "task-1", @@ -792,7 +802,9 @@ describe("TaskProcessorService", () => { await failTask(makeTask()); - expect(postIssueComment.mock.calls as unknown[]).toEqual([["o", "r", 7, MSG]]); + expect(postIssueComment.mock.calls as unknown[]).toEqual([ + ["o", "r", 7, MSG, expect.any(AbortSignal)], + ]); expect(releaseLease).toHaveBeenCalledWith({ taskId: "task-1", ownerId: "pod-test", @@ -811,7 +823,7 @@ describe("TaskProcessorService", () => { await failTask(makeTask()); expect(emitError).not.toHaveBeenCalled(); - expect(postIssueComment).toHaveBeenCalledWith("o", "r", 7, MSG); + expect(postIssueComment).toHaveBeenCalledWith("o", "r", 7, MSG, expect.any(AbortSignal)); expect(failTaskInbox).toHaveBeenCalled(); expect(releaseLease).toHaveBeenCalledWith({ taskId: "task-1", @@ -1145,6 +1157,118 @@ describe("TaskProcessorService", () => { expect(requeueForRetry).not.toHaveBeenCalled(); }); + it("a hold stops before the cursor advances and is not a reconcile error", async () => { + const task = makeTask({ sandboxId: "sbx-1" }); + listInFlightOwnedBy.mockResolvedValueOnce([task]).mockResolvedValue([]); + readEvents + .mockResolvedValueOnce([ + { seq: 1, ts: "t", type: "agent_thought", inbox_id: "i", target_id: "s", body: "x" }, + { seq: 2, ts: "t", type: "agent_thought", inbox_id: "i", target_id: "s", body: "y" }, + ] as HarnessEvent[]) + .mockResolvedValue([]); + dispatcherHandle.mockResolvedValueOnce("hold"); + + service.start(); + await waitFor(() => dispatcherHandle.mock.calls.length >= 1); + await new Promise((r) => setTimeout(r, 30)); + await service.stop(); + + expect(dispatcherHandle).toHaveBeenCalledTimes(1); + expect(recordEventCursor).not.toHaveBeenCalled(); + expect(getSandboxPodStatus).not.toHaveBeenCalled(); + }); + + it("skips a held task's events while it waits out Retry-After (the sweep still runs)", async () => { + const task = makeTask({ sandboxId: "sbx-1" }); + listInFlightOwnedBy.mockResolvedValue([task]); + holds.hold("task-1", { seq: 1, inbox_id: "i" }, "github", new RetryableError("429", 60_000)); + + service.start(); + await waitFor(() => sweepUndelivered.mock.calls.length >= 3); + await service.stop(); + + expect(readEvents).not.toHaveBeenCalled(); + expect(holds.stats().github?.heldTasks).toBe(1); + }); + + it("a dropped send advances the cursor, then ends this task's events for the tick", async () => { + const task = makeTask({ sandboxId: "sbx-1" }); + listInFlightOwnedBy.mockResolvedValueOnce([task]).mockResolvedValue([]); + readEvents + .mockResolvedValueOnce([ + { seq: 1, ts: "t", type: "agent_thought", inbox_id: "i", target_id: "s", body: "x" }, + { seq: 2, ts: "t", type: "agent_thought", inbox_id: "i", target_id: "s", body: "y" }, + ] as HarnessEvent[]) + .mockResolvedValue([]); + dispatcherHandle.mockResolvedValueOnce("dropped"); + + service.start(); + await waitFor(() => recordEventCursor.mock.calls.length >= 1); + await new Promise((r) => setTimeout(r, 30)); + await service.stop(); + + expect(dispatcherHandle).toHaveBeenCalledTimes(1); + const seqs = recordEventCursor.mock.calls.map( + (c: unknown[]) => (c[0] as { event: { seq: number } }).event.seq, + ); + expect(seqs).toEqual([1]); + }); + + it("drops a hold once the cursor moves past its event, or once the task leaves this pod", async () => { + const task = makeTask({ sandboxId: "sbx-1" }); + // task-1 stays owned every tick, so only the cursor advance can drop its hold. + listInFlightOwnedBy.mockResolvedValue([task]); + readEvents + .mockResolvedValueOnce([ + { seq: 1, ts: "t", type: "agent_thought", inbox_id: "i", target_id: "s", body: "x" }, + ] as HarnessEvent[]) + .mockResolvedValue([]); + const err = new RetryableError("token fetch failed"); + holds.hold("task-1", { seq: 1, inbox_id: "i" }, "linear", err); + holds.hold("task-gone", { seq: 3, inbox_id: "j" }, "linear", err); + // The retry of task-1's held event succeeds; task-gone is not owned. + service.start(); + await waitFor(() => recordEventCursor.mock.calls.length >= 1); + await new Promise((r) => setTimeout(r, 30)); + await service.stop(); + + expect(holds.stats()).toEqual({}); + }); + + it("renews each task's lease right before its reconcile; a lost lease skips that task", async () => { + const t1 = makeTask({ id: "task-1", sandboxId: "sbx-1" }); + const t2 = makeTask({ id: "task-2", sandboxId: "sbx-2" }); + listInFlightOwnedBy.mockResolvedValueOnce([t1, t2]).mockResolvedValue([]); + const order: string[] = []; + heartbeatTask.mockImplementation((p: { taskId: string }) => { + order.push(`heartbeat ${p.taskId}`); + return Promise.resolve(p.taskId === "task-2" ? [] : [{ id: p.taskId }]); + }); + getSandboxInfo.mockImplementation((id: string) => { + order.push(`reconcile ${id}`); + return Promise.resolve(makeSandboxInfo()); + }); + + service.start(); + await waitFor(() => order.length >= 3); + await new Promise((r) => setTimeout(r, 30)); + await service.stop(); + + expect(order).toEqual(["heartbeat task-1", "reconcile sbx-1", "heartbeat task-2"]); + expect(heartbeatTask).toHaveBeenCalledWith({ taskId: "task-1", ownerId: "pod-test" }); + }); + + it("claims and lists only the configured task sources", async () => { + service.start(); + await waitFor(() => claimNext.mock.calls.length > 0); + await service.stop(); + + expect(listInFlightOwnedBy).toHaveBeenCalledWith("pod-test", ["linear", "api"]); + expect(claimNext).toHaveBeenCalledWith( + expect.objectContaining({ sources: ["linear", "api"] }), + ); + }); + it("calls sweepUndelivered before reading events", async () => { const task = makeTask({ sandboxId: "sbx-1" }); listInFlightOwnedBy.mockResolvedValueOnce([task]).mockResolvedValue([]); diff --git a/control-plane/src/services/eventDispatcher.ts b/control-plane/src/services/eventDispatcher.ts index db2342d..a668d0d 100644 --- a/control-plane/src/services/eventDispatcher.ts +++ b/control-plane/src/services/eventDispatcher.ts @@ -9,6 +9,7 @@ import { import { attachUrlToIssue, getIssueHumanAssignee, uploadIssueAttachment } from "../clients/linear"; import { config } from "../config"; import type { Task } from "../db/schema"; +import { SEND_TIMEOUT_MS, withTimeout } from "../integrations/http"; import { type Outbound, RetryableError } from "../integrations/outbound"; import type { Integration, IntegrationName } from "../integrations/types"; import { logger } from "../logger"; @@ -16,10 +17,19 @@ import type { InboxRepository, InboxRow } from "../repositories/inbox.repository import type { TaskRepository } from "../repositories/task.repository"; import type { AgentEmit, HarnessEvent } from "../types/harnessEvent"; import type { LinearTokenProvider } from "./linearTokenProvider"; +import type { ReplyHolds } from "./replyHolds"; import type { SandboxManager } from "./sandboxManager.service"; import type { SlackNotifierService } from "./slackNotifier.service"; // Outcome of dispatching a single event. +// - `ok`: handled; reconcileOne advances the cursor and goes on. +// - `dropped`: the event's send timed out or failed for good. reconcileOne +// advances past it, then stops this task's events for the tick, so a slow +// or failing platform can't eat the tick (§4 I14). +// - `hold`: a send failed before anything went out (RetryableError); the hold +// is recorded in ReplyHolds. reconcileOne stops this task's events BEFORE +// advancing the cursor, so the event is retried until it is sent or fails +// for good. // - `self_evict`: an owner-guarded write returned 0 rows — the lease was taken // over mid-tick — so reconcileOne must stop processing this task BEFORE // advancing the cursor (matching recordEventCursor's 0-row return). @@ -29,7 +39,7 @@ import type { SlackNotifierService } from "./slackNotifier.service"; // advancing the cursor and WITHOUT applying the side effect. We deliberately // stop-and-alarm rather than skip-and-continue, so a foreign event is visible // and cannot trigger a side effect. -export type DispatchResult = "ok" | "self_evict" | "abort"; +export type DispatchResult = "ok" | "dropped" | "hold" | "self_evict" | "abort"; // One-time onboarding comment posted on every PR harness opens — comment // interactivity is invisible unless users know it exists. Built per-call from @@ -59,9 +69,23 @@ export class EventDispatcher { private linearTokenProvider: Pick, private slackNotifier: SlackNotifierService, private integrations: Record, + private holds: ReplyHolds, ) {} async handle(event: HarnessEvent, task: Task): Promise { + try { + return await this.apply(event, task); + } catch (err) { + // A task-scoped call (the Linear client on pr_opened / completed) failed + // before anything was sent: hold on the task's own integration. deliver + // holds on the round's integration itself. + if (!(err instanceof RetryableError)) throw err; + this.holds.hold(task.id, event, task.source, err); + return "hold"; + } + } + + private async apply(event: HarnessEvent, task: Task): Promise { // event.inbox_id is sandbox-written; resolve it task-scoped. A null row // means this task's event stream is carrying an event whose inbox row it // does NOT own — a cross-task LEAK (a sibling's event in our stream, our own @@ -102,13 +126,12 @@ export class EventDispatcher { case "agent_action": case "agent_response": case "agent_error": - await this.deliver(event, inboxRow, task); - break; + return this.deliver(event, inboxRow, task); case "pr_opened": { // The Linear attach and assignee mirror below need a client. Fetched - // first, so a Linear token failure rejects before any side effect and - // the cursor holds. + // first, so a Linear token failure holds the event before any side + // effect. const linearClient = await this.linearClientFor(task); // appendPrUrl is owner-guarded; a 0-row return means the lease was // taken over (stale-lease takeover). Self-evict BEFORE attachUrlToIssue @@ -152,7 +175,7 @@ export class EventDispatcher { case "completed": { // The Slack notify below needs a Linear client for a Linear task's // assignment round. Fetched before marking, so a Linear token failure - // rejects with the row still open and the cursor held. + // holds the event with the row still open. const linearClient = inboxRow?.kind === "assignment" ? await this.linearClientFor(task) : null; // Bare terminal marker — the user-facing message already went out via @@ -199,23 +222,35 @@ export class EventDispatcher { return "ok"; } + // Nothing has been sent when the token fetch fails, so handle() holds. private async linearClientFor(task: Task): Promise { - return task.source === "linear" ? this.linearTokenProvider.getAuthorizedLinearClient() : null; + if (task.source !== "linear") return null; + try { + return await this.linearTokenProvider.getAuthorizedLinearClient(); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + throw new RetryableError(`Linear token fetch failed: ${msg}`); + } } // Delivers one agent emit to one thread: looks up the target on the inbox // row's reply_targets[] by target_id, labels the emit, and hands both to the // round's integration, which decides where (or whether) it shows up. Core - // never checks what an integration can show. A RetryableError (nothing was - // sent) propagates, so handle() rejects and the cursor holds; any other - // failure is logged and the event dropped. - private async deliver(event: AgentEmit, inboxRow: InboxRow | null, task: Task): Promise { + // never checks what an integration can show. The send runs under + // SEND_TIMEOUT_MS. A RetryableError (nothing was sent) holds the event on + // the round's integration; a timeout or any other failure is logged and the + // event dropped. + private async deliver( + event: AgentEmit, + inboxRow: InboxRow | null, + task: Task, + ): Promise { if (!inboxRow) { logger.warn(`${event.type} references missing inbox row`, { taskId: task.id, inboxId: event.inbox_id, }); - return; + return "ok"; } const target = inboxRow.reply_targets.find((t) => t.id === event.target_id); if (!target) { @@ -224,18 +259,25 @@ export class EventDispatcher { inboxId: inboxRow.id, targetId: event.target_id, }); - return; // cursor still advances — drop + return "ok"; // cursor still advances — drop } try { - await this.integrations[inboxRow.source].deliver( - { signal: new AbortController().signal, log: logger }, - target, - labelAgentEmit(event, inboxRow), + await withTimeout(SEND_TIMEOUT_MS, (signal) => + this.integrations[inboxRow.source].deliver( + { signal, log: logger }, + target, + labelAgentEmit(event, inboxRow), + ), ); + return "ok"; } catch (err) { - if (err instanceof RetryableError) throw err; + if (err instanceof RetryableError) { + this.holds.hold(task.id, event, inboxRow.source, err); + return "hold"; + } logErr(`deliver ${event.type} failed for target ${target.id}`, err); + return "dropped"; } } diff --git a/control-plane/src/services/inbox.service.ts b/control-plane/src/services/inbox.service.ts index 4cebd67..08d0baa 100644 --- a/control-plane/src/services/inbox.service.ts +++ b/control-plane/src/services/inbox.service.ts @@ -1,5 +1,6 @@ import { config } from "../config"; import type { Task } from "../db/schema"; +import { SEND_TIMEOUT_MS, withTimeout } from "../integrations/http"; import type { Integration, IntegrationName } from "../integrations/types"; import { logger } from "../logger"; import type { @@ -342,13 +343,13 @@ export class InboxService { }); return; } - await integration - .deliver({ signal: new AbortController().signal, log: logger }, target, { + await withTimeout(SEND_TIMEOUT_MS, (signal) => + integration.deliver({ signal, log: logger }, target, { type: "failure", cause: "workspace_missing", round: entry, - }) - .catch((err) => logErr(`workspace_missing notice failed for inbox ${entry.id}`, err)); + }), + ).catch((err) => logErr(`workspace_missing notice failed for inbox ${entry.id}`, err)); } } diff --git a/control-plane/src/services/linearTokenProvider.ts b/control-plane/src/services/linearTokenProvider.ts index 2b898f9..af1afec 100644 --- a/control-plane/src/services/linearTokenProvider.ts +++ b/control-plane/src/services/linearTokenProvider.ts @@ -77,9 +77,11 @@ export class LinearTokenProvider { // A LinearClient that, when Linear answers 401, renews the token and retries // the call once. Covers SDK calls (`request`) and raw queries (`rawRequest`). - async getAuthorizedLinearClient(): Promise { + // `signal` aborts this client's requests only; the shared token request + // never gets it, so one caller's timeout can't fail another's refresh. + async getAuthorizedLinearClient(signal?: AbortSignal): Promise { let current = await this.getAccessToken(); - const client = new LinearClient({ accessToken: current, apiUrl: this.apiUrl }); + const client = new LinearClient({ accessToken: current, apiUrl: this.apiUrl, signal }); // LinearClient sends every query through this instance's `request`, read at // call time, so wrapping it covers all calls made through the client. const gql = client.client; diff --git a/control-plane/src/services/replyHolds.ts b/control-plane/src/services/replyHolds.ts new file mode 100644 index 0000000..a8e2bce --- /dev/null +++ b/control-plane/src/services/replyHolds.ts @@ -0,0 +1,82 @@ +import type { RetryableError } from "../integrations/outbound"; +import type { IntegrationName } from "../integrations/types"; +import { logger } from "../logger"; +import type { HarnessEvent } from "../types/harnessEvent"; + +const WARN_EVERY_MS = 10 * 60 * 1000; + +type Hold = { + inboxId: string; + seq: number; + integration: IntegrationName; + since: number; + nextAttemptAt: number; + warnedAt: number; +}; + +// Events whose send failed before anything went out (§4 I14). A held task +// stops at that event, so it holds at most one: keyed by task, the hold is +// (taskId, inboxId, seq) and a different event starts a new one. In memory and +// with no time limit: the reconcile loop drops a hold when the cursor moves +// past its event or the task leaves this pod. +export class ReplyHolds { + private readonly byTask = new Map(); + + // Records (or renews) a hold. The next attempt honours Retry-After, else it + // is the next tick. Warns `reply_held` at most every 10 min per task. + hold( + taskId: string, + event: Pick, + integration: IntegrationName, + err: RetryableError, + ): void { + const now = Date.now(); + const { inbox_id: inboxId, seq } = event; + const prev = this.byTask.get(taskId); + const h = + prev?.inboxId === inboxId && prev.seq === seq + ? prev + : { inboxId, seq, integration, since: now, nextAttemptAt: now, warnedAt: -1 }; + h.nextAttemptAt = now + (err.retryAfterMs ?? 0); + if (h.warnedAt < 0 || now - h.warnedAt >= WARN_EVERY_MS) { + h.warnedAt = now; + logger.warn("reply held: its send failed before anything was sent; retrying", { + event: "reply_held", + taskId, + integration, + heldForMin: Math.floor((now - h.since) / 60_000), + reason: err.message, + }); + } + this.byTask.set(taskId, h); + } + + // True while the task's held event waits out its Retry-After. + waiting(taskId: string): boolean { + const h = this.byTask.get(taskId); + return h !== undefined && Date.now() < h.nextAttemptAt; + } + + release(taskId: string): void { + this.byTask.delete(taskId); + } + + // Drops the holds of tasks this pod no longer owns. + retain(ownedIds: Set): void { + for (const id of this.byTask.keys()) if (!ownedIds.has(id)) this.byTask.delete(id); + } + + // Held tasks and the oldest hold's age, per integration (for phase 2's + // status routes). + stats(): Partial> { + const now = Date.now(); + const out: Partial> = {}; + for (const h of this.byTask.values()) { + const s = out[h.integration] ?? { heldTasks: 0, oldestHoldMs: 0 }; + s.heldTasks += 1; + s.oldestHoldMs = Math.max(s.oldestHoldMs, now - h.since); + out[h.integration] = s; + } + return out; + } +} diff --git a/control-plane/src/services/taskProcessor.service.ts b/control-plane/src/services/taskProcessor.service.ts index 570d50f..cfbca5b 100644 --- a/control-plane/src/services/taskProcessor.service.ts +++ b/control-plane/src/services/taskProcessor.service.ts @@ -3,6 +3,7 @@ import { emitAction, emitThought, setIssueStarted } from "../clients/linear"; import { config } from "../config"; import type { Task } from "../db/schema"; import { PvcNotFoundError, SandboxCreateError } from "../errors"; +import { SEND_TIMEOUT_MS, withTimeout } from "../integrations/http"; import type { Integration, IntegrationName } from "../integrations/types"; import { logger } from "../logger"; import type { InboxRepository } from "../repositories/inbox.repository"; @@ -11,6 +12,7 @@ import type { EventDispatcher } from "./eventDispatcher"; import { extractExperimentalFlags } from "./experimentalFlags"; import type { InboxService } from "./inbox.service"; import type { LinearTokenProvider } from "./linearTokenProvider"; +import type { ReplyHolds } from "./replyHolds"; import { type PodStatus, SandboxManager } from "./sandboxManager.service"; import type { SlackNotifierService } from "./slackNotifier.service"; @@ -42,6 +44,10 @@ export class TaskProcessorService { private readonly inboxService: InboxService; private readonly slackNotifier: SlackNotifierService; private readonly integrations: Record; + private readonly holds: ReplyHolds; + // Sources this build has an integration for; claimNext and + // listInFlightOwnedBy see only these tasks (§10). + private readonly taskSources: Task["source"][]; constructor( taskRepo: TaskRepository, @@ -52,6 +58,8 @@ export class TaskProcessorService { inboxService: InboxService, slackNotifier: SlackNotifierService, integrations: Record, + holds: ReplyHolds, + taskSources: Task["source"][], ) { this.taskRepo = taskRepo; this.inboxRepo = inboxRepo; @@ -61,6 +69,8 @@ export class TaskProcessorService { this.inboxService = inboxService; this.slackNotifier = slackNotifier; this.integrations = integrations; + this.holds = holds; + this.taskSources = taskSources; } start(): void { @@ -109,9 +119,16 @@ export class TaskProcessorService { // 2. Idle reaper — kill sandboxes idle past the ceiling, release leases. await this.reapIdle().catch((err) => logErr("reapIdle failed", err)); - // 3. Reconcile owned tasks with live sandboxes. - const owned = await this.taskRepo.listInFlightOwnedBy(config.podId); + // 3. Reconcile owned tasks with live sandboxes. Each task's lease is renewed + // right before its reconcile, so slow sends on earlier tasks can't let it + // outlive the lease mid-tick; 0 rows means a sibling took it over. + const owned = await this.taskRepo.listInFlightOwnedBy(config.podId, this.taskSources); for (const task of owned) { + const renewed = await this.taskRepo.heartbeatTask({ taskId: task.id, ownerId: config.podId }); + if (renewed.length === 0) { + logger.info("lease lost before reconcile; skipping", { taskId: task.id }); + continue; + } await this.reconcileOne(task) .then(() => { logger.debug("reconcileOne: succeeded", { taskId: task.id, sandboxId: task.sandboxId }); @@ -127,16 +144,18 @@ export class TaskProcessorService { }); } - // Drop grace/window timers for any task this pod no longer owns (lease handed - // off, retried away, or terminally failed). Keyed by task.id with no other GC, - // these Maps would otherwise leak for the process lifetime AND let a stale - // `since` from a prior ownership epoch skip the grace/window on a re-claimed - // task's first fault. `owned` is this tick's owned, sandbox-bound set. + // Drop grace/window timers and reply holds for any task this pod no longer + // owns (lease handed off, retried away, or terminally failed). Keyed by + // task.id with no other GC, these Maps would otherwise leak for the process + // lifetime AND let a stale `since` from a prior ownership epoch skip the + // grace/window on a re-claimed task's first fault. `owned` is this tick's + // owned, sandbox-bound set. const ownedIds = new Set(owned.map((t) => t.id)); for (const id of this.sandboxAbsentSince.keys()) if (!ownedIds.has(id)) this.sandboxAbsentSince.delete(id); for (const id of this.reconcileFilesUnavailableSince.keys()) if (!ownedIds.has(id)) this.reconcileFilesUnavailableSince.delete(id); + this.holds.retain(ownedIds); // 4. Claim new work (fresh-pending OR comment-pickup OR stale-lease // takeover — single predicate). @@ -144,6 +163,7 @@ export class TaskProcessorService { ownerId: config.podId, leaseTimeoutMs: config.leaseTimeoutMs, maxConcurrentSandboxes: config.maxConcurrentSandboxes, + sources: this.taskSources, }); if (claimed) this.dispatch(claimed); } @@ -364,6 +384,10 @@ export class TaskProcessorService { // Sweep: deliver any undelivered inbox rows to the sandbox. await this.inboxService.sweepUndelivered(task); + // A held event waiting out its Retry-After: skip this task's events this + // tick (the sweep above still hands new rounds to the sandbox). + if (this.holds.waiting(task.id)) return; + // Read events.jsonl since cursor; dispatch each new event. Re-fetch the // task between events because dispatcher cases mutate task state // (appendPrUrl, setSandboxId). @@ -372,33 +396,45 @@ export class TaskProcessorService { for (const event of events) { if (currentTask === null) break; // self-evict: lease lost or task gone - // handle() returns "self_evict" when an owner-guarded write inside the - // dispatcher (e.g. appendPrUrl) returned 0 rows — the lease was taken - // over mid-tick. Stop BEFORE advancing the cursor: a sibling pod owns - // this task and will resume from its own cursor. const result = await this.eventDispatcher.handle(event, currentTask); - if (result === "self_evict") { - logger.info("dispatcher self-evicted (lease lost); stopping reconcile", { - taskId: currentTask.id, - seq: event.seq, - }); - break; - } - if (result === "abort") { - // The dispatcher found a foreign durable event in this task's stream (a - // cross-task leak). Stop here WITHOUT advancing the cursor — we neither - // apply nor skip past the leaked event. The task intentionally halts - // (and re-logs each tick) so the leak surfaces for an operator rather - // than being silently absorbed. Detail is in `cross_task_leak_detected`. - logger.error("reconcile halted: cross-task leak detected; not advancing cursor", { - event: "reconcile_halted_cross_task_leak", - taskId: currentTask.id, - taskExternalId: currentTask.externalId, - sandboxId: currentTask.sandboxId, - podId: config.podId, - seq: event.seq, - }); - break; + switch (result) { + case "ok": + case "dropped": + break; + case "hold": + // A send failed before anything went out. Stop BEFORE advancing the + // cursor and return normally (not a reconcile error): the event is + // retried on a later tick, and nothing after it runs ahead of it. + return; + case "self_evict": + // An owner-guarded write inside the dispatcher (e.g. appendPrUrl) + // returned 0 rows — the lease was taken over mid-tick. Stop BEFORE + // advancing the cursor: a sibling pod owns this task and will resume + // from its own cursor. + logger.info("dispatcher self-evicted (lease lost); stopping reconcile", { + taskId: currentTask.id, + seq: event.seq, + }); + return; + case "abort": + // The dispatcher found a foreign durable event in this task's stream (a + // cross-task leak). Stop here WITHOUT advancing the cursor — we neither + // apply nor skip past the leaked event. The task intentionally halts + // (and re-logs each tick) so the leak surfaces for an operator rather + // than being silently absorbed. Detail is in `cross_task_leak_detected`. + logger.error("reconcile halted: cross-task leak detected; not advancing cursor", { + event: "reconcile_halted_cross_task_leak", + taskId: currentTask.id, + taskExternalId: currentTask.externalId, + sandboxId: currentTask.sandboxId, + podId: config.podId, + seq: event.seq, + }); + return; + default: { + const _exhaustive: never = result; + void _exhaustive; + } } const advanced = await this.taskRepo.recordEventCursor({ @@ -415,6 +451,11 @@ export class TaskProcessorService { }); break; } + // The cursor is past any held event now. + this.holds.release(currentTask.id); + // A dropped send (timed out or failed for good) ends this task's events + // for the tick, so a slow or failing platform can't eat the tick. + if (result === "dropped") break; // Re-fetch (owner-scoped) to pick up dispatcher mutations (pr_urls // append, etc.). An owner-scoped fetch returns null on a takeover in the @@ -781,14 +822,14 @@ export class TaskProcessorService { for (const t of row.reply_targets) { if (seen.has(t.id)) continue; seen.add(t.id); - await integration - .deliver({ signal: new AbortController().signal, log: logger }, t, { + await withTimeout(SEND_TIMEOUT_MS, (signal) => + integration.deliver({ signal, log: logger }, t, { type: "failure", cause: "task_failed", round: row, summary: message, - }) - .catch((e) => logErr(`failTask: post to ${t.spec.kind} target ${t.id} failed`, e)); + }), + ).catch((e) => logErr(`failTask: post to ${t.spec.kind} target ${t.id} failed`, e)); } } From cf8fa6ca9574b58c1b7f8ff39bbf8cb13597ac54 Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Fri, 30 Oct 2026 18:32:39 +0530 Subject: [PATCH 06/22] feat(control-plane): run integration side effects as lifecycle hooks The Linear PR attach, run-archive upload and assignee lookup, and GitHub's review re-request, move into integration hooks run with timeouts; detached work is drained on shutdown. Replayed events no longer repeat the onboarding comment, assignee mirroring or archive upload, which now run only on the first transition. The "Sandbox provisioned" thought is gone, and the chart and Compose get a shutdown grace period covering one task's reconcile plus the drain. Co-Authored-By: Claude Opus 5.5 --- control-plane/docker-compose.yml | 4 + control-plane/src/app.ts | 7 +- control-plane/src/clients/githubApi.ts | 14 +- .../src/integrations/github/index.ts | 84 ++++- .../src/integrations/linear/index.ts | 69 +++- control-plane/src/integrations/types.ts | 28 +- .../__tests__/inbox.repository.test.ts | 16 + .../__tests__/task.repository.test.ts | 9 +- .../src/repositories/inbox.repository.ts | 10 + .../src/repositories/task.repository.ts | 10 +- .../__tests__/eventDispatcher.test.ts | 91 +++-- .../services/__tests__/inbox.service.test.ts | 11 +- .../__tests__/integrationHooks.test.ts | 118 +++++++ .../__tests__/taskProcessor.service.test.ts | 29 +- control-plane/src/services/eventDispatcher.ts | 318 ++++++------------ control-plane/src/services/inbox.service.ts | 13 +- .../src/services/integrationHooks.ts | 154 +++++++++ .../src/services/taskProcessor.service.ts | 34 +- deploy/helm/devbox/README.md | 1 + .../helm/devbox/templates/control-plane.yaml | 2 + .../helm/devbox/tests/golden/byo-secrets.json | 1 + deploy/helm/devbox/tests/golden/defaults.json | 1 + .../helm/devbox/tests/golden/external-db.json | 1 + .../tests/golden/github-app-external.json | 1 + .../helm/devbox/tests/golden/github-app.json | 1 + .../helm/devbox/tests/golden/ingress-api.json | 1 + .../devbox/tests/golden/ingress-both.json | 1 + .../devbox/tests/golden/ingress-webhooks.json | 1 + deploy/helm/devbox/tests/golden/linear.json | 1 + .../helm/devbox/tests/golden/llm-stages.json | 1 + deploy/helm/devbox/tests/golden/replicas.json | 1 + .../devbox/tests/golden/sandbox-options.json | 1 + deploy/helm/devbox/tests/golden/slack.json | 1 + deploy/helm/devbox/tests/values/replicas.yaml | 2 +- deploy/helm/devbox/values.schema.json | 3 +- deploy/helm/devbox/values.yaml | 4 + 36 files changed, 757 insertions(+), 287 deletions(-) create mode 100644 control-plane/src/services/__tests__/integrationHooks.test.ts create mode 100644 control-plane/src/services/integrationHooks.ts diff --git a/control-plane/docker-compose.yml b/control-plane/docker-compose.yml index 6fdf7f9..0118c79 100644 --- a/control-plane/docker-compose.yml +++ b/control-plane/docker-compose.yml @@ -93,6 +93,10 @@ services: image: devbox-control-plane build: *control-plane-build restart: unless-stopped + # Shutdown finishes the current reconcile tick (up to 90s when a platform + # is slow), then drains for SHUTDOWN_DRAIN_MS (10s); Compose's default 10s + # would cut both short. + stop_grace_period: 120s # Explicit, with no env_file. Only allowlisted HARNESS_* settings cross # into sandboxes, and a forwarded name must never hold a real secret. environment: diff --git a/control-plane/src/app.ts b/control-plane/src/app.ts index 6347050..5fb4b26 100644 --- a/control-plane/src/app.ts +++ b/control-plane/src/app.ts @@ -27,6 +27,7 @@ import { ApiTaskService } from "./services/apiTask.service"; import { EventDispatcher } from "./services/eventDispatcher"; import { GithubStatusService } from "./services/githubStatus.service"; import { InboxService } from "./services/inbox.service"; +import { IntegrationHooks } from "./services/integrationHooks"; import { LinearStatusService } from "./services/linearStatus.service"; import { LinearTokenProvider } from "./services/linearTokenProvider"; import { ReplyHolds } from "./services/replyHolds"; @@ -92,8 +93,10 @@ export function createApp(): AppBundle { // Sends held because they failed before anything went out; the status // routes will report them per integration (phase 2). const replyHolds = new ReplyHolds(); + // Integrations' lifecycle hooks; TaskProcessor.stop() drains detached ones. + const hooks = new IntegrationHooks(integrations, inboxRepository); - const inboxService = new InboxService(inboxRepository, sandboxManager, integrations); + const inboxService = new InboxService(inboxRepository, sandboxManager, integrations, hooks); const agentSessionService = new AgentSessionService( taskRepository, @@ -117,6 +120,7 @@ export function createApp(): AppBundle { slackNotifier, integrations, replyHolds, + hooks, ); const taskProcessor = new TaskProcessorService( @@ -130,6 +134,7 @@ export function createApp(): AppBundle { integrations, replyHolds, taskSources, + hooks, ); const linearStatus = new LinearStatusService( diff --git a/control-plane/src/clients/githubApi.ts b/control-plane/src/clients/githubApi.ts index 3d08cfd..34a2b82 100644 --- a/control-plane/src/clients/githubApi.ts +++ b/control-plane/src/clients/githubApi.ts @@ -132,11 +132,13 @@ export async function requestReviewers( repo: string, prNumber: number, reviewers: string[], + signal?: AbortSignal, ): Promise { await githubFetch( "POST", `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/pulls/${prNumber}/requested_reviewers`, { reviewers }, + signal, ); } @@ -315,8 +317,16 @@ export async function addPrAssignees( // GET /user/{id} — resolve an immutable numeric GitHub account id to the // current login. Ids survive renames and are never re-issued, so this is the // rename-proof lookup. Throws on 404 (githubFetch); callers catch. -export async function getUserById(id: string | number): Promise<{ login: string; id: number }> { - const { json } = await githubFetch("GET", `/user/${encodeURIComponent(String(id))}`); +export async function getUserById( + id: string | number, + signal?: AbortSignal, +): Promise<{ login: string; id: number }> { + const { json } = await githubFetch( + "GET", + `/user/${encodeURIComponent(String(id))}`, + undefined, + signal, + ); const r = asRecord(json); return { login: String(r.login ?? ""), id: Number(r.id) }; } diff --git a/control-plane/src/integrations/github/index.ts b/control-plane/src/integrations/github/index.ts index db0f905..4730635 100644 --- a/control-plane/src/integrations/github/index.ts +++ b/control-plane/src/integrations/github/index.ts @@ -1,15 +1,18 @@ import { addIssueCommentReaction, addReviewCommentReaction, + parsePrUrl, postIssueComment, postReviewCommentReply, + requestReviewers, } from "../../clients/github"; import { config } from "../../config"; +import type { Task } from "../../db/schema"; import type { InboxRow, ReplyTargetSpec } from "../../repositories/inbox.repository"; import { WORKSPACE_MISSING_MESSAGE } from "../../services/inbox.service"; import { retryable } from "../http"; import type { Outbound, SendCtx } from "../outbound"; -import type { Integration } from "../types"; +import type { HookCtx, Integration } from "../types"; type GithubSpec = Extract< ReplyTargetSpec, @@ -36,8 +39,87 @@ export const githubIntegration: Integration = { return; } }, + + // A finished review re-requests review from its reviewer, once (a replayed + // `completed` doesn't). Best-effort: a failure is logged, nothing waits on it. + async onRoundFinished(ctx, task, round) { + if (ctx.firstTransition && round.kind === "review") { + await requestReviewFromOriginalReviewer(ctx, round, task); + } + }, }; +async function requestReviewFromOriginalReviewer( + ctx: HookCtx, + inboxRow: Extract, + task: Task, +): Promise { + const payload = inboxRow.payload as Record | null | undefined; + const reviewerLoginRaw = payload?.reviewer_login; + const reviewerLogin = typeof reviewerLoginRaw === "string" ? reviewerLoginRaw.trim() : ""; + if (!reviewerLogin) { + ctx.log.info("review re-request skipped", { + event: "review_rerequest.result", + status: "skipped", + reason: "missing_reviewer_login", + taskId: task.id, + inboxId: inboxRow.id, + }); + return; + } + + const pr = reviewPrCoordinates(inboxRow); + if (!pr) { + ctx.log.warn("review re-request skipped", { + event: "review_rerequest.result", + status: "skipped", + reason: "missing_pr_coordinates", + taskId: task.id, + inboxId: inboxRow.id, + reviewerLogin, + }); + return; + } + + try { + await requestReviewers(pr.owner, pr.repo, pr.prNumber, [reviewerLogin], ctx.signal); + } catch (err) { + const e = err instanceof Error ? err : new Error(String(err)); + ctx.log.error("request reviewers failed", { + taskId: task.id, + inboxId: inboxRow.id, + owner: pr.owner, + repo: pr.repo, + prNumber: pr.prNumber, + reviewerLogin, + message: e.message, + stack: e.stack, + }); + } +} + +function reviewPrCoordinates( + inboxRow: Extract, +): { owner: string; repo: string; prNumber: number } | null { + const pr = inboxRow.payload.pr as Record | undefined; + const owner = typeof pr?.owner === "string" ? pr.owner : ""; + const repo = typeof pr?.repo === "string" ? pr.repo : ""; + const prNumber = typeof pr?.prNumber === "number" ? pr.prNumber : Number.NaN; + if (owner && repo && Number.isFinite(prNumber)) { + return { owner, repo, prNumber }; + } + + const target = inboxRow.reply_targets.find( + (t) => t.spec.kind === "review_body" || t.spec.kind === "review_comment", + ); + if (target && "owner" in target.spec) { + return { owner: target.spec.owner, repo: target.spec.repo, prNumber: target.spec.prNumber }; + } + + const prUrl = typeof pr?.pr_url === "string" ? pr.pr_url : ""; + return prUrl ? parsePrUrl(prUrl) : null; +} + async function deliverFailure( ctx: SendCtx, spec: GithubSpec, diff --git a/control-plane/src/integrations/linear/index.ts b/control-plane/src/integrations/linear/index.ts index 3a73e4c..6195ed4 100644 --- a/control-plane/src/integrations/linear/index.ts +++ b/control-plane/src/integrations/linear/index.ts @@ -1,11 +1,21 @@ import { type LinearClient, LinearError, RatelimitedLinearError } from "@linear/sdk"; -import { emitAction, emitError, emitResponse, emitThought } from "../../clients/linear"; -import type { ReplyTarget } from "../../repositories/inbox.repository"; +import { getUserById } from "../../clients/github"; +import { + attachUrlToIssue, + emitAction, + emitError, + emitResponse, + emitThought, + getIssueHumanAssignee, + uploadIssueAttachment, +} from "../../clients/linear"; +import type { Task } from "../../db/schema"; +import type { InboxRow, ReplyTarget } from "../../repositories/inbox.repository"; import { WORKSPACE_MISSING_MESSAGE } from "../../services/inbox.service"; import type { LinearTokenProvider } from "../../services/linearTokenProvider"; import { retryable } from "../http"; import { type Outbound, RetryableError, type SendCtx } from "../outbound"; -import type { Integration } from "../types"; +import type { HookCtx, Integration, Person, PrRef } from "../types"; // Linear rounds post to the agent session. target.id IS the agent session id // by construction (the enqueue paths stamp the live session id). The session @@ -55,6 +65,59 @@ export class LinearIntegration implements Integration { } } + // Links the PR on the issue. Idempotent (Linear updates an existing link), + // so it runs on every pr_opened dispatch; a RetryableError holds the event. + async onPrOpened(ctx: HookCtx, task: Task, pr: PrRef): Promise { + const client = await this.client(ctx.signal); + try { + await attachUrlToIssue(client, task.externalId, pr.url, pr.repo); + } catch (err) { + throw linearRetryable(err) ?? err; + } + } + + // Uploads the run archive to the issue once, when this call finished the + // assignment (a replayed `completed` doesn't upload again). The sandbox is + // still alive here (it lives until the idle reaper). The archive is a + // forensic nicety: core logs a failure and nothing else depends on it. + async onRoundFinished( + ctx: HookCtx & { firstTransition: boolean; archive(): Promise }, + task: Task, + round: InboxRow, + ): Promise { + if (!ctx.firstTransition || round.kind !== "assignment") return; + const bytes = await ctx.archive(); + if (!bytes || bytes.byteLength === 0) return; + const client = await this.client(ctx.signal); + await uploadIssueAttachment(client, { + issueId: task.externalId, + title: "Harness run archive", + subtitle: `Task ${task.id}`, + filename: `harness-run-${task.id}.tar.gz`, + contentType: "application/gzip", + bytes, + metadata: { taskId: task.id }, + signal: ctx.signal, + }); + } + + // The issue's human assignee. Identity comes from Linear's own user↔GitHub + // mapping (User.gitHubUserId, set when a member connects GitHub under Linear + // Settings → Connected accounts), never email matching. + async assignee(ctx: HookCtx, task: Task): Promise { + const client = await this.client(ctx.signal); + const human = await getIssueHumanAssignee(client, task.externalId); + if (!human) return null; + const person = { displayName: human.displayName, email: human.email }; + if (!human.gitHubUserId) return { ...person, codeHost: "unlinked" }; + // Defensive about the field's format: numeric → GitHub account id + // (resolve to the current login), anything else → already a login. + const githubLogin = /^\d+$/.test(human.gitHubUserId) + ? (await getUserById(human.gitHubUserId, ctx.signal)).login + : human.gitHubUserId; + return { ...person, codeHost: { githubLogin } }; + } + // Nothing has been sent when the token fetch fails, so the caller may retry. // A per-call client: the signal bounds this send's requests only. private async client(signal: AbortSignal): Promise { diff --git a/control-plane/src/integrations/types.ts b/control-plane/src/integrations/types.ts index 2a37133..6c62797 100644 --- a/control-plane/src/integrations/types.ts +++ b/control-plane/src/integrations/types.ts @@ -1,6 +1,10 @@ -import type { ReplyTarget } from "../repositories/inbox.repository"; +import type { Task } from "../db/schema"; +import type { logger } from "../logger"; +import type { InboxRow, ReplyTarget } from "../repositories/inbox.repository"; import type { Outbound, SendCtx } from "./outbound"; +type Logger = typeof logger; + // One per task/round source (today's InboxSource). export type IntegrationName = "linear" | "api" | "github"; @@ -10,4 +14,26 @@ export interface Integration { // Outgoing. Core labels each message; the integration decides where (or // whether) it shows up. Throws RetryableError only before anything was sent. deliver(ctx: SendCtx, target: ReplyTarget, message: Outbound): Promise; + + // Lifecycle hooks. Core runs each through IntegrationHooks (catch, log, + // timeout, abort signal); see services/integrationHooks.ts for where and how. + onTaskStarted?(ctx: HookCtx, task: Task, target: ReplyTarget): Promise; + onPrOpened?(ctx: HookCtx & { firstTime: boolean }, task: Task, pr: PrRef): Promise; + onRoundFinished?( + ctx: HookCtx & { firstTransition: boolean; archive(): Promise }, + task: Task, + round: InboxRow, + outcome: Outcome | null, + ): Promise; + assignee?(ctx: HookCtx, task: Task): Promise; // throws when the lookup itself fails } + +// isCurrent() is false once a restart rotated the task's assignment id. +export type HookCtx = { signal: AbortSignal; log: Logger; isCurrent(): Promise }; +export type Person = { + displayName: string; + email?: string; + codeHost: { githubLogin: string } | "unlinked" | "unsupported"; +}; +export type PrRef = { url: string; repo: string; branch: string }; +export type Outcome = "completed" | "partial_failure" | "no_action" | "failed"; diff --git a/control-plane/src/repositories/__tests__/inbox.repository.test.ts b/control-plane/src/repositories/__tests__/inbox.repository.test.ts index fe0e137..99bcc2f 100644 --- a/control-plane/src/repositories/__tests__/inbox.repository.test.ts +++ b/control-plane/src/repositories/__tests__/inbox.repository.test.ts @@ -363,6 +363,22 @@ describe("InboxRepository", () => { }); }); + describe("getAssignment", () => { + it("returns the task's assignment row, never another round or another task's", async () => { + const task = await insertTask(); + const other = await insertTask(); + await insertRow(task.id, { source: "github", kind: "comment", status: "pending" }); + const assignment = await insertRow(task.id, { + source: "linear", + kind: "assignment", + status: "delivered", + }); + + expect((await repo.getAssignment(task.id))?.id).toBe(assignment.id); + expect(await repo.getAssignment(other.id)).toBeNull(); + }); + }); + describe("markResponded", () => { it("owner match → status 'responded' + 1 row; owner mismatch / re-fire → 0 rows", async () => { const task = await insertTask({ ownerId: "pod-a" }); diff --git a/control-plane/src/repositories/__tests__/task.repository.test.ts b/control-plane/src/repositories/__tests__/task.repository.test.ts index 15df5b1..ce73a45 100644 --- a/control-plane/src/repositories/__tests__/task.repository.test.ts +++ b/control-plane/src/repositories/__tests__/task.repository.test.ts @@ -968,7 +968,7 @@ describe("TaskRepository", () => { branch: "b", commitSha: "abc", }); - expect(first).toEqual([{ id: t.id }]); + expect(first).toEqual([{ id: t.id, firstTime: true }]); // Re-append same url — should not duplicate. await repo.appendPrUrl({ taskId: t.id, @@ -992,8 +992,7 @@ describe("TaskRepository", () => { expect(urls).toEqual(["https://github.com/o/r/pull/1", "https://github.com/o/r/pull/2"]); }); - it("re-appending a known url still returns the row (no first-time signal)", async () => { - // characterization: changes in phase 1b (spec §15 bug 4, I11 firstTime) + it("re-appending a known url still returns the row, with firstTime false", async () => { const t = await insertTask({ ownerId: "pod-a" }); const pr = { taskId: t.id, @@ -1003,8 +1002,8 @@ describe("TaskRepository", () => { branch: "b", commitSha: "abc", }; - await repo.appendPrUrl(pr); - expect(await repo.appendPrUrl(pr)).toEqual([{ id: t.id }]); + expect(await repo.appendPrUrl(pr)).toEqual([{ id: t.id, firstTime: true }]); + expect(await repo.appendPrUrl(pr)).toEqual([{ id: t.id, firstTime: false }]); }); it("owner-guarded: returns 0 rows on stale-lease takeover", async () => { diff --git a/control-plane/src/repositories/inbox.repository.ts b/control-plane/src/repositories/inbox.repository.ts index aa21715..3b3e26a 100644 --- a/control-plane/src/repositories/inbox.repository.ts +++ b/control-plane/src/repositories/inbox.repository.ts @@ -219,6 +219,16 @@ export class InboxRepository { return row ? mapRow(row) : null; } + // The task's assignment row. A restart re-pends it under a new id, which is + // how hooks tell whether their run is still the current one. + async getAssignment(taskId: string): Promise { + const result = await db.execute(sql` + SELECT * FROM inbox WHERE task_id = ${taskId} AND kind = 'assignment' LIMIT 1; + `); + const row = asRows(result)[0]; + return row ? mapRow(row) : null; + } + // Reply target id for task-scoped CP-originated emits (provisioning // lifecycle). Reads the first entry of the assignment row's reply_targets — // the canonical store, written on `created` and overwritten on diff --git a/control-plane/src/repositories/task.repository.ts b/control-plane/src/repositories/task.repository.ts index dfa487c..4ee0cd7 100644 --- a/control-plane/src/repositories/task.repository.ts +++ b/control-plane/src/repositories/task.repository.ts @@ -481,7 +481,8 @@ export class TaskRepository { // Owner-guarded append to pr_urls; dedupes on (url) within the array. // Returns 0 rows on stale-lease takeover — the caller (EventDispatcher's // pr_opened case) treats that as a self-eviction signal, matching every - // other state-advancing repo method. + // other state-advancing repo method. A URL already there still returns the + // row; `firstTime` (PG18 `old.`) says whether this call added it. async appendPrUrl(params: { taskId: string; ownerId: string; @@ -489,7 +490,7 @@ export class TaskRepository { repo: string; branch: string; commitSha: string; - }): Promise<{ id: string }[]> { + }): Promise<{ id: string; firstTime: boolean }[]> { const entry = sql`jsonb_build_object( 'repo', ${params.repo}::text, 'url', ${params.prUrl}::text, @@ -512,7 +513,10 @@ export class TaskRepository { updatedAt: new Date(), }) .where(this.ownedByUs(params.taskId, params.ownerId)) - .returning({ id: schema.tasks.id }); + .returning({ + id: schema.tasks.id, + firstTime: sql`old.pr_urls IS DISTINCT FROM new.pr_urls`, + }); } // GitHub webhook PR → task lookup. GIN-indexed. LIMIT 2 (not 1) so that a PR diff --git a/control-plane/src/services/__tests__/eventDispatcher.test.ts b/control-plane/src/services/__tests__/eventDispatcher.test.ts index 78079a6..bc5a217 100644 --- a/control-plane/src/services/__tests__/eventDispatcher.test.ts +++ b/control-plane/src/services/__tests__/eventDispatcher.test.ts @@ -83,6 +83,7 @@ const { githubIntegration } = await import("../../integrations/github"); const { apiIntegration } = await import("../../integrations/api"); const { HttpError } = await import("../../integrations/http"); const { ReplyHolds } = await import("../replyHolds"); +const { IntegrationHooks } = await import("../integrationHooks"); const { config: mockedConfig } = (await import("../../config")) as unknown as { config: { github: { webhooks: { botLogin: string } | undefined }; @@ -186,7 +187,7 @@ describe("EventDispatcher", () => { requestReviewers.mockReset(); requestReviewers.mockResolvedValue(undefined); - appendPrUrl = mock(() => Promise.resolve([{ id: "task-1" }])); + appendPrUrl = mock(() => Promise.resolve([{ id: "task-1", firstTime: true }])); taskRepo = { appendPrUrl } as unknown as TaskRepository; getByIdForTask = mock(() => Promise.resolve(linearRow())); @@ -194,6 +195,7 @@ describe("EventDispatcher", () => { inboxRepo = { getByIdForTask, markResponded, + getAssignment: mock(() => Promise.resolve(linearRow())), } as unknown as InboxRepository; extractArchive = mock(() => Promise.resolve(null as Uint8Array | null)); @@ -206,18 +208,20 @@ describe("EventDispatcher", () => { slackNotifier = { notifyTerminal } as unknown as SlackNotifierService; holds = new ReplyHolds(); + const integrations = { + linear: new LinearIntegration(linearTokenProvider), + api: apiIntegration, + github: githubIntegration, + }; dispatcher = new EventDispatcher( taskRepo, inboxRepo, sandboxManager, linearTokenProvider, slackNotifier, - { - linear: new LinearIntegration(linearTokenProvider), - api: apiIntegration, - github: githubIntegration, - }, + integrations, holds, + new IntegrationHooks(integrations, inboxRepo), ); }); @@ -588,14 +592,20 @@ describe("EventDispatcher", () => { delivered.push([name, target, message]); }, }); + const integrations = { + linear: record("linear"), + api: record("api"), + github: record("github"), + }; const d = new EventDispatcher( taskRepo, inboxRepo, sandboxManager, linearTokenProvider, slackNotifier, - { linear: record("linear"), api: record("api"), github: record("github") }, + integrations, new ReplyHolds(), + new IntegrationHooks(integrations, inboxRepo), ); const assignment = linearRow(); const ghComment = { @@ -678,12 +688,13 @@ describe("EventDispatcher", () => { ); }); - it("Linear task: a token fetch failure holds the event before appendPrUrl", async () => { + it("Linear task: a token fetch failure holds the event at onPrOpened, after appendPrUrl", async () => { (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( new Error("token refresh failed"), ); expect(await dispatcher.handle(prEvent(), makeTask())).toBe("hold"); - expect(appendPrUrl).not.toHaveBeenCalled(); + expect(appendPrUrl).toHaveBeenCalledTimes(1); + expect(attachUrlToIssue).not.toHaveBeenCalled(); }); it("self-evicts when appendPrUrl returns 0 rows (lease lost); skips attachUrlToIssue", async () => { @@ -745,7 +756,7 @@ describe("EventDispatcher", () => { expect(result).toBe("ok"); expect(getIssueHumanAssignee).toHaveBeenCalledWith(expect.anything(), "issue-1"); - expect(getUserById).toHaveBeenCalledWith("583231"); + expect(getUserById).toHaveBeenCalledWith("583231", expect.any(AbortSignal)); expect(addPrAssignees).toHaveBeenCalledWith("o", "r", 1, ["octocat"]); }); @@ -1154,6 +1165,7 @@ describe("EventDispatcher", () => { contentType: "application/gzip", bytes: new Uint8Array([1, 2]), metadata: { taskId: "task-1" }, + signal: expect.any(AbortSignal), }); }); @@ -1205,7 +1217,13 @@ describe("EventDispatcher", () => { await flushMicrotasks(); expect(requestReviewers).toHaveBeenCalledTimes(1); - expect(requestReviewers).toHaveBeenCalledWith("o", "r", 7, ["reviewer"]); + expect(requestReviewers).toHaveBeenCalledWith( + "o", + "r", + 7, + ["reviewer"], + expect.any(AbortSignal), + ); }); it("falls back to review reply_targets for historical review rows without payload PR coordinates", async () => { @@ -1216,7 +1234,13 @@ describe("EventDispatcher", () => { await dispatcher.handle(completedEvent(), makeTask()); await flushMicrotasks(); - expect(requestReviewers).toHaveBeenCalledWith("o", "r", 7, ["reviewer"]); + expect(requestReviewers).toHaveBeenCalledWith( + "o", + "r", + 7, + ["reviewer"], + expect.any(AbortSignal), + ); }); it("falls back to a review_comment reply_target when review_body is unavailable", async () => { @@ -1242,7 +1266,13 @@ describe("EventDispatcher", () => { await dispatcher.handle(completedEvent(), makeTask()); await flushMicrotasks(); - expect(requestReviewers).toHaveBeenCalledWith("oc", "rc", 8, ["reviewer"]); + expect(requestReviewers).toHaveBeenCalledWith( + "oc", + "rc", + 8, + ["reviewer"], + expect.any(AbortSignal), + ); }); it("falls back to parsing review payload pr_url when reply_targets are unavailable", async () => { @@ -1254,7 +1284,13 @@ describe("EventDispatcher", () => { await dispatcher.handle(completedEvent(), makeTask()); await flushMicrotasks(); - expect(requestReviewers).toHaveBeenCalledWith("fallback", "repo", 9, ["reviewer"]); + expect(requestReviewers).toHaveBeenCalledWith( + "fallback", + "repo", + 9, + ["reviewer"], + expect.any(AbortSignal), + ); }); it("does not re-request review on an idempotent review completion re-fire", async () => { @@ -1318,7 +1354,13 @@ describe("EventDispatcher", () => { await flushMicrotasks(); expect(result).toBe("ok"); - expect(requestReviewers).toHaveBeenCalledWith("o", "r", 7, ["reviewer"]); + expect(requestReviewers).toHaveBeenCalledWith( + "o", + "r", + 7, + ["reviewer"], + expect.any(AbortSignal), + ); }); it("does not crash when markResponded returns 0 rows (lease lost; warned)", async () => { @@ -1349,8 +1391,7 @@ describe("EventDispatcher", () => { // reconcileOne advances the cursor only after handle() returns, so a lost // lease or a crash in between dispatches the same events again. describe("replay (same event dispatched again)", () => { - it("completed: the run archive upload re-fires on the already-responded row", async () => { - // characterization: changes in phase 1b (spec §15 bug 3, I11 firstTransition) + it("completed: the run archive uploads once; the already-responded replay skips it", async () => { extractArchive.mockResolvedValue(new Uint8Array([1])); const done: HarnessEvent = { seq: 5, @@ -1365,7 +1406,8 @@ describe("EventDispatcher", () => { await flushMicrotasks(); expect(markResponded).toHaveBeenCalledTimes(1); - expect(uploadIssueAttachment).toHaveBeenCalledTimes(2); + expect(extractArchive).toHaveBeenCalledTimes(1); + expect(uploadIssueAttachment).toHaveBeenCalledTimes(1); }); it("pr_opened: the Linear attach re-runs (appendPrUrl returns the row for a known URL)", async () => { @@ -1375,8 +1417,11 @@ describe("EventDispatcher", () => { expect(attachUrlToIssue).toHaveBeenCalledTimes(2); }); - it("pr_opened: the onboarding comment and the PR assignee mirror re-run", async () => { - // characterization: changes in phase 1b (spec §13, §15 bug 4, I11 firstTime) + it("pr_opened: the onboarding comment and the PR assignee mirror run only the first time", async () => { + // appendPrUrl finds the URL already there on the replay. + appendPrUrl + .mockResolvedValueOnce([{ id: "task-1", firstTime: true }]) + .mockResolvedValueOnce([{ id: "task-1", firstTime: false }]); getIssueHumanAssignee.mockResolvedValue({ id: "usr-1", name: "Ada Lovelace", @@ -1389,8 +1434,10 @@ describe("EventDispatcher", () => { await dispatcher.handle(prEvent(), makeTask()); await flushMicrotasks(); - expect(postIssueComment).toHaveBeenCalledTimes(2); - expect(addPrAssignees).toHaveBeenCalledTimes(2); + expect(postIssueComment).toHaveBeenCalledTimes(1); + expect(addPrAssignees).toHaveBeenCalledTimes(1); + // The owner's onPrOpened (the idempotent Linear attach) runs every time. + expect(attachUrlToIssue).toHaveBeenCalledTimes(2); }); it("agent_response and agent_error post again on every dispatch", async () => { diff --git a/control-plane/src/services/__tests__/inbox.service.test.ts b/control-plane/src/services/__tests__/inbox.service.test.ts index ca46f8e..52909ef 100644 --- a/control-plane/src/services/__tests__/inbox.service.test.ts +++ b/control-plane/src/services/__tests__/inbox.service.test.ts @@ -71,6 +71,7 @@ const { InboxService, WORKSPACE_MISSING_MESSAGE } = await import("../inbox.servi const { LinearIntegration } = await import("../../integrations/linear"); const { githubIntegration } = await import("../../integrations/github"); const { apiIntegration } = await import("../../integrations/api"); +const { IntegrationHooks } = await import("../integrationHooks"); const makeTask = (overrides: Partial = {}): Task => ({ id: "task-1", @@ -179,11 +180,17 @@ describe("InboxService", () => { getAuthorizedLinearClient: mock(() => Promise.resolve({})), } as unknown as Pick; - service = new InboxService(inboxRepo, sandboxManager, { + const integrations = { linear: new LinearIntegration(linearTokenProvider), api: apiIntegration, github: githubIntegration, - }); + }; + service = new InboxService( + inboxRepo, + sandboxManager, + integrations, + new IntegrationHooks(integrations, inboxRepo), + ); }); describe("sweepUndelivered", () => { diff --git a/control-plane/src/services/__tests__/integrationHooks.test.ts b/control-plane/src/services/__tests__/integrationHooks.test.ts new file mode 100644 index 0000000..f4b0021 --- /dev/null +++ b/control-plane/src/services/__tests__/integrationHooks.test.ts @@ -0,0 +1,118 @@ +import { describe, expect, it, mock, spyOn } from "bun:test"; +import type { Task } from "../../db/schema"; +import type { HookCtx, Integration } from "../../integrations/types"; +import type { InboxRepository, InboxRow } from "../../repositories/inbox.repository"; + +const { IntegrationHooks } = await import("../integrationHooks"); +const { RetryableError } = await import("../../integrations/outbound"); +const { logger } = await import("../../logger"); + +const task = { id: "task-1", source: "linear" } as Task; +const row = (id: string, kind = "assignment", source = "linear") => + ({ + id, + kind, + source, + reply_targets: [{ id: "sess-1", spec: { kind: "linear_session" } }], + }) as unknown as InboxRow; +const noop: Integration = { name: "api", async deliver() {} }; +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +function hooksWith(linear: Partial, github: Partial = {}) { + const inboxRepo = { getAssignment: mock(() => Promise.resolve(row("a-1"))) }; + const integrations = { + linear: { ...noop, name: "linear" as const, ...linear }, + api: noop, + github: { ...noop, name: "github" as const, ...github }, + }; + return { + hooks: new IntegrationHooks(integrations, inboxRepo as unknown as InboxRepository), + inboxRepo, + }; +} + +describe("IntegrationHooks", () => { + it("never throws: a failing hook is logged and yields undefined", async () => { + const error = spyOn(logger, "error").mockImplementation(() => logger); + try { + const { hooks } = hooksWith({ + assignee: () => Promise.reject(new RetryableError("token fetch failed")), + }); + expect(await hooks.assignee(task, row("a-1"))).toBeUndefined(); + expect(error).toHaveBeenCalledWith( + "assignee hook failed", + expect.objectContaining({ taskId: "task-1", integration: "linear" }), + ); + } finally { + error.mockRestore(); + } + }); + + it("onPrOpened alone lets a RetryableError escape, so pr_opened holds", async () => { + const error = spyOn(logger, "error").mockImplementation(() => logger); + try { + const pr = { url: "u", repo: "r", branch: "b" }; + const retry = hooksWith({ onPrOpened: () => Promise.reject(new RetryableError("429")) }); + await expect(retry.hooks.prOpened(task, row("a-1"), pr, true)).rejects.toThrow("429"); + const other = hooksWith({ onPrOpened: () => Promise.reject(new Error("HTTP 500")) }); + expect(await other.hooks.prOpened(task, row("a-1"), pr, true)).toBeUndefined(); + } finally { + error.mockRestore(); + } + }); + + it("isCurrent() turns false once a restart rotated the assignment id", async () => { + let ctx: HookCtx | undefined; + const { hooks, inboxRepo } = hooksWith({ + async assignee(c) { + ctx = c; + return null; + }, + }); + // A comment round: the baseline is the task's assignment id at hook start. + await hooks.assignee(task, row("c-1", "comment")); + expect(await ctx?.isCurrent()).toBe(true); + inboxRepo.getAssignment.mockResolvedValue(row("a-2")); + expect(await ctx?.isCurrent()).toBe(false); + }); + + it("onTaskStarted runs at most once per assignment id", async () => { + const onTaskStarted = mock(() => Promise.resolve()); + const { hooks, inboxRepo } = hooksWith({ onTaskStarted }); + await hooks.taskStarted(task); + await hooks.taskStarted(task); + expect(onTaskStarted).toHaveBeenCalledTimes(1); + // A restart re-pends the assignment under a new id: a new run starts. + inboxRepo.getAssignment.mockResolvedValue(row("a-2")); + await hooks.taskStarted(task); + expect(onTaskStarted).toHaveBeenCalledTimes(2); + }); + + it("onRoundFinished routes by round.source; drain logs hook_abandoned for what is still running", async () => { + const warn = spyOn(logger, "warn").mockImplementation(() => logger); + try { + const finished: string[] = []; + const record = async (_ctx: unknown, _task: Task, round: InboxRow) => { + finished.push(round.id); + }; + const { hooks } = hooksWith( + // Outlives the drain deadline below. + { onRoundFinished: (ctx, t, round) => sleep(200).then(() => record(ctx, t, round)) }, + { onRoundFinished: record }, + ); + hooks.roundFinished(task, row("gh-1", "review", "github"), null, true); + hooks.roundFinished(task, row("a-1"), "failed", true); + await hooks.drain(sleep(50)); + + expect(finished).toEqual(["gh-1"]); + expect(warn.mock.calls).toEqual([ + [ + "hook abandoned at shutdown", + { event: "hook_abandoned", taskId: "task-1", hook: "onRoundFinished" }, + ], + ]); + } finally { + warn.mockRestore(); + } + }); +}); diff --git a/control-plane/src/services/__tests__/taskProcessor.service.test.ts b/control-plane/src/services/__tests__/taskProcessor.service.test.ts index 23d5d40..08f5981 100644 --- a/control-plane/src/services/__tests__/taskProcessor.service.test.ts +++ b/control-plane/src/services/__tests__/taskProcessor.service.test.ts @@ -95,6 +95,7 @@ const { githubIntegration } = await import("../../integrations/github"); const { apiIntegration } = await import("../../integrations/api"); const { logger } = await import("../../logger"); const { ReplyHolds } = await import("../replyHolds"); +const { IntegrationHooks } = await import("../integrationHooks"); const { RetryableError } = await import("../../integrations/outbound"); const makeTask = (overrides: Partial = {}): Task => ({ @@ -278,6 +279,11 @@ describe("TaskProcessorService", () => { slackNotifier = { notifyTerminal } as unknown as SlackNotifierService; holds = new ReplyHolds(); + const integrations = { + linear: new LinearIntegration(linearTokenProvider), + api: apiIntegration, + github: githubIntegration, + }; service = new TaskProcessorService( taskRepo, inboxRepo, @@ -286,13 +292,10 @@ describe("TaskProcessorService", () => { eventDispatcher, inboxService, slackNotifier, - { - linear: new LinearIntegration(linearTokenProvider), - api: apiIntegration, - github: githubIntegration, - }, + integrations, holds, ["linear", "api"], + new IntegrationHooks(integrations, inboxRepo), ); }); @@ -351,12 +354,12 @@ describe("TaskProcessorService", () => { }); describe("runTask — fresh provisioning", () => { - it("creates a sandbox, stamps sandbox_id, asserts InProgress + emits IDs thought", async () => { + it("creates a sandbox, stamps sandbox_id, asserts InProgress, emits no IDs thought", async () => { const task = makeTask({ sandboxId: null }); claimNext.mockResolvedValueOnce(task).mockResolvedValue(null); service.start(); - await waitFor(() => emitThought.mock.calls.length > 0); + await waitFor(() => attachSandboxResources.mock.calls.length > 0); await service.stop(); expect(createSandboxWithPvc).toHaveBeenCalledWith( @@ -370,12 +373,8 @@ describe("TaskProcessorService", () => { }); expect(setIssueStarted).toHaveBeenCalledWith(expect.anything(), "issue-1"); expect(emitAction).toHaveBeenCalledWith(expect.anything(), "sess-1", "Provisioning", "infra"); - // characterization: changes in phase 1b (spec §13: the "Sandbox provisioned" thought is removed) - expect(emitThought).toHaveBeenCalledWith( - expect.anything(), - "sess-1", - "Sandbox provisioned.\n- sandboxId: `sbx-1`\n- pvcName: `devbox-pvc-task-1-aaaaaa`", - ); + // The post-create "Sandbox provisioned" thought is gone (spec §13). + expect(emitThought).not.toHaveBeenCalled(); }); it("logs and still provisions when the team has no started state", async () => { @@ -388,7 +387,7 @@ describe("TaskProcessorService", () => { try { service.start(); - await waitFor(() => emitThought.mock.calls.length > 0); + await waitFor(() => attachSandboxResources.mock.calls.length > 0); await service.stop(); expect(error).toHaveBeenCalledTimes(1); @@ -1133,7 +1132,7 @@ describe("TaskProcessorService", () => { expect(dispatcherHandle.mock.calls.length).toBe(1); }); - it("a dispatcher throw (e.g. a Linear token outage) holds the cursor and fails nothing", async () => { + it("a dispatcher throw (e.g. a database error) holds the cursor and fails nothing", async () => { const task = makeTask({ sandboxId: "sbx-1" }); listInFlightOwnedBy.mockResolvedValueOnce([task]).mockResolvedValue([]); readEvents diff --git a/control-plane/src/services/eventDispatcher.ts b/control-plane/src/services/eventDispatcher.ts index a668d0d..2768c09 100644 --- a/control-plane/src/services/eventDispatcher.ts +++ b/control-plane/src/services/eventDispatcher.ts @@ -1,12 +1,5 @@ import type { LinearClient } from "@linear/sdk"; -import { - addPrAssignees, - getUserById, - parsePrUrl, - postIssueComment, - requestReviewers, -} from "../clients/github"; -import { attachUrlToIssue, getIssueHumanAssignee, uploadIssueAttachment } from "../clients/linear"; +import { addPrAssignees, parsePrUrl, postIssueComment } from "../clients/github"; import { config } from "../config"; import type { Task } from "../db/schema"; import { SEND_TIMEOUT_MS, withTimeout } from "../integrations/http"; @@ -16,6 +9,7 @@ import { logger } from "../logger"; import type { InboxRepository, InboxRow } from "../repositories/inbox.repository"; import type { TaskRepository } from "../repositories/task.repository"; import type { AgentEmit, HarnessEvent } from "../types/harnessEvent"; +import type { IntegrationHooks } from "./integrationHooks"; import type { LinearTokenProvider } from "./linearTokenProvider"; import type { ReplyHolds } from "./replyHolds"; import type { SandboxManager } from "./sandboxManager.service"; @@ -26,10 +20,10 @@ import type { SlackNotifierService } from "./slackNotifier.service"; // - `dropped`: the event's send timed out or failed for good. reconcileOne // advances past it, then stops this task's events for the tick, so a slow // or failing platform can't eat the tick (§4 I14). -// - `hold`: a send failed before anything went out (RetryableError); the hold -// is recorded in ReplyHolds. reconcileOne stops this task's events BEFORE -// advancing the cursor, so the event is retried until it is sent or fails -// for good. +// - `hold`: a send (or the owner's onPrOpened) failed before anything went out +// (RetryableError); the hold is recorded in ReplyHolds. reconcileOne stops +// this task's events BEFORE advancing the cursor, so the event is retried +// until it is sent or fails for good. // - `self_evict`: an owner-guarded write returned 0 rows — the lease was taken // over mid-tick — so reconcileOne must stop processing this task BEFORE // advancing the cursor (matching recordEventCursor's 0-row return). @@ -60,7 +54,9 @@ function onboardingCommentBody(botLogin: string, taskKey: string): string { // // At-least-once dispatch semantics: the cursor advances AFTER handle() // completes. A crash between a side-effect and the cursor commit re-fires the -// event on the next reconcile tick — all side-effects are idempotent. +// event on the next reconcile tick: pr_opened and completed side effects are +// idempotent or run only the first time (appendPrUrl's firstTime, +// markResponded's first transition). export class EventDispatcher { constructor( private taskRepo: TaskRepository, @@ -70,15 +66,16 @@ export class EventDispatcher { private slackNotifier: SlackNotifierService, private integrations: Record, private holds: ReplyHolds, + private hooks: IntegrationHooks, ) {} async handle(event: HarnessEvent, task: Task): Promise { try { return await this.apply(event, task); } catch (err) { - // A task-scoped call (the Linear client on pr_opened / completed) failed - // before anything was sent: hold on the task's own integration. deliver - // holds on the round's integration itself. + // A task-scoped call (the owner's onPrOpened, or the Linear client for + // Slack on completed) failed before anything was sent: hold on the + // task's own integration. deliver holds on the round's integration itself. if (!(err instanceof RetryableError)) throw err; this.holds.hold(task.id, event, task.source, err); return "hold"; @@ -129,14 +126,10 @@ export class EventDispatcher { return this.deliver(event, inboxRow, task); case "pr_opened": { - // The Linear attach and assignee mirror below need a client. Fetched - // first, so a Linear token failure holds the event before any side - // effect. - const linearClient = await this.linearClientFor(task); // appendPrUrl is owner-guarded; a 0-row return means the lease was - // taken over (stale-lease takeover). Self-evict BEFORE attachUrlToIssue + // taken over (stale-lease takeover). Self-evict BEFORE any side effect // and before the cursor advance — a sibling pod now owns this task. - const appended = await this.taskRepo.appendPrUrl({ + const [appended] = await this.taskRepo.appendPrUrl({ taskId: task.id, ownerId: config.podId, prUrl: event.pr_url, @@ -144,38 +137,43 @@ export class EventDispatcher { branch: event.branch, commitSha: event.commit_sha, }); - if (appended.length === 0) { + if (!appended) { logger.warn("appendPrUrl returned 0 rows (lease lost); self-evicting", { taskId: task.id, prUrl: event.pr_url, }); return "self_evict"; } - if (linearClient) { - try { - await attachUrlToIssue(linearClient, task.externalId, event.pr_url, event.repo); - } catch (err) { - logErr(`attachUrlToIssue failed for ${event.repo}`, err); - } + // Only the dispatch that added the URL posts the onboarding comment + // and mirrors the assignee; a replay (the cursor advances only after + // handle() returns) finds the URL already there. + if (appended.firstTime) { + // Onboarding comment so users discover they can drive harness with + // @harness comments + reviews. Best-effort, not load-bearing. + await this.postOnboardingComment(event, task); + // Detached (not awaited): up to 4 upstream round-trips must not + // stall tick() toward the lease window — the same hazard + // reconcileFilesTimeoutMs exists for. Nothing downstream reads the + // result, and the method never rejects (see its contract). + this.hooks.detach(task.id, "assignee", this.mirrorPrAssignee(event, task, inboxRow)); } - // Onboarding comment so users discover they can drive harness with - // @harness comments + reviews. Best-effort, no dedupe: at-least-once - // dispatch can re-fire on cursor-replay → duplicate help comments are - // possible. Acceptable: it's a help message, not load-bearing. - await this.postOnboardingComment(event, task); - // Mirror the Linear ticket's human assignee onto the PR. Detached - // (not awaited): up to 4 unbounded upstream round-trips (2× Linear, - // 2× GitHub) must not stall tick() toward the lease window — the same - // hazard reconcileFilesTimeoutMs exists for. Nothing downstream reads - // the result, and the method never rejects (see its contract). - void this.assignPrToLinearAssignee(event, task, linearClient); + // The owner's onPrOpened (Linear: attach the PR to the issue) runs on + // every dispatch and is idempotent. Unlike other hooks, its + // RetryableError escapes to handle(), which holds pr_opened. + await this.hooks.prOpened( + task, + inboxRow, + { url: event.pr_url, repo: event.repo, branch: event.branch }, + appended.firstTime, + ); break; } case "completed": { // The Slack notify below needs a Linear client for a Linear task's - // assignment round. Fetched before marking, so a Linear token failure - // holds the event with the row still open. + // assignment round (until Slack moves to assignee()). Fetched before + // marking, so a Linear token failure holds the event with the row + // still open. const linearClient = inboxRow?.kind === "assignment" ? await this.linearClientFor(task) : null; // Bare terminal marker — the user-facing message already went out via @@ -184,28 +182,26 @@ export class EventDispatcher { // No inbox row → nothing to mark. `marked` is true only on the FIRST // transition (a re-fire on cursor-replay returns false), which gates // at-most-once completion side effects below. - const marked = inboxRow - ? await this.markInboxResponded(inboxRow, event.status ?? null) - : false; - // Kind-specific side effect: assignment rounds upload a run archive to - // the Linear issue while the sandbox is still alive (it lives until - // the idle reaper). Fire-and-forget — the dispatch loop must not - // block on an in-sandbox archive command (~tens of seconds). - if (inboxRow?.kind === "assignment") { - void this.attachRunArchiveToLinearIssue(task); - // Slack completion update to the configured channel. - // Assignment-only (the top-level task, not comment/review rounds) and - // gated on the first inbox transition so a cursor-replay can't double- - // post. Fire-and-forget: a Slack outage must not stall the dispatch - // loop. The notifier no-ops when Slack is disabled. - if (marked && linearClient) { - void this.slackNotifier - .notifyTerminal(linearClient, task, event.status ?? "completed") - .catch((err) => logErr("slack completion notify failed", err)); - } - } - if (marked && inboxRow?.kind === "review") { - this.requestReviewFromOriginalReviewer(inboxRow, task); + if (!inboxRow) break; + const outcome = event.status ?? null; + const marked = await this.markInboxResponded(inboxRow, outcome); + // The round's integration finishes it (Linear: the run archive; GitHub: + // the review re-request), detached: the archive is an in-sandbox + // command (~tens of seconds), run only if the hook asks for it while + // the sandbox is still alive (it lives until the idle reaper). + const sandboxId = task.sandboxId; + this.hooks.roundFinished(task, inboxRow, outcome, marked, async () => + sandboxId ? this.sandboxManager.extractArchive(sandboxId) : null, + ); + // Slack completion update to the configured channel. + // Assignment-only (the top-level task, not comment/review rounds) and + // gated on the first inbox transition so a cursor-replay can't double- + // post. Fire-and-forget: a Slack outage must not stall the dispatch + // loop. The notifier no-ops when Slack is disabled. + if (marked && linearClient) { + void this.slackNotifier + .notifyTerminal(linearClient, task, event.status ?? "completed") + .catch((err) => logErr("slack completion notify failed", err)); } break; } @@ -293,7 +289,7 @@ export class EventDispatcher { // Returns true iff THIS call transitioned the row to 'responded' (0-row // no-ops and already-responded re-fires return false). Callers use the // return as a first-transition gate for at-most-once side effects (the - // Slack completion notify). + // Slack completion notify, and onRoundFinished's firstTransition). private async markInboxResponded(inboxRow: InboxRow, outcome: string | null): Promise { if (inboxRow.status === "responded") return false; // already done — idempotent re-fire try { @@ -320,49 +316,6 @@ export class EventDispatcher { } } - private requestReviewFromOriginalReviewer( - inboxRow: Extract, - task: Task, - ): void { - const payload = inboxRow.payload as Record | null | undefined; - const reviewerLoginRaw = payload?.reviewer_login; - const reviewerLogin = typeof reviewerLoginRaw === "string" ? reviewerLoginRaw.trim() : ""; - if (!reviewerLogin) { - logger.info("review re-request skipped", { - event: "review_rerequest.result", - status: "skipped", - reason: "missing_reviewer_login", - taskId: task.id, - inboxId: inboxRow.id, - }); - return; - } - - const pr = reviewPrCoordinates(inboxRow); - if (!pr) { - logger.warn("review re-request skipped", { - event: "review_rerequest.result", - status: "skipped", - reason: "missing_pr_coordinates", - taskId: task.id, - inboxId: inboxRow.id, - reviewerLogin, - }); - return; - } - - void requestReviewers(pr.owner, pr.repo, pr.prNumber, [reviewerLogin]).catch((err) => - logErr("request reviewers failed", err, { - taskId: task.id, - inboxId: inboxRow.id, - owner: pr.owner, - repo: pr.repo, - prNumber: pr.prNumber, - reviewerLogin, - }), - ); - } - private async postOnboardingComment( event: Extract, task: Task, @@ -380,68 +333,60 @@ export class EventDispatcher { ).catch((err) => logErr("onboarding comment post failed", err)); } - // Mirrors the Linear ticket's human assignee onto the GitHub PR as assignee. - // Assignee is the right primitive here: harness PRs open as DRAFTS and GitHub - // rejects requested_reviewers on drafts. Identity comes from Linear's own - // user↔GitHub mapping (User.gitHubUserId, populated when a member connects - // GitHub under Linear Settings → Connected accounts) — never email matching. + // Mirrors the task's assignee (the owner integration's assignee()) onto the + // GitHub PR as assignee. Assignee is the right primitive here: harness PRs + // open as DRAFTS and GitHub rejects requested_reviewers on drafts. // // Best-effort by contract: never throws, a failure must not block dispatch or - // the cursor. Idempotent on cursor-replay (POST assignees re-add is a GitHub - // no-op) — with the accepted quirk that a human who manually unassigned - // themselves gets re-added on a rare replay. - private async assignPrToLinearAssignee( + // the cursor. Runs once per PR (firstTime). + private async mirrorPrAssignee( event: Extract, task: Task, - linearClient: LinearClient | null, + round: InboxRow | null, ): Promise { - if (!linearClient) return; // GH-sourced task — no Linear identity to mirror. const pr = parsePrUrl(event.pr_url); if (!pr) return; // not a github.com PR URL — skip + // undefined: no assignee() (API tasks), or the lookup failed (logged). + const person = await this.hooks.assignee(task, round); + if (person === undefined) return; + if (!person) { + logger.info("PR assignee mirror skipped", { + event: "pr_assignee.result", + status: "skipped", + reason: "no_human_assignee", + taskId: task.id, + prUrl: event.pr_url, + }); + return; + } + if (typeof person.codeHost !== "object") { + // Linear's fix path: the member connects GitHub in Linear Settings → + // Connected accounts. Log the display name, not the email. + logger.info("PR assignee mirror skipped", { + event: "pr_assignee.result", + status: "skipped", + reason: person.codeHost === "unlinked" ? "no_github_mapping" : "unsupported", + taskId: task.id, + prUrl: event.pr_url, + displayName: person.displayName, + }); + return; + } + const login = person.codeHost.githubLogin; + if (!login) { + // getUserById normalizes a malformed 2xx body to login: "" — don't POST + // an empty assignee and misreport it as no_push_access. + logger.warn("PR assignee mirror skipped", { + event: "pr_assignee.result", + status: "skipped", + reason: "unresolved_login", + taskId: task.id, + prUrl: event.pr_url, + displayName: person.displayName, + }); + return; + } try { - const human = await getIssueHumanAssignee(linearClient, task.externalId); - if (!human) { - logger.info("PR assignee mirror skipped", { - event: "pr_assignee.result", - status: "skipped", - reason: "no_human_assignee", - taskId: task.id, - prUrl: event.pr_url, - }); - return; - } - if (!human.gitHubUserId) { - // Fix path: the member connects GitHub in Linear Settings → Connected - // accounts. Log id + displayName, not email. - logger.info("PR assignee mirror skipped", { - event: "pr_assignee.result", - status: "skipped", - reason: "no_github_mapping", - taskId: task.id, - prUrl: event.pr_url, - linearUserId: human.id, - linearDisplayName: human.displayName, - }); - return; - } - // Defensive about the field's format: numeric → GitHub account id - // (resolve to the current login), anything else → already a login. - const login = /^\d+$/.test(human.gitHubUserId) - ? (await getUserById(human.gitHubUserId)).login - : human.gitHubUserId; - if (!login) { - // getUserById normalizes a malformed 2xx body to login: "" — don't POST - // an empty assignee and misreport it as no_push_access. - logger.warn("PR assignee mirror skipped", { - event: "pr_assignee.result", - status: "skipped", - reason: "unresolved_login", - taskId: task.id, - prUrl: event.pr_url, - linearUserId: human.id, - }); - return; - } const { assigned } = await addPrAssignees(pr.owner, pr.repo, pr.prNumber, [login]); // GitHub logins are case-insensitive but the response echoes canonical // casing — compare case-folded so a cased mapping isn't a false drop. @@ -467,32 +412,7 @@ export class EventDispatcher { assigned, }); } catch (err) { - logErr("assignPrToLinearAssignee failed", err, { taskId: task.id, prUrl: event.pr_url }); - } - } - - // Best-effort run-archive upload. Runs the harness's `archive` command in the - // still-live sandbox, then uploads the tar.gz to the Linear issue as an - // attachment. Fully swallowed — the archive is a forensic nicety, never - // load-bearing; a failure must not affect dispatch. GH-source tasks have no - // Linear issue to attach to, so they are skipped. - private async attachRunArchiveToLinearIssue(task: Task): Promise { - if (!task.sandboxId || task.source !== "linear") return; - try { - const bytes = await this.sandboxManager.extractArchive(task.sandboxId); - if (!bytes || bytes.byteLength === 0) return; - const client = await this.linearTokenProvider.getAuthorizedLinearClient(); - await uploadIssueAttachment(client, { - issueId: task.externalId, - title: "Harness run archive", - subtitle: `Task ${task.id}`, - filename: `harness-run-${task.id}.tar.gz`, - contentType: "application/gzip", - bytes, - metadata: { taskId: task.id }, - }); - } catch (err) { - logErr(`attachRunArchiveToLinearIssue failed for task ${task.id}`, err); + logErr("PR assignee mirror failed", err, { taskId: task.id, prUrl: event.pr_url }); } } } @@ -521,28 +441,6 @@ function labelAgentEmit(event: AgentEmit, round: InboxRow): Outbound { } } -function reviewPrCoordinates( - inboxRow: Extract, -): { owner: string; repo: string; prNumber: number } | null { - const pr = inboxRow.payload.pr as Record | undefined; - const owner = typeof pr?.owner === "string" ? pr.owner : ""; - const repo = typeof pr?.repo === "string" ? pr.repo : ""; - const prNumber = typeof pr?.prNumber === "number" ? pr.prNumber : Number.NaN; - if (owner && repo && Number.isFinite(prNumber)) { - return { owner, repo, prNumber }; - } - - const target = inboxRow.reply_targets.find( - (t) => t.spec.kind === "review_body" || t.spec.kind === "review_comment", - ); - if (target && "owner" in target.spec) { - return { owner: target.spec.owner, repo: target.spec.repo, prNumber: target.spec.prNumber }; - } - - const prUrl = typeof pr?.pr_url === "string" ? pr.pr_url : ""; - return prUrl ? parsePrUrl(prUrl) : null; -} - // Untrusted values (e.g. sandbox-written pr_url) belong in `meta`, never in // `msg`: metadata is JSON-stringified by the logger while the message is // emitted raw, so a newline in an interpolated message would forge log lines. diff --git a/control-plane/src/services/inbox.service.ts b/control-plane/src/services/inbox.service.ts index 08d0baa..21ec14c 100644 --- a/control-plane/src/services/inbox.service.ts +++ b/control-plane/src/services/inbox.service.ts @@ -12,6 +12,7 @@ import type { } from "../repositories/inbox.repository"; import { taskIdentity } from "../repositories/inbox.repository"; import type { InboxMessage } from "../types/inboxMessage"; +import type { IntegrationHooks } from "./integrationHooks"; import type { SandboxManager } from "./sandboxManager.service"; // InboxService owns inbox enqueue (comment/review/Linear-comment), the sweep @@ -26,6 +27,7 @@ export class InboxService { private inboxRepo: InboxRepository, private sandboxManager: SandboxManager, private integrations: Record, + private hooks: IntegrationHooks, ) {} // ──── Enqueue paths ──── @@ -307,7 +309,7 @@ export class InboxService { for (const row of rows) { const latest = await this.inboxRepo.getById(row.id); if (!latest || (latest.status !== "pending" && latest.status !== "delivered")) continue; - await this.handlePvcMissing(latest); + await this.handlePvcMissing(task, latest); } } @@ -318,8 +320,9 @@ export class InboxService { // dedupe (unlike failTask); reply_targets[0] is the round's thread, and the // integration words the notice by round kind. At most once: there is no // cursor to hold, so a delivery error (RetryableError included) is logged and - // dropped, and the task's remaining rows still dead-letter. - private async handlePvcMissing(entry: InboxRow): Promise { + // dropped, and the task's remaining rows still dead-letter. A dead-lettered + // assignment also finishes its round as failed (onRoundFinished, detached). + private async handlePvcMissing(task: Task, entry: InboxRow): Promise { const marked = await this.inboxRepo .markFailed({ inboxId: entry.id, @@ -331,8 +334,10 @@ export class InboxService { logErr(`markFailed failed for inbox ${entry.id}`, err); return []; }); + if (marked.length === 0) return; + if (entry.kind === "assignment") this.hooks.roundFinished(task, entry, "failed", true); const target = entry.reply_targets[0]; - if (marked.length === 0 || !target) return; + if (!target) return; // A source this build has no integration for (say, rows written by a newer // build before a rollback) gets no notice; its row is still failed. const integration: Integration | undefined = this.integrations[entry.source]; diff --git a/control-plane/src/services/integrationHooks.ts b/control-plane/src/services/integrationHooks.ts new file mode 100644 index 0000000..d7ffc0f --- /dev/null +++ b/control-plane/src/services/integrationHooks.ts @@ -0,0 +1,154 @@ +import type { Task } from "../db/schema"; +import { withTimeout } from "../integrations/http"; +import { RetryableError } from "../integrations/outbound"; +import type { + HookCtx, + Integration, + IntegrationName, + Outcome, + Person, + PrRef, +} from "../integrations/types"; +import { logger } from "../logger"; +import type { InboxRepository, InboxRow } from "../repositories/inbox.repository"; + +// Upper bound on one hook call (§4). +const HOOK_TIMEOUT_MS = { + onTaskStarted: 20_000, + onPrOpened: 30_000, + onRoundFinished: 120_000, + assignee: 15_000, +} as const; +type HookName = keyof typeof HOOK_TIMEOUT_MS; + +// Runs integrations' lifecycle hooks (§4). onTaskStarted, onPrOpened and +// assignee go to the task's own integration; onRoundFinished goes to the +// round's. Each call runs under its hook's timeout with an abort signal and +// never throws: a failure is logged and the caller goes on. The one exception +// is a RetryableError from onPrOpened, which escapes so the dispatcher holds +// pr_opened as it holds a send (the hook must be idempotent). Detached work is +// tracked so TaskProcessor.stop() can drain it. +export class IntegrationHooks { + private readonly detached = new Map, { taskId: string; hook: HookName }>(); + // Assignment ids whose onTaskStarted already ran on this pod: a provision + // deferred by a transient error runs again for the same assignment. + // ponytail: one id per started task for the pod's lifetime; prune if that ever matters. + private readonly started = new Set(); + + constructor( + private integrations: Record, + private inboxRepo: Pick, + ) {} + + // Fresh provision, before the sandbox is created. Awaited; no hold. + async taskStarted(task: Task): Promise { + const owner = this.integrations[task.source]; + if (!owner.onTaskStarted) return; + const round = await this.inboxRepo.getAssignment(task.id).catch((err) => { + logErr("onTaskStarted skipped: assignment lookup failed", err, { taskId: task.id }); + return null; + }); + const target = round?.reply_targets[0]; + if (!round || !target || this.started.has(round.id)) return; + this.started.add(round.id); + await this.callHook("onTaskStarted", owner, task, round, (ctx) => + owner.onTaskStarted?.(ctx, task, target), + ); + } + + // pr_opened, after appendPrUrl, awaited before the cursor advances. Throws + // RetryableError (nothing was sent), which the dispatcher turns into a hold. + async prOpened(task: Task, round: InboxRow | null, pr: PrRef, firstTime: boolean): Promise { + const owner = this.integrations[task.source]; + await this.callHook("onPrOpened", owner, task, round, (ctx) => + owner.onPrOpened?.({ ...ctx, firstTime }, task, pr), + ); + } + + // The task's assignee, for detached work. undefined when the integration + // has no assignee or the lookup failed (logged here). + assignee(task: Task, round: InboxRow | null): Promise { + const owner = this.integrations[task.source]; + return this.callHook("assignee", owner, task, round, (ctx) => owner.assignee?.(ctx, task)); + } + + // Detached. Called (a) on `completed`, with firstTransition = whether that + // call marked the row responded, and (b) when a task failure or a missing + // workspace dead-letters an assignment row: outcome failed, first + // transition, no archive. Integrations gate their posts on firstTransition. + roundFinished( + task: Task, + round: InboxRow, + outcome: Outcome | null, + firstTransition: boolean, + archive: () => Promise = async () => null, + ): void { + // A source this build has no integration for (rows written by a newer + // build before a rollback) has no hook. + const owner: Integration | undefined = this.integrations[round.source]; + if (!owner?.onRoundFinished) return; + this.detach( + task.id, + "onRoundFinished", + this.callHook("onRoundFinished", owner, task, round, (ctx) => + owner.onRoundFinished?.({ ...ctx, firstTransition, archive }, task, round, outcome), + ), + ); + } + + // Tracks detached work (a hook, or core work that calls one) for drain(). + detach(taskId: string, hook: HookName, work: Promise): void { + const p: Promise = work + .then( + () => {}, + (err) => logErr(`detached ${hook} work failed`, err, { taskId }), + ) + .finally(() => this.detached.delete(p)); + this.detached.set(p, { taskId, hook }); + } + + // Waits for detached work until `deadline` settles, then logs what is still + // running; the process exits right after. + async drain(deadline: Promise): Promise { + await Promise.race([Promise.allSettled(this.detached.keys()), deadline]); + for (const { taskId, hook } of this.detached.values()) { + logger.warn("hook abandoned at shutdown", { event: "hook_abandoned", taskId, hook }); + } + } + + private async callHook( + hook: HookName, + owner: Integration, + task: Task, + round: InboxRow | null, + run: (ctx: HookCtx) => Promise | undefined, + ): Promise { + if (!owner[hook]) return undefined; + try { + return await withTimeout(HOOK_TIMEOUT_MS[hook], async (signal) => { + const startedOn = await this.assignmentId(task.id, round); + return run({ + signal, + log: logger, + isCurrent: async () => (await this.assignmentId(task.id)) === startedOn, + }); + }); + } catch (err) { + if (hook === "onPrOpened" && err instanceof RetryableError) throw err; + logErr(`${hook} hook failed`, err, { taskId: task.id, integration: owner.name }); + return undefined; + } + } + + // The task's assignment id: a restart re-pends the assignment row under a + // new id. When the round in hand is the assignment, its id is that id. + private async assignmentId(taskId: string, round?: InboxRow | null): Promise { + if (round?.kind === "assignment") return round.id; + return (await this.inboxRepo.getAssignment(taskId))?.id ?? null; + } +} + +function logErr(msg: string, err: unknown, meta: Record): void { + const e = err instanceof Error ? err : new Error(String(err)); + logger.error(msg, { ...meta, message: e.message, stack: e.stack }); +} diff --git a/control-plane/src/services/taskProcessor.service.ts b/control-plane/src/services/taskProcessor.service.ts index cfbca5b..a7cef05 100644 --- a/control-plane/src/services/taskProcessor.service.ts +++ b/control-plane/src/services/taskProcessor.service.ts @@ -1,5 +1,5 @@ import type { SandboxInfo } from "@alibaba-group/opensandbox"; -import { emitAction, emitThought, setIssueStarted } from "../clients/linear"; +import { emitAction, setIssueStarted } from "../clients/linear"; import { config } from "../config"; import type { Task } from "../db/schema"; import { PvcNotFoundError, SandboxCreateError } from "../errors"; @@ -11,6 +11,7 @@ import type { TaskRepository } from "../repositories/task.repository"; import type { EventDispatcher } from "./eventDispatcher"; import { extractExperimentalFlags } from "./experimentalFlags"; import type { InboxService } from "./inbox.service"; +import type { IntegrationHooks } from "./integrationHooks"; import type { LinearTokenProvider } from "./linearTokenProvider"; import type { ReplyHolds } from "./replyHolds"; import { type PodStatus, SandboxManager } from "./sandboxManager.service"; @@ -48,6 +49,7 @@ export class TaskProcessorService { // Sources this build has an integration for; claimNext and // listInFlightOwnedBy see only these tasks (§10). private readonly taskSources: Task["source"][]; + private readonly hooks: IntegrationHooks; constructor( taskRepo: TaskRepository, @@ -60,6 +62,7 @@ export class TaskProcessorService { integrations: Record, holds: ReplyHolds, taskSources: Task["source"][], + hooks: IntegrationHooks, ) { this.taskRepo = taskRepo; this.inboxRepo = inboxRepo; @@ -71,6 +74,7 @@ export class TaskProcessorService { this.integrations = integrations; this.holds = holds; this.taskSources = taskSources; + this.hooks = hooks; } start(): void { @@ -88,9 +92,12 @@ export class TaskProcessorService { await this.loopPromise.catch(() => {}); } + // In-flight provisions, then detached hooks (some started by those + // provisions), share one shutdownDrainMs budget. const drain = Promise.allSettled([...this.inflight]); const timeout = new Promise((resolve) => setTimeout(resolve, config.shutdownDrainMs)); await Promise.race([drain, timeout]); + await this.hooks.drain(timeout); // Backdate own heartbeat so any survivors become immediately reclaimable // instead of waiting the full LEASE_TIMEOUT_MS. @@ -288,6 +295,10 @@ export class TaskProcessorService { } } + // The owner integration's onTaskStarted (none today), at most once per + // assignment on this pod. Never throws. + await this.hooks.taskStarted(task); + // WORKSPACE is the only task-specific input passed at creation; the daemon // derives its .harness dir from it and reads everything else from the inbox file // payload. @@ -324,20 +335,7 @@ export class TaskProcessorService { }); return; } - - if (isLinear) { - try { - const client = await this.linearTokenProvider.getAuthorizedLinearClient(); - await emitThought( - client, - replyTargetId, - `Sandbox provisioned.\n- sandboxId: \`${sandboxId}\`\n- pvcName: \`${pvcName}\``, - ); - } catch (err) { - logErr(`emitThought(provisioned-ids) failed for task ${task.id}`, err); - } - } - // No further Linear emit here — the daemon emits the Planning action via + // No Linear emit here — the daemon emits the Planning action via // events.jsonl once it starts processing the inbox file. } catch (err) { // A throw after a successful create (e.g. attachSandboxResources hit a @@ -794,6 +792,7 @@ export class TaskProcessorService { }); const failedIds = new Set(failed.map((r) => r.id)); const failedRows = nonTerminal.filter((r) => failedIds.has(r.id)); + const assignment = failedRows.find((r) => r.kind === "assignment"); if (failed.length > 0) { logger.warn("inbox rows dead-lettered on task failure", { taskId: task.id, @@ -833,6 +832,9 @@ export class TaskProcessorService { } } + // The assignment's integration finishes its round as failed (detached). + if (assignment) this.hooks.roundFinished(task, assignment, "failed", true); + // Kill the sandbox the task ACTUALLY holds, read from the DB — NOT the // dispatch-time `task` closure, whose sandbox_id can be stale (a recreate // persists a fresh sandbox_id via setSandboxId after dispatch; the closure @@ -852,7 +854,7 @@ export class TaskProcessorService { // ever notify Slack. Tying it to this failTaskInbox transition also keeps it // at-most-once. Fire-and-forget: a Slack outage must not affect teardown; // the notifier no-ops when Slack is disabled. - if (failedRows.some((r) => r.kind === "assignment") && task.source === "linear") { + if (assignment && task.source === "linear") { const client = await this.linearTokenProvider.getAuthorizedLinearClient().catch((e) => { logErr("failTask: getAuthorizedLinearClient failed", e); return null; diff --git a/deploy/helm/devbox/README.md b/deploy/helm/devbox/README.md index bdc9b84..b0ff926 100644 --- a/deploy/helm/devbox/README.md +++ b/deploy/helm/devbox/README.md @@ -25,6 +25,7 @@ docker run --rm -v "$PWD/deploy/helm/devbox:/helm-docs" jnorwood/helm-docs:v1.14 | controlPlane.resources.limits.memory | string | `"1Gi"` | | | controlPlane.resources.requests.cpu | string | `"100m"` | | | controlPlane.resources.requests.memory | string | `"256Mi"` | | +| controlPlane.terminationGracePeriodSeconds | int | `120` | Seconds between SIGTERM and SIGKILL. At least one reconcile tick (up to 90s when a platform is slow) plus SHUTDOWN_DRAIN_MS (10s) for in-flight provisions and integration hooks; raise it with SHUTDOWN_DRAIN_MS. | | database.existingSecret | string | `""` | With postgresql.enabled false: a Secret with key `DATABASE_URL`. | | extraEnv | list | `[]` | Extra control-plane env. Only explicitly allowlisted HARNESS_* settings reach sandboxes; never put a real secret in a forwarded name. | | extraManifests | list | `[]` | Extra manifests, rendered with tpl. | diff --git a/deploy/helm/devbox/templates/control-plane.yaml b/deploy/helm/devbox/templates/control-plane.yaml index 0cda6f4..f05e30b 100644 --- a/deploy/helm/devbox/templates/control-plane.yaml +++ b/deploy/helm/devbox/templates/control-plane.yaml @@ -62,6 +62,8 @@ spec: serviceAccountName: devbox-control-plane {{ include "devbox.controlPlaneImagePullSecrets" . | indent 6 }} securityContext: { runAsNonRoot: true, runAsUser: 1000, runAsGroup: 1000 } + # Shutdown finishes the current tick, then drains for SHUTDOWN_DRAIN_MS. + terminationGracePeriodSeconds: {{ .Values.controlPlane.terminationGracePeriodSeconds }} containers: - name: control-plane image: {{ include "devbox.controlPlaneImage" . }} diff --git a/deploy/helm/devbox/tests/golden/byo-secrets.json b/deploy/helm/devbox/tests/golden/byo-secrets.json index 1ac8a3b..d173989 100644 --- a/deploy/helm/devbox/tests/golden/byo-secrets.json +++ b/deploy/helm/devbox/tests/golden/byo-secrets.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 120, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/defaults.json b/deploy/helm/devbox/tests/golden/defaults.json index 9b71ca6..481da19 100644 --- a/deploy/helm/devbox/tests/golden/defaults.json +++ b/deploy/helm/devbox/tests/golden/defaults.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 120, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/external-db.json b/deploy/helm/devbox/tests/golden/external-db.json index a3c5f7c..2fe8982 100644 --- a/deploy/helm/devbox/tests/golden/external-db.json +++ b/deploy/helm/devbox/tests/golden/external-db.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 120, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/github-app-external.json b/deploy/helm/devbox/tests/golden/github-app-external.json index 97d7894..9e27ad1 100644 --- a/deploy/helm/devbox/tests/golden/github-app-external.json +++ b/deploy/helm/devbox/tests/golden/github-app-external.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 120, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/github-app.json b/deploy/helm/devbox/tests/golden/github-app.json index ceea984..2e4226e 100644 --- a/deploy/helm/devbox/tests/golden/github-app.json +++ b/deploy/helm/devbox/tests/golden/github-app.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 120, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/ingress-api.json b/deploy/helm/devbox/tests/golden/ingress-api.json index 44d601d..a881601 100644 --- a/deploy/helm/devbox/tests/golden/ingress-api.json +++ b/deploy/helm/devbox/tests/golden/ingress-api.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 120, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/ingress-both.json b/deploy/helm/devbox/tests/golden/ingress-both.json index dcea67e..e050ec1 100644 --- a/deploy/helm/devbox/tests/golden/ingress-both.json +++ b/deploy/helm/devbox/tests/golden/ingress-both.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 120, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/ingress-webhooks.json b/deploy/helm/devbox/tests/golden/ingress-webhooks.json index e628228..b69a1a0 100644 --- a/deploy/helm/devbox/tests/golden/ingress-webhooks.json +++ b/deploy/helm/devbox/tests/golden/ingress-webhooks.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 120, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/linear.json b/deploy/helm/devbox/tests/golden/linear.json index 1a51e61..31a36a5 100644 --- a/deploy/helm/devbox/tests/golden/linear.json +++ b/deploy/helm/devbox/tests/golden/linear.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 120, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/llm-stages.json b/deploy/helm/devbox/tests/golden/llm-stages.json index 13d8c9c..7ecfe9c 100644 --- a/deploy/helm/devbox/tests/golden/llm-stages.json +++ b/deploy/helm/devbox/tests/golden/llm-stages.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 120, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/replicas.json b/deploy/helm/devbox/tests/golden/replicas.json index 9a34f4b..018083d 100644 --- a/deploy/helm/devbox/tests/golden/replicas.json +++ b/deploy/helm/devbox/tests/golden/replicas.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 180, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/sandbox-options.json b/deploy/helm/devbox/tests/golden/sandbox-options.json index 71465c5..348d5ef 100644 --- a/deploy/helm/devbox/tests/golden/sandbox-options.json +++ b/deploy/helm/devbox/tests/golden/sandbox-options.json @@ -607,6 +607,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 120, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/slack.json b/deploy/helm/devbox/tests/golden/slack.json index 389362a..87cdd03 100644 --- a/deploy/helm/devbox/tests/golden/slack.json +++ b/deploy/helm/devbox/tests/golden/slack.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 120, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/values/replicas.yaml b/deploy/helm/devbox/tests/values/replicas.yaml index d801db2..3499f0c 100644 --- a/deploy/helm/devbox/tests/values/replicas.yaml +++ b/deploy/helm/devbox/tests/values/replicas.yaml @@ -1,2 +1,2 @@ -controlPlane: { replicas: 2 } +controlPlane: { replicas: 2, terminationGracePeriodSeconds: 180 } ironProxy: { replicas: 2 } diff --git a/deploy/helm/devbox/values.schema.json b/deploy/helm/devbox/values.schema.json index d28cf9e..ee98c75 100644 --- a/deploy/helm/devbox/values.schema.json +++ b/deploy/helm/devbox/values.schema.json @@ -149,7 +149,8 @@ "properties": { "replicas": { "type": "integer", "minimum": 1 }, "publicBaseUrl": { "type": "string" }, - "resources": { "type": "object" } + "resources": { "type": "object" }, + "terminationGracePeriodSeconds": { "type": "integer", "minimum": 0 } } }, "postgresql": { diff --git a/deploy/helm/devbox/values.yaml b/deploy/helm/devbox/values.yaml index 4001deb..29c47df 100644 --- a/deploy/helm/devbox/values.yaml +++ b/deploy/helm/devbox/values.yaml @@ -149,6 +149,10 @@ controlPlane: resources: requests: { cpu: 100m, memory: 256Mi } limits: { memory: 1Gi } + # -- Seconds between SIGTERM and SIGKILL. At least one reconcile tick (up to 90s + # when a platform is slow) plus SHUTDOWN_DRAIN_MS (10s) for in-flight provisions + # and integration hooks; raise it with SHUTDOWN_DRAIN_MS. + terminationGracePeriodSeconds: 120 postgresql: # -- A minimal bundled Postgres for trying devbox: no backups, HA or automated upgrades. From 17e77a85a5c9b91fff9b52561ca7bb4cb7f3386e Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Fri, 30 Oct 2026 18:32:47 +0530 Subject: [PATCH 07/22] feat(harness): channel-neutral follow-up prompt; pin agent_error formats The follow-up prompt no longer assumes Linear. A test pins the two agent_error formats the control plane routes on: a failed assignment's user-safe summary, and a crashed round's exception text. Co-Authored-By: Claude Opus 5.5 --- harness/src/harness/daemon.py | 10 +- .../src/harness/deepagents/agents/followup.py | 8 +- harness/src/harness/handlers/assignment.py | 5 +- harness/tests/test_daemon.py | 97 ++++++++++++++++++- 4 files changed, 108 insertions(+), 12 deletions(-) diff --git a/harness/src/harness/daemon.py b/harness/src/harness/daemon.py index 3d549a5..64244fd 100644 --- a/harness/src/harness/daemon.py +++ b/harness/src/harness/daemon.py @@ -274,10 +274,12 @@ def _emit_round_failure( # comment / review — the round died before (or part-way through) the agent # posting its replies. Derive the primary target_id from `payload` (the # daemon's inbox file carries no reply_targets — that is a CP-only column). - # Failures ride on agent_error (not agent_response): CP routes them to the - # Linear session via emitError, the same lane the assignment-stage terminal - # failure uses — keeps "this round broke" as a structured error block - # rather than masquerading as a normal reply. + # Failures ride on agent_error (not agent_response), so "this round broke" + # never masquerades as a normal reply. CP labels it a round failure with + # this exception text as its raw detail, and the round's integration + # decides what to show: Linear shows it on the agent session; GitHub posts a + # generic retry message instead (a PR is public); a CI round stays silent. + # test_daemon pins this format (TestAgentErrorContract). if entry["kind"] == "review": primary_target_id = (payload.get("review_body") or {}).get("target_id") else: # comment / ci_result diff --git a/harness/src/harness/deepagents/agents/followup.py b/harness/src/harness/deepagents/agents/followup.py index d4b1e60..51a0595 100644 --- a/harness/src/harness/deepagents/agents/followup.py +++ b/harness/src/harness/deepagents/agents/followup.py @@ -61,7 +61,7 @@ Each round, the user message hands you one of: -- a **single comment**, from GitHub or Linear; or +- a **single comment**, from any channel; or - a **review**: a review summary plus zero or more inline comments, each pinned to a file and a line or line range. @@ -77,8 +77,10 @@ - **GitHub**: pull request comments and reviews. The reviewer sees your reply on the comment or review it answers. - **Linear**: comments on the agent session. The user may be watching live. +- **Any other channel**: comments or messages on the task, where it was filed. + The user sees your reply there. -Post progress updates as you work, on either channel. +Post progress updates as you work, on any channel. ## What "address it" means @@ -123,7 +125,7 @@ task did not already complete planning and implementation, even if it is checked out in the workspace. Never create or clone a repository. Instead, reply that you can't take on new work in another repository here, and ask the - user to re-delegate it as a new task from Linear with a proper description. + user to re-delegate it as a new task with a proper description. 7. **Reply once per item, when its work is done.** For a review summary, stay silent by default: skip it when its body is empty, whitespace-only or just an diff --git a/harness/src/harness/handlers/assignment.py b/harness/src/harness/handlers/assignment.py index 3992344..7a0c496 100644 --- a/harness/src/harness/handlers/assignment.py +++ b/harness/src/harness/handlers/assignment.py @@ -605,8 +605,9 @@ def _format_completion_body( Two sections: a per-repo "Planned / Done" recap — the plan.json repo `goal` and the implementation.json repo `summary`, matched by repo name; a repo with neither is dropped from the recap — then a "Pull requests" list of - every opened PR. CP relays this body verbatim — see CP eventDispatcher - handleAssignmentCompleted. + every opened PR. CP relays this body verbatim; for a failed run it is the + failure's user-safe summary, so it must never carry exception text (see the + control plane's labelAgentEmit; test_daemon's TestAgentErrorContract pins it). """ plan_by_name = { r["name"]: r diff --git a/harness/tests/test_daemon.py b/harness/tests/test_daemon.py index c59c21e..e518cf2 100644 --- a/harness/tests/test_daemon.py +++ b/harness/tests/test_daemon.py @@ -820,9 +820,8 @@ def test_stage_exception_emits_error_and_exits_for_assignment_retry(self) -> Non def test_comment_round_failure_emits_agent_error_not_response(self) -> None: # Round-failure messages ride on agent_error, not agent_response — CP - # routes agent_error through emitError on the Linear session - # (structured error block) rather than as a normal targeted reply. - # Lines up with the assignment-stage terminal-failure path. + # labels it a round failure (Linear shows it as a structured error + # block) rather than a normal targeted reply. entry = { "id": "inbox-cf", "kind": "comment", @@ -1107,6 +1106,98 @@ def test_assignment_reads_task_key(self) -> None: self.assertTrue((self.processed / f.name).exists()) +class TestAgentErrorContract(unittest.TestCase): + """The two non-retryable agent_error formats the control plane routes on. + + The control plane's labelAgentEmit relies on them: on an assignment row + the body is the user-safe summary (shown on public channels as is); on a + comment, review or CI row it is exception text (shown only on private + channels). Changing either format requires changing that routing. + """ + + setUp = TestProcessOneInboxFile.setUp + tearDown = TestProcessOneInboxFile.tearDown + _write_inbox = TestProcessOneInboxFile._write_inbox + _events = TestProcessOneInboxFile._events + + def test_failed_assignment_is_the_summary_without_exception_text(self) -> None: + entry = { + "id": "inbox-f", + "kind": "assignment", + "source": "linear", + "payload": {"title": "T", "task_key": "ABC-9", "target_id": "sess-9"}, + } + f = self._write_inbox(entry) + plan_data = {"repos": [{"name": "repo-a", "path": "/tmp/repo-a", "goal": "g"}]} + impl_data = {"repos": []} + # Exception text the commit stage might carry must not reach the body. + commit_repos = [ + { + "name": "repo-a", + "pr_url": None, + "pr_status": "push_failed", + "error": "RuntimeError: push rejected for token=abc", + } + ] + plan_md = self.dir / "plan.md" + plan_md.write_text("# Plan", encoding="utf-8") + + with ( + patch.object(assignment_module, "setup_workspace"), + patch.object( + assignment_module, "_run_planning_stage", return_value=plan_data + ), + patch.object( + assignment_module, "_run_implementation_stage", return_value=impl_data + ), + patch.object( + assignment_module, "run_self_review_loop", return_value=({}, impl_data) + ), + patch.object(assignment_module, "run_distillation_stage"), + patch.object( + assignment_module, + "run_commit_stage", + return_value=("failed", commit_repos), + ), + patch.object(assignment_module, "PLAN_MD", plan_md), + ): + process_one_inbox_file(f, self.state) + + errors = [e for e in self._events() if e["type"] == "agent_error"] + self.assertEqual(len(errors), 1) + self.assertNotIn(RETRYABLE_ASSIGNMENT_ERROR_EVENT_FIELD, errors[0]) + self.assertEqual( + errors[0]["body"], + assignment_module._format_completion_body( + commit_repos, plan_data, impl_data, "failed" + ), + ) + self.assertNotIn("RuntimeError", errors[0]["body"]) + + def test_round_crash_carries_the_exception_text(self) -> None: + entry = { + "id": "inbox-c", + "kind": "comment", + "source": "github", + "payload": {"target_id": "555", "body": "fix it", "task_key": "ABC-9"}, + } + f = self._write_inbox(entry) + + with patch( + "harness.daemon.process_comment", + side_effect=RuntimeError("tool call blew up"), + ): + process_one_inbox_file(f, self.state) + + errors = [e for e in self._events() if e["type"] == "agent_error"] + self.assertEqual(len(errors), 1) + self.assertNotIn(RETRYABLE_ASSIGNMENT_ERROR_EVENT_FIELD, errors[0]) + self.assertEqual( + errors[0]["body"], + "⚠ Something went wrong — RuntimeError: tool call blew up", + ) + + class TestArchiveMoveFailureDoesNotCrashLoop(unittest.TestCase): """An FS error from the post-dispatch archive move must not escape.""" From f6a2adfcc1e4d74cac943ed8324c949433bd43b1 Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Fri, 30 Oct 2026 19:17:10 +0530 Subject: [PATCH 08/22] fix(control-plane): renew lease per event and tighten holds Each cursor advance renews the task's lease, so a run of slow sends can't outlive it. A rate limit without Retry-After waits for the platform's reset time, at least 60 s, instead of retrying every tick, and a GitHub 403 carrying rate-limit headers holds like a 429. Shutdown stops reconciling and claiming new tasks, the onboarding comment gets the send timeout, assignee logs carry the account id, and timeouts, the drain and the hooks get tests. Co-Authored-By: Claude Opus 5.5 --- control-plane/docker-compose.yml | 4 +- .../src/clients/__tests__/github.test.ts | 23 +- control-plane/src/clients/githubApi.ts | 16 +- .../src/integrations/__tests__/http.test.ts | 33 +++ control-plane/src/integrations/http.ts | 21 +- .../src/integrations/linear/index.ts | 13 +- control-plane/src/integrations/types.ts | 1 + .../__tests__/task.repository.test.ts | 8 + .../src/repositories/task.repository.ts | 4 +- .../__tests__/eventDispatcher.test.ts | 18 +- .../services/__tests__/inbox.service.test.ts | 39 +++- .../__tests__/integrationHooks.test.ts | 56 ++++- .../__tests__/taskProcessor.service.test.ts | 207 +++++++++++++++++- control-plane/src/services/eventDispatcher.ts | 31 +-- .../src/services/integrationHooks.ts | 34 ++- .../src/services/taskProcessor.service.ts | 17 +- deploy/helm/devbox/README.md | 2 +- .../helm/devbox/templates/control-plane.yaml | 2 +- deploy/helm/devbox/values.yaml | 6 +- 19 files changed, 478 insertions(+), 57 deletions(-) diff --git a/control-plane/docker-compose.yml b/control-plane/docker-compose.yml index 0118c79..3175e50 100644 --- a/control-plane/docker-compose.yml +++ b/control-plane/docker-compose.yml @@ -93,8 +93,8 @@ services: image: devbox-control-plane build: *control-plane-build restart: unless-stopped - # Shutdown finishes the current reconcile tick (up to 90s when a platform - # is slow), then drains for SHUTDOWN_DRAIN_MS (10s); Compose's default 10s + # Shutdown finishes the task it is reconciling (up to 110s when platforms + # are slow), then drains for SHUTDOWN_DRAIN_MS (10s); Compose's default 10s # would cut both short. stop_grace_period: 120s # Explicit, with no env_file. Only allowlisted HARNESS_* settings cross diff --git a/control-plane/src/clients/__tests__/github.test.ts b/control-plane/src/clients/__tests__/github.test.ts index c32c031..e0cd43d 100644 --- a/control-plane/src/clients/__tests__/github.test.ts +++ b/control-plane/src/clients/__tests__/github.test.ts @@ -28,7 +28,8 @@ const { postIssueComment, requestReviewers, } = (await import("../githubApi.ts" + "?github-impl-test")) as typeof import("../githubApi"); -const { HttpError } = await import("../../integrations/http"); +const { HttpError, retryable } = await import("../../integrations/http"); +const { RetryableError } = await import("../../integrations/outbound"); // Build a fetch stub that returns each page in `pages` as a 200 JSON response, // recording the requested URLs. githubFetch does resp.text() → JSON.parse, so @@ -378,6 +379,26 @@ describe("githubFetch errors and signal", () => { ); }); + it("a rate-limit 403 holds like a 429, waiting for x-ratelimit-reset and at least a minute; any other 403 drops", async () => { + const classify = async (status: number, headers: Record) => { + globalThis.fetch = (() => + Promise.resolve(new Response("{}", { status, headers }))) as unknown as typeof fetch; + return retryable(await postIssueComment("o", "r", 1, "hi").catch((e: unknown) => e)); + }; + const inFiveMinutes = Math.floor(Date.now() / 1000) + 300; // epoch seconds + const exhausted = await classify(403, { + "x-ratelimit-remaining": "0", + "x-ratelimit-reset": String(inFiveMinutes), + }); + expect(exhausted).toBeInstanceOf(RetryableError); + expect(exhausted?.retryAfterMs).toBeGreaterThan(290_000); + expect(exhausted?.retryAfterMs).toBeLessThanOrEqual(300_000); + expect((await classify(403, { "retry-after": "30" }))?.retryAfterMs).toBe(30_000); + // No Retry-After and no reset ahead: at least a minute, never the next tick. + expect((await classify(429, {}))?.retryAfterMs).toBe(60_000); + expect(await classify(403, { "x-ratelimit-remaining": "12" })).toBeUndefined(); + }); + it("passes the caller's signal to fetch", async () => { let seen: AbortSignal | null | undefined; globalThis.fetch = ((_url: string, init?: RequestInit) => { diff --git a/control-plane/src/clients/githubApi.ts b/control-plane/src/clients/githubApi.ts index 34a2b82..71e66d3 100644 --- a/control-plane/src/clients/githubApi.ts +++ b/control-plane/src/clients/githubApi.ts @@ -13,7 +13,7 @@ import { readFileSync } from "node:fs"; import { config } from "../config"; import { githubApiBaseUrl } from "../githubToken/mint"; -import { HttpError, parseRetryAfter, TokenFetchError } from "../integrations/http"; +import { HttpError, parseRetryAfter, rateLimitWaitMs, TokenFetchError } from "../integrations/http"; import { logger } from "../logger"; import { type GithubTokenProvider, @@ -83,11 +83,21 @@ async function githubFetch( const text = await resp.text(); if (!resp.ok) { const snippet = text.slice(0, 500); + // `?.`: the client tests stub fetch with header-less partial responses. + const header = (name: string) => resp.headers?.get(name) ?? null; + // GitHub's rate limit is a 429, or a 403 carrying x-ratelimit-remaining: 0 + // or Retry-After; any other 403 is a refusal. Without Retry-After, a rate + // limit waits for x-ratelimit-reset (epoch seconds), and at least a minute. + const rateLimited = + resp.status === 429 || + (resp.status === 403 && + (header("x-ratelimit-remaining") === "0" || header("retry-after") !== null)); throw new HttpError( `GitHub ${method} ${path} failed: HTTP ${resp.status} ${snippet}`, resp.status, - // `?.`: the client tests stub fetch with header-less partial responses. - parseRetryAfter(resp.headers?.get("retry-after") ?? null), + parseRetryAfter(header("retry-after")) ?? + (rateLimited ? rateLimitWaitMs(Number(header("x-ratelimit-reset")) * 1000) : undefined), + rateLimited, ); } let json: unknown = null; diff --git a/control-plane/src/integrations/__tests__/http.test.ts b/control-plane/src/integrations/__tests__/http.test.ts index 852ca0f..34ad881 100644 --- a/control-plane/src/integrations/__tests__/http.test.ts +++ b/control-plane/src/integrations/__tests__/http.test.ts @@ -115,6 +115,13 @@ describe("retryable: typed errors", () => { } }); + it("a certificate failure in the TLS handshake (DEPTH_ZERO_SELF_SIGNED_CERT) is retryable", () => { + const err = Object.assign(new TypeError("self signed certificate"), { + code: "DEPTH_ZERO_SELF_SIGNED_CERT", + }); + expect(retryable(err)).toBeInstanceOf(RetryableError); + }); + it("a token-fetch failure is retryable, Linear's included", () => { expect(retryable(new TokenFetchError("no token"))).toBeInstanceOf(RetryableError); expect(retryable(new LinearTokenError("Linear token request failed"))).toBeInstanceOf( @@ -233,6 +240,32 @@ describe("Linear deliver: SDK failures", () => { } }); + it("a rate limit without Retry-After waits for Linear's reset time, at least a minute", async () => { + const ratelimited = (headers: Record) => + server(() => + Response.json( + { errors: [{ message: "Rate limit exceeded", extensions: { type: "ratelimited" } }] }, + { status: 400, headers }, + ), + ); + const waitOf = async (headers: Record) => { + const s = ratelimited(headers); + try { + const err = await deliverTo(`http://127.0.0.1:${s.port}/graphql`); + expect(err).toBeInstanceOf(RetryableError); + return (err as InstanceType).retryAfterMs; + } finally { + s.stop(true); + } + }; + const inTwoMinutes = await waitOf({ + "x-ratelimit-requests-reset": String(Date.now() + 120_000), // epoch ms + }); + expect(inTwoMinutes).toBeGreaterThan(110_000); + expect(inTwoMinutes).toBeLessThanOrEqual(120_000); + expect(await waitOf({})).toBe(60_000); + }); + it("a 500 is not retryable", async () => { const s = server(() => new Response("boom", { status: 500 })); try { diff --git a/control-plane/src/integrations/http.ts b/control-plane/src/integrations/http.ts index 4254f75..f1bdd29 100644 --- a/control-plane/src/integrations/http.ts +++ b/control-plane/src/integrations/http.ts @@ -7,17 +7,29 @@ import { RetryableError } from "./outbound"; export const SEND_TIMEOUT_MS = 20_000; // A non-2xx answer. The message keeps the client's own wording (tests and logs -// match on it); status and Retry-After ride along for classification. +// match on it); status, the wait before a retry and whether the platform +// rate-limited the call ride along for classification. export class HttpError extends Error { constructor( message: string, readonly status: number, readonly retryAfterMs?: number, + // GitHub also answers a rate limit with a 403 (see githubFetch). + readonly rateLimited = status === 429, ) { super(message); } } +// How long a rate limit without Retry-After waits: until the platform's reset +// time (epoch ms), and at least a minute. GitHub may ban integrations that +// retry sooner (§4). +export function rateLimitWaitMs(resetAtMs?: number): number { + const untilReset = + resetAtMs !== undefined && Number.isFinite(resetAtMs) ? resetAtMs - Date.now() : 0; + return Math.max(untilReset, 60_000); +} + // The client had no credential to send with, so nothing was sent. export class TokenFetchError extends Error {} @@ -39,13 +51,14 @@ const NOTHING_SENT = new Set([ ]); // The failure as a RetryableError when it is one of the exact retryable cases -// (§4): 429, connection refused, DNS, TLS handshake, token fetch. Anything else -// (timeouts, 5xx, other 4xx) is undefined: the caller logs and drops it. +// (§4): a rate limit (429), connection refused, DNS, TLS handshake, token +// fetch. Anything else (timeouts, 5xx, other 4xx) is undefined: the caller +// logs and drops it. export function retryable(err: unknown): RetryableError | undefined { if (err instanceof RetryableError) return err; if (err instanceof TokenFetchError) return new RetryableError(err.message); if (err instanceof HttpError) { - return err.status === 429 ? new RetryableError(err.message, err.retryAfterMs) : undefined; + return err.rateLimited ? new RetryableError(err.message, err.retryAfterMs) : undefined; } const code = (err as { code?: unknown } | null)?.code; if (err instanceof TypeError && typeof code === "string" && NOTHING_SENT.has(code)) { diff --git a/control-plane/src/integrations/linear/index.ts b/control-plane/src/integrations/linear/index.ts index 6195ed4..d2ce11a 100644 --- a/control-plane/src/integrations/linear/index.ts +++ b/control-plane/src/integrations/linear/index.ts @@ -13,7 +13,7 @@ import type { Task } from "../../db/schema"; import type { InboxRow, ReplyTarget } from "../../repositories/inbox.repository"; import { WORKSPACE_MISSING_MESSAGE } from "../../services/inbox.service"; import type { LinearTokenProvider } from "../../services/linearTokenProvider"; -import { retryable } from "../http"; +import { rateLimitWaitMs, retryable } from "../http"; import { type Outbound, RetryableError, type SendCtx } from "../outbound"; import type { HookCtx, Integration, Person, PrRef } from "../types"; @@ -108,7 +108,7 @@ export class LinearIntegration implements Integration { const client = await this.client(ctx.signal); const human = await getIssueHumanAssignee(client, task.externalId); if (!human) return null; - const person = { displayName: human.displayName, email: human.email }; + const person = { id: human.id, displayName: human.displayName, email: human.email }; if (!human.gitHubUserId) return { ...person, codeHost: "unlinked" }; // Defensive about the field's format: numeric → GitHub account id // (resolve to the current login), anything else → already a login. @@ -133,11 +133,16 @@ export class LinearIntegration implements Integration { // The SDK wraps every failure in a LinearError whose `raw` is what fetch (or // the token renewal after a 401) threw. A rate limit is the SDK's // RatelimitedLinearError: an HTTP 429 or a "ratelimited" GraphQL error, with -// Retry-After in seconds. +// Retry-After in seconds. Without it, the wait runs to the later of Linear's +// request and complexity reset times (epoch ms), and at least a minute. function linearRetryable(err: unknown): RetryableError | undefined { if (err instanceof RatelimitedLinearError) { const s = err.retryAfter; - return new RetryableError(err.message, s !== undefined && s >= 0 ? s * 1000 : undefined); + const wait = + s !== undefined && s >= 0 + ? s * 1000 + : rateLimitWaitMs(Math.max(err.requestsResetAt ?? 0, err.complexityResetAt ?? 0)); + return new RetryableError(err.message, wait); } return retryable(err instanceof LinearError ? err.raw : err); } diff --git a/control-plane/src/integrations/types.ts b/control-plane/src/integrations/types.ts index 6c62797..b1da3cd 100644 --- a/control-plane/src/integrations/types.ts +++ b/control-plane/src/integrations/types.ts @@ -31,6 +31,7 @@ export interface Integration { // isCurrent() is false once a restart rotated the task's assignment id. export type HookCtx = { signal: AbortSignal; log: Logger; isCurrent(): Promise }; export type Person = { + id: string; // the platform account id: logs carry it, never the name or email displayName: string; email?: string; codeHost: { githubLogin: string } | "unlinked" | "unsupported"; diff --git a/control-plane/src/repositories/__tests__/task.repository.test.ts b/control-plane/src/repositories/__tests__/task.repository.test.ts index ce73a45..964a002 100644 --- a/control-plane/src/repositories/__tests__/task.repository.test.ts +++ b/control-plane/src/repositories/__tests__/task.repository.test.ts @@ -854,6 +854,14 @@ describe("TaskRepository", () => { expect(await events()).toEqual([]); }); + it("renews the lease with each advance", async () => { + const stale = new Date(Date.now() - 100_000); + const t = await insertTask({ ownerId: "pod-a", heartbeatAt: stale }); + await repo.recordEventCursor({ taskId: t.id, ownerId: "pod-a", event: ev(1) }); + const [row] = await db.select().from(schema.tasks); + expect(row?.heartbeatAt?.getTime()).toBeGreaterThan(stale.getTime() + 90_000); + }); + it("API task: one log row per event, holding the full event", async () => { const t = await insertTask({ source: "api", ownerId: "pod-a" }); await repo.recordEventCursor({ taskId: t.id, ownerId: "pod-a", event: ev(1) }); diff --git a/control-plane/src/repositories/task.repository.ts b/control-plane/src/repositories/task.repository.ts index 4ee0cd7..3f0cae1 100644 --- a/control-plane/src/repositories/task.repository.ts +++ b/control-plane/src/repositories/task.repository.ts @@ -455,6 +455,8 @@ export class TaskRepository { // a replay of a passed event, a pod without the lease, and the old owner // after a restart (owner_id cleared) all change 0 rows and log nothing. // Returns 0 rows when the lease is lost OR the cursor is already past seq. + // Each advance also renews the lease, so a long run of slow but successful + // sends can't outlive it mid-reconcile. async recordEventCursor(params: { taskId: string; ownerId: string; @@ -464,7 +466,7 @@ export class TaskRepository { return db.transaction(async (tx) => { const result = await tx.execute<{ id: string; source: string }>(sql` UPDATE tasks - SET last_observed_event_seq = ${event.seq}, updated_at = now() + SET last_observed_event_seq = ${event.seq}, heartbeat_at = now(), updated_at = now() WHERE id = ${params.taskId} AND owner_id = ${params.ownerId} AND last_observed_event_seq < ${event.seq} diff --git a/control-plane/src/services/__tests__/eventDispatcher.test.ts b/control-plane/src/services/__tests__/eventDispatcher.test.ts index bc5a217..332091d 100644 --- a/control-plane/src/services/__tests__/eventDispatcher.test.ts +++ b/control-plane/src/services/__tests__/eventDispatcher.test.ts @@ -1,5 +1,5 @@ import { beforeEach, describe, expect, it, mock } from "bun:test"; -import type { LinearClient } from "@linear/sdk"; +import { type LinearClient, LinearError, RatelimitedLinearError } from "@linear/sdk"; // Type-only import — erased at link time, so it can't collide with the // mock.module barrel factories below, and it pins the real contract. import type { LinearHumanUser } from "../../clients/linearApi"; @@ -287,6 +287,7 @@ describe("EventDispatcher", () => { "Mention `@harness` and I'll act on it — on a top-level PR comment,", "in a review summary, or on an inline comment on the diff. Without a mention I won't see it.", ].join("\n"), + expect.any(AbortSignal), ); }); @@ -697,6 +698,20 @@ describe("EventDispatcher", () => { expect(attachUrlToIssue).not.toHaveBeenCalled(); }); + it("Linear task: a 429 or a refused connection on the attach holds the event", async () => { + const rateLimited = new RatelimitedLinearError({ + response: { headers: new Headers({ "retry-after": "5" }) }, + } as never); + const refused = Object.assign(new TypeError("Unable to connect"), { code: "ECONNREFUSED" }); + attachUrlToIssue + .mockRejectedValueOnce(rateLimited) + .mockRejectedValueOnce(new LinearError(refused as never)); + expect(await dispatcher.handle(prEvent(), makeTask())).toBe("hold"); + expect(await dispatcher.handle(prEvent(), makeTask())).toBe("hold"); + expect(attachUrlToIssue).toHaveBeenCalledTimes(2); + expect(holds.stats().linear?.heldTasks).toBe(1); + }); + it("self-evicts when appendPrUrl returns 0 rows (lease lost); skips attachUrlToIssue", async () => { appendPrUrl.mockResolvedValue([]); const result = await dispatcher.handle(prEvent(), makeTask()); @@ -722,6 +737,7 @@ describe("EventDispatcher", () => { "Mention `@harness` and I'll act on it — on a top-level PR comment,", "in a review summary, or on an inline comment on the diff. Without a mention I won't see it.", ].join("\n"), + expect.any(AbortSignal), ); }); diff --git a/control-plane/src/services/__tests__/inbox.service.test.ts b/control-plane/src/services/__tests__/inbox.service.test.ts index 52909ef..d7a2f9e 100644 --- a/control-plane/src/services/__tests__/inbox.service.test.ts +++ b/control-plane/src/services/__tests__/inbox.service.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, mock } from "bun:test"; +import { beforeEach, describe, expect, it, mock, spyOn } from "bun:test"; import type { Task } from "../../db/schema"; import type { InboxRepository, InboxRow } from "../../repositories/inbox.repository"; import type { LinearTokenProvider } from "../linearTokenProvider"; @@ -133,6 +133,7 @@ describe("InboxService", () => { let writeHarnessFile: ReturnType; let killSandbox: ReturnType; let getById: ReturnType; + let linear: InstanceType; let service: InstanceType; beforeEach(() => { @@ -180,11 +181,8 @@ describe("InboxService", () => { getAuthorizedLinearClient: mock(() => Promise.resolve({})), } as unknown as Pick; - const integrations = { - linear: new LinearIntegration(linearTokenProvider), - api: apiIntegration, - github: githubIntegration, - }; + linear = new LinearIntegration(linearTokenProvider); + const integrations = { linear, api: apiIntegration, github: githubIntegration }; service = new InboxService( inboxRepo, sandboxManager, @@ -255,6 +253,35 @@ describe("InboxService", () => { }); }); + it("finishes a dead-lettered assignment's round: outcome failed, first transition, no archive", async () => { + const onRoundFinished = spyOn(linear, "onRoundFinished"); + const assignment = { + ...ghCommentRow("as"), + source: "linear", + kind: "assignment", + reply_targets: [{ id: "sess-1", spec: { kind: "linear_session" } }], + } as InboxRow; + listNonTerminalByTask.mockResolvedValue([assignment, ghCommentRow("c")]); + getById.mockImplementation((id: string) => + Promise.resolve(id === "as" ? assignment : ghCommentRow(id)), + ); + + await service.deadLetterWorkspaceMissing(makeTask({ sandboxId: null })); + await new Promise((r) => setTimeout(r, 0)); + + expect(onRoundFinished).toHaveBeenCalledTimes(1); + const [ctx, , round, outcome] = onRoundFinished.mock.calls[0] as unknown as [ + { firstTransition: boolean; archive(): Promise }, + Task, + InboxRow, + string, + ]; + expect(round.id).toBe("as"); + expect(outcome).toBe("failed"); + expect(ctx.firstTransition).toBe(true); + expect(await ctx.archive()).toBeNull(); + }); + it("API rows: no Linear or GitHub post, row failed with the user message", async () => { const row = { ...ghCommentRow("r1"), diff --git a/control-plane/src/services/__tests__/integrationHooks.test.ts b/control-plane/src/services/__tests__/integrationHooks.test.ts index f4b0021..a011612 100644 --- a/control-plane/src/services/__tests__/integrationHooks.test.ts +++ b/control-plane/src/services/__tests__/integrationHooks.test.ts @@ -18,15 +18,24 @@ const row = (id: string, kind = "assignment", source = "linear") => const noop: Integration = { name: "api", async deliver() {} }; const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); -function hooksWith(linear: Partial, github: Partial = {}) { +function hooksWith( + linear: Partial, + github: Partial = {}, + timeoutMs?: number, +) { const inboxRepo = { getAssignment: mock(() => Promise.resolve(row("a-1"))) }; const integrations = { linear: { ...noop, name: "linear" as const, ...linear }, api: noop, github: { ...noop, name: "github" as const, ...github }, }; + const t = timeoutMs; + const timeouts = + t === undefined + ? undefined + : { onTaskStarted: t, onPrOpened: t, onRoundFinished: t, assignee: t }; return { - hooks: new IntegrationHooks(integrations, inboxRepo as unknown as InboxRepository), + hooks: new IntegrationHooks(integrations, inboxRepo as unknown as InboxRepository, timeouts), inboxRepo, }; } @@ -38,7 +47,7 @@ describe("IntegrationHooks", () => { const { hooks } = hooksWith({ assignee: () => Promise.reject(new RetryableError("token fetch failed")), }); - expect(await hooks.assignee(task, row("a-1"))).toBeUndefined(); + expect(await hooks.assignee(task, row("a-1"), { prUrl: "u" })).toBeUndefined(); expect(error).toHaveBeenCalledWith( "assignee hook failed", expect.objectContaining({ taskId: "task-1", integration: "linear" }), @@ -61,6 +70,45 @@ describe("IntegrationHooks", () => { } }); + it("cuts a hung hook at its timeout: the call resolves and the hook's signal aborts", async () => { + const error = spyOn(logger, "error").mockImplementation(() => logger); + try { + let signal: AbortSignal | undefined; + const { hooks } = hooksWith( + { + assignee: (ctx) => { + signal = ctx.signal; + return new Promise(() => {}); + }, + }, + {}, + 20, + ); + expect(await hooks.assignee(task, row("a-1"), {})).toBeUndefined(); + expect(signal?.aborted).toBe(true); + } finally { + error.mockRestore(); + } + }); + + it("a failed assignment lookup still runs the hook, with isCurrent() false", async () => { + const error = spyOn(logger, "error").mockImplementation(() => logger); + try { + let ctx: HookCtx | undefined; + const { hooks, inboxRepo } = hooksWith({ + async assignee(c) { + ctx = c; + return null; + }, + }); + inboxRepo.getAssignment.mockRejectedValue(new Error("connection reset")); + expect(await hooks.assignee(task, row("c-1", "comment"), {})).toBeNull(); + expect(await ctx?.isCurrent()).toBe(false); + } finally { + error.mockRestore(); + } + }); + it("isCurrent() turns false once a restart rotated the assignment id", async () => { let ctx: HookCtx | undefined; const { hooks, inboxRepo } = hooksWith({ @@ -70,7 +118,7 @@ describe("IntegrationHooks", () => { }, }); // A comment round: the baseline is the task's assignment id at hook start. - await hooks.assignee(task, row("c-1", "comment")); + await hooks.assignee(task, row("c-1", "comment"), {}); expect(await ctx?.isCurrent()).toBe(true); inboxRepo.getAssignment.mockResolvedValue(row("a-2")); expect(await ctx?.isCurrent()).toBe(false); diff --git a/control-plane/src/services/__tests__/taskProcessor.service.test.ts b/control-plane/src/services/__tests__/taskProcessor.service.test.ts index 08f5981..8e1abbf 100644 --- a/control-plane/src/services/__tests__/taskProcessor.service.test.ts +++ b/control-plane/src/services/__tests__/taskProcessor.service.test.ts @@ -4,6 +4,7 @@ import { SandboxApiException } from "@alibaba-group/opensandbox"; import type { LinearClient } from "@linear/sdk"; import type { Task } from "../../db/schema"; import { PvcNotFoundError, SandboxCreateError } from "../../errors"; +import type { Integration } from "../../integrations/types"; import type { InboxRepository } from "../../repositories/inbox.repository"; import type { TaskRepository } from "../../repositories/task.repository"; import type { HarnessEvent } from "../../types/harnessEvent"; @@ -96,6 +97,11 @@ const { apiIntegration } = await import("../../integrations/api"); const { logger } = await import("../../logger"); const { ReplyHolds } = await import("../replyHolds"); const { IntegrationHooks } = await import("../integrationHooks"); +const { EventDispatcher: RealEventDispatcher } = await import("../eventDispatcher"); +const { config: mockedConfig } = (await import("../../config")) as unknown as { + config: { shutdownDrainMs: number }; +}; +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); const { RetryableError } = await import("../../integrations/outbound"); const makeTask = (overrides: Partial = {}): Task => ({ @@ -182,6 +188,12 @@ describe("TaskProcessorService", () => { let notifyTerminal: ReturnType; let slackNotifier: SlackNotifierService; let holds: InstanceType; + let integrations: { + linear: InstanceType; + api: Integration; + github: Integration; + }; + let hooks: InstanceType; let service: InstanceType; beforeEach(() => { @@ -279,11 +291,12 @@ describe("TaskProcessorService", () => { slackNotifier = { notifyTerminal } as unknown as SlackNotifierService; holds = new ReplyHolds(); - const integrations = { + integrations = { linear: new LinearIntegration(linearTokenProvider), api: apiIntegration, github: githubIntegration, }; + hooks = new IntegrationHooks(integrations, inboxRepo); service = new TaskProcessorService( taskRepo, inboxRepo, @@ -295,7 +308,7 @@ describe("TaskProcessorService", () => { integrations, holds, ["linear", "api"], - new IntegrationHooks(integrations, inboxRepo), + hooks, ); }); @@ -409,6 +422,42 @@ describe("TaskProcessorService", () => { expect(failTaskInbox).not.toHaveBeenCalled(); }); + it("runs the owner's onTaskStarted once per assignment id, before the sandbox is created", async () => { + const order: string[] = []; + const onTaskStarted = mock(async () => { + order.push("started"); + }); + (integrations.linear as Integration).onTaskStarted = onTaskStarted; + const assignment = survivingNonTerminalRow(); + (inboxRepo as { getAssignment?: unknown }).getAssignment = mock(() => + Promise.resolve(assignment), + ); + // The first provision defers on a transient 503; the re-claim provisions + // the same assignment again. + createSandboxWithPvc + .mockImplementationOnce(() => { + order.push("create"); + return Promise.reject(new SandboxApiException({ statusCode: 503 })); + }) + .mockImplementation(() => { + order.push("create"); + return Promise.resolve({ sandboxId: "sbx-1", pvcName: "devbox-pvc-task-1-aaaaaa" }); + }); + const task = makeTask({ sandboxId: null }); + claimNext.mockResolvedValueOnce(task).mockResolvedValueOnce(task).mockResolvedValue(null); + + service.start(); + await waitFor(() => attachSandboxResources.mock.calls.length > 0); + await service.stop(); + + expect(order).toEqual(["started", "create", "create"]); + expect(onTaskStarted).toHaveBeenCalledWith( + expect.anything(), + task, + assignment.reply_targets[0], + ); + }); + it("threads experimental flags from task metadata into sandbox creation", async () => { const task = makeTask({ sandboxId: null, @@ -792,6 +841,30 @@ describe("TaskProcessorService", () => { expect(postIssueComment).not.toHaveBeenCalled(); }); + it("finishes a dead-lettered assignment's round: outcome failed, first transition, no archive", async () => { + const onRoundFinished = spyOn(integrations.linear, "onRoundFinished"); + (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValue([ + row("a", "assignment", "linear", [{ id: "sess-1", spec: { kind: "linear_session" } }]), + row("gc", "comment", "github", [issueComment("555", 7, 555)]), + ]); + failTaskInbox.mockResolvedValue([{ id: "a" }, { id: "gc" }]); + + await failTask(makeTask()); + await waitFor(() => onRoundFinished.mock.calls.length > 0); + + expect(onRoundFinished).toHaveBeenCalledTimes(1); + const [ctx, , round, outcome] = onRoundFinished.mock.calls[0] as unknown as [ + { firstTransition: boolean; archive(): Promise }, + Task, + { id: string }, + string, + ]; + expect(round.id).toBe("a"); + expect(outcome).toBe("failed"); + expect(ctx.firstTransition).toBe(true); + expect(await ctx.archive()).toBeNull(); + }); + it("a row whose source has no integration is skipped; other rows post and the lease is released", async () => { (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValue([ row("x", "comment", "unknown-source", [issueComment("777", 7, 777)]), @@ -1156,6 +1229,105 @@ describe("TaskProcessorService", () => { expect(requeueForRetry).not.toHaveBeenCalled(); }); + it("a hung send is cut at the send timeout: dropped, cursor advanced, the task stops, the tick goes on", async () => { + const error = spyOn(logger, "error").mockImplementation(() => logger); + try { + const deliver = mock(() => new Promise(() => {})); + const round = { ...survivingNonTerminalRow(), source: "github", status: "delivered" }; + const dispatcher = new RealEventDispatcher( + taskRepo, + { getByIdForTask: mock(() => Promise.resolve(round)) } as unknown as InboxRepository, + sandboxManager, + linearTokenProvider, + slackNotifier, + { ...integrations, github: { name: "github", deliver } }, + holds, + hooks, + 20, // send timeout (ms) + ); + service = new TaskProcessorService( + taskRepo, + inboxRepo, + linearTokenProvider, + sandboxManager, + dispatcher, + inboxService, + slackNotifier, + integrations, + holds, + ["linear", "api"], + hooks, + ); + const thought = (seq: number) => ({ + seq, + ts: "t", + type: "agent_thought", + inbox_id: "inbox-1", + target_id: "sess-1", + body: "x", + }); + listInFlightOwnedBy + .mockResolvedValueOnce([ + makeTask({ id: "task-1", sandboxId: "sbx-1" }), + makeTask({ id: "task-2", sandboxId: "sbx-2" }), + ]) + .mockResolvedValue([]); + readEvents.mockImplementation((sandboxId: string) => + Promise.resolve(sandboxId === "sbx-1" ? [thought(1), thought(2)] : [thought(1)]), + ); + + service.start(); + await waitFor(() => recordEventCursor.mock.calls.length >= 2); + await service.stop(); + + // task-1's seq 2 is never sent: the drop ended task-1's events; task-2 ran. + const cursors = recordEventCursor.mock.calls.map((c: unknown[]) => { + const p = c[0] as { taskId: string; event: { seq: number } }; + return [p.taskId, p.event.seq]; + }); + expect(cursors).toEqual([ + ["task-1", 1], + ["task-2", 1], + ]); + expect(deliver).toHaveBeenCalledTimes(2); + } finally { + error.mockRestore(); + } + }); + + it("releases a stale hold when this pod claims the task", async () => { + const task = makeTask({ sandboxId: "sbx-1" }); + // Still owned, so only the claim can drop the hold. + listInFlightOwnedBy.mockResolvedValue([task]); + holds.hold("task-1", { seq: 1, inbox_id: "i" }, "github", new RetryableError("429", 60_000)); + claimNext.mockResolvedValueOnce(task).mockResolvedValue(null); + + service.start(); + await waitFor(() => attachSandbox.mock.calls.length > 0); + await service.stop(); + + expect(holds.stats()).toEqual({}); + }); + + it("once shutdown starts, reconciles no further owned task and claims nothing", async () => { + listInFlightOwnedBy.mockResolvedValue([ + makeTask({ id: "task-1", sandboxId: "sbx-1" }), + makeTask({ id: "task-2", sandboxId: "sbx-2" }), + ]); + let stopping: Promise | undefined; + getSandboxInfo.mockImplementation(() => { + stopping ??= service.stop(); + return Promise.resolve(makeSandboxInfo()); + }); + + service.start(); + await waitFor(() => stopping !== undefined); + await stopping; + + expect(getSandboxInfo).toHaveBeenCalledTimes(1); + expect(claimNext).not.toHaveBeenCalled(); + }); + it("a hold stops before the cursor advances and is not a reconcile error", async () => { const task = makeTask({ sandboxId: "sbx-1" }); listInFlightOwnedBy.mockResolvedValueOnce([task]).mockResolvedValue([]); @@ -1577,6 +1749,37 @@ describe("TaskProcessorService", () => { expect(backdateHeartbeatsOwnedBy).not.toHaveBeenCalled(); }); + it("stop() waits for detached hooks and logs hook_abandoned for one that outlives the drain", async () => { + const warn = spyOn(logger, "warn").mockImplementation(() => logger); + const drainMs = mockedConfig.shutdownDrainMs; + mockedConfig.shutdownDrainMs = 100; + try { + let finished = false; + service.start(); + await waitFor(() => heartbeatOwnedBy.mock.calls.length > 0); + hooks.detach( + "task-1", + "onRoundFinished", + sleep(20).then(() => { + finished = true; + }), + ); + hooks.detach("task-2", "assignee", sleep(400)); + await service.stop(); + + expect(finished).toBe(true); + expect(warn.mock.calls.filter(([msg]) => msg === "hook abandoned at shutdown")).toEqual([ + [ + "hook abandoned at shutdown", + { event: "hook_abandoned", taskId: "task-2", hook: "assignee" }, + ], + ]); + } finally { + mockedConfig.shutdownDrainMs = drainMs; + warn.mockRestore(); + } + }); + it("stop() backdates heartbeats owned by this pod before resolving", async () => { service.start(); await waitFor(() => heartbeatOwnedBy.mock.calls.length > 0); diff --git a/control-plane/src/services/eventDispatcher.ts b/control-plane/src/services/eventDispatcher.ts index 2768c09..aa1438c 100644 --- a/control-plane/src/services/eventDispatcher.ts +++ b/control-plane/src/services/eventDispatcher.ts @@ -67,6 +67,7 @@ export class EventDispatcher { private integrations: Record, private holds: ReplyHolds, private hooks: IntegrationHooks, + private sendTimeoutMs = SEND_TIMEOUT_MS, ) {} async handle(event: HarnessEvent, task: Task): Promise { @@ -233,9 +234,9 @@ export class EventDispatcher { // row's reply_targets[] by target_id, labels the emit, and hands both to the // round's integration, which decides where (or whether) it shows up. Core // never checks what an integration can show. The send runs under - // SEND_TIMEOUT_MS. A RetryableError (nothing was sent) holds the event on - // the round's integration; a timeout or any other failure is logged and the - // event dropped. + // sendTimeoutMs (SEND_TIMEOUT_MS). A RetryableError (nothing was sent) holds + // the event on the round's integration; a timeout or any other failure is + // logged and the event dropped. private async deliver( event: AgentEmit, inboxRow: InboxRow | null, @@ -259,7 +260,7 @@ export class EventDispatcher { } try { - await withTimeout(SEND_TIMEOUT_MS, (signal) => + await withTimeout(this.sendTimeoutMs, (signal) => this.integrations[inboxRow.source].deliver( { signal, log: logger }, target, @@ -325,11 +326,15 @@ export class EventDispatcher { if (!webhooks) return; const pr = parsePrUrl(event.pr_url); if (!pr) return; // not a github.com PR URL — skip - await postIssueComment( - pr.owner, - pr.repo, - pr.prNumber, - onboardingCommentBody(webhooks.botLogin, task.key), + // Awaited inside the tick, so bounded like a send. + await withTimeout(this.sendTimeoutMs, (signal) => + postIssueComment( + pr.owner, + pr.repo, + pr.prNumber, + onboardingCommentBody(webhooks.botLogin, task.key), + signal, + ), ).catch((err) => logErr("onboarding comment post failed", err)); } @@ -347,7 +352,7 @@ export class EventDispatcher { const pr = parsePrUrl(event.pr_url); if (!pr) return; // not a github.com PR URL — skip // undefined: no assignee() (API tasks), or the lookup failed (logged). - const person = await this.hooks.assignee(task, round); + const person = await this.hooks.assignee(task, round, { prUrl: event.pr_url }); if (person === undefined) return; if (!person) { logger.info("PR assignee mirror skipped", { @@ -361,14 +366,14 @@ export class EventDispatcher { } if (typeof person.codeHost !== "object") { // Linear's fix path: the member connects GitHub in Linear Settings → - // Connected accounts. Log the display name, not the email. + // Connected accounts. Log the account id, not the name or email. logger.info("PR assignee mirror skipped", { event: "pr_assignee.result", status: "skipped", reason: person.codeHost === "unlinked" ? "no_github_mapping" : "unsupported", taskId: task.id, prUrl: event.pr_url, - displayName: person.displayName, + linearUserId: person.id, }); return; } @@ -382,7 +387,7 @@ export class EventDispatcher { reason: "unresolved_login", taskId: task.id, prUrl: event.pr_url, - displayName: person.displayName, + linearUserId: person.id, }); return; } diff --git a/control-plane/src/services/integrationHooks.ts b/control-plane/src/services/integrationHooks.ts index d7ffc0f..5c11fdd 100644 --- a/control-plane/src/services/integrationHooks.ts +++ b/control-plane/src/services/integrationHooks.ts @@ -38,6 +38,7 @@ export class IntegrationHooks { constructor( private integrations: Record, private inboxRepo: Pick, + private timeoutsMs: Record = HOOK_TIMEOUT_MS, ) {} // Fresh provision, before the sandbox is created. Awaited; no hold. @@ -66,10 +67,21 @@ export class IntegrationHooks { } // The task's assignee, for detached work. undefined when the integration - // has no assignee or the lookup failed (logged here). - assignee(task: Task, round: InboxRow | null): Promise { + // has no assignee or the lookup failed (logged here, with `logMeta`). + assignee( + task: Task, + round: InboxRow | null, + logMeta: Record, + ): Promise { const owner = this.integrations[task.source]; - return this.callHook("assignee", owner, task, round, (ctx) => owner.assignee?.(ctx, task)); + return this.callHook( + "assignee", + owner, + task, + round, + (ctx) => owner.assignee?.(ctx, task), + logMeta, + ); } // Detached. Called (a) on `completed`, with firstTransition = whether that @@ -122,20 +134,28 @@ export class IntegrationHooks { task: Task, round: InboxRow | null, run: (ctx: HookCtx) => Promise | undefined, + logMeta: Record = {}, ): Promise { if (!owner[hook]) return undefined; + const meta = { ...logMeta, taskId: task.id, integration: owner.name }; try { - return await withTimeout(HOOK_TIMEOUT_MS[hook], async (signal) => { - const startedOn = await this.assignmentId(task.id, round); + return await withTimeout(this.timeoutsMs[hook], async (signal) => { + // A failed lookup still runs the hook; isCurrent() then answers false. + const startedOn = await this.assignmentId(task.id, round).catch((err) => { + logErr(`${hook}: assignment lookup failed; isCurrent() is false`, err, meta); + return undefined; + }); return run({ signal, log: logger, - isCurrent: async () => (await this.assignmentId(task.id)) === startedOn, + isCurrent: async () => + startedOn !== undefined && + (await this.assignmentId(task.id).catch(() => undefined)) === startedOn, }); }); } catch (err) { if (hook === "onPrOpened" && err instanceof RetryableError) throw err; - logErr(`${hook} hook failed`, err, { taskId: task.id, integration: owner.name }); + logErr(`${hook} hook failed`, err, meta); return undefined; } } diff --git a/control-plane/src/services/taskProcessor.service.ts b/control-plane/src/services/taskProcessor.service.ts index a7cef05..01c62de 100644 --- a/control-plane/src/services/taskProcessor.service.ts +++ b/control-plane/src/services/taskProcessor.service.ts @@ -127,10 +127,14 @@ export class TaskProcessorService { await this.reapIdle().catch((err) => logErr("reapIdle failed", err)); // 3. Reconcile owned tasks with live sandboxes. Each task's lease is renewed - // right before its reconcile, so slow sends on earlier tasks can't let it - // outlive the lease mid-tick; 0 rows means a sibling took it over. + // right before its reconcile (and with each cursor advance inside it), so + // slow sends can't let it outlive the lease mid-tick; 0 rows means a + // sibling took it over. const owned = await this.taskRepo.listInFlightOwnedBy(config.podId, this.taskSources); for (const task of owned) { + // Shutting down: leave the rest to siblings (stop() backdates our + // heartbeats), so shutdown waits for one task's reconcile at most (§10). + if (!this.running) break; const renewed = await this.taskRepo.heartbeatTask({ taskId: task.id, ownerId: config.podId }); if (renewed.length === 0) { logger.info("lease lost before reconcile; skipping", { taskId: task.id }); @@ -165,14 +169,19 @@ export class TaskProcessorService { this.holds.retain(ownedIds); // 4. Claim new work (fresh-pending OR comment-pickup OR stale-lease - // takeover — single predicate). + // takeover — single predicate). None while shutting down. + if (!this.running) return; const claimed = await this.taskRepo.claimNext({ ownerId: config.podId, leaseTimeoutMs: config.leaseTimeoutMs, maxConcurrentSandboxes: config.maxConcurrentSandboxes, sources: this.taskSources, }); - if (claimed) this.dispatch(claimed); + if (!claimed) return; + // A hold from before (say, a restart claimed in this same tick) belongs to + // the old run; it must not delay the new one. + this.holds.release(claimed.id); + this.dispatch(claimed); } private async reapIdle(): Promise { diff --git a/deploy/helm/devbox/README.md b/deploy/helm/devbox/README.md index b0ff926..ad1577c 100644 --- a/deploy/helm/devbox/README.md +++ b/deploy/helm/devbox/README.md @@ -25,7 +25,7 @@ docker run --rm -v "$PWD/deploy/helm/devbox:/helm-docs" jnorwood/helm-docs:v1.14 | controlPlane.resources.limits.memory | string | `"1Gi"` | | | controlPlane.resources.requests.cpu | string | `"100m"` | | | controlPlane.resources.requests.memory | string | `"256Mi"` | | -| controlPlane.terminationGracePeriodSeconds | int | `120` | Seconds between SIGTERM and SIGKILL. At least one reconcile tick (up to 90s when a platform is slow) plus SHUTDOWN_DRAIN_MS (10s) for in-flight provisions and integration hooks; raise it with SHUTDOWN_DRAIN_MS. | +| controlPlane.terminationGracePeriodSeconds | int | `120` | Seconds between SIGTERM and SIGKILL. At least one task's reconcile (up to 110s when platforms are slow) plus SHUTDOWN_DRAIN_MS (10s) for in-flight provisions and integration hooks; raise it with SHUTDOWN_DRAIN_MS. | | database.existingSecret | string | `""` | With postgresql.enabled false: a Secret with key `DATABASE_URL`. | | extraEnv | list | `[]` | Extra control-plane env. Only explicitly allowlisted HARNESS_* settings reach sandboxes; never put a real secret in a forwarded name. | | extraManifests | list | `[]` | Extra manifests, rendered with tpl. | diff --git a/deploy/helm/devbox/templates/control-plane.yaml b/deploy/helm/devbox/templates/control-plane.yaml index f05e30b..6def2ec 100644 --- a/deploy/helm/devbox/templates/control-plane.yaml +++ b/deploy/helm/devbox/templates/control-plane.yaml @@ -62,7 +62,7 @@ spec: serviceAccountName: devbox-control-plane {{ include "devbox.controlPlaneImagePullSecrets" . | indent 6 }} securityContext: { runAsNonRoot: true, runAsUser: 1000, runAsGroup: 1000 } - # Shutdown finishes the current tick, then drains for SHUTDOWN_DRAIN_MS. + # Shutdown finishes the task it is reconciling, then drains for SHUTDOWN_DRAIN_MS. terminationGracePeriodSeconds: {{ .Values.controlPlane.terminationGracePeriodSeconds }} containers: - name: control-plane diff --git a/deploy/helm/devbox/values.yaml b/deploy/helm/devbox/values.yaml index 29c47df..44d1afa 100644 --- a/deploy/helm/devbox/values.yaml +++ b/deploy/helm/devbox/values.yaml @@ -149,9 +149,9 @@ controlPlane: resources: requests: { cpu: 100m, memory: 256Mi } limits: { memory: 1Gi } - # -- Seconds between SIGTERM and SIGKILL. At least one reconcile tick (up to 90s - # when a platform is slow) plus SHUTDOWN_DRAIN_MS (10s) for in-flight provisions - # and integration hooks; raise it with SHUTDOWN_DRAIN_MS. + # -- Seconds between SIGTERM and SIGKILL. At least one task's reconcile (up to + # 110s when platforms are slow) plus SHUTDOWN_DRAIN_MS (10s) for in-flight + # provisions and integration hooks; raise it with SHUTDOWN_DRAIN_MS. terminationGracePeriodSeconds: 120 postgresql: From 221f73768fe7640b85a0f20a57cadaf7a82a8f01 Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Fri, 30 Oct 2026 19:18:57 +0530 Subject: [PATCH 09/22] chore(deploy): raise the control-plane grace period to 150s Bounding the onboarding post made one task's reconcile up to 110s, which with the 10s drain left the 120s default no headroom. Co-Authored-By: Claude Opus 5.5 --- control-plane/docker-compose.yml | 6 +++--- deploy/helm/devbox/README.md | 2 +- deploy/helm/devbox/tests/golden/byo-secrets.json | 2 +- deploy/helm/devbox/tests/golden/defaults.json | 2 +- deploy/helm/devbox/tests/golden/external-db.json | 2 +- deploy/helm/devbox/tests/golden/github-app-external.json | 2 +- deploy/helm/devbox/tests/golden/github-app.json | 2 +- deploy/helm/devbox/tests/golden/ingress-api.json | 2 +- deploy/helm/devbox/tests/golden/ingress-both.json | 2 +- deploy/helm/devbox/tests/golden/ingress-webhooks.json | 2 +- deploy/helm/devbox/tests/golden/linear.json | 2 +- deploy/helm/devbox/tests/golden/llm-stages.json | 2 +- deploy/helm/devbox/tests/golden/sandbox-options.json | 2 +- deploy/helm/devbox/tests/golden/slack.json | 2 +- deploy/helm/devbox/values.yaml | 7 ++++--- 15 files changed, 20 insertions(+), 19 deletions(-) diff --git a/control-plane/docker-compose.yml b/control-plane/docker-compose.yml index 3175e50..c7c4b39 100644 --- a/control-plane/docker-compose.yml +++ b/control-plane/docker-compose.yml @@ -94,9 +94,9 @@ services: build: *control-plane-build restart: unless-stopped # Shutdown finishes the task it is reconciling (up to 110s when platforms - # are slow), then drains for SHUTDOWN_DRAIN_MS (10s); Compose's default 10s - # would cut both short. - stop_grace_period: 120s + # are slow, plus 50s per extra PR it opens), then drains for + # SHUTDOWN_DRAIN_MS (10s); Compose's default 10s would cut both short. + stop_grace_period: 150s # Explicit, with no env_file. Only allowlisted HARNESS_* settings cross # into sandboxes, and a forwarded name must never hold a real secret. environment: diff --git a/deploy/helm/devbox/README.md b/deploy/helm/devbox/README.md index ad1577c..4946ed3 100644 --- a/deploy/helm/devbox/README.md +++ b/deploy/helm/devbox/README.md @@ -25,7 +25,7 @@ docker run --rm -v "$PWD/deploy/helm/devbox:/helm-docs" jnorwood/helm-docs:v1.14 | controlPlane.resources.limits.memory | string | `"1Gi"` | | | controlPlane.resources.requests.cpu | string | `"100m"` | | | controlPlane.resources.requests.memory | string | `"256Mi"` | | -| controlPlane.terminationGracePeriodSeconds | int | `120` | Seconds between SIGTERM and SIGKILL. At least one task's reconcile (up to 110s when platforms are slow) plus SHUTDOWN_DRAIN_MS (10s) for in-flight provisions and integration hooks; raise it with SHUTDOWN_DRAIN_MS. | +| controlPlane.terminationGracePeriodSeconds | int | `150` | Seconds between SIGTERM and SIGKILL. At least one task's reconcile (up to 110s when platforms are slow, plus 50s per extra PR opened in that reconcile) plus SHUTDOWN_DRAIN_MS (10s) for in-flight provisions and integration hooks; raise it with SHUTDOWN_DRAIN_MS. | | database.existingSecret | string | `""` | With postgresql.enabled false: a Secret with key `DATABASE_URL`. | | extraEnv | list | `[]` | Extra control-plane env. Only explicitly allowlisted HARNESS_* settings reach sandboxes; never put a real secret in a forwarded name. | | extraManifests | list | `[]` | Extra manifests, rendered with tpl. | diff --git a/deploy/helm/devbox/tests/golden/byo-secrets.json b/deploy/helm/devbox/tests/golden/byo-secrets.json index d173989..cf318a4 100644 --- a/deploy/helm/devbox/tests/golden/byo-secrets.json +++ b/deploy/helm/devbox/tests/golden/byo-secrets.json @@ -597,7 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, - "terminationGracePeriodSeconds": 120, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/defaults.json b/deploy/helm/devbox/tests/golden/defaults.json index 481da19..70060f0 100644 --- a/deploy/helm/devbox/tests/golden/defaults.json +++ b/deploy/helm/devbox/tests/golden/defaults.json @@ -597,7 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, - "terminationGracePeriodSeconds": 120, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/external-db.json b/deploy/helm/devbox/tests/golden/external-db.json index 2fe8982..c307850 100644 --- a/deploy/helm/devbox/tests/golden/external-db.json +++ b/deploy/helm/devbox/tests/golden/external-db.json @@ -597,7 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, - "terminationGracePeriodSeconds": 120, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/github-app-external.json b/deploy/helm/devbox/tests/golden/github-app-external.json index 9e27ad1..a3245f9 100644 --- a/deploy/helm/devbox/tests/golden/github-app-external.json +++ b/deploy/helm/devbox/tests/golden/github-app-external.json @@ -597,7 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, - "terminationGracePeriodSeconds": 120, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/github-app.json b/deploy/helm/devbox/tests/golden/github-app.json index 2e4226e..3431690 100644 --- a/deploy/helm/devbox/tests/golden/github-app.json +++ b/deploy/helm/devbox/tests/golden/github-app.json @@ -597,7 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, - "terminationGracePeriodSeconds": 120, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/ingress-api.json b/deploy/helm/devbox/tests/golden/ingress-api.json index a881601..8312390 100644 --- a/deploy/helm/devbox/tests/golden/ingress-api.json +++ b/deploy/helm/devbox/tests/golden/ingress-api.json @@ -597,7 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, - "terminationGracePeriodSeconds": 120, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/ingress-both.json b/deploy/helm/devbox/tests/golden/ingress-both.json index e050ec1..a92a867 100644 --- a/deploy/helm/devbox/tests/golden/ingress-both.json +++ b/deploy/helm/devbox/tests/golden/ingress-both.json @@ -597,7 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, - "terminationGracePeriodSeconds": 120, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/ingress-webhooks.json b/deploy/helm/devbox/tests/golden/ingress-webhooks.json index b69a1a0..4a567a0 100644 --- a/deploy/helm/devbox/tests/golden/ingress-webhooks.json +++ b/deploy/helm/devbox/tests/golden/ingress-webhooks.json @@ -597,7 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, - "terminationGracePeriodSeconds": 120, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/linear.json b/deploy/helm/devbox/tests/golden/linear.json index 31a36a5..8b00fbe 100644 --- a/deploy/helm/devbox/tests/golden/linear.json +++ b/deploy/helm/devbox/tests/golden/linear.json @@ -597,7 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, - "terminationGracePeriodSeconds": 120, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/llm-stages.json b/deploy/helm/devbox/tests/golden/llm-stages.json index 7ecfe9c..4d585bf 100644 --- a/deploy/helm/devbox/tests/golden/llm-stages.json +++ b/deploy/helm/devbox/tests/golden/llm-stages.json @@ -597,7 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, - "terminationGracePeriodSeconds": 120, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/sandbox-options.json b/deploy/helm/devbox/tests/golden/sandbox-options.json index 348d5ef..67f2843 100644 --- a/deploy/helm/devbox/tests/golden/sandbox-options.json +++ b/deploy/helm/devbox/tests/golden/sandbox-options.json @@ -607,7 +607,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, - "terminationGracePeriodSeconds": 120, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/slack.json b/deploy/helm/devbox/tests/golden/slack.json index 87cdd03..d538f36 100644 --- a/deploy/helm/devbox/tests/golden/slack.json +++ b/deploy/helm/devbox/tests/golden/slack.json @@ -597,7 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, - "terminationGracePeriodSeconds": 120, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/values.yaml b/deploy/helm/devbox/values.yaml index 44d1afa..deb695e 100644 --- a/deploy/helm/devbox/values.yaml +++ b/deploy/helm/devbox/values.yaml @@ -150,9 +150,10 @@ controlPlane: requests: { cpu: 100m, memory: 256Mi } limits: { memory: 1Gi } # -- Seconds between SIGTERM and SIGKILL. At least one task's reconcile (up to - # 110s when platforms are slow) plus SHUTDOWN_DRAIN_MS (10s) for in-flight - # provisions and integration hooks; raise it with SHUTDOWN_DRAIN_MS. - terminationGracePeriodSeconds: 120 + # 110s when platforms are slow, plus 50s per extra PR opened in that reconcile) + # plus SHUTDOWN_DRAIN_MS (10s) for in-flight provisions and integration hooks; + # raise it with SHUTDOWN_DRAIN_MS. + terminationGracePeriodSeconds: 150 postgresql: # -- A minimal bundled Postgres for trying devbox: no backups, HA or automated upgrades. From 7c5c617b76937f7734d0d620a705f5027df67cac Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Fri, 30 Oct 2026 19:46:27 +0530 Subject: [PATCH 10/22] feat(control-plane): case-insensitive task keys and a per-kind inbox index Task keys become unique regardless of case, and inbox rows dedupe on (source, kind, external_id), so ids of different kinds from one source can't collide. The old (source, external_id) index stays for a release so older pods keep working during a rolling deploy. Co-Authored-By: Claude Opus 5.5 --- .../drizzle/0001_tasks_key_lower.sql | 3 + .../0002_inbox_source_kind_external.sql | 2 + control-plane/drizzle/meta/0001_snapshot.json | 580 +++++++++++++++++ control-plane/drizzle/meta/0002_snapshot.json | 607 ++++++++++++++++++ control-plane/drizzle/meta/_journal.json | 16 +- .../db/__tests__/migration.baseline.test.ts | 9 + control-plane/src/db/schema.ts | 7 +- 7 files changed, 1222 insertions(+), 2 deletions(-) create mode 100644 control-plane/drizzle/0001_tasks_key_lower.sql create mode 100644 control-plane/drizzle/0002_inbox_source_kind_external.sql create mode 100644 control-plane/drizzle/meta/0001_snapshot.json create mode 100644 control-plane/drizzle/meta/0002_snapshot.json diff --git a/control-plane/drizzle/0001_tasks_key_lower.sql b/control-plane/drizzle/0001_tasks_key_lower.sql new file mode 100644 index 0000000..049627c --- /dev/null +++ b/control-plane/drizzle/0001_tasks_key_lower.sql @@ -0,0 +1,3 @@ +SET LOCAL lock_timeout = '5s';--> statement-breakpoint +CREATE UNIQUE INDEX "tasks_key_lower_uniq" ON "tasks" USING btree (lower("key"));--> statement-breakpoint +DROP INDEX "tasks_key_uniq"; \ No newline at end of file diff --git a/control-plane/drizzle/0002_inbox_source_kind_external.sql b/control-plane/drizzle/0002_inbox_source_kind_external.sql new file mode 100644 index 0000000..ad5cb6b --- /dev/null +++ b/control-plane/drizzle/0002_inbox_source_kind_external.sql @@ -0,0 +1,2 @@ +SET LOCAL lock_timeout = '5s';--> statement-breakpoint +CREATE UNIQUE INDEX "inbox_source_kind_external_uniq" ON "inbox" USING btree ("source","kind","external_id"); \ No newline at end of file diff --git a/control-plane/drizzle/meta/0001_snapshot.json b/control-plane/drizzle/meta/0001_snapshot.json new file mode 100644 index 0000000..da73214 --- /dev/null +++ b/control-plane/drizzle/meta/0001_snapshot.json @@ -0,0 +1,580 @@ +{ + "id": "f6737fb5-1099-49c5-be13-f3c095f0156c", + "prevId": "08e0c675-47da-4dc6-bb2e-1329f1a39eb5", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.api_tokens": { + "name": "api_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuidv7()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_hash": { + "name": "secret_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "github_login": { + "name": "github_login", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_tokens_name_unique": { + "name": "api_tokens_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.inbox": { + "name": "inbox", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuidv7()" + }, + "task_id": { + "name": "task_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "reply_targets": { + "name": "reply_targets", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "inbox_source_external_uniq": { + "name": "inbox_source_external_uniq", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_pending_idx": { + "name": "inbox_pending_idx", + "columns": [ + { + "expression": "task_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"inbox\".\"status\" = 'pending'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_task_created_idx": { + "name": "inbox_task_created_idx", + "columns": [ + { + "expression": "task_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "inbox_task_id_tasks_id_fk": { + "name": "inbox_task_id_tasks_id_fk", + "tableFrom": "inbox", + "tableTo": "tasks", + "columnsFrom": [ + "task_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.task_events": { + "name": "task_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "task_id": { + "name": "task_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "inbox_id": { + "name": "inbox_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "seq": { + "name": "seq", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "task_events_task_id_id_idx": { + "name": "task_events_task_id_id_idx", + "columns": [ + { + "expression": "task_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "task_events_task_id_tasks_id_fk": { + "name": "task_events_task_id_tasks_id_fk", + "tableFrom": "task_events", + "tableTo": "tasks", + "columnsFrom": [ + "task_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tasks": { + "name": "tasks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuidv7()" + }, + "source": { + "name": "source", + "type": "task_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_url": { + "name": "external_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "heartbeat_at": { + "name": "heartbeat_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "sandbox_id": { + "name": "sandbox_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pvc_name": { + "name": "pvc_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pr_urls": { + "name": "pr_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "last_observed_event_seq": { + "name": "last_observed_event_seq", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "tasks_source_external_id_uniq": { + "name": "tasks_source_external_id_uniq", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tasks_key_lower_uniq": { + "name": "tasks_key_lower_uniq", + "columns": [ + { + "expression": "lower(\"key\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tasks_lease_idx": { + "name": "tasks_lease_idx", + "columns": [ + { + "expression": "heartbeat_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"tasks\".\"owner_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "tasks_pr_urls_gin_idx": { + "name": "tasks_pr_urls_gin_idx", + "columns": [ + { + "expression": "pr_urls", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "jsonb_path_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "tasks_owner_sandbox_idx": { + "name": "tasks_owner_sandbox_idx", + "columns": [ + { + "expression": "owner_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"tasks\".\"sandbox_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tasks_created_by_api_tokens_id_fk": { + "name": "tasks_created_by_api_tokens_id_fk", + "tableFrom": "tasks", + "tableTo": "api_tokens", + "columnsFrom": [ + "created_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.task_source": { + "name": "task_source", + "schema": "public", + "values": [ + "linear", + "api" + ] + } + }, + "schemas": {}, + "sequences": { + "public.api_task_key_seq": { + "name": "api_task_key_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/control-plane/drizzle/meta/0002_snapshot.json b/control-plane/drizzle/meta/0002_snapshot.json new file mode 100644 index 0000000..e8c4e1b --- /dev/null +++ b/control-plane/drizzle/meta/0002_snapshot.json @@ -0,0 +1,607 @@ +{ + "id": "4f733928-3a33-4bef-890a-703ae5d73d88", + "prevId": "f6737fb5-1099-49c5-be13-f3c095f0156c", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.api_tokens": { + "name": "api_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuidv7()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_hash": { + "name": "secret_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "github_login": { + "name": "github_login", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_tokens_name_unique": { + "name": "api_tokens_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.inbox": { + "name": "inbox", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuidv7()" + }, + "task_id": { + "name": "task_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "reply_targets": { + "name": "reply_targets", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "inbox_source_kind_external_uniq": { + "name": "inbox_source_kind_external_uniq", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_source_external_uniq": { + "name": "inbox_source_external_uniq", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_pending_idx": { + "name": "inbox_pending_idx", + "columns": [ + { + "expression": "task_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"inbox\".\"status\" = 'pending'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_task_created_idx": { + "name": "inbox_task_created_idx", + "columns": [ + { + "expression": "task_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "inbox_task_id_tasks_id_fk": { + "name": "inbox_task_id_tasks_id_fk", + "tableFrom": "inbox", + "tableTo": "tasks", + "columnsFrom": [ + "task_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.task_events": { + "name": "task_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "task_id": { + "name": "task_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "inbox_id": { + "name": "inbox_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "seq": { + "name": "seq", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "task_events_task_id_id_idx": { + "name": "task_events_task_id_id_idx", + "columns": [ + { + "expression": "task_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "task_events_task_id_tasks_id_fk": { + "name": "task_events_task_id_tasks_id_fk", + "tableFrom": "task_events", + "tableTo": "tasks", + "columnsFrom": [ + "task_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tasks": { + "name": "tasks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuidv7()" + }, + "source": { + "name": "source", + "type": "task_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_url": { + "name": "external_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "heartbeat_at": { + "name": "heartbeat_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "sandbox_id": { + "name": "sandbox_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pvc_name": { + "name": "pvc_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pr_urls": { + "name": "pr_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "last_observed_event_seq": { + "name": "last_observed_event_seq", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "tasks_source_external_id_uniq": { + "name": "tasks_source_external_id_uniq", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tasks_key_lower_uniq": { + "name": "tasks_key_lower_uniq", + "columns": [ + { + "expression": "lower(\"key\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tasks_lease_idx": { + "name": "tasks_lease_idx", + "columns": [ + { + "expression": "heartbeat_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"tasks\".\"owner_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "tasks_pr_urls_gin_idx": { + "name": "tasks_pr_urls_gin_idx", + "columns": [ + { + "expression": "pr_urls", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "jsonb_path_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "tasks_owner_sandbox_idx": { + "name": "tasks_owner_sandbox_idx", + "columns": [ + { + "expression": "owner_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"tasks\".\"sandbox_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tasks_created_by_api_tokens_id_fk": { + "name": "tasks_created_by_api_tokens_id_fk", + "tableFrom": "tasks", + "tableTo": "api_tokens", + "columnsFrom": [ + "created_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.task_source": { + "name": "task_source", + "schema": "public", + "values": [ + "linear", + "api" + ] + } + }, + "schemas": {}, + "sequences": { + "public.api_task_key_seq": { + "name": "api_task_key_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/control-plane/drizzle/meta/_journal.json b/control-plane/drizzle/meta/_journal.json index a637ac5..9e21983 100644 --- a/control-plane/drizzle/meta/_journal.json +++ b/control-plane/drizzle/meta/_journal.json @@ -8,6 +8,20 @@ "when": 1792183649849, "tag": "0000_init", "breakpoints": true + }, + { + "idx": 1, + "version": "7", + "when": 1793368811788, + "tag": "0001_tasks_key_lower", + "breakpoints": true + }, + { + "idx": 2, + "version": "7", + "when": 1793368820991, + "tag": "0002_inbox_source_kind_external", + "breakpoints": true } ] -} +} \ No newline at end of file diff --git a/control-plane/src/db/__tests__/migration.baseline.test.ts b/control-plane/src/db/__tests__/migration.baseline.test.ts index 06e9aa6..ecc78ae 100644 --- a/control-plane/src/db/__tests__/migration.baseline.test.ts +++ b/control-plane/src/db/__tests__/migration.baseline.test.ts @@ -15,6 +15,15 @@ const journal = JSON.parse(readFileSync("drizzle/meta/_journal.json", "utf8")) a entries: Array<{ tag: string; when: number }>; }; +describe("drizzle journal", () => { + it("has strictly increasing timestamps", () => { + // The migrator applies a migration only when its `when` is past the last + // applied one, so a file out of order would be skipped for good. + const whens = journal.entries.map((e) => e.when); + expect(whens.every((w, i) => i === 0 || w > (whens[i - 1] ?? Number.NaN))).toBe(true); + }); +}); + async function applyBaseline(client: SQL): Promise { const migration = readFileSync(`drizzle/${journal.entries[0]?.tag}.sql`, "utf8"); for (const statement of migration.split("--> statement-breakpoint")) { diff --git a/control-plane/src/db/schema.ts b/control-plane/src/db/schema.ts index 8a55e56..a51c4ed 100644 --- a/control-plane/src/db/schema.ts +++ b/control-plane/src/db/schema.ts @@ -74,7 +74,8 @@ export const tasks = pgTable( }, (t) => [ uniqueIndex("tasks_source_external_id_uniq").on(t.source, t.externalId), - uniqueIndex("tasks_key_uniq").on(t.key), + // Keys are unique case-insensitively; readers match lower(key) = lower($1). + uniqueIndex("tasks_key_lower_uniq").on(sql`lower(${t.key})`), // Heartbeat scans only care whether a pod owns the task. index("tasks_lease_idx").on(t.heartbeatAt).where(sql`${t.ownerId} IS NOT NULL`), // GitHub webhook PR→task lookup. @@ -124,6 +125,10 @@ export const inbox = pgTable( }, (t) => [ // Idempotency anchor: webhook retries / LB reroutes collapse via this. + // Every ON CONFLICT targets (source, kind, external_id). + uniqueIndex("inbox_source_kind_external_uniq").on(t.source, t.kind, t.externalId), + // The stricter pre-phase-2 index, kept for one release so older pods' + // ON CONFLICT (source, external_id) still has its arbiter. Phase 3 drops it. uniqueIndex("inbox_source_external_uniq").on(t.source, t.externalId), // Pending = not yet delivered or dead-lettered. Sweep + claimNext filter on this. index("inbox_pending_idx").on(t.taskId).where(sql`${t.status} = 'pending'`), From fab93aa3d66fe735823f99b31fb39f40dfa9db47 Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Fri, 30 Oct 2026 19:46:46 +0530 Subject: [PATCH 11/22] feat(control-plane): create and follow up tasks through TaskIntake Linear, the API and GitHub create tasks and follow-ups only through TaskIntake. A re-delegation is compared with the stored one under the task's row lock, so concurrent webhooks restart a task at most once, and a restart refreshes title, URL, flags and requester. A duplicate Linear delegation now changes nothing. Follow-ups lock the task, drop rounds for PRs the task no longer owns, and run the CI loop breaker in the same transaction. Hard-coded source lists are derived from the task-source enum, with compile-time checks. Co-Authored-By: Claude Opus 5.5 --- control-plane/src/app.ts | 36 +- .../githubWebhook.controller.test.ts | 14 +- .../src/controllers/api.controller.ts | 15 +- .../controllers/githubWebhook.controller.ts | 12 +- .../src/integrations/__tests__/intake.test.ts | 393 ++++++++++++++++ control-plane/src/integrations/intake.ts | 167 +++++++ control-plane/src/integrations/types.ts | 19 +- .../__tests__/apiTask.repository.test.ts | 62 ++- .../__tests__/inbox.repository.test.ts | 4 +- .../__tests__/task.repository.test.ts | 203 +++------ .../src/repositories/apiTask.repository.ts | 112 +---- .../src/repositories/inbox.repository.ts | 151 +++++-- .../src/repositories/task.repository.ts | 424 +++++++++++------- .../src/repositories/taskEvent.repository.ts | 2 + .../__tests__/agentSession.service.test.ts | 159 ++----- .../__tests__/apiTask.service.test.ts | 18 +- .../services/__tests__/inbox.service.test.ts | 139 +++--- .../src/services/agentSession.service.ts | 117 ++--- control-plane/src/services/apiTask.service.ts | 81 ++-- control-plane/src/services/inbox.service.ts | 100 ++--- .../src/services/integrationHooks.ts | 9 +- control-plane/src/types/api.ts | 2 + control-plane/src/types/inboxMessage.ts | 2 +- 23 files changed, 1335 insertions(+), 906 deletions(-) create mode 100644 control-plane/src/integrations/__tests__/intake.test.ts create mode 100644 control-plane/src/integrations/intake.ts diff --git a/control-plane/src/app.ts b/control-plane/src/app.ts index 5fb4b26..489f87f 100644 --- a/control-plane/src/app.ts +++ b/control-plane/src/app.ts @@ -10,6 +10,7 @@ import { db } from "./db"; import { taskSource } from "./db/schema"; import { apiIntegration } from "./integrations/api"; import { githubIntegration } from "./integrations/github"; +import { Intake } from "./integrations/intake"; import { LinearIntegration } from "./integrations/linear"; import type { Integration, IntegrationName } from "./integrations/types"; import { logger } from "./logger"; @@ -95,17 +96,23 @@ export function createApp(): AppBundle { const replyHolds = new ReplyHolds(); // Integrations' lifecycle hooks; TaskProcessor.stop() drains detached ones. const hooks = new IntegrationHooks(integrations, inboxRepository); + // The only way tasks and rounds are created; a restart's preempted sandbox + // is killed in the hooks' tracked detached set. + const intake = new Intake(taskRepository, inboxRepository, sandboxManager, hooks, { + apiKeyPrefix: config.api.taskKeyPrefix, + ciResultMaxPerTask: config.github.ciResultMaxPerTask, + }); - const inboxService = new InboxService(inboxRepository, sandboxManager, integrations, hooks); - - const agentSessionService = new AgentSessionService( - taskRepository, + const inboxService = new InboxService( inboxRepository, - linearTokenProvider, sandboxManager, - inboxService, + integrations, + hooks, + intake, ); + const agentSessionService = new AgentSessionService(intake, linearTokenProvider, inboxService); + const slackNotifier = new SlackNotifierService( taskRepository, config.slack?.botToken ?? "", @@ -147,7 +154,7 @@ export function createApp(): AppBundle { ? new LinearWebhookController(agentSessionService, (at) => linearStatus.recordWebhook(at)) : undefined; const githubWebhookController = config.github.webhooks - ? new GitHubWebhookController(inboxService, taskRepository, config.github.webhooks) + ? new GitHubWebhookController(inboxService, intake, config.github.webhooks) : undefined; const githubStatus = new GithubStatusService(githubTokenState, { enabled: githubWebhookController !== undefined, @@ -167,17 +174,10 @@ export function createApp(): AppBundle { // internal ingress exists must expose nothing. let stopApiStreams = () => {}; if (config.api.enabled) { - const apiTaskService = new ApiTaskService( - apiTaskRepository, - taskRepository, - taskEventRepository, - sandboxManager, - { - keyPrefix: config.api.taskKeyPrefix, - publicBaseUrl: config.api.publicBaseUrl, - now: () => new Date(), - }, - ); + const apiTaskService = new ApiTaskService(apiTaskRepository, intake, taskEventRepository, { + publicBaseUrl: config.api.publicBaseUrl, + now: () => new Date(), + }); const apiController = new ApiController(apiTaskService, { authEnabled: config.api.authEnabled, ssePollMs: config.api.ssePollMs, diff --git a/control-plane/src/controllers/__tests__/githubWebhook.controller.test.ts b/control-plane/src/controllers/__tests__/githubWebhook.controller.test.ts index da566ad..8ab5956 100644 --- a/control-plane/src/controllers/__tests__/githubWebhook.controller.test.ts +++ b/control-plane/src/controllers/__tests__/githubWebhook.controller.test.ts @@ -2,8 +2,8 @@ import { beforeEach, describe, expect, it, mock, spyOn } from "bun:test"; import { createHmac } from "node:crypto"; import { Hono } from "hono"; import type { Task } from "../../db/schema"; +import type { TaskIntake } from "../../integrations/intake"; import { logger } from "../../logger"; -import type { TaskRepository } from "../../repositories/task.repository"; import type { InboxService } from "../../services/inbox.service"; const WEBHOOK_SECRET = "gh-test-secret"; @@ -373,7 +373,7 @@ describe("GitHubWebhookController.handleWebhook", () => { enqueueReview, enqueueCiResult, } as unknown as InboxServiceLike as InboxService; - const taskRepo = { findByPrUrl } as unknown as TaskRepository; + const intake = { findTaskByPrUrl: findByPrUrl } as unknown as TaskIntake; // Reset the module-mocked fetchOrgMembers so per-test mockImplementationOnce // overrides don't leak across cases. Default: resolves to the trusted-set // logins used across positive-path bodies. The bot login `acme-bot[bot]` is @@ -384,7 +384,7 @@ describe("GitHubWebhookController.handleWebhook", () => { getUserIdByLogin.mockReset(); getUserIdByLogin.mockResolvedValue(null); clock = { now: 1_000_000 }; - controller = new GitHubWebhookController(inboxService, taskRepo, SETTINGS, () => clock.now); + controller = new GitHubWebhookController(inboxService, intake, SETTINGS, () => clock.now); const a = new Hono(); a.post("/webhook/github", (c) => controller.handleWebhook(c)); return a; @@ -968,11 +968,11 @@ describe("GitHubWebhookController.handleWebhook", () => { enqueueReview: mock(() => Promise.resolve()), enqueueCiResult: flagOffEnqueueCiResult, } as unknown as InboxService; - const flagOffRepo = { findByPrUrl: flagOffFindByPrUrl } as unknown as TaskRepository; + const flagOffIntake = { findTaskByPrUrl: flagOffFindByPrUrl } as unknown as TaskIntake; fetchOrgMembers.mockResolvedValue(new Set([1, 2, 3]) as ReadonlySet); const flagOffApp = new Hono(); flagOffApp.post("/webhook/github", (c) => - new Ctrl(flagOffService, flagOffRepo, SETTINGS).handleWebhook(c), + new Ctrl(flagOffService, flagOffIntake, SETTINGS).handleWebhook(c), ); const body = workflowRunBody(); @@ -1136,10 +1136,10 @@ describe("GitHubWebhookController.handleWebhook", () => { enqueueReview, enqueueCiResult, } as unknown as InboxService; - const taskRepo = { findByPrUrl } as unknown as TaskRepository; + const intake = { findTaskByPrUrl: findByPrUrl } as unknown as TaskIntake; controller = new GitHubWebhookController( inboxService, - taskRepo, + intake, { ...SETTINGS, ...settings }, () => clock.now, ); diff --git a/control-plane/src/controllers/api.controller.ts b/control-plane/src/controllers/api.controller.ts index 1ddd67a..0b92c9b 100644 --- a/control-plane/src/controllers/api.controller.ts +++ b/control-plane/src/controllers/api.controller.ts @@ -1,7 +1,9 @@ import type { Context } from "hono"; import { type SSEStreamingApi, streamSSE } from "hono/streaming"; import type { Clock } from "../clock"; +import { taskSource } from "../db/schema"; import { ApiError } from "../errors"; +import type { TaskSource } from "../integrations/types"; import { logger } from "../logger"; import type { ApiTaskService, @@ -37,6 +39,7 @@ const TEXT_MAX_BYTES = 64 * 1024; const FLAGS_MAX = 16; const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; const STATES: readonly ApiTaskState[] = ["queued", "running", "idle", "failed"]; +const SOURCES: readonly string[] = taskSource.enumValues; export function errorBody(code: ApiErrorCode, message: string): ApiErrorBody { return { error: { code, message } }; @@ -142,7 +145,7 @@ export class ApiController { // Last-Event-ID / ?after=. Errors before the first byte are JSON; errors // after are logged and close the stream (no message to the client). async streamEvents(c: Context): Promise { - let target: { id: string; source: "linear" | "api" }; + let target: { id: string; source: TaskSource }; let start: number; try { start = parseStreamStart(c.req.header("last-event-id"), c.req.query("after")); @@ -205,7 +208,7 @@ export class ApiController { private async runStream( stream: SSEStreamingApi, - target: { id: string; source: "linear" | "api" }, + target: { id: string; source: TaskSource }, start: number, signal: AbortSignal, ): Promise { @@ -369,8 +372,12 @@ function parseListQuery(c: Context): ListTasksInput { }; const source = c.req.query("source"); if (source !== undefined) { - if (source !== "linear" && source !== "api") throw invalid("source must be linear or api"); - q.source = source; + if (!SOURCES.includes(source)) { + throw invalid( + `source must be ${SOURCES.slice(0, -1).join(", ")} or ${SOURCES[SOURCES.length - 1]}`, + ); + } + q.source = source as TaskSource; } const state = c.req.query("state"); if (state !== undefined) { diff --git a/control-plane/src/controllers/githubWebhook.controller.ts b/control-plane/src/controllers/githubWebhook.controller.ts index 9dfa505..4d4cc81 100644 --- a/control-plane/src/controllers/githubWebhook.controller.ts +++ b/control-plane/src/controllers/githubWebhook.controller.ts @@ -8,10 +8,10 @@ import { getUserIdByLogin, } from "../clients/github"; import { config } from "../config"; +import type { TaskIntake } from "../integrations/intake"; import type { GithubWebhooksConfig } from "../integrationsConfig"; import { logger } from "../logger"; import type { ReplyTargetSpec } from "../repositories/inbox.repository"; -import type { TaskRepository } from "../repositories/task.repository"; import type { InboxService } from "../services/inbox.service"; // ──── Parsed event surface ──── @@ -159,7 +159,7 @@ export function extractPrOpenerLogin(eventHeader: string | undefined, payload: u } // Canonical PR URL — built deterministically so it is byte-identical to the -// html_url the harness stores via pr_opened (findByPrUrl does an exact jsonb +// html_url the harness stores via pr_opened (findTaskByPrUrl does an exact jsonb // containment match). Never trust the payload's html_url formatting. function prUrlOf(owner: string, repo: string, prNumber: number): string { return `https://github.com/${owner}/${repo}/pull/${prNumber}`; @@ -341,7 +341,7 @@ export class GitHubWebhookController { constructor( private inboxService: InboxService, - private taskRepo: TaskRepository, + private intake: Pick, private readonly settings: GithubWebhooksConfig, private readonly now: () => number = Date.now, ) { @@ -453,7 +453,7 @@ export class GitHubWebhookController { // trust gate and DB lookup so unrelated org activity doesn't fan out // into API calls. // workflow_run events have no single PR opener field. The task-lookup - // (findByPrUrl) already scopes to bot-opened PRs, so the pre-filter is + // (findTaskByPrUrl) already scopes to bot-opened PRs, so the pre-filter is // redundant for this event type. if (eventHeader !== "workflow_run" && prOpenerLogin !== this.botPayloadLogin) { this.reject(c, "not_bot_pr"); @@ -510,7 +510,7 @@ export class GitHubWebhookController { // and logs each trip — nothing to surface in the response here. for (const pr of parsed.pullRequests) { const prUrl = prUrlOf(parsed.owner, parsed.repo, pr.number); - const task = await this.taskRepo.findByPrUrl(prUrl); + const task = await this.intake.findTaskByPrUrl(prUrl); if (!task) continue; logger.debug("workflow_run matched a bot-opened task; enqueueing ci_result", { taskId: task.id, @@ -550,7 +550,7 @@ export class GitHubWebhookController { // comment / review — task has no status; every comment enqueues an inbox // row. The sweep recreates the sandbox from PVC if needed and surfaces a // PVC-missing error if the workspace has already been pruned. - const task = await this.taskRepo.findByPrUrl(parsed.prUrl); + const task = await this.intake.findTaskByPrUrl(parsed.prUrl); if (!task) return c.json({ status: "ignored" }, 200); if (parsed.kind === "comment") { diff --git a/control-plane/src/integrations/__tests__/intake.test.ts b/control-plane/src/integrations/__tests__/intake.test.ts new file mode 100644 index 0000000..6e4af0e --- /dev/null +++ b/control-plane/src/integrations/__tests__/intake.test.ts @@ -0,0 +1,393 @@ +import { beforeAll, beforeEach, describe, expect, it, mock } from "bun:test"; +import { SQL } from "bun"; +import { sql } from "drizzle-orm"; + +const TEST_DATABASE_URL = + process.env.TEST_DATABASE_URL ?? "postgres://devbox:devbox@localhost:5432/devbox"; +mock.module("../../config", () => ({ config: { databaseUrl: TEST_DATABASE_URL } })); + +const { db } = await import("../../db"); +const { TaskRepository } = await import("../../repositories/task.repository"); +const { InboxRepository } = await import("../../repositories/inbox.repository"); +const { Intake } = await import("../intake"); +const { migrate } = await import("drizzle-orm/bun-sql/migrator"); + +import type { ReplyTarget } from "../../repositories/inbox.repository"; +import type { InboxMessageDraft, TaskIntake } from "../intake"; + +const session = (id: string, order?: number): ReplyTarget => ({ + id, + spec: { kind: "linear_session" }, + ...(order === undefined ? {} : { order }), +}); + +const PR = "https://github.com/o/r/pull/7"; +const ghComment = (id: string): InboxMessageDraft => ({ + source: "github", + kind: "comment", + external_id: id, + payload: { target_id: id, body: "@bot fix", pr: { repo: "r", pr_url: PR }, created_at: "" }, +}); +const ghTarget = (id: string): ReplyTarget => ({ + id, + spec: { kind: "issue_comment", owner: "o", repo: "r", prNumber: 7, commentId: Number(id) }, +}); +const ciResult = (run: number): InboxMessageDraft => ({ + source: "github", + kind: "ci_result", + external_id: `workflow_run:${run}:1:7`, + payload: { + target_id: String(run), + workflow_name: "CI", + workflow_path: ".github/workflows/ci.yml", + conclusion: "failure", + run_url: "u", + head_sha: "abc", + head_branch: "b", + pr: { repo: "r", pr_url: PR }, + }, +}); + +async function insertTask(o: { + source?: string; + externalId?: string; + key: string; + prs?: string[]; +}) { + const prUrls = (o.prs ?? []).map((url) => ({ repo: "r", url, branch: "b", commit_sha: "c" })); + const [row] = await db.execute<{ id: string; key: string }>(sql` + INSERT INTO tasks (source, external_id, key, title, external_url, pr_urls) + VALUES (${o.source ?? "api"}, ${o.externalId ?? crypto.randomUUID()}, ${o.key}, 't', 'u', + ${JSON.stringify(prUrls)}::text::jsonb) + RETURNING id, key;`); + return row as { id: string; key: string }; +} + +describe("Intake", () => { + let killSandbox: ReturnType; + let detach: ReturnType; + let intake: TaskIntake; + + // A Linear `created`: issue-1 / ISS-1, delegated through `target`. + const assignLinear = ( + target: ReplyTarget, + o: Partial[0]> = {}, + ) => + intake.assign({ + source: "linear", + externalId: "issue-1", + key: { hint: "ISS-1" }, + title: "Fix auth", + description: "desc", + flags: [], + url: () => "https://linear.app/x/issue/ISS-1", + target: () => target, + ...o, + }); + + beforeAll(async () => { + await migrate(db, { migrationsFolder: "./drizzle" }); + }); + beforeEach(async () => { + await db.execute( + sql`TRUNCATE TABLE task_events, inbox, tasks, api_tokens RESTART IDENTITY CASCADE`, + ); + killSandbox = mock(() => Promise.resolve()); + detach = mock(() => {}); + intake = new Intake( + new TaskRepository(), + new InboxRepository(), + { killSandbox }, + { detach }, + { apiKeyPrefix: "DBX-API-", ciResultMaxPerTask: 20 }, + ); + }); + + describe("assign", () => { + it("creates the task and its assignment round; the platform id is the round's external id", async () => { + const requester = { id: "u-1", displayName: "Ann", codeHost: "unlinked" as const }; + const res = await assignLinear(session("sess-1"), { flags: ["FOO"], requester }); + expect(res).toEqual({ taskId: expect.any(String), key: "ISS-1", result: "created" }); + + const [task] = await db.execute( + sql`SELECT source, external_id, key, title, external_url, metadata FROM tasks`, + ); + expect(task).toEqual({ + source: "linear", + external_id: "issue-1", + key: "ISS-1", + title: "Fix auth", + external_url: "https://linear.app/x/issue/ISS-1", + metadata: { experimentalFlags: ["FOO"], requester }, + }); + const rounds = await db.execute( + sql`SELECT source, kind, external_id, payload, reply_targets FROM inbox`, + ); + expect([...rounds]).toEqual([ + { + source: "linear", + kind: "assignment", + external_id: "issue-1", + payload: { + title: "Fix auth", + description: "desc", + external_url: "https://linear.app/x/issue/ISS-1", + target_id: "sess-1", + task_id: res.taskId, + task_key: "ISS-1", + }, + reply_targets: [session("sess-1")], + }, + ]); + // task_events is API-only. + expect(await db.execute(sql`SELECT 1 FROM task_events`)).toHaveLength(0); + }); + + it("a hint another task holds, in any case, gets the source prefix", async () => { + await insertTask({ key: "iss-1" }); + expect((await assignLinear(session("sess-1"))).key).toBe("LINEAR-ISS-1"); + }); + + it("makes the hint branch-safe", async () => { + expect((await assignLinear(session("sess-1"), { key: { hint: "ISS 1/x" } })).key).toBe( + "ISS-1-x", + ); + }); + + it("retries once when a concurrent insert takes the key", async () => { + // Another transaction holds an uncommitted task with the key: the check + // doesn't see it, the insert waits on the index, then fails with 23505 + // once it commits, and the retry picks the prefixed key. + const other = new SQL(TEST_DATABASE_URL); + try { + let release!: () => void; + const gate = new Promise((r) => { + release = r; + }); + let inserted!: () => void; + const ready = new Promise((r) => { + inserted = r; + }); + const holder = other.begin(async (tx) => { + await tx`INSERT INTO tasks (source, external_id, key, title, external_url) + VALUES ('api', 'other', 'ISS-1', 't', 'u')`; + inserted(); + await gate; + }); + await ready; + const pending = assignLinear(session("sess-1")); + await new Promise((r) => setTimeout(r, 150)); + release(); + await holder; + expect((await pending).key).toBe("LINEAR-ISS-1"); + } finally { + await other.close(); + } + }); + + it("compares delegation order when both targets carry one", async () => { + await assignLinear(session("d:5", 5)); + expect((await assignLinear(session("d:3", 3))).result).toBe("duplicate"); + expect((await assignLinear(session("d:5", 5))).result).toBe("duplicate"); + expect((await assignLinear(session("d:7", 7))).result).toBe("restarted"); + const [round] = await db.execute<{ reply_targets: ReplyTarget[] }>( + sql`SELECT reply_targets FROM inbox`, + ); + expect(round?.reply_targets).toEqual([session("d:7", 7)]); + }); + + it("a duplicate kills nothing", async () => { + const { taskId } = await assignLinear(session("sess-1")); + await db.execute(sql`UPDATE tasks SET sandbox_id = 'sb-1' WHERE id = ${taskId}`); + expect((await assignLinear(session("sess-1"))).result).toBe("duplicate"); + expect(killSandbox).not.toHaveBeenCalled(); + expect(detach).not.toHaveBeenCalled(); + }); + + // Moved from agentSession.service.test ("kills the preempted sandbox when a + // re-delegation restarts a live run"): intake now does the kill. + it("kills the preempted sandbox when a re-delegation restarts a live run", async () => { + const ann = { id: "u-1", displayName: "Ann", codeHost: "unlinked" as const }; + const { taskId } = await assignLinear(session("sess-1"), { flags: ["OLD"], requester: ann }); + await db.execute( + sql`UPDATE tasks SET sandbox_id = 'sb-old', pvc_name = 'pvc-old' WHERE id = ${taskId}`, + ); + + const res = await assignLinear(session("sess-2"), { + title: "Fix auth v2", + url: () => "https://linear.app/x/issue/ISS-1/v2", + flags: ["NEW"], + }); + + expect(res).toEqual({ taskId, key: "ISS-1", result: "restarted" }); + expect(killSandbox).toHaveBeenCalledTimes(1); + expect(killSandbox).toHaveBeenCalledWith("sb-old"); + // In the tracked detached set, so shutdown drains it. + expect(detach).toHaveBeenCalledWith(taskId, "killSandbox", expect.any(Promise)); + const [task] = await db.execute( + sql`SELECT title, external_url, metadata, sandbox_id, pvc_name FROM tasks`, + ); + // The old workspace volume is only unlinked, not deleted (§15 bug 6). + expect(task).toEqual({ + title: "Fix auth v2", + external_url: "https://linear.app/x/issue/ISS-1/v2", + metadata: { experimentalFlags: ["NEW"] }, + sandbox_id: null, + pvc_name: null, + }); + }); + + it("rejects a target of another integration, and a hint without an external id", async () => { + await expect(assignLinear({ id: "t", spec: { kind: "api_session" } })).rejects.toThrow( + "a linear round can't carry a api_session reply target", + ); + await expect(assignLinear(session("sess-1"), { externalId: undefined })).rejects.toThrow( + "a hint key needs externalId", + ); + expect(await db.execute(sql`SELECT 1 FROM tasks`)).toHaveLength(0); + }); + }); + + describe("followUp", () => { + it("inserts the round with the task's identity; a redelivery is a duplicate", async () => { + const task = await insertTask({ key: "DBX-API-1", prs: [PR] }); + const first = await intake.followUp({ + task, + message: ghComment("555"), + targets: [ghTarget("555")], + prUrl: PR, + }); + const [row] = await db.execute<{ id: string; payload: unknown }>( + sql`SELECT id, payload FROM inbox`, + ); + expect(first).toEqual({ roundId: (row as { id: string }).id }); + expect(row?.payload).toEqual({ + ...ghComment("555").payload, + task_id: task.id, + task_key: "DBX-API-1", + }); + expect( + await intake.followUp({ task, message: ghComment("555"), targets: [ghTarget("555")] }), + ).toEqual({ dropped: "duplicate" }); + // A GitHub round on an API task writes nothing to the log. + expect(await db.execute(sql`SELECT 1 FROM task_events`)).toHaveLength(0); + }); + + it("a review whose id equals a stored comment id is a duplicate while the old index exists", async () => { + // changes in phase 3 (spec I9: the old (source, external_id) index goes) + const task = await insertTask({ key: "DBX-API-1" }); + await intake.followUp({ task, message: ghComment("555"), targets: [ghTarget("555")] }); + const review = await intake.followUp({ + task, + message: { + source: "github", + kind: "review", + external_id: "555", + payload: { + review_body: { target_id: "555", body: "@bot" }, + inline_comments: [], + pr: { repo: "r", pr_url: PR }, + created_at: "", + }, + }, + targets: [ + { + id: "555", + spec: { kind: "review_body", owner: "o", repo: "r", prNumber: 7, reviewId: 555 }, + }, + ], + }); + expect(review).toEqual({ dropped: "duplicate" }); + }); + + it("drops a round whose PR is no longer on the task", async () => { + const task = await insertTask({ key: "DBX-API-1" }); + expect( + await intake.followUp({ + task, + message: ghComment("555"), + targets: [ghTarget("555")], + prUrl: PR, + }), + ).toEqual({ dropped: "stale" }); + expect(await db.execute(sql`SELECT 1 FROM inbox`)).toHaveLength(0); + }); + + it("writes message_queued for an inserted API message only", async () => { + const task = await insertTask({ key: "DBX-API-1" }); + const message: InboxMessageDraft = { + source: "api", + kind: "comment", + external_id: "m-1", + payload: { target_id: task.id, body: "more", pr: null, created_at: "" }, + }; + const targets: ReplyTarget[] = [{ id: task.id, spec: { kind: "api_session" } }]; + const first = await intake.followUp({ task, message, targets }); + expect(await intake.followUp({ task, message, targets })).toEqual({ dropped: "duplicate" }); + const events = await db.execute(sql`SELECT type, inbox_id, data FROM task_events`); + expect([...events]).toEqual([ + { + type: "message_queued", + inbox_id: (first as { roundId: string }).roundId, + data: { body: "more" }, + }, + ]); + }); + + // Moved from inbox.service.test ("enqueues when under the loop-breaker + // threshold" / "drops the CI failure once the loop-breaker threshold is + // reached"): the count now runs in the follow-up transaction. + it("enqueues when under the loop-breaker threshold", async () => { + const task = await insertTask({ key: "DBX-API-1" }); + for (let run = 1; run <= 19; run++) { + await intake.followUp({ task, message: ciResult(run), targets: [ghTarget(String(run))] }); + } + const res = await intake.followUp({ task, message: ciResult(20), targets: [ghTarget("20")] }); + expect(res).toEqual({ roundId: expect.any(String) }); + }); + + it("drops the CI failure once the loop-breaker threshold is reached", async () => { + const task = await insertTask({ key: "DBX-API-1" }); + for (let run = 1; run <= 20; run++) { + await intake.followUp({ task, message: ciResult(run), targets: [ghTarget(String(run))] }); + } + const res = await intake.followUp({ task, message: ciResult(21), targets: [ghTarget("21")] }); + expect(res).toEqual({ dropped: "ci_limit" }); + const [n] = await db.execute(sql`SELECT count(*)::int AS n FROM inbox`); + expect(n).toEqual({ n: 20 }); + }); + + it("checks payload.target_id against targets[0] (reviews exempt) and target kinds", async () => { + const task = await insertTask({ key: "DBX-API-1" }); + await expect( + intake.followUp({ task, message: ghComment("555"), targets: [ghTarget("556")] }), + ).rejects.toThrow("payload.target_id must equal targets[0].id"); + await expect( + intake.followUp({ task, message: ghComment("555"), targets: [session("555")] }), + ).rejects.toThrow("a github round can't carry a linear_session reply target"); + // A review's targets[0] is its first inline comment; it has no target_id. + const review = await intake.followUp({ + task, + message: { + source: "github", + kind: "review", + external_id: "88", + payload: { + review_body: { target_id: "88", body: "@bot" }, + inline_comments: [{ target_id: "11", path: "a.ts", line: 3, body: "nit" }], + pr: { repo: "r", pr_url: PR }, + created_at: "", + }, + }, + targets: [ghTarget("11")], + }); + expect(review).toEqual({ roundId: expect.any(String) }); + }); + }); + + it("findTask matches on source and external id", async () => { + const t = await insertTask({ source: "linear", externalId: "issue-9", key: "ISS-9" }); + expect((await intake.findTask("linear", "issue-9"))?.id).toBe(t.id); + expect(await intake.findTask("api", "issue-9")).toBeNull(); + }); +}); diff --git a/control-plane/src/integrations/intake.ts b/control-plane/src/integrations/intake.ts new file mode 100644 index 0000000..7d4912c --- /dev/null +++ b/control-plane/src/integrations/intake.ts @@ -0,0 +1,167 @@ +import type { Task } from "../db/schema"; +import { logger } from "../logger"; +import { + apiSessionTarget, + type InboxRepository, + type NewInboxMessage, + type ReplyTarget, + type ReplyTargetSpec, + taskIdentity, +} from "../repositories/inbox.repository"; +import type { TaskRepository } from "../repositories/task.repository"; +import type { IntegrationHooks } from "../services/integrationHooks"; +import type { SandboxManager } from "../services/sandboxManager.service"; +import type { InboxMessage, TaskIdentity } from "../types/inboxMessage"; +import type { IntegrationName, Person, TaskSource } from "./types"; + +// The only way modules create work (§3, §5). + +export type TaskRef = { id: string; key: string }; +type Draft = M extends InboxMessage + ? Pick & { payload: Omit } + : never; +export type InboxMessageDraft = Draft>; + +export interface TaskIntake { + assign(p: { + source: TaskSource; + externalId?: string; // required with a hint key; with a sequence key the new key is used + key: { hint: string } | { sequence: "api" }; + title: string; + description: string | null; // Markdown/plain; flags already stripped; bot mention stripped + flags: string[]; // modules parse with parseExperimentalFlags; intake never parses + url: (t: TaskRef) => string; + target: (t: TaskRef) => ReplyTarget; // target.id = delegation id; optional target.order (monotonic number) + requester?: Person; // tasks.metadata.requester + createdBy?: string; // API token id + }): Promise<{ taskId: string; key: string; result: "created" | "restarted" | "duplicate" }>; + + restart( + taskId: string, + def: { title: string; description: string | null; flags: string[] }, + ): Promise; + + followUp(p: { + task: Pick; + message: InboxMessageDraft; + targets: ReplyTarget[]; + prUrl?: string; + }): Promise<{ roundId: string } | { dropped: "duplicate" | "stale" | "ci_limit" }>; + + findTask(source: TaskSource, externalId: string): Promise; + findTaskByPrUrl(prUrl: string): Promise; +} + +// Which integration each reply-target kind belongs to. Exists only for the +// check that a round's targets belong to the round's own integration. +const TARGET_OWNER: Record = { + linear_session: "linear", + api_session: "api", + issue_comment: "github", + review_comment: "github", + review_body: "github", +}; + +export class Intake implements TaskIntake { + constructor( + private taskRepo: TaskRepository, + private inboxRepo: InboxRepository, + private sandboxManager: Pick, + private hooks: Pick, + private opts: { apiKeyPrefix: string; ciResultMaxPerTask: number }, + ) {} + + async assign(p: Parameters[0]): ReturnType { + if ("hint" in p.key && !p.externalId) throw new Error("assign: a hint key needs externalId"); + const { prevSandboxId, ...t } = await this.taskRepo.assign({ + ...p, + key: + "hint" in p.key + ? { hint: p.key.hint.replace(/[^A-Za-z0-9-]/g, "-") } // branch-safe + : { sequencePrefix: this.opts.apiKeyPrefix }, + target: (ref) => { + const target = p.target(ref); + checkTargets(p.source, [target]); + return target; + }, + }); + this.killPreempted(t.id, prevSandboxId); + return { taskId: t.id, key: t.key, result: t.result }; + } + + // API only: today's API restart, with the definition replacing the current + // title, description and flags. + async restart( + taskId: string, + def: { title: string; description: string | null; flags: string[] }, + ): Promise { + const task = await this.taskRepo.getById(taskId); + if (!task) throw new Error(`restart: task ${taskId} not found`); + const { prevSandboxId } = await this.taskRepo.restartTaskWithInbox({ + taskId, + inboxPayload: { + title: def.title, + description: def.description, + external_url: task.externalUrl, + target_id: taskId, + }, + inboxReplyTargets: [apiSessionTarget(taskId)], + api: { title: def.title, description: def.description, experimentalFlags: def.flags }, + }); + this.killPreempted(taskId, prevSandboxId); + } + + async followUp(p: Parameters[0]): ReturnType { + const { task, message, targets } = p; + // A review's payload carries one target id per thread instead. + if ("target_id" in message.payload && message.payload.target_id !== targets[0]?.id) { + throw new Error("followUp: payload.target_id must equal targets[0].id"); + } + checkTargets(message.source, targets); + return this.inboxRepo.appendFollowUp({ + taskId: task.id, + message: { + ...message, + payload: { ...message.payload, ...taskIdentity(task) }, + } as NewInboxMessage, + replyTargets: targets, + prUrl: p.prUrl, + ciResultMax: this.opts.ciResultMaxPerTask, + }); + } + + findTask(source: TaskSource, externalId: string): Promise { + return this.taskRepo.findByExternalId(source, externalId); + } + + findTaskByPrUrl(prUrl: string): Promise { + return this.taskRepo.findByPrUrl(prUrl); + } + + // A restart's preempted sandbox, killed after commit in the tracked detached + // set so shutdown drains it. The old workspace volume is kept (§15 bug 6). + private killPreempted(taskId: string, sandboxId: string | null): void { + if (!sandboxId) return; + this.hooks.detach( + taskId, + "killSandbox", + this.sandboxManager.killSandbox(sandboxId).catch((err) => { + const e = err instanceof Error ? err : new Error(String(err)); + logger.error("killSandbox(restart) failed", { + taskId, + sandboxId, + message: e.message, + stack: e.stack, + }); + }), + ); + } +} + +function checkTargets(source: IntegrationName, targets: ReplyTarget[]): void { + for (const t of targets) { + if (TARGET_OWNER[t.spec.kind] !== source) { + throw new Error(`a ${source} round can't carry a ${t.spec.kind} reply target`); + } + } +} diff --git a/control-plane/src/integrations/types.ts b/control-plane/src/integrations/types.ts index b1da3cd..1a20acc 100644 --- a/control-plane/src/integrations/types.ts +++ b/control-plane/src/integrations/types.ts @@ -1,13 +1,28 @@ -import type { Task } from "../db/schema"; +import type { Task, taskSource } from "../db/schema"; import type { logger } from "../logger"; import type { InboxRow, ReplyTarget } from "../repositories/inbox.repository"; +import type { ApiTaskSource } from "../types/api"; +import type { InboxSource } from "../types/inboxMessage"; import type { Outbound, SendCtx } from "./outbound"; type Logger = typeof logger; -// One per task/round source (today's InboxSource). +// The sources a task can have (the task_source enum). +export type TaskSource = (typeof taskSource.enumValues)[number]; +// One per task/round source (InboxSource). export type IntegrationName = "linear" | "api" | "github"; +// Compile-time checks: the literal unions kept where db/schema can't be +// imported (types/api.ts is bundled into the CLI) match what they mirror, so a +// new source fails the build until each is updated. +type Equal = + (() => T extends A ? 1 : 2) extends () => T extends B ? 1 : 2 ? true : false; +type Assert = T; +export type SourceChecks = [ + Assert>, + Assert>, +]; + export interface Integration { name: IntegrationName; diff --git a/control-plane/src/repositories/__tests__/apiTask.repository.test.ts b/control-plane/src/repositories/__tests__/apiTask.repository.test.ts index 1ad712d..a3ae765 100644 --- a/control-plane/src/repositories/__tests__/apiTask.repository.test.ts +++ b/control-plane/src/repositories/__tests__/apiTask.repository.test.ts @@ -9,7 +9,7 @@ mock.module("../../config", () => ({ config: { databaseUrl: TEST_DATABASE_URL } const { db } = await import("../../db"); const { ApiTaskRepository } = await import("../apiTask.repository"); const { TaskRepository } = await import("../task.repository"); -const { InboxRepository } = await import("../inbox.repository"); +const { InboxRepository, apiSessionTarget } = await import("../inbox.repository"); const { migrate } = await import("drizzle-orm/bun-sql/migrator"); const repo = new ApiTaskRepository(); @@ -17,16 +17,44 @@ const taskRepo = new TaskRepository(); const inboxRepo = new InboxRepository(); const T0 = "2026-10-07T12:00:00.000Z"; -const create = (o: Partial[0]> = {}) => - repo.createApiTask({ - keyPrefix: "DBX-API-", - publicBaseUrl: "https://api.example", +// An API create: assign with a sequence key, as TaskIntake does it. +const create = async (o: { createdBy?: string } = {}) => { + const t = await taskRepo.assign({ + source: "api", + key: { sequencePrefix: "DBX-API-" }, title: "Add a thing", description: "Please", - experimentalFlags: ["FOO"], - createdBy: null, - ...o, + flags: ["FOO"], + url: (ref) => `https://api.example/api/tasks/${ref.key}`, + target: (ref) => apiSessionTarget(ref.id), + createdBy: o.createdBy, }); + return { taskId: t.id, key: t.key }; +}; + +// An API message: a comment round through the follow-up transaction. +const appendMessage = async (p: { task: { id: string; key: string }; body: string }) => { + const result = await inboxRepo.appendFollowUp({ + taskId: p.task.id, + message: { + source: "api", + kind: "comment", + external_id: crypto.randomUUID(), + payload: { + target_id: p.task.id, + body: p.body, + pr: null, + created_at: T0, + task_id: p.task.id, + task_key: p.task.key, + }, + }, + replyTargets: [apiSessionTarget(p.task.id)], + ciResultMax: 0, + }); + if (!("roundId" in result)) throw new Error(`dropped: ${result.dropped}`); + return result; +}; // Pins every DB timestamp of a task to T0 (+ offsets for inbox rows) so whole // records compare exactly. @@ -64,7 +92,7 @@ describe("ApiTaskRepository", () => { await db.execute(sql`ALTER SEQUENCE api_task_key_seq RESTART`); }); - describe("createApiTask", () => { + describe("assign (API create)", () => { it("inserts the task, the assignment row and task_created in one transaction", async () => { const { taskId, key } = await create(); expect(key).toBe("DBX-API-1"); @@ -157,14 +185,10 @@ describe("ApiTaskRepository", () => { }); }); - describe("appendApiMessage", () => { + describe("appendFollowUp (API message)", () => { it("inserts a comment row and message_queued atomically", async () => { const { taskId, key } = await create(); - const { roundId } = await repo.appendApiMessage({ - task: { id: taskId, key }, - body: "more", - createdAt: T0, - }); + const { roundId } = await appendMessage({ task: { id: taskId, key }, body: "more" }); const [row] = await db.execute( sql`SELECT source, kind, payload, reply_targets FROM inbox WHERE id = ${roundId}`, ); @@ -195,9 +219,7 @@ describe("ApiTaskRepository", () => { sql`ALTER TABLE task_events ADD CONSTRAINT test_block CHECK (type <> 'message_queued')`, ); try { - await expect( - repo.appendApiMessage({ task: { id: taskId, key }, body: "x", createdAt: T0 }), - ).rejects.toThrow(); + await expect(appendMessage({ task: { id: taskId, key }, body: "x" })).rejects.toThrow(); const [n] = await db.execute<{ n: number }>( sql`SELECT count(*)::int AS n FROM inbox WHERE kind = 'comment'`, ); @@ -428,11 +450,11 @@ describe("ApiTaskRepository", () => { expect(await logTypes(taskId)).toEqual(["task_created", "agent_action", cpType]); } - it("appendApiMessage waits for a concurrent per-task writer to commit", async () => { + it("an API message waits for a concurrent per-task writer to commit", async () => { const { taskId, key } = await create(); await expectSerialized( taskId, - () => repo.appendApiMessage({ task: { id: taskId, key }, body: "more", createdAt: T0 }), + () => appendMessage({ task: { id: taskId, key }, body: "more" }), "message_queued", ); }); diff --git a/control-plane/src/repositories/__tests__/inbox.repository.test.ts b/control-plane/src/repositories/__tests__/inbox.repository.test.ts index 99bcc2f..a86c032 100644 --- a/control-plane/src/repositories/__tests__/inbox.repository.test.ts +++ b/control-plane/src/repositories/__tests__/inbox.repository.test.ts @@ -150,7 +150,9 @@ describe("InboxRepository", () => { }); it("dedupes across kinds: a review whose id equals a stored comment id is dropped", async () => { - // characterization: changes in phase 2 (spec I9: unique on (source, kind, external_id)) + // characterization: changes in phase 3 (spec I9: phase 2 adds the + // (source, kind, external_id) index, but the old (source, external_id) + // one still rejects this row until phase 3 drops it) const task = await insertTask(); await repo.appendInbox({ taskId: task.id, message: ghComment("555"), replyTargets: [] }); diff --git a/control-plane/src/repositories/__tests__/task.repository.test.ts b/control-plane/src/repositories/__tests__/task.repository.test.ts index 964a002..997650d 100644 --- a/control-plane/src/repositories/__tests__/task.repository.test.ts +++ b/control-plane/src/repositories/__tests__/task.repository.test.ts @@ -39,6 +39,22 @@ const assignmentContent = { target_id: "agent-session-1", }; +// A Linear `created` through assign: issue-1 / ISS-1. The delegation (agent +// session) defaults to assignmentContent's target_id. +const assignLinear = ( + o: { session?: string; title?: string; url?: string; key?: string; flags?: string[] } = {}, +) => + repo.assign({ + source: "linear", + externalId: "issue-1", + key: { hint: o.key ?? "ISS-1" }, + title: o.title ?? "Fix auth", + description: "desc", + flags: o.flags ?? [], + url: () => o.url ?? "https://linear.app/x/issue/ISS-1", + target: () => linearTarget(o.session ?? "agent-session-1"), + }); + // Inserts a bare tasks row directly. Returns the synthetic id. async function insertTask(overrides: Record = {}): Promise<{ id: string }> { const [row] = await db @@ -102,19 +118,11 @@ describe("TaskRepository", () => { await db.execute(sql`ALTER SEQUENCE api_task_key_seq RESTART`); }); - describe("upsertFromAgentSessionWithInbox", () => { + describe("assign (Linear created)", () => { it("inserts task + inbox row in one tx, created=true on fresh insert", async () => { - const result = await repo.upsertFromAgentSessionWithInbox({ - taskKey: "ISS-1", - taskExternalId: "issue-1", - taskTitle: "Fix auth", - taskExternalUrl: "https://linear.app/x/issue/ISS-1", - inboxPayload: assignmentContent, - inboxReplyTargets: [linearTarget("sess-1")], - metadata: { experimentalFlags: ["FAST_PATH"] }, - }); + const result = await assignLinear({ session: "sess-1", flags: ["FAST_PATH"] }); - expect(result.created).toBe(true); + expect(result.result).toBe("created"); const tasks = await db.select().from(schema.tasks); expect(tasks).toHaveLength(1); @@ -130,26 +138,14 @@ describe("TaskRepository", () => { }); it("sets tasks.key on insert and never changes it on conflict", async () => { - const first = await repo.upsertFromAgentSessionWithInbox({ - taskExternalId: "issue-1", - taskKey: "ISS-1", - taskTitle: "Fix auth", - taskExternalUrl: "https://linear.app/x/issue/ISS-1", - inboxPayload: assignmentContent, - inboxReplyTargets: [linearTarget("sess-1")], - metadata: {}, - }); + const first = await assignLinear({ session: "sess-1" }); // The issue moved team: new URL, new identifier. The stored key stays. - await repo.upsertFromAgentSessionWithInbox({ - taskExternalId: "issue-1", - taskKey: "XYZ-9", - taskTitle: "Fix auth", - taskExternalUrl: "https://linear.app/x/issue/XYZ-9", - inboxPayload: assignmentContent, - inboxReplyTargets: [linearTarget("sess-2")], - metadata: {}, + await assignLinear({ + session: "sess-2", + key: "XYZ-9", + url: "https://linear.app/x/issue/XYZ-9", }); - const row = await repo.getById(first.taskId); + const row = await repo.getById(first.id); expect({ key: row?.key, createdBy: row?.createdBy }).toEqual({ key: "ISS-1", createdBy: null, @@ -157,45 +153,28 @@ describe("TaskRepository", () => { }); it("created=false on conflict; inbox INSERT is a no-op; existing reply_targets persist", async () => { - await repo.upsertFromAgentSessionWithInbox({ - taskKey: "ISS-1", - taskExternalId: "issue-1", - taskTitle: "Fix auth", - taskExternalUrl: "https://linear.app/x/issue/ISS-1", - inboxPayload: assignmentContent, - inboxReplyTargets: [linearTarget("sess-1")], - metadata: {}, - }); + await assignLinear({ session: "sess-1" }); // Seed a durable retry budget on the existing row. A duplicate `created` - // redelivery passes metadata={}; the upsert MERGES (metadata || EXCLUDED), - // so the budget must be PRESERVED — a blind overwrite would silently reset + // redelivery must PRESERVE it — a blind overwrite would silently reset // the sandbox-retry cap. await db.execute(sql` UPDATE tasks SET metadata = '{"retryCount":2}'::jsonb WHERE external_id = 'issue-1'; `); - const result = await repo.upsertFromAgentSessionWithInbox({ - taskKey: "ISS-1", - taskExternalId: "issue-1", - taskTitle: "Fix auth v2", - taskExternalUrl: "https://linear.app/x/issue/ISS-1", - inboxPayload: assignmentContent, - inboxReplyTargets: [linearTarget("sess-2")], - metadata: {}, - }); + // The same delegation (session) redelivered. + const result = await assignLinear({ session: "sess-1", title: "Fix auth v2" }); - expect(result.created).toBe(false); + expect(result.result).toBe("duplicate"); const [task] = await db.select().from(schema.tasks); - expect(task?.title).toBe("Fix auth v2"); - // retryCount survives the duplicate-redelivery upsert (merge, not replace). + // A duplicate changes nothing, not even the title (spec §13). + expect(task?.title).toBe("Fix auth"); + // retryCount survives the duplicate redelivery. expect((task?.metadata as { retryCount?: number })?.retryCount).toBe(2); - // Still exactly one inbox row — INSERT was DO NOTHING on conflict, so - // the existing reply_targets carrying "sess-1" are preserved (the - // caller reads this via inboxRepo.getAssignmentReplyTargetId for - // duplicate-webhook detection). + // Still exactly one inbox row, and the existing reply_targets carrying + // "sess-1" are preserved (assign compares them for duplicate detection). const rows = await db.execute<{ id: string; reply_targets: Array<{ id: string }>; @@ -204,20 +183,9 @@ describe("TaskRepository", () => { expect(rows[0]?.reply_targets[0]?.id).toBe("sess-1"); }); - // The Linear `created` handler runs this upsert first and restarts only when - // the stored session id differs, so a same-session redelivery ends here. - it("a duplicate created (same session) refreshes title, URL and flags but leaves the run alone", async () => { - // characterization: changes in phase 2 (spec §13: a duplicate created no longer refreshes title and URL) - // Only the title / URL / flags assertions flip; the run stays untouched. - const { taskId } = await repo.upsertFromAgentSessionWithInbox({ - taskKey: "ISS-1", - taskExternalId: "issue-1", - taskTitle: "Fix auth", - taskExternalUrl: "https://linear.app/x/issue/ISS-1", - inboxPayload: assignmentContent, - inboxReplyTargets: [linearTarget("sess-1")], - metadata: { experimentalFlags: ["OLD"] }, - }); + it("a duplicate created (same session) changes nothing: not the title, URL or flags, nor the run", async () => { + // Flipped in phase 2 (spec §13): it used to refresh title, URL and flags. + const { id: taskId } = await assignLinear({ session: "sess-1", flags: ["OLD"] }); await db.execute(sql` UPDATE tasks SET owner_id = 'pod-a', heartbeat_at = now(), sandbox_id = 'sb-1', pvc_name = 'pvc-1', @@ -228,21 +196,24 @@ describe("TaskRepository", () => { await db.execute(sql`UPDATE inbox SET status = 'delivered'`); const [before] = await db.execute<{ id: string }>(sql`SELECT id FROM inbox`); - await repo.upsertFromAgentSessionWithInbox({ - taskKey: "ISS-1", - taskExternalId: "issue-1", - taskTitle: "Fix auth (edited)", - taskExternalUrl: "https://linear.app/x/issue/ISS-1/fix-auth-edited", - inboxPayload: { ...assignmentContent, title: "Fix auth (edited)" }, - inboxReplyTargets: [linearTarget("sess-1")], - metadata: { experimentalFlags: ["NEW"] }, + const result = await assignLinear({ + session: "sess-1", + title: "Fix auth (edited)", + url: "https://linear.app/x/issue/ISS-1/fix-auth-edited", + flags: ["NEW"], }); + expect(result).toEqual({ + id: taskId, + key: "ISS-1", + result: "duplicate", + prevSandboxId: null, + }); const task = await repo.getById(taskId); expect({ title: task?.title, url: task?.externalUrl, metadata: task?.metadata }).toEqual({ - title: "Fix auth (edited)", - url: "https://linear.app/x/issue/ISS-1/fix-auth-edited", - metadata: { experimentalFlags: ["NEW"] }, + title: "Fix auth", + url: "https://linear.app/x/issue/ISS-1", + metadata: { experimentalFlags: ["OLD"] }, }); expect({ ownerId: task?.ownerId, @@ -270,34 +241,17 @@ describe("TaskRepository", () => { }); describe("restartTaskWithInbox", () => { - it("re-delegation: the conflict upsert refreshes title, URL and flags; the restart drops retryCount", async () => { - const { taskId } = await repo.upsertFromAgentSessionWithInbox({ - taskKey: "ISS-1", - taskExternalId: "issue-1", - taskTitle: "Fix auth", - taskExternalUrl: "https://linear.app/x/issue/ISS-1", - inboxPayload: assignmentContent, - inboxReplyTargets: [linearTarget("sess-1")], - metadata: { experimentalFlags: ["OLD"] }, - }); + it("re-delegation: assign restarts, refreshing title, URL and flags and dropping retryCount", async () => { + const { id: taskId } = await assignLinear({ session: "sess-1", flags: ["OLD"] }); await db.execute(sql` UPDATE tasks SET metadata = metadata || '{"retryCount":2}'::jsonb WHERE id = ${taskId}; `); - const next = { - taskKey: "ISS-1", - taskExternalId: "issue-1", - taskTitle: "Fix auth v2", - taskExternalUrl: "https://linear.app/x/issue/ISS-1/v2", - inboxPayload: { ...assignmentContent, title: "Fix auth v2", target_id: "sess-2" }, - inboxReplyTargets: [linearTarget("sess-2")], - metadata: { experimentalFlags: ["NEW"] }, - }; - await repo.upsertFromAgentSessionWithInbox(next); - await repo.restartTaskWithInbox({ - taskId, - inboxPayload: next.inboxPayload, - inboxReplyTargets: next.inboxReplyTargets, + await assignLinear({ + session: "sess-2", + title: "Fix auth v2", + url: "https://linear.app/x/issue/ISS-1/v2", + flags: ["NEW"], }); const task = await repo.getById(taskId); @@ -309,15 +263,7 @@ describe("TaskRepository", () => { }); it("stamps task_id/task_key on the assignment payload from the task row", async () => { - const { taskId } = await repo.upsertFromAgentSessionWithInbox({ - taskExternalId: "issue-1", - taskKey: "ISS-1", - taskTitle: "Fix auth", - taskExternalUrl: "https://linear.app/x/issue/ISS-1", - inboxPayload: assignmentContent, - inboxReplyTargets: [linearTarget("sess-1")], - metadata: {}, - }); + const { id: taskId } = await assignLinear(); const created = await db.execute<{ payload: unknown }>( sql`SELECT payload FROM inbox WHERE task_id = ${taskId}`, ); @@ -345,15 +291,7 @@ describe("TaskRepository", () => { it("rotates and re-pends the task's single assignment inbox row, dead-letters open siblings, and resets the task", async () => { // Seed a task + assignment inbox row, then stage an in-flight run. - const { taskId } = await repo.upsertFromAgentSessionWithInbox({ - taskKey: "ISS-1", - taskExternalId: "issue-1", - taskTitle: "Fix auth", - taskExternalUrl: "https://linear.app/x/issue/ISS-1", - inboxPayload: assignmentContent, - inboxReplyTargets: [linearTarget("sess-1")], - metadata: {}, - }); + const { id: taskId } = await assignLinear({ session: "sess-1" }); await db.execute(sql` UPDATE tasks SET owner_id = 'pod-a', heartbeat_at = now(), sandbox_id = 'sb-old', @@ -449,15 +387,7 @@ describe("TaskRepository", () => { describe("requeueForRetry", () => { it("owner-guarded RESUME: keeps pvc/cursor/pr_urls + inbox rows, only drops sandbox_id + releases the lease", async () => { - const { taskId } = await repo.upsertFromAgentSessionWithInbox({ - taskKey: "ISS-1", - taskExternalId: "issue-1", - taskTitle: "Fix auth", - taskExternalUrl: "https://linear.app/x/issue/ISS-1", - inboxPayload: assignmentContent, - inboxReplyTargets: [linearTarget("sess-1")], - metadata: {}, - }); + const { id: taskId } = await assignLinear(); const prUrls = [ { repo: "api", @@ -1031,7 +961,7 @@ describe("TaskRepository", () => { }); }); - describe("setSandboxId / getById / findByLinearIssueId / findByPrUrl", () => { + describe("setSandboxId / getById / findByExternalId / findByPrUrl", () => { it("setSandboxId owner-guarded", async () => { const t = await insertTask({ ownerId: "pod-a" }); expect(await repo.setSandboxId({ taskId: t.id, ownerId: "pod-x", sandboxId: "s" })).toEqual( @@ -1055,10 +985,11 @@ describe("TaskRepository", () => { expect(await repo.getByIdOwnedBy(t.id, "pod-other")).toBeNull(); }); - it("findByLinearIssueId matches on source + external_id", async () => { + it("findByExternalId matches on source + external_id", async () => { const t = await insertTask({ externalId: "linear-uuid-1" }); - expect((await repo.findByLinearIssueId("linear-uuid-1"))?.id).toBe(t.id); - expect(await repo.findByLinearIssueId("nope")).toBeNull(); + expect((await repo.findByExternalId("linear", "linear-uuid-1"))?.id).toBe(t.id); + expect(await repo.findByExternalId("linear", "nope")).toBeNull(); + expect(await repo.findByExternalId("api", "linear-uuid-1")).toBeNull(); }); it("findByPrUrl matches via the GIN-indexed jsonb predicate", async () => { diff --git a/control-plane/src/repositories/apiTask.repository.ts b/control-plane/src/repositories/apiTask.repository.ts index 67fc707..f19a498 100644 --- a/control-plane/src/repositories/apiTask.repository.ts +++ b/control-plane/src/repositories/apiTask.repository.ts @@ -1,16 +1,14 @@ import { type SQL, sql } from "drizzle-orm"; import { db } from "../db"; +import type { TaskSource } from "../integrations/types"; import type { ApiRound, ApiTaskState } from "../types/api"; -import type { AssignmentPayload, CommentPayload } from "../types/inboxMessage"; -import { apiSessionTarget, taskIdentity } from "./inbox.repository"; -import { insertCpEventIfApi } from "./taskEvent.repository"; -// API read model and the API task write transactions. +// The API read model. API tasks are written through TaskIntake. export type ApiTaskRecord = { id: string; key: string; - source: "linear" | "api"; + source: TaskSource; title: string; description: string | null; external_url: string; @@ -26,7 +24,7 @@ export type ApiTaskRecord = { }; export type ApiTaskListFilter = { - source?: "linear" | "api"; + source?: TaskSource; // undefined = no filter; null = matches nothing (created_by=me with auth off). createdBy?: string | null; state?: ApiTaskState; @@ -98,8 +96,11 @@ export class ApiTaskRepository { return asRows(rows)[0] ?? null; } + // Case-insensitive, like the key's unique index (lower(key)). async findByKey(key: string): Promise { - const rows = await db.execute(sql`${RESOURCE_SELECT} WHERE t.key = ${key}`); + const rows = await db.execute( + sql`${RESOURCE_SELECT} WHERE lower(t.key) = lower(${key})`, + ); return asRows(rows)[0] ?? null; } @@ -126,103 +127,6 @@ export class ApiTaskRepository { WHERE t.id = ${taskId}`); return asRows<{ state: ApiTaskState; sandbox_live: boolean }>(rows)[0] ?? null; } - - // POST /api/tasks: the tasks row, the assignment inbox row and task_created - // in ONE transaction. The key comes from api_task_key_seq. - async createApiTask(p: { - keyPrefix: string; - publicBaseUrl: string; - title: string; - description: string | null; - experimentalFlags: string[]; - createdBy: string | null; - }): Promise<{ taskId: string; key: string }> { - return db.transaction(async (tx) => { - const [seq] = asRows<{ n: string }>( - await tx.execute(sql`SELECT nextval('api_task_key_seq')::text AS n`), - ); - if (!seq) throw new Error("createApiTask: nextval returned no row"); - const key = `${p.keyPrefix}${seq.n}`; - const externalUrl = `${p.publicBaseUrl}/api/tasks/${key}`; - const metadataJson = JSON.stringify({ experimentalFlags: p.experimentalFlags }); - const [task] = asRows<{ id: string }>( - await tx.execute(sql` - INSERT INTO tasks (source, external_id, key, title, external_url, metadata, created_by) - VALUES ('api', ${key}, ${key}, ${p.title}, ${externalUrl}, - ${metadataJson}::text::jsonb, ${p.createdBy}) - RETURNING id;`), - ); - if (!task) throw new Error("createApiTask: tasks insert returned no row"); - - const payload: AssignmentPayload = { - title: p.title, - description: p.description, - external_url: externalUrl, - target_id: task.id, - ...taskIdentity({ id: task.id, key }), - }; - const [round] = asRows<{ id: string }>( - await tx.execute(sql` - INSERT INTO inbox (task_id, source, kind, external_id, payload, reply_targets) - VALUES (${task.id}, 'api', 'assignment', ${key}, - ${JSON.stringify(payload)}::text::jsonb, - ${JSON.stringify([apiSessionTarget(task.id)])}::text::jsonb) - RETURNING id;`), - ); - if (!round) throw new Error("createApiTask: inbox insert returned no row"); - - await insertCpEventIfApi(tx, { - taskId: task.id, - type: "task_created", - inboxId: round.id, - data: { - title: p.title, - description: p.description, - experimental_flags: p.experimentalFlags, - }, - }); - return { taskId: task.id, key }; - }); - } - - // POST /api/tasks/:ref/messages: a comment row (external_id = a new UUID) - // and message_queued in ONE transaction. A round in progress doesn't block - // it: the new row queues behind it, as on Linear. - async appendApiMessage(p: { - task: { id: string; key: string }; - body: string; - createdAt: string; - }): Promise<{ roundId: string }> { - const payload: CommentPayload = { - target_id: p.task.id, - body: p.body, - pr: null, - created_at: p.createdAt, - ...taskIdentity(p.task), - }; - return db.transaction(async (tx) => { - // Lock the task row first: task_events ids must be handed out in commit - // order per task, or an event cursor can skip past a still-open insert. - // Every other per-task log writer starts with UPDATE tasks. - await tx.execute(sql`SELECT 1 FROM tasks WHERE id = ${p.task.id} FOR NO KEY UPDATE`); - const [round] = asRows<{ id: string }>( - await tx.execute(sql` - INSERT INTO inbox (task_id, source, kind, external_id, payload, reply_targets) - VALUES (${p.task.id}, 'api', 'comment', ${crypto.randomUUID()}, - ${JSON.stringify(payload)}::text::jsonb, - ${JSON.stringify([apiSessionTarget(p.task.id)])}::text::jsonb) - RETURNING id;`), - ); - if (!round) throw new Error("appendApiMessage: inbox insert returned no row"); - await insertCpEventIfApi(tx, { - taskId: p.task.id, - type: "message_queued", - inboxId: round.id, - data: { body: p.body }, - }); - return { roundId: round.id }; - }); - } } export const apiTaskRepository = new ApiTaskRepository(); diff --git a/control-plane/src/repositories/inbox.repository.ts b/control-plane/src/repositories/inbox.repository.ts index 3b3e26a..036b2df 100644 --- a/control-plane/src/repositories/inbox.repository.ts +++ b/control-plane/src/repositories/inbox.repository.ts @@ -1,6 +1,7 @@ import { sql } from "drizzle-orm"; import { db } from "../db"; import type { Task } from "../db/schema"; +import { logger } from "../logger"; import type { AssignmentPayload, InboxKind, @@ -8,7 +9,7 @@ import type { InboxSource, TaskIdentity, } from "../types/inboxMessage"; -import { insertCpEventIfApi } from "./taskEvent.repository"; +import { insertCpEventIfApi, type SqlExecutor, type Tx } from "./taskEvent.repository"; // Re-export contract types so existing repo-import sites stay valid without // reaching into types/ directly. @@ -65,8 +66,24 @@ export type ReplyTarget = { // event from the daemon routes to the right thread. id: string; spec: ReplyTargetSpec; + // An assignment's delegation order, when the platform has one: a delegation + // whose order is not above the stored one is a duplicate (TaskIntake.assign). + order?: number; }; +// A row to insert: the wire message without its id, per (source, kind) arm so +// source, kind and payload stay paired. +type WithoutId = M extends InboxMessage ? Omit : never; +export type NewInboxMessage = WithoutId; + +// A Postgres unique violation (23505) on `constraint`. drizzle wraps the +// driver's error in `cause`. +export function isUniqueViolation(err: unknown, constraint: string): boolean { + const e = (err as { cause?: unknown })?.cause ?? err; + const pg = e as { errno?: unknown; constraint?: unknown }; + return pg.errno === "23505" && pg.constraint === constraint; +} + // Assignment content as callers build it. The identity fields are stamped by // the repository from the tasks row (see TaskRepository). export type AssignmentContent = Omit; @@ -137,44 +154,116 @@ function asRows(result: unknown): T[] { export class InboxRepository { // INSERT (synthetic uuid PK auto-generated). Idempotent on - // (source, external_id) via the unique index. Returns the synthetic id on - // insert, null on conflict. - // - // `message` is the wire contract sans `id` — the tagged union forbids - // invalid (source, kind) combos at the type level. - async appendInbox(params: { - taskId: string; - message: Omit; - replyTargets: ReplyTarget[]; - }): Promise<{ id: string } | null> { + // (source, kind, external_id) via the unique index. Returns the synthetic id + // on insert, null on conflict. Given a transaction, it inserts in a + // savepoint, so a unique violation leaves that transaction usable. + async appendInbox( + params: { + taskId: string; + message: NewInboxMessage; + replyTargets: ReplyTarget[]; + }, + tx?: Tx, + ): Promise<{ id: string } | null> { const { source, kind, external_id, payload } = params.message; // bun-sql JSON-encodes a JS string param, so `${jsonStr}::jsonb` would // store a JSON *string scalar*, not the array/object. Casting via // `::text::jsonb` forces PG to parse the raw text into proper jsonb. const payloadJson = JSON.stringify(payload); const replyTargetsJson = JSON.stringify(params.replyTargets ?? []); - const result = await db.execute<{ id: string }>(sql` - INSERT INTO inbox (task_id, source, kind, external_id, payload, reply_targets) - VALUES (${params.taskId}, ${source}, ${kind}, ${external_id}, - ${payloadJson}::text::jsonb, ${replyTargetsJson}::text::jsonb) - ON CONFLICT (source, external_id) DO NOTHING - RETURNING id; - `); - return asRows<{ id: string }>(result)[0] ?? null; + const insert = (ex: SqlExecutor) => + ex.execute<{ id: string }>(sql` + INSERT INTO inbox (task_id, source, kind, external_id, payload, reply_targets) + VALUES (${params.taskId}, ${source}, ${kind}, ${external_id}, + ${payloadJson}::text::jsonb, ${replyTargetsJson}::text::jsonb) + ON CONFLICT (source, kind, external_id) DO NOTHING + RETURNING id; + `); + try { + const result = await (tx ? tx.transaction(insert) : insert(db)); + return asRows<{ id: string }>(result)[0] ?? null; + } catch (err) { + // Until phase 3 drops it, the old (source, external_id) index also + // rejects a row whose id another kind of the same source already uses, + // with a 23505 instead of DO NOTHING. Dedupe it as before; phase 3 + // removes this with the index. + if (isUniqueViolation(err, "inbox_source_external_uniq")) return null; + throw err; + } } - // Count of ci_result rows already forwarded for a task — the loop breaker's - // denominator. Counts every ci_result row regardless of status: a 'failed' - // or 'responded' row still represents a CI failure harness already saw, so - // it must count toward the runaway threshold. Enqueue is idempotent on - // (source, external_id) = workflow_run:run:attempt:pr, so a redelivered - // webhook does not inflate this count. - async countCiResultsByTask(taskId: string): Promise { - const result = await db.execute<{ n: number }>(sql` - SELECT COUNT(*)::int AS n FROM inbox - WHERE task_id = ${taskId} AND source = 'github' AND kind = 'ci_result'; - `); - return asRows<{ n: number }>(result)[0]?.n ?? 0; + // TaskIntake.followUp's transaction. Locks the task row first: tasks before + // inbox, and task_events ids handed out in commit order per task. Drops the + // round when `prUrl` is no longer one of the task's PRs (a restart cleared + // it) or when the CI loop breaker trips, and as a duplicate when the round is + // already stored. message_queued is written for an inserted API message. + async appendFollowUp(p: { + taskId: string; + message: NewInboxMessage; + replyTargets: ReplyTarget[]; + prUrl?: string; + ciResultMax: number; + }): Promise<{ roundId: string } | { dropped: "duplicate" | "stale" | "ci_limit" }> { + const { message } = p; + const hasPr = + p.prUrl === undefined + ? sql`true` + : sql`pr_urls @> jsonb_build_array(jsonb_build_object('url', ${p.prUrl}::text))`; + return db.transaction(async (tx) => { + const [task] = asRows<{ has_pr: boolean }>( + await tx.execute( + sql`SELECT ${hasPr} AS has_pr FROM tasks WHERE id = ${p.taskId} FOR NO KEY UPDATE`, + ), + ); + if (!task) throw new Error(`appendFollowUp: task ${p.taskId} not found`); + if (!task.has_pr) { + logger.info("follow-up dropped: the PR is no longer on the task", { + taskId: p.taskId, + prUrl: p.prUrl, + }); + return { dropped: "stale" }; + } + + // Loop breaker. harness fixes CI → pushes → CI re-runs → may fail again, + // ad infinitum. Once a task has accumulated `ciResultMax` ci_result rows, + // stop forwarding new failures. Every row counts regardless of status: a + // 'failed' or 'responded' row is still a CI failure the harness saw. The + // count runs under the task lock, so concurrent webhooks can't overshoot. + // 0 / negative disables the breaker. + if (message.kind === "ci_result" && p.ciResultMax > 0) { + const [count] = asRows<{ n: number }>( + await tx.execute(sql` + SELECT COUNT(*)::int AS n FROM inbox + WHERE task_id = ${p.taskId} AND source = 'github' AND kind = 'ci_result'; + `), + ); + const seen = count?.n ?? 0; + if (seen >= p.ciResultMax) { + logger.warn("ci_result loop breaker tripped — dropping CI failure", { + taskId: p.taskId, + seen, + max: p.ciResultMax, + externalId: message.external_id, + }); + return { dropped: "ci_limit" }; + } + } + + const row = await this.appendInbox( + { taskId: p.taskId, message, replyTargets: p.replyTargets }, + tx, + ); + if (!row) return { dropped: "duplicate" }; + if (message.source === "api" && message.kind === "comment") { + await insertCpEventIfApi(tx, { + taskId: p.taskId, + type: "message_queued", + inboxId: row.id, + data: { body: message.payload.body }, + }); + } + return { roundId: row.id }; + }); } // Rows still awaiting delivery to the sandbox (status = 'pending'). diff --git a/control-plane/src/repositories/task.repository.ts b/control-plane/src/repositories/task.repository.ts index 3f0cae1..22fbdab 100644 --- a/control-plane/src/repositories/task.repository.ts +++ b/control-plane/src/repositories/task.repository.ts @@ -1,91 +1,173 @@ import { and, eq, inArray, sql } from "drizzle-orm"; import { db, schema } from "../db"; import type { Task } from "../db/schema"; +import type { TaskRef } from "../integrations/intake"; +import type { Person, TaskSource } from "../integrations/types"; import { logger } from "../logger"; import type { HarnessEvent } from "../types/harnessEvent"; -import type { AssignmentContent, ReplyTarget } from "./inbox.repository"; -import { insertCpEventIfApi, insertDaemonEvent } from "./taskEvent.repository"; +import type { AssignmentPayload } from "../types/inboxMessage"; +import { + type AssignmentContent, + isUniqueViolation, + type ReplyTarget, + taskIdentity, +} from "./inbox.repository"; +import { + insertCpEventIfApi, + insertDaemonEvent, + type SqlExecutor, + type Tx, +} from "./taskEvent.repository"; -export class TaskRepository { - // Single tx for a Linear `created` event. UPSERT the tasks row (infra) + - // INSERT the assignment inbox row (work content + reply destinations). - // Returns the task id and an insert-vs-update bit (`created`). - // - // A re-delegation conflict refreshes the display fields + metadata only — the - // inbox INSERT keys off the Linear issue UUID, so it DO-NOTHINGs and the - // assignment row is left untouched. The caller branches on `created` and - // routes a re-delegation to `restartTaskWithInbox` — duplicate-webhook - // detection (same session id) is the caller's job via - // `inboxRepo.getAssignmentReplyTargetId(taskId)`. - async upsertFromAgentSessionWithInbox(params: { - taskExternalId: string; - taskKey: string; - taskTitle: string; - taskExternalUrl: string; - inboxPayload: AssignmentContent; - inboxReplyTargets: ReplyTarget[]; - metadata: unknown; - }): Promise<{ taskId: string; created: boolean }> { - // bun-sql JSON-encodes a JS string param, so `${jsonStr}::jsonb` would - // store a JSON *string scalar* rather than the array/object. Casting via - // `::text::jsonb` forces PG to parse the raw text into proper jsonb. - // `?? {}` / `?? []` guard against `JSON.stringify(undefined)` returning - // `undefined`, which would bind as SQL NULL and violate NOT NULL. - const metadataJson = JSON.stringify(params.metadata ?? {}); - const payloadJson = JSON.stringify(params.inboxPayload); - const replyTargetsJson = JSON.stringify(params.inboxReplyTargets ?? []); +function asRows(result: unknown): T[] { + return Array.isArray(result) ? (result as T[]) : []; +} +export class TaskRepository { + // TaskIntake.assign's transaction (§5). Inserts the task, or finds it by + // (source, external_id) and locks it, then compares the stored delegation + // (the assignment row's reply target) with the incoming one: when both carry + // an order, one not above the stored order is a duplicate; otherwise the + // same id is. A duplicate changes nothing; anything else restarts the task. + // A new task gets its assignment row and, for API tasks, task_created. + // Returns the preempted sandbox for the caller to kill after commit. + async assign(p: { + source: TaskSource; + externalId?: string; // required with a hint key; a sequence key is its own + key: { hint: string } | { sequencePrefix: string }; + title: string; + description: string | null; + flags: string[]; + url: (t: TaskRef) => string; + target: (t: TaskRef) => ReplyTarget; + requester?: Person; + createdBy?: string; + }): Promise< + TaskRef & { result: "created" | "restarted" | "duplicate"; prevSandboxId: string | null } + > { return db.transaction(async (tx) => { - // 1. UPSERT tasks. RETURNING (xmax = 0) AS created: xmax=0 means a fresh - // INSERT, non-zero means an ON CONFLICT UPDATE path was taken. - const taskRows = await tx.execute<{ id: string; key: string; created: boolean }>(sql` - INSERT INTO tasks (source, external_id, key, title, external_url, metadata) - VALUES ('linear', ${params.taskExternalId}, ${params.taskKey}, ${params.taskTitle}, - ${params.taskExternalUrl}, ${metadataJson}::text::jsonb) - ON CONFLICT (source, external_id) DO UPDATE - SET title = EXCLUDED.title, - external_url = EXCLUDED.external_url, - -- MERGE, don't replace. A duplicate created-webhook redelivery - -- passes metadata={}; a blind = EXCLUDED.metadata would wipe the - -- durable retry budget (metadata.retryCount), silently resetting - -- the sandbox-retry cap. The jsonb || keeps existing keys while - -- still letting a real EXCLUDED value overlay. A genuine - -- re-delegation resets the budget via restartTaskWithInbox. - metadata = tasks.metadata || EXCLUDED.metadata, - updated_at = now() - RETURNING id, key, (xmax = 0) AS created; - `); - const taskRow = Array.isArray(taskRows) ? taskRows[0] : undefined; - if (!taskRow) - throw new Error("upsertFromAgentSessionWithInbox: tasks upsert returned no row"); - - // 2. INSERT inbox. external_id is the Linear issue UUID, so a re-delegation - // conflicts here and DO NOTHING leaves the assignment row untouched — the - // caller routes the re-delegation to restartTaskWithInbox instead. - await tx.execute(sql` - INSERT INTO inbox (task_id, source, kind, external_id, payload, reply_targets) - VALUES (${taskRow.id}, 'linear', 'assignment', ${params.taskExternalId}, - ${payloadJson}::text::jsonb || jsonb_build_object( - 'task_id', ${taskRow.id}::text, - 'task_key', ${taskRow.key}::text), - ${replyTargetsJson}::text::jsonb) - ON CONFLICT (source, external_id) DO NOTHING; - `); + // In a savepoint so a key collision leaves the transaction usable: a + // concurrent insert can take the key between the check and the insert, + // and the retry then sees it. + const insert = () => tx.transaction((sp) => this.insertTask(sp, p)); + const inserted = await insert().catch((err) => { + if (!isUniqueViolation(err, "tasks_key_lower_uniq")) throw err; + return insert(); + }); + + if (inserted) { + const t = { id: inserted.id, key: inserted.key }; + const target = p.target(t); + const payload: AssignmentPayload = { + title: p.title, + description: p.description, + external_url: inserted.url, + target_id: target.id, + ...taskIdentity(t), + }; + // external_id is the task's: the platform's id, or the API task key. + const [round] = asRows<{ id: string }>( + await tx.execute(sql` + INSERT INTO inbox (task_id, source, kind, external_id, payload, reply_targets) + VALUES (${t.id}, ${p.source}, 'assignment', ${inserted.externalId}, + ${JSON.stringify(payload)}::text::jsonb, + ${JSON.stringify([target])}::text::jsonb) + RETURNING id;`), + ); + if (!round) throw new Error("assign: inbox insert returned no row"); + await insertCpEventIfApi(tx, { + taskId: t.id, + type: "task_created", + inboxId: round.id, + data: { title: p.title, description: p.description, experimental_flags: p.flags }, + }); + return { ...t, result: "created", prevSandboxId: null }; + } - return { taskId: taskRow.id, created: taskRow.created }; + const [t] = asRows( + await tx.execute(sql` + SELECT id, key FROM tasks + WHERE source = ${p.source} AND external_id = ${p.externalId ?? null} + FOR NO KEY UPDATE`), + ); + if (!t) throw new Error("assign: task not found after a conflicting insert"); + // Read after the lock, so a restart that committed meanwhile is seen. + const [stored] = asRows<{ target: ReplyTarget | null }>( + await tx.execute(sql` + SELECT reply_targets->0 AS target FROM inbox + WHERE task_id = ${t.id} AND kind = 'assignment' LIMIT 1`), + ); + const prev = stored?.target ?? null; + const next = p.target(t); + const duplicate = + prev !== null && + (typeof prev.order === "number" && typeof next.order === "number" + ? next.order <= prev.order + : next.id === prev.id); + if (duplicate) return { ...t, result: "duplicate", prevSandboxId: null }; + + const url = p.url(t); + const { prevSandboxId } = await this.restartTaskWithInbox( + { + taskId: t.id, + inboxPayload: { + title: p.title, + description: p.description, + external_url: url, + target_id: next.id, + }, + inboxReplyTargets: [next], + delegation: { flags: p.flags, requester: p.requester }, + }, + tx, + ); + return { ...t, result: "restarted", prevSandboxId }; }); } - // Re-delegation restart. The caller routes here when - // upsertFromAgentSessionWithInbox returned `created: false` AND the prior - // session id differs from the event's. PREEMPTS any in-flight run and - // restarts from scratch, in one tx: + // assign's insert. Picks the key: a hint is used as is unless another task + // holds it (case-insensitively), then `${SOURCE}-${hint}`; a sequence key is + // the prefix plus api_task_key_seq. Returns null when (source, external_id) + // already exists. + private async insertTask( + ex: SqlExecutor, + p: Parameters[0], + ): Promise<(TaskRef & { externalId: string; url: string }) | null> { + const picked = + "hint" in p.key + ? sql`CASE WHEN EXISTS ( + SELECT 1 FROM tasks + WHERE lower(key) = lower(${p.key.hint}) + AND NOT (source = ${p.source} AND external_id = ${p.externalId ?? null}) + ) THEN ${`${p.source.toUpperCase()}-${p.key.hint}`}::text ELSE ${p.key.hint}::text END` + : sql`${p.key.sequencePrefix}::text || nextval('api_task_key_seq')::text`; + const [pick] = asRows( + await ex.execute(sql`SELECT uuidv7()::text AS id, ${picked} AS key`), + ); + if (!pick) throw new Error("assign: key selection returned no row"); + const externalId = p.externalId ?? pick.key; + const url = p.url(pick); + const metadataJson = JSON.stringify({ experimentalFlags: p.flags, requester: p.requester }); + const [row] = asRows<{ id: string }>( + await ex.execute(sql` + INSERT INTO tasks (id, source, external_id, key, title, external_url, metadata, created_by) + VALUES (${pick.id}, ${p.source}, ${externalId}, ${pick.key}, ${p.title}, ${url}, + ${metadataJson}::text::jsonb, ${p.createdBy ?? null}) + ON CONFLICT (source, external_id) DO NOTHING + RETURNING id;`), + ); + return row ? { ...pick, externalId, url } : null; + } + + // Restart: assign's newer delegation, or TaskIntake.restart (API). PREEMPTS + // any in-flight run and restarts from scratch, in one tx (the caller's, when + // given: assign already holds the task row lock): // 1. Reset the task to a clean slate — pvc_name / sandbox_id / owner_id / // heartbeat_at cleared, cursor reset. The old owner pod self-evicts on // its next owner-guarded write (owner_id gone). // 2. Re-pend the task's single `assignment` inbox row (one per task, created - // by upsertFromAgentSessionWithInbox) with a fresh id plus the new - // event's payload + reply_targets. The inbox id is the daemon-visible run + // by assign) with a fresh id plus the new event's payload + + // reply_targets. The inbox id is the daemon-visible run // identity (`inbox/.json`) and feeds the harness PR branch suffixes; // the logical task row id and assignment external_id remain stable. // 3. Dead-letter every still-open NON-assignment inbox row (pending @@ -93,108 +175,119 @@ export class TaskRepository { // re-delivered. // Not owner-guarded — it forcibly supersedes whatever pod ran the old round. // 4. With `api`: also set the new title and experimental flags in step 1, - // and write `task_restarted` to the log. + // and write `task_restarted` to the log. With `delegation`: also set the + // payload's title and URL and the new flags and requester in step 1. // // Returns `prevSandboxId` — the preempted sandbox the caller must kill; // `assignmentId` — the re-pended assignment row's new id (null if the task // has none); and `supersededRoundIds` — the rows dead-lettered in step 3. - // Duplicate webhook deliveries do not route here; the service checks the - // prior session first. - async restartTaskWithInbox(params: { - taskId: string; - inboxPayload: AssignmentContent; - inboxReplyTargets: ReplyTarget[]; - api?: { title: string; description: string | null; experimentalFlags: string[] }; - }): Promise<{ + // Duplicate deliveries do not route here; assign compares delegations first. + async restartTaskWithInbox( + params: { + taskId: string; + inboxPayload: AssignmentContent; + inboxReplyTargets: ReplyTarget[]; + api?: { title: string; description: string | null; experimentalFlags: string[] }; + delegation?: { flags: string[]; requester?: Person }; + }, + tx?: Tx, + ): Promise<{ prevSandboxId: string | null; assignmentId: string | null; supersededRoundIds: string[]; }> { + if (!tx) return db.transaction((t) => this.restartTaskWithInbox(params, t)); + const { api, delegation } = params; const payloadJson = JSON.stringify(params.inboxPayload); - // An API restart also applies the new title and flags. The + // An API restart or a re-delegation also applies the new title and flags + // (a re-delegation its URL and requester too, dropping a stale one). The // description lives on the assignment payload (step 2). - const apiSet = params.api - ? sql`, title = ${params.api.title}, - metadata = jsonb_set(metadata - 'retryCount', '{experimentalFlags}', - ${JSON.stringify(params.api.experimentalFlags)}::text::jsonb)` - : sql`, metadata = metadata - 'retryCount'`; + const flags = api?.experimentalFlags ?? delegation?.flags; + const definition = JSON.stringify({ + experimentalFlags: flags, + requester: delegation?.requester, + }); + const definitionSet = + flags === undefined + ? sql`, metadata = metadata - 'retryCount'` + : sql`, title = ${params.inboxPayload.title}, + metadata = (metadata - 'retryCount' - 'requester') || ${definition}::text::jsonb + ${delegation ? sql`, external_url = ${params.inboxPayload.external_url}` : sql``}`; const replyTargetsJson = JSON.stringify(params.inboxReplyTargets ?? []); - return db.transaction(async (tx) => { - // 1. Reset the task to a clean slate. PG18 `RETURNING old.sandbox_id` is - // the preempted sandbox the caller kills. - const taskRows = await tx.execute<{ prev_sandbox_id: string | null }>(sql` - UPDATE tasks - SET pvc_name = NULL, - sandbox_id = NULL, - owner_id = NULL, - heartbeat_at = NULL, - last_observed_event_seq = 0, - pr_urls = '[]'::jsonb - -- Clean slate also resets the sandbox-retry budget (retryCount - -- is dropped in the metadata fragment) — a genuine - -- re-delegation is a fresh attempt, not a continuation. - ${apiSet}, - updated_at = now() - WHERE id = ${params.taskId} - RETURNING old.sandbox_id AS prev_sandbox_id; - `); - const taskRow = Array.isArray(taskRows) ? taskRows[0] : undefined; - if (!taskRow) throw new Error("restartTaskWithInbox: task not found"); - - // 2. Re-pend the task's single `assignment` inbox row with a new id plus - // the new event's payload + reply targets — this drives the fresh run. - // The row remains the one assignment for this task / Linear issue, but - // the daemon sees a new run identity via `inbox/.json`. - const reRows = await tx.execute<{ id: string }>(sql` - UPDATE inbox i - SET id = uuidv7(), - status = 'pending', - failure_reason = NULL, - -- a re-run never inherits the previous run's outcome - outcome = NULL, - payload = ${payloadJson}::text::jsonb || jsonb_build_object( - 'task_id', t.id::text, 'task_key', t.key), - reply_targets = ${replyTargetsJson}::text::jsonb, - updated_at = now() - FROM tasks t - WHERE t.id = i.task_id - AND i.task_id = ${params.taskId} - AND i.kind = 'assignment' - RETURNING i.id; - `); - const assignmentId = (Array.isArray(reRows) ? reRows[0]?.id : undefined) ?? null; - - // 3. Dead-letter every still-open NON-assignment inbox row (pending - // comment / review rows). The re-pended `assignment` row (step 2) - // is excluded by `kind <> 'assignment'`. - const supRows = await tx.execute<{ id: string }>(sql` - UPDATE inbox - SET status = 'failed', - failure_reason = 'superseded by re-delegation', - updated_at = now() - WHERE task_id = ${params.taskId} - AND status IN ('pending', 'delivered') - AND kind <> 'assignment' - RETURNING id; - `); - const supersededRoundIds = (Array.isArray(supRows) ? supRows : []).map((r) => r.id); + // 1. Reset the task to a clean slate. PG18 `RETURNING old.sandbox_id` is + // the preempted sandbox the caller kills. + const taskRows = await tx.execute<{ prev_sandbox_id: string | null }>(sql` + UPDATE tasks + SET pvc_name = NULL, + sandbox_id = NULL, + owner_id = NULL, + heartbeat_at = NULL, + last_observed_event_seq = 0, + pr_urls = '[]'::jsonb + -- Clean slate also resets the sandbox-retry budget (retryCount + -- is dropped in the metadata fragment) — a genuine + -- re-delegation is a fresh attempt, not a continuation. + ${definitionSet}, + updated_at = now() + WHERE id = ${params.taskId} + RETURNING old.sandbox_id AS prev_sandbox_id; + `); + const taskRow = Array.isArray(taskRows) ? taskRows[0] : undefined; + if (!taskRow) throw new Error("restartTaskWithInbox: task not found"); - if (params.api) { - await insertCpEventIfApi(tx, { - taskId: params.taskId, - type: "task_restarted", - inboxId: assignmentId, - data: { - title: params.api.title, - description: params.api.description, - experimental_flags: params.api.experimentalFlags, - superseded_round_ids: supersededRoundIds, - }, - }); - } - return { prevSandboxId: taskRow.prev_sandbox_id, assignmentId, supersededRoundIds }; - }); + // 2. Re-pend the task's single `assignment` inbox row with a new id plus + // the new event's payload + reply targets — this drives the fresh run. + // The row remains the one assignment for this task / Linear issue, but + // the daemon sees a new run identity via `inbox/.json`. + const reRows = await tx.execute<{ id: string }>(sql` + UPDATE inbox i + SET id = uuidv7(), + status = 'pending', + failure_reason = NULL, + -- a re-run never inherits the previous run's outcome + outcome = NULL, + payload = ${payloadJson}::text::jsonb || jsonb_build_object( + 'task_id', t.id::text, 'task_key', t.key), + reply_targets = ${replyTargetsJson}::text::jsonb, + updated_at = now() + FROM tasks t + WHERE t.id = i.task_id + AND i.task_id = ${params.taskId} + AND i.kind = 'assignment' + RETURNING i.id; + `); + const assignmentId = (Array.isArray(reRows) ? reRows[0]?.id : undefined) ?? null; + + // 3. Dead-letter every still-open NON-assignment inbox row (pending + // comment / review rows). The re-pended `assignment` row (step 2) + // is excluded by `kind <> 'assignment'`. + const supRows = await tx.execute<{ id: string }>(sql` + UPDATE inbox + SET status = 'failed', + failure_reason = 'superseded by re-delegation', + updated_at = now() + WHERE task_id = ${params.taskId} + AND status IN ('pending', 'delivered') + AND kind <> 'assignment' + RETURNING id; + `); + const supersededRoundIds = (Array.isArray(supRows) ? supRows : []).map((r) => r.id); + + if (params.api) { + await insertCpEventIfApi(tx, { + taskId: params.taskId, + type: "task_restarted", + inboxId: assignmentId, + data: { + title: params.api.title, + description: params.api.description, + experimental_flags: params.api.experimentalFlags, + superseded_round_ids: supersededRoundIds, + }, + }); + } + return { prevSandboxId: taskRow.prev_sandbox_id, assignmentId, supersededRoundIds }; } // Retry reset — RESUME, not restart. The backing sandbox failed (terminal / @@ -545,12 +638,13 @@ export class TaskRepository { return row ? mapTaskRow(row) : null; } - // tasks.external_id IS the Linear issue UUID. Replaces findByAgentSessionId. - async findByLinearIssueId(issueId: string): Promise { + // tasks.external_id is the platform's id (the Linear issue UUID, the API + // task key). + async findByExternalId(source: TaskSource, externalId: string): Promise { const rows = await db .select() .from(schema.tasks) - .where(and(eq(schema.tasks.source, "linear"), eq(schema.tasks.externalId, issueId))) + .where(and(eq(schema.tasks.source, source), eq(schema.tasks.externalId, externalId))) .limit(1); return rows[0] ?? null; } @@ -666,7 +760,7 @@ export class TaskRepository { // Raw row shape from db.execute (snake_case). type RawTaskRow = { id: string; - source: "linear" | "api"; + source: TaskSource; external_id: string; key: string; created_by: string | null; diff --git a/control-plane/src/repositories/taskEvent.repository.ts b/control-plane/src/repositories/taskEvent.repository.ts index e4b98c1..10c56ee 100644 --- a/control-plane/src/repositories/taskEvent.repository.ts +++ b/control-plane/src/repositories/taskEvent.repository.ts @@ -7,6 +7,8 @@ import type { HarnessEvent } from "../types/harnessEvent"; // Both are no-ops for non-API tasks; Linear tasks do not use this event log. export type SqlExecutor = Pick; +// A transaction handle, for writes that also run inside a caller's transaction. +export type Tx = Parameters[0]>[0]; export type CpEventType = | "task_created" diff --git a/control-plane/src/services/__tests__/agentSession.service.test.ts b/control-plane/src/services/__tests__/agentSession.service.test.ts index 27358e2..2566ee0 100644 --- a/control-plane/src/services/__tests__/agentSession.service.test.ts +++ b/control-plane/src/services/__tests__/agentSession.service.test.ts @@ -1,12 +1,11 @@ import { beforeEach, describe, expect, it, mock } from "bun:test"; import type { LinearClient } from "@linear/sdk"; import type { Task } from "../../db/schema"; -import type { InboxRepository } from "../../repositories/inbox.repository"; -import type { TaskRepository } from "../../repositories/task.repository"; +import type { TaskIntake, TaskRef } from "../../integrations/intake"; +import type { ReplyTarget } from "../../repositories/inbox.repository"; import type { AgentSessionCreated, AgentSessionPrompted } from "../../types/agentSession"; import type { InboxService } from "../inbox.service"; import type { LinearTokenProvider } from "../linearTokenProvider"; -import type { SandboxManager } from "../sandboxManager.service"; const emitResponse = mock(() => Promise.resolve()); const emitAction = mock(() => Promise.resolve()); @@ -63,45 +62,36 @@ const foundTask = { } as Task; describe("AgentSessionService", () => { - let taskRepo: TaskRepository; - let inboxRepo: InboxRepository; + let intake: TaskIntake; let linearTokenProvider: Pick; - let sandboxManager: SandboxManager; let inboxService: InboxService; - let upsertFromAgentSessionWithInbox: ReturnType; - let restartTaskWithInbox: ReturnType; - let findByLinearIssueId: ReturnType; - let getAssignmentReplyTargetId: ReturnType; + let assign: ReturnType; + let findTask: ReturnType; let enqueueLinearComment: ReturnType; let getAuthorizedLinearClient: ReturnType; - let killSandbox: ReturnType; let service: InstanceType; + type AssignArg = { + source: string; + externalId: string; + key: { hint: string }; + title: string; + description: string | null; + flags: string[]; + url: (t: TaskRef) => string; + target: (t: TaskRef) => ReplyTarget; + }; + const ref = { id: "task-1", key: "ISS-1" }; + beforeEach(() => { emitResponse.mockReset(); emitResponse.mockResolvedValue(undefined); emitAction.mockReset(); emitAction.mockResolvedValue(undefined); - upsertFromAgentSessionWithInbox = mock(() => - Promise.resolve({ taskId: "task-1", created: true }), - ); - restartTaskWithInbox = mock(() => - Promise.resolve({ - prevSandboxId: null, - assignmentId: null, - supersededRoundIds: [] as string[], - }), - ); - findByLinearIssueId = mock(() => Promise.resolve(foundTask)); - taskRepo = { - upsertFromAgentSessionWithInbox, - restartTaskWithInbox, - findByLinearIssueId, - } as unknown as TaskRepository; - - getAssignmentReplyTargetId = mock(() => Promise.resolve(null as string | null)); - inboxRepo = { getAssignmentReplyTargetId } as unknown as InboxRepository; + assign = mock(() => Promise.resolve({ taskId: "task-1", key: "ISS-1", result: "created" })); + findTask = mock(() => Promise.resolve(foundTask)); + intake = { assign, findTask } as unknown as TaskIntake; getAuthorizedLinearClient = mock(() => Promise.resolve({} as unknown as LinearClient)); linearTokenProvider = { getAuthorizedLinearClient } as unknown as Pick< @@ -109,55 +99,28 @@ describe("AgentSessionService", () => { "getAuthorizedLinearClient" >; - killSandbox = mock(() => Promise.resolve()); - sandboxManager = { killSandbox } as unknown as SandboxManager; - enqueueLinearComment = mock(() => Promise.resolve()); inboxService = { enqueueLinearComment } as unknown as InboxService; - service = new AgentSessionService( - taskRepo, - inboxRepo, - linearTokenProvider, - sandboxManager, - inboxService, - ); + service = new AgentSessionService(intake, linearTokenProvider, inboxService); }); describe("handleEvent created", () => { it("writes task + inbox row and emits Evaluating on fresh insert", async () => { - upsertFromAgentSessionWithInbox.mockResolvedValue({ - taskId: "task-1", - created: true, - }); - const result = await service.handleEvent(createdEvent()); expect(result).toEqual({ status: "ok" }); - expect(upsertFromAgentSessionWithInbox).toHaveBeenCalledTimes(1); - const arg = upsertFromAgentSessionWithInbox.mock.calls[0]?.[0] as { - taskExternalId: string; - taskKey: string; - taskTitle: string; - taskExternalUrl: string; - inboxPayload: { - title: string; - description: string | null; - target_id: string; - }; - inboxReplyTargets: Array<{ id: string; spec: { kind: string } }>; - metadata: { experimentalFlags: string[] }; - }; - expect(arg.taskExternalId).toBe("issue-1"); - expect(arg.taskKey).toBe("ISS-1"); - expect(arg.taskExternalUrl).toBe("https://linear.app/x/issue/ISS-1"); - expect(arg.inboxPayload.description).toBe("desc"); - expect(arg.metadata.experimentalFlags).toEqual([]); - // Agent session id is carried by the assignment row's reply_targets + - // payload.target_id — the canonical store after dropping the column. - expect(arg.inboxPayload.target_id).toBe("sess-1"); - expect(arg.inboxReplyTargets[0]?.id).toBe("sess-1"); - expect(arg.inboxReplyTargets[0]?.spec.kind).toBe("linear_session"); + expect(assign).toHaveBeenCalledTimes(1); + const arg = assign.mock.calls[0]?.[0] as AssignArg; + expect(arg.source).toBe("linear"); + expect(arg.externalId).toBe("issue-1"); + expect(arg.key).toEqual({ hint: "ISS-1" }); + expect(arg.url(ref)).toBe("https://linear.app/x/issue/ISS-1"); + expect(arg.description).toBe("desc"); + expect(arg.flags).toEqual([]); + // The agent session id is the delegation id: the assignment row's reply + // target (intake stamps it as payload.target_id too). + expect(arg.target(ref)).toEqual({ id: "sess-1", spec: { kind: "linear_session" } }); await flushMicrotasks(); expect(emitResponse).not.toHaveBeenCalled(); @@ -179,30 +142,18 @@ describe("AgentSessionService", () => { const result = await service.handleEvent(event); expect(result).toEqual({ status: "ok" }); - const arg = upsertFromAgentSessionWithInbox.mock.calls[0]?.[0] as { - inboxPayload: { description: string | null }; - metadata: { experimentalFlags: string[] }; - }; - expect(arg.metadata.experimentalFlags).toEqual(["FAST_PATH", "TRACE2"]); - expect(arg.inboxPayload.description).toBe( - "Please validate the new path.\n\n\n#devbox:exp:lowercase\n", - ); + const arg = assign.mock.calls[0]?.[0] as AssignArg; + expect(arg.flags).toEqual(["FAST_PATH", "TRACE2"]); + expect(arg.description).toBe("Please validate the new path.\n\n\n#devbox:exp:lowercase\n"); }); - it("restarts on conflict (new agent session id): calls restartTaskWithInbox, acks restart", async () => { - upsertFromAgentSessionWithInbox.mockResolvedValue({ - taskId: "task-1", - created: false, - }); - getAssignmentReplyTargetId.mockResolvedValue("sess-old"); - restartTaskWithInbox.mockResolvedValue({ prevSandboxId: null }); + it("restarts on a new agent session id: acks the restart", async () => { + assign.mockResolvedValue({ taskId: "task-1", key: "ISS-1", result: "restarted" }); const result = await service.handleEvent(createdEvent()); expect(result).toEqual({ status: "ok" }); - expect(restartTaskWithInbox).toHaveBeenCalledTimes(1); - const restartArg = restartTaskWithInbox.mock.calls[0]?.[0] as { taskId: string }; - expect(restartArg.taskId).toBe("task-1"); + expect(assign).toHaveBeenCalledTimes(1); await flushMicrotasks(); const respArgs = emitResponse.mock.calls[0] as unknown as [unknown, string, string]; @@ -213,48 +164,22 @@ describe("AgentSessionService", () => { expect(actArgs[3]).toBe("task"); }); - it("kills the preempted sandbox when a re-delegation restarts a live run", async () => { - upsertFromAgentSessionWithInbox.mockResolvedValue({ - taskId: "task-1", - created: false, - }); - getAssignmentReplyTargetId.mockResolvedValue("sess-old"); - restartTaskWithInbox.mockResolvedValue({ prevSandboxId: "sb-old" }); - - const result = await service.handleEvent(createdEvent()); - - expect(result).toEqual({ status: "ok" }); - await flushMicrotasks(); - expect(killSandbox).toHaveBeenCalledTimes(1); - expect(killSandbox.mock.calls[0]?.[0]).toBe("sb-old"); - }); - it("ignores a duplicate webhook redelivery (same agent session id): no restart", async () => { // Linear redelivers the same `created` event — the existing assignment // row's reply_targets already carries the event's session id, so it is - // not a new delegation. No restart, no sandbox kill, no re-emit. - upsertFromAgentSessionWithInbox.mockResolvedValue({ - taskId: "task-1", - created: false, - }); - getAssignmentReplyTargetId.mockResolvedValue("sess-1"); + // not a new delegation. No restart, no re-emit. + assign.mockResolvedValue({ taskId: "task-1", key: "ISS-1", result: "duplicate" }); const result = await service.handleEvent(createdEvent()); expect(result).toEqual({ status: "ok" }); - expect(restartTaskWithInbox).not.toHaveBeenCalled(); await flushMicrotasks(); expect(emitResponse).not.toHaveBeenCalled(); expect(emitAction).not.toHaveBeenCalled(); - expect(killSandbox).not.toHaveBeenCalled(); }); it("does not throw when linear emit fails (fire-and-forget)", async () => { - upsertFromAgentSessionWithInbox.mockResolvedValue({ - taskId: "task-1", - created: true, - }); getAuthorizedLinearClient.mockImplementation(() => Promise.reject(new Error("boom"))); const result = await service.handleEvent(createdEvent()); @@ -271,7 +196,7 @@ describe("AgentSessionService", () => { const result = await service.handleEvent(promptedEvent()); expect(result).toEqual({ status: "ok" }); - expect(findByLinearIssueId).toHaveBeenCalledWith("issue-1"); + expect(findTask).toHaveBeenCalledWith("linear", "issue-1"); expect(enqueueLinearComment).toHaveBeenCalledWith({ task: foundTask, agentActivityId: "activity-1", @@ -295,7 +220,7 @@ describe("AgentSessionService", () => { }); it("fires the Evaluating ack even when the task is not yet known (session_not_found)", async () => { - findByLinearIssueId.mockResolvedValue(null); + findTask.mockResolvedValue(null); const result = await service.handleEvent(promptedEvent()); diff --git a/control-plane/src/services/__tests__/apiTask.service.test.ts b/control-plane/src/services/__tests__/apiTask.service.test.ts index bdadb1d..6da9930 100644 --- a/control-plane/src/services/__tests__/apiTask.service.test.ts +++ b/control-plane/src/services/__tests__/apiTask.service.test.ts @@ -7,7 +7,9 @@ mock.module("../../config", () => ({ config: { databaseUrl: TEST_DATABASE_URL } const { db } = await import("../../db"); const { ApiTaskRepository } = await import("../../repositories/apiTask.repository"); +const { InboxRepository } = await import("../../repositories/inbox.repository"); const { TaskRepository } = await import("../../repositories/task.repository"); +const { Intake } = await import("../../integrations/intake"); const { TaskEventRepository } = await import("../../repositories/taskEvent.repository"); const { ApiTaskService } = await import("../apiTask.service"); const { ApiError } = await import("../../errors"); @@ -41,17 +43,17 @@ describe("ApiTaskService", () => { ); principal.tokenId = (tok as { id: string }).id; killSandbox = mock(() => Promise.resolve()); - service = new ApiTaskService( - new ApiTaskRepository(), + const intake = new Intake( new TaskRepository(), - new TaskEventRepository(), + new InboxRepository(), { killSandbox }, - { - keyPrefix: "DBX-API-", - publicBaseUrl: "https://api.example", - now: () => new Date(T0), - }, + { detach: () => {} }, + { apiKeyPrefix: "DBX-API-", ciResultMaxPerTask: 20 }, ); + service = new ApiTaskService(new ApiTaskRepository(), intake, new TaskEventRepository(), { + publicBaseUrl: "https://api.example", + now: () => new Date(T0), + }); }); it("create returns the whole task resource", async () => { diff --git a/control-plane/src/services/__tests__/inbox.service.test.ts b/control-plane/src/services/__tests__/inbox.service.test.ts index d7a2f9e..e68dad7 100644 --- a/control-plane/src/services/__tests__/inbox.service.test.ts +++ b/control-plane/src/services/__tests__/inbox.service.test.ts @@ -4,12 +4,11 @@ import type { InboxRepository, InboxRow } from "../../repositories/inbox.reposit import type { LinearTokenProvider } from "../linearTokenProvider"; import type { SandboxManager } from "../sandboxManager.service"; -// Pin the two config values InboxService reads (podId, the ci_result cap) so -// results do not depend on the environment the suite runs in. +// Pin the config value InboxService reads (podId) so results do not depend on +// the environment the suite runs in. mock.module("../../config", () => ({ config: { podId: "pod-test", - github: { ciResultMaxPerTask: 20 }, }, })); @@ -128,8 +127,7 @@ describe("InboxService", () => { let listNonTerminalByTask: ReturnType; let markDelivered: ReturnType; let markFailed: ReturnType; - let appendInbox: ReturnType; - let countCiResultsByTask: ReturnType; + let followUp: ReturnType; let writeHarnessFile: ReturnType; let killSandbox: ReturnType; let getById: ReturnType; @@ -157,8 +155,7 @@ describe("InboxService", () => { listNonTerminalByTask = mock(() => Promise.resolve([] as InboxRow[])); markDelivered = mock(() => Promise.resolve()); markFailed = mock(() => Promise.resolve([{ id: "x" }])); - appendInbox = mock(() => Promise.resolve({ id: "inbox-new" })); - countCiResultsByTask = mock(() => Promise.resolve(0)); + followUp = mock(() => Promise.resolve({ roundId: "inbox-new" })); getById = mock((id: string) => Promise.resolve(ghCommentRow(id))); inboxRepo = { listPendingByTask, @@ -166,8 +163,6 @@ describe("InboxService", () => { getById, markDelivered, markFailed, - appendInbox, - countCiResultsByTask, } as unknown as InboxRepository; writeHarnessFile = mock(() => Promise.resolve()); @@ -188,6 +183,7 @@ describe("InboxService", () => { sandboxManager, integrations, new IntegrationHooks(integrations, inboxRepo), + { followUp }, ); }); @@ -526,26 +522,28 @@ describe("InboxService", () => { createdAt: "2026-04-23T10:00:00Z", }); - const arg = appendInbox.mock.calls[0]?.[0] as { + const arg = followUp.mock.calls[0]?.[0] as { + task: { id: string; key: string }; message: { source: string; kind: string; external_id: string; - payload: { target_id: string; task_key: string; created_at: string }; + payload: { target_id: string; created_at: string }; }; - replyTargets: Array<{ id: string }>; + targets: Array<{ id: string }>; + prUrl: string; }; expect(arg.message.source).toBe("github"); expect(arg.message.kind).toBe("comment"); expect(arg.message.external_id).toBe("555"); expect(arg.message.payload.target_id).toBe("555"); - expect(arg.message.payload).toMatchObject({ - task_id: "task-1", - task_key: "XYZ-1", - }); + // followUp stamps task_id / task_key from the task. + expect(arg.task).toMatchObject({ id: "task-1", key: "XYZ-1" }); // The comment's created_at is carried into the payload for temporal scoping. expect(arg.message.payload.created_at).toBe("2026-04-23T10:00:00Z"); - expect(arg.replyTargets[0]?.id).toBe("555"); + expect(arg.targets[0]?.id).toBe("555"); + // Dropped as stale if a restart has since cleared the PR from the task. + expect(arg.prUrl).toBe("https://github.com/o/r/pull/7"); }); }); @@ -566,7 +564,7 @@ describe("InboxService", () => { createdAt: "2026-04-23T10:00:00Z", }); - const arg = appendInbox.mock.calls[0]?.[0] as { + const arg = followUp.mock.calls[0]?.[0] as { message: { kind: string; payload: { @@ -582,7 +580,7 @@ describe("InboxService", () => { }>; }; }; - replyTargets: Array<{ + targets: Array<{ id: string; spec: { kind: string; @@ -601,7 +599,7 @@ describe("InboxService", () => { pr_url: "https://github.com/o/r/pull/7", }); // 2 inline + 1 review_body - expect(arg.replyTargets).toHaveLength(3); + expect(arg.targets).toHaveLength(3); expect(arg.message.payload.inline_comments[0]).toMatchObject({ target_id: "11", path: "a.ts", @@ -609,11 +607,11 @@ describe("InboxService", () => { start_line: 1, }); expect(arg.message.payload.inline_comments[1]?.start_line).toBeUndefined(); - expect(arg.replyTargets[0]).toMatchObject({ + expect(arg.targets[0]).toMatchObject({ id: "11", spec: { kind: "review_comment", startLine: 1 }, }); - expect(arg.replyTargets[1]?.spec.startLine).toBeUndefined(); + expect(arg.targets[1]?.spec.startLine).toBeUndefined(); }); }); @@ -626,24 +624,22 @@ describe("InboxService", () => { body: "any update?", }); - const arg = appendInbox.mock.calls[0]?.[0] as { + const arg = followUp.mock.calls[0]?.[0] as { + task: { id: string; key: string }; message: { source: string; external_id: string; - payload: { task_key: string; target_id: string }; + payload: { target_id: string }; }; - replyTargets: Array<{ id: string; spec: { kind: string } }>; + targets: Array<{ id: string; spec: { kind: string } }>; }; expect(arg.message.source).toBe("linear"); expect(arg.message.external_id).toBe("act-1"); - expect(arg.message.payload).toMatchObject({ - task_id: "task-1", - task_key: "XYZ-1", - }); + expect(arg.task).toMatchObject({ id: "task-1", key: "XYZ-1" }); // Replies route to the session, not the activity. expect(arg.message.payload.target_id).toBe("sess-1"); - expect(arg.replyTargets[0]?.id).toBe("sess-1"); - expect(arg.replyTargets[0]?.spec.kind).toBe("linear_session"); + expect(arg.targets[0]?.id).toBe("sess-1"); + expect(arg.targets[0]?.spec.kind).toBe("linear_session"); }); }); @@ -664,7 +660,8 @@ describe("InboxService", () => { prNumber: 42, }); - const arg = appendInbox.mock.calls[0]?.[0] as { + const arg = followUp.mock.calls[0]?.[0] as { + task: { id: string; key: string }; message: { source: string; kind: string; @@ -677,11 +674,10 @@ describe("InboxService", () => { run_url: string; head_sha: string; head_branch: string; - task_key: string; pr: { repo: string; pr_url: string }; }; }; - replyTargets: Array<{ + targets: Array<{ id: string; spec: { kind: string; owner: string; repo: string; prNumber: number; commentId: number }; }>; @@ -696,57 +692,27 @@ describe("InboxService", () => { expect(arg.message.payload.run_url).toBe("https://github.com/o/r/actions/runs/123456789"); expect(arg.message.payload.head_sha).toBe("abc123"); expect(arg.message.payload.head_branch).toBe("devbox/XYZ-1234-fix"); - expect(arg.message.payload).toMatchObject({ - task_id: "task-1", - task_key: "XYZ-1", - }); + expect(arg.task).toMatchObject({ id: "task-1", key: "XYZ-1" }); expect(arg.message.payload.pr).toEqual({ repo: "r", pr_url: "https://github.com/o/r/pull/42", }); - expect(arg.replyTargets).toHaveLength(1); - expect(arg.replyTargets[0]).toMatchObject({ + expect(arg.targets).toHaveLength(1); + expect(arg.targets[0]).toMatchObject({ id: "123456789", spec: { kind: "issue_comment", owner: "o", repo: "r", prNumber: 42, commentId: 0 }, }); }); - - const ciParams = { - task: makeTask(), - runId: 123456789, - runAttempt: 2, - workflowName: "CI", - workflowPath: ".github/workflows/ci.yml", - conclusion: "failure", - runUrl: "https://github.com/o/r/actions/runs/123456789", - headSha: "abc123", - headBranch: "devbox/XYZ-1234-fix", - owner: "o", - repo: "r", - prNumber: 42, - }; - - it("enqueues when under the loop-breaker threshold", async () => { - countCiResultsByTask.mockResolvedValue(19); // < 20 - await service.enqueueCiResult(ciParams); - expect(appendInbox).toHaveBeenCalledTimes(1); - }); - - it("drops the CI failure once the loop-breaker threshold is reached", async () => { - countCiResultsByTask.mockResolvedValue(20); // >= 20 - await service.enqueueCiResult(ciParams); - expect(appendInbox).not.toHaveBeenCalled(); - }); }); - describe("enqueue paths carry task_id and task_key from the task row", () => { + // followUp stamps task_id and task_key from the task it is handed. + describe("enqueue paths hand followUp the task row", () => { const apiTask = makeTask({ id: "task-api", source: "api", key: "DBX-API-7", externalUrl: "https://cp.example/api/tasks/DBX-API-7", }); - const identity = { task_id: "task-api", task_key: "DBX-API-7" }; const spec = { kind: "issue_comment" as const, owner: "o", @@ -763,8 +729,8 @@ describe("InboxService", () => { spec, createdAt: "2026-10-07T12:00:00Z", }); - expect(appendInbox).toHaveBeenCalledWith({ - taskId: "task-api", + expect(followUp).toHaveBeenCalledWith({ + task: apiTask, message: { source: "github", kind: "comment", @@ -774,10 +740,10 @@ describe("InboxService", () => { body: "@harness fix", pr: { repo: "r", pr_url: "https://github.com/o/r/pull/7" }, created_at: "2026-10-07T12:00:00Z", - ...identity, }, }, - replyTargets: [{ id: "555", spec }], + targets: [{ id: "555", spec }], + prUrl: "https://github.com/o/r/pull/7", }); }); @@ -793,8 +759,8 @@ describe("InboxService", () => { reviewerLogin: "rev", createdAt: "2026-10-07T12:00:00Z", }); - expect(appendInbox).toHaveBeenCalledWith({ - taskId: "task-api", + expect(followUp).toHaveBeenCalledWith({ + task: apiTask, message: { source: "github", kind: "review", @@ -807,10 +773,10 @@ describe("InboxService", () => { pr: { owner: "o", repo: "r", prNumber: 7, pr_url: "https://github.com/o/r/pull/7" }, reviewer_login: "rev", created_at: "2026-10-07T12:00:00Z", - ...identity, }, }, - replyTargets: [ + prUrl: "https://github.com/o/r/pull/7", + targets: [ { id: "11", spec: { @@ -848,8 +814,8 @@ describe("InboxService", () => { repo: "r", prNumber: 7, }); - expect(appendInbox).toHaveBeenCalledWith({ - taskId: "task-api", + expect(followUp).toHaveBeenCalledWith({ + task: apiTask, message: { source: "github", kind: "ci_result", @@ -863,10 +829,10 @@ describe("InboxService", () => { head_sha: "abc", head_branch: "devbox/DBX-API-7-x", pr: { repo: "r", pr_url: "https://github.com/o/r/pull/7" }, - ...identity, }, }, - replyTargets: [ + prUrl: "https://github.com/o/r/pull/7", + targets: [ { id: "9", spec: { kind: "issue_comment", owner: "o", repo: "r", prNumber: 7, commentId: 0 }, @@ -876,14 +842,15 @@ describe("InboxService", () => { }); it("enqueueLinearComment", async () => { + const task = makeTask({ id: "task-1", key: "XYZ-1" }); await service.enqueueLinearComment({ - task: makeTask({ id: "task-1", key: "XYZ-1" }), + task, agentActivityId: "act-1", agentSessionId: "sess-2", body: "follow-up", }); - expect(appendInbox).toHaveBeenCalledWith({ - taskId: "task-1", + expect(followUp).toHaveBeenCalledWith({ + task, message: { source: "linear", kind: "comment", @@ -893,11 +860,9 @@ describe("InboxService", () => { body: "follow-up", pr: null, created_at: "", - task_id: "task-1", - task_key: "XYZ-1", }, }, - replyTargets: [{ id: "sess-2", spec: { kind: "linear_session" } }], + targets: [{ id: "sess-2", spec: { kind: "linear_session" } }], }); }); }); diff --git a/control-plane/src/services/agentSession.service.ts b/control-plane/src/services/agentSession.service.ts index 2fc1c26..c1e5e49 100644 --- a/control-plane/src/services/agentSession.service.ts +++ b/control-plane/src/services/agentSession.service.ts @@ -1,16 +1,11 @@ import { emitAction, emitResponse, requireLinearTicketId } from "../clients/linear"; +import type { TaskIntake } from "../integrations/intake"; import { logger } from "../logger"; -import type { - AssignmentContent, - InboxRepository, - ReplyTarget, -} from "../repositories/inbox.repository"; -import type { TaskRepository } from "../repositories/task.repository"; +import type { ReplyTarget } from "../repositories/inbox.repository"; import type { HandleAgentSessionResult, ParsedAgentSessionEvent } from "../types/agentSession"; import { parseExperimentalFlags } from "./experimentalFlags"; import type { InboxService } from "./inbox.service"; import type { LinearTokenProvider } from "./linearTokenProvider"; -import type { SandboxManager } from "./sandboxManager.service"; const RESTARTED_BODY = [ "🔄 **Restarting from scratch.**", @@ -21,101 +16,57 @@ const RESTARTED_BODY = [ const STOP_NOT_SUPPORTED_BODY = "Stop is not supported yet — work will continue."; export class AgentSessionService { - private readonly taskRepo: TaskRepository; - private readonly inboxRepo: InboxRepository; + private readonly intake: Pick; private readonly linearTokenProvider: Pick; - private readonly sandboxManager: SandboxManager; private readonly inboxService: InboxService; constructor( - taskRepo: TaskRepository, - inboxRepo: InboxRepository, + intake: Pick, linearTokenProvider: Pick, - sandboxManager: SandboxManager, inboxService: InboxService, ) { - this.taskRepo = taskRepo; - this.inboxRepo = inboxRepo; + this.intake = intake; this.linearTokenProvider = linearTokenProvider; - this.sandboxManager = sandboxManager; this.inboxService = inboxService; } async handleEvent(event: ParsedAgentSessionEvent): Promise { switch (event.action) { case "created": { - const { flags: experimentalFlags, strippedDescription } = parseExperimentalFlags( - event.issue.description, - ); - const taskKey = requireLinearTicketId(event.issue.url); - const inboxPayload: AssignmentContent = { + const { flags, strippedDescription } = parseExperimentalFlags(event.issue.description); + // The agent session is the delegation id: Linear mints a new one per + // delegation and redelivers a `created` with the same one. + const target: ReplyTarget = { id: event.agentSessionId, spec: { kind: "linear_session" } }; + + // One transaction: insert the task and its assignment row, or compare + // the stored session with this one. The same session is a redelivery + // of a processed delegation (no change, no ack: a restart would kill + // the freshly-provisioned sandbox); a new one is a re-delegation, which + // preempts any in-flight run and restarts from a clean slate (intake + // kills the preempted sandbox). + const { taskId, result } = await this.intake.assign({ + source: "linear", + externalId: event.issue.id, + key: { hint: requireLinearTicketId(event.issue.url) }, title: event.issue.title, description: strippedDescription, - external_url: event.issue.url, - target_id: event.agentSessionId, - }; - const inboxReplyTargets: ReplyTarget[] = [ - { id: event.agentSessionId, spec: { kind: "linear_session" } }, - ]; - - // Single tx: upsert the tasks row (infra) + insert the assignment inbox - // row. The inbox row's reply_targets[0].id is the canonical Linear - // agent session id; on a re-delegation `restartTaskWithInbox` - // overwrites it with the freshly-minted session and rotates the inbox - // id. Inbox row uses the Linear issue UUID as external_id; its id is the - // daemon-visible run identity. - const { taskId, created } = await this.taskRepo.upsertFromAgentSessionWithInbox({ - taskExternalId: event.issue.id, - taskKey, - taskTitle: event.issue.title, - taskExternalUrl: event.issue.url, - inboxPayload, - inboxReplyTargets, - metadata: { experimentalFlags }, + flags, + url: () => event.issue.url, + target: () => target, }); - if (created) { + if (result === "created") { // Fresh task — Linear sees Evaluating to satisfy the 10s SLA. this.fireAndForgetEmitAction(event.agentSessionId, "Evaluating", "task", "created-ack"); - return { status: "ok" }; - } - - // Conflict path — distinguish a duplicate webhook redelivery (same - // session id as the row already carries) from a genuine re-delegation - // (new session id). The upsert above DO-NOTHING'd the inbox INSERT, so - // the assignment row's existing reply_targets still carries the prior - // session id. - const prevSessionId = await this.inboxRepo.getAssignmentReplyTargetId(taskId); - if (prevSessionId !== event.agentSessionId) { - // Genuine re-delegation — Linear minted a NEW agent session for a new - // `created` event. Preempt any in-flight run and restart from scratch: - // restartTaskWithInbox resets the task + re-pends the `assignment` inbox - // row (with the new session id in reply_targets). Kill the preempted - // sandbox (if the run was still live) so the slot frees right away; - // the old owner pod self-evicts on its next owner-guarded write. Then - // ack the restart. + } else if (result === "restarted") { logger.info("re-delegation detected; restarting task from clean slate", { taskId, - previousSessionId: prevSessionId, currentSessionId: event.agentSessionId, }); - const { prevSandboxId } = await this.taskRepo.restartTaskWithInbox({ - taskId, - inboxPayload, - inboxReplyTargets, - }); - if (prevSandboxId) { - this.fireAndForgetKillSandbox(prevSandboxId); - } this.fireAndForgetEmitResponse(event.agentSessionId, RESTARTED_BODY, "restart-ack"); this.fireAndForgetEmitAction(event.agentSessionId, "Evaluating", "task", "restart-ack"); } - // else: prevSessionId === event.agentSessionId — a duplicate webhook - // delivery of an already-processed delegation (Linear redelivers the same - // `created` event, same agent session id). The run is already (re)started - // and acked by the first delivery; no-op rather than run the restart - // twice (which would kill the freshly-provisioned sandbox). Fall through - // to the 200 below so Linear stops retrying. + // A duplicate falls through to the 200 so Linear stops retrying. return { status: "ok" }; } @@ -133,10 +84,10 @@ export class AgentSessionService { // (Linear mints a fresh agent session per re-prompt). Unknown issue → // session_not_found so the webhook returns 503 and Linear retries after // the `created` event lands. - const task = await this.taskRepo.findByLinearIssueId(event.issue.id); + const task = await this.intake.findTask("linear", event.issue.id); if (!task) return { status: "session_not_found" }; - // enqueueLinearComment INSERTs the inbox row (idempotent on activityId). + // enqueueLinearComment queues the round (idempotent on activityId). // The new comment row's reply_targets carry the fresh session id, so // replies route correctly; the assignment row's reply_targets is the // canonical session id for task-scoped lifecycle / failure emits and @@ -205,16 +156,4 @@ export class AgentSessionService { }); }); } - - // Fire-and-forget — the webhook handler must not block on the sandbox API. - // Kills a preempted sandbox when a re-delegation restarts a task. - private fireAndForgetKillSandbox(sandboxId: string): void { - void this.sandboxManager.killSandbox(sandboxId).catch((err) => { - const msg = err instanceof Error ? err.message : String(err); - logger.error("killSandbox(preempted re-delegation) failed", { - sandboxId, - message: msg, - }); - }); - } } diff --git a/control-plane/src/services/apiTask.service.ts b/control-plane/src/services/apiTask.service.ts index 415d268..9c5185b 100644 --- a/control-plane/src/services/apiTask.service.ts +++ b/control-plane/src/services/apiTask.service.ts @@ -1,8 +1,8 @@ import { ApiError } from "../errors"; -import { logger } from "../logger"; +import type { TaskIntake } from "../integrations/intake"; +import type { TaskSource } from "../integrations/types"; import type { ApiTaskRecord, ApiTaskRepository } from "../repositories/apiTask.repository"; import { apiSessionTarget } from "../repositories/inbox.repository"; -import type { TaskRepository } from "../repositories/task.repository"; import type { TaskEventRepository, TaskEventRow } from "../repositories/taskEvent.repository"; import type { ApiEventEnvelope, @@ -15,7 +15,6 @@ import type { SseStateData, } from "../types/api"; import { extractExperimentalFlags } from "./experimentalFlags"; -import type { SandboxManager } from "./sandboxManager.service"; export type Principal = { tokenId: string; name: string }; export type CreateTaskInput = { @@ -24,14 +23,13 @@ export type CreateTaskInput = { experimentalFlags: string[]; }; export type ListTasksInput = { - source?: "linear" | "api"; + source?: TaskSource; createdBy?: "me" | string; state?: ApiTaskState; before?: string; limit: number; }; export type ApiTaskServiceOptions = { - keyPrefix: string; publicBaseUrl: string; now: () => Date; }; @@ -41,26 +39,30 @@ const NOT_FOUND = () => new ApiError(404, "task_not_found", "Task not found"); const NOT_API = () => new ApiError(409, "not_api_task", "Only API tasks can be changed through the API"); -// The /api use cases read every task and write only API tasks. All settings -// arrive through the constructor so this module never imports ../config; that -// keeps process-global mock.module replacements from leaking into these tests. +// The /api use cases read every task and write only API tasks, through +// TaskIntake. All settings arrive through the constructor so this module never +// imports ../config; that keeps process-global mock.module replacements from +// leaking into these tests. export class ApiTaskService { constructor( private apiTaskRepo: ApiTaskRepository, - private taskRepo: TaskRepository, + private intake: Pick, private taskEventRepo: TaskEventRepository, - private sandboxManager: Pick, private opts: ApiTaskServiceOptions, ) {} + // The task row, its assignment round and task_created, in one transaction. + // The key comes from api_task_key_seq and is also the task's external id. async create(principal: Principal | null, input: CreateTaskInput): Promise { - const { taskId } = await this.apiTaskRepo.createApiTask({ - keyPrefix: this.opts.keyPrefix, - publicBaseUrl: this.opts.publicBaseUrl, + const { taskId } = await this.intake.assign({ + source: "api", + key: { sequence: "api" }, title: input.title, description: input.description, - experimentalFlags: input.experimentalFlags, - createdBy: principal?.tokenId ?? null, + flags: input.experimentalFlags, + url: (t) => `${this.opts.publicBaseUrl}/api/tasks/${t.key}`, + target: (t) => apiSessionTarget(t.id), + createdBy: principal?.tokenId, }); return this.get(taskId); } @@ -85,39 +87,36 @@ export class ApiTaskService { return this.toApiTask(await this.resolve(ref)); } + // A comment round (external_id = a new UUID) and message_queued in one + // transaction. A round in progress doesn't block it: the new row queues + // behind it, as on Linear. async sendMessage(ref: string, body: string): Promise { const task = await this.resolveApiTask(ref); - const { roundId } = await this.apiTaskRepo.appendApiMessage({ + const result = await this.intake.followUp({ task: { id: task.id, key: task.key }, - body, - createdAt: this.opts.now().toISOString(), + message: { + source: "api", + kind: "comment", + external_id: crypto.randomUUID(), + payload: { target_id: task.id, body, pr: null, created_at: this.opts.now().toISOString() }, + }, + targets: [apiSessionTarget(task.id)], }); - return { round_id: roundId }; + // A fresh external id and no PR guard: nothing can drop it. + if (!("roundId" in result)) throw new Error(`sendMessage: round dropped (${result.dropped})`); + return { round_id: result.roundId }; } - // Same transaction as Linear re-delegation (restartTaskWithInbox), plus - // the full new definition and task_restarted. The input replaces the current - // title/description/flags; nothing is merged. The previous sandbox is killed - // fire-and-forget afterwards. + // Same restart as a re-delegation, plus the full new definition and + // task_restarted. The input replaces the current title/description/flags; + // nothing is merged. The previous sandbox is killed afterwards. async restart(ref: string, input: CreateTaskInput): Promise { const task = await this.resolveApiTask(ref); - const { title, description, experimentalFlags } = input; - const { prevSandboxId } = await this.taskRepo.restartTaskWithInbox({ - taskId: task.id, - inboxPayload: { title, description, external_url: task.external_url, target_id: task.id }, - inboxReplyTargets: [apiSessionTarget(task.id)], - api: { title, description, experimentalFlags }, + await this.intake.restart(task.id, { + title: input.title, + description: input.description, + flags: input.experimentalFlags, }); - if (prevSandboxId) { - void this.sandboxManager.killSandbox(prevSandboxId).catch((err) => { - const e = err instanceof Error ? err : new Error(String(err)); - logger.error("killSandbox(API restart) failed", { - sandboxId: prevSandboxId, - message: e.message, - stack: e.stack, - }); - }); - } return this.get(task.id); } @@ -128,7 +127,7 @@ export class ApiTaskService { return { events, next_after: events[events.length - 1]?.id ?? after, logging: "available" }; } - async streamTarget(ref: string): Promise<{ id: string; source: "linear" | "api" }> { + async streamTarget(ref: string): Promise<{ id: string; source: TaskSource }> { const task = await this.resolve(ref); return { id: task.id, source: task.source }; } @@ -147,7 +146,7 @@ export class ApiTaskService { private async resolve(ref: string): Promise { const task = UUID_RE.test(ref) ? await this.apiTaskRepo.findById(ref) - : await this.apiTaskRepo.findByKey(ref.toUpperCase()); + : await this.apiTaskRepo.findByKey(ref); if (!task) throw NOT_FOUND(); return task; } diff --git a/control-plane/src/services/inbox.service.ts b/control-plane/src/services/inbox.service.ts index 21ec14c..02fb6d1 100644 --- a/control-plane/src/services/inbox.service.ts +++ b/control-plane/src/services/inbox.service.ts @@ -1,6 +1,7 @@ import { config } from "../config"; import type { Task } from "../db/schema"; import { SEND_TIMEOUT_MS, withTimeout } from "../integrations/http"; +import type { TaskIntake } from "../integrations/intake"; import type { Integration, IntegrationName } from "../integrations/types"; import { logger } from "../logger"; import type { @@ -8,16 +9,16 @@ import type { InboxRow, ReplyTarget, ReplyTargetSpec, - ReviewPayload, } from "../repositories/inbox.repository"; -import { taskIdentity } from "../repositories/inbox.repository"; import type { InboxMessage } from "../types/inboxMessage"; import type { IntegrationHooks } from "./integrationHooks"; import type { SandboxManager } from "./sandboxManager.service"; -// InboxService owns inbox enqueue (comment/review/Linear-comment), the sweep -// that delivers pending rows to the sandbox (recreating it from the preserved -// PVC when it has been reaped), and the PVC-missing per-row dead-letter. +// InboxService builds the GitHub and Linear follow-up rounds (comment, review, +// CI result, Linear comment) and queues them through TaskIntake.followUp; it +// owns the sweep that delivers pending rows to the sandbox (recreating it from +// the preserved PVC when it has been reaped), and the PVC-missing per-row +// dead-letter. export const WORKSPACE_MISSING_MESSAGE = "The workspace for this task has been cleaned up and can no longer be resumed. " + "Please reach out to the harness team or open a new ticket to continue."; @@ -28,6 +29,7 @@ export class InboxService { private sandboxManager: SandboxManager, private integrations: Record, private hooks: IntegrationHooks, + private intake: Pick, ) {} // ──── Enqueue paths ──── @@ -42,8 +44,9 @@ export class InboxService { createdAt: string; }): Promise { const externalId = String(params.commentId); - await this.inboxRepo.appendInbox({ - taskId: params.task.id, + const prUrl = `https://github.com/${params.spec.owner}/${params.spec.repo}/pull/${params.spec.prNumber}`; + await this.intake.followUp({ + task: params.task, message: { source: "github", kind: "comment", @@ -51,15 +54,12 @@ export class InboxService { payload: { target_id: externalId, // matches reply_targets[0].id body: params.body, - ...taskIdentity(params.task), - pr: { - repo: params.spec.repo, - pr_url: `https://github.com/${params.spec.owner}/${params.spec.repo}/pull/${params.spec.prNumber}`, - }, + pr: { repo: params.spec.repo, pr_url: prUrl }, created_at: params.createdAt, }, }, - replyTargets: [{ id: externalId, spec: params.spec }], + targets: [{ id: externalId, spec: params.spec }], + prUrl, }); } @@ -84,8 +84,9 @@ export class InboxService { createdAt: string; }): Promise { const reviewIdStr = String(params.reviewId); + const prUrl = `https://github.com/${params.owner}/${params.repo}/pull/${params.prNumber}`; - const payload: ReviewPayload = { + const payload = { review_body: { target_id: reviewIdStr, body: params.reviewBody }, inline_comments: params.inlineComments.map((c) => ({ target_id: String(c.id), @@ -94,13 +95,7 @@ export class InboxService { start_line: c.startLine, body: c.body, })), - ...taskIdentity(params.task), - pr: { - owner: params.owner, - repo: params.repo, - prNumber: params.prNumber, - pr_url: `https://github.com/${params.owner}/${params.repo}/pull/${params.prNumber}`, - }, + pr: { owner: params.owner, repo: params.repo, prNumber: params.prNumber, pr_url: prUrl }, reviewer_login: params.reviewerLogin, created_at: params.createdAt, }; @@ -135,40 +130,35 @@ export class InboxService { }, ]; - await this.inboxRepo.appendInbox({ - taskId: params.task.id, - message: { - source: "github", - kind: "review", - external_id: reviewIdStr, - payload, - }, - replyTargets, + await this.intake.followUp({ + task: params.task, + message: { source: "github", kind: "review", external_id: reviewIdStr, payload }, + targets: replyTargets, + prUrl, }); } // Linear `prompted` — a follow-up comment on the agent session. Same inbox // path as a GitHub comment; the single reply target routes to the Linear // session. Idempotent on the Linear activity UUID — a duplicate redelivery - // is a no-op insert. + // is dropped. async enqueueLinearComment(params: { task: Pick; agentActivityId: string; agentSessionId: string; body: string; }): Promise { - await this.inboxRepo.appendInbox({ - taskId: params.task.id, + await this.intake.followUp({ + task: params.task, message: { source: "linear", kind: "comment", - // Linear activity UUID — drives idempotency at (source, external_id). + // Linear activity UUID — drives idempotency at (source, kind, external_id). external_id: params.agentActivityId, payload: { // Replies route to the Linear agent session — matches reply_targets[0].id. target_id: params.agentSessionId, body: params.body, - ...taskIdentity(params.task), // Linear comment isn't PR-scoped — lives on the agent session. pr: null, // Linear-sourced → the daemon never injects PR discussion (gated on @@ -176,14 +166,14 @@ export class InboxService { created_at: "", }, }, - replyTargets: [{ id: params.agentSessionId, spec: { kind: "linear_session" } }], + targets: [{ id: params.agentSessionId, spec: { kind: "linear_session" } }], }); } // GitHub `workflow_run` completed — one row per task whose PR URL matches. // external_id includes runAttempt (re-runs share runId but bump attempt) and - // prNumber (one run can touch multiple PRs) for the global (source, external_id) - // unique index. + // prNumber (one run can touch multiple PRs). followUp applies the per-task + // loop breaker. async enqueueCiResult(params: { task: Pick; runId: number; @@ -198,30 +188,11 @@ export class InboxService { repo: string; prNumber: number; }): Promise { - // Loop breaker. harness fixes CI → pushes → CI re-runs → may fail again, - // ad infinitum. Once a task has accumulated `ciResultMaxPerTask` ci_result - // rows, stop forwarding new failures to the harness. The count/insert is - // not transactional, so concurrent webhooks for one task may overshoot by a - // few — fine for a circuit breaker. 0 / negative disables the breaker. - const max = config.github.ciResultMaxPerTask; - if (max > 0) { - const seen = await this.inboxRepo.countCiResultsByTask(params.task.id); - if (seen >= max) { - logger.warn("ci_result loop breaker tripped — dropping CI failure", { - taskId: params.task.id, - seen, - max, - runId: params.runId, - prNumber: params.prNumber, - }); - return; - } - } - const runIdStr = String(params.runId); const externalId = `workflow_run:${params.runId}:${params.runAttempt}:${params.prNumber}`; - await this.inboxRepo.appendInbox({ - taskId: params.task.id, + const prUrl = `https://github.com/${params.owner}/${params.repo}/pull/${params.prNumber}`; + await this.intake.followUp({ + task: params.task, message: { source: "github", kind: "ci_result", @@ -234,14 +205,10 @@ export class InboxService { run_url: params.runUrl, head_sha: params.headSha, head_branch: params.headBranch, - ...taskIdentity(params.task), - pr: { - repo: params.repo, - pr_url: `https://github.com/${params.owner}/${params.repo}/pull/${params.prNumber}`, - }, + pr: { repo: params.repo, pr_url: prUrl }, }, }, - replyTargets: [ + targets: [ { id: runIdStr, spec: { @@ -255,6 +222,7 @@ export class InboxService { }, }, ], + prUrl, }); } diff --git a/control-plane/src/services/integrationHooks.ts b/control-plane/src/services/integrationHooks.ts index 5c11fdd..cdaa166 100644 --- a/control-plane/src/services/integrationHooks.ts +++ b/control-plane/src/services/integrationHooks.ts @@ -20,6 +20,8 @@ const HOOK_TIMEOUT_MS = { assignee: 15_000, } as const; type HookName = keyof typeof HOOK_TIMEOUT_MS; +// What a detached promise is doing, for the shutdown log. +type DetachedWork = HookName | "killSandbox"; // Runs integrations' lifecycle hooks (§4). onTaskStarted, onPrOpened and // assignee go to the task's own integration; onRoundFinished goes to the @@ -29,7 +31,7 @@ type HookName = keyof typeof HOOK_TIMEOUT_MS; // pr_opened as it holds a send (the hook must be idempotent). Detached work is // tracked so TaskProcessor.stop() can drain it. export class IntegrationHooks { - private readonly detached = new Map, { taskId: string; hook: HookName }>(); + private readonly detached = new Map, { taskId: string; hook: DetachedWork }>(); // Assignment ids whose onTaskStarted already ran on this pod: a provision // deferred by a transient error runs again for the same assignment. // ponytail: one id per started task for the pod's lifetime; prune if that ever matters. @@ -108,8 +110,9 @@ export class IntegrationHooks { ); } - // Tracks detached work (a hook, or core work that calls one) for drain(). - detach(taskId: string, hook: HookName, work: Promise): void { + // Tracks detached work (a hook, core work that calls one, or a restart's + // sandbox kill) for drain(). + detach(taskId: string, hook: DetachedWork, work: Promise): void { const p: Promise = work .then( () => {}, diff --git a/control-plane/src/types/api.ts b/control-plane/src/types/api.ts index a303728..f4a6547 100644 --- a/control-plane/src/types/api.ts +++ b/control-plane/src/types/api.ts @@ -2,6 +2,8 @@ // values, so the devbox CLI can import it without pulling // in server modules. Guarded by src/types/__tests__/api.test.ts. +// The task_source enum, as a literal union: this file can't import db/schema. +// integrations/types.ts checks the two stay equal. export type ApiTaskSource = "linear" | "api"; export type ApiTaskState = "queued" | "running" | "idle" | "failed"; export type ApiRoundStatus = "pending" | "delivered" | "responded" | "failed"; diff --git a/control-plane/src/types/inboxMessage.ts b/control-plane/src/types/inboxMessage.ts index f71896d..64e96a4 100644 --- a/control-plane/src/types/inboxMessage.ts +++ b/control-plane/src/types/inboxMessage.ts @@ -107,7 +107,7 @@ export type CiResultPayload = TaskIdentity & { // external_id — Upstream identifier (GH comment/review id stringified, // Linear activity UUID, Linear issue UUID for assignment, the task key for an API assignment; // a new UUID for an API comment). -// Drives idempotency at CP's `(source, external_id)` unique +// Drives idempotency at CP's `(source, kind, external_id)` unique // index AND flows through to event reply-target routing. // payload — Kind-specific body. See the payload types above. export type InboxMessage = From a9d7173529b27af4a639f4c6b3a4f9d4bf8db27f Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Fri, 30 Oct 2026 19:46:51 +0530 Subject: [PATCH 12/22] feat(cli): derive ls --source choices from ApiTaskSource The choices come from the shared type, so a new source can't be forgotten. Co-Authored-By: Claude Opus 5.5 --- cli/interface/commands/ls.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cli/interface/commands/ls.ts b/cli/interface/commands/ls.ts index e9a71f5..af6cf80 100644 --- a/cli/interface/commands/ls.ts +++ b/cli/interface/commands/ls.ts @@ -15,6 +15,8 @@ export type LsArgs = CommonArgs & { }; const HEADER = ["KEY", "STATE", "TITLE", "DESCRIPTION", "CREATOR", "AGE"]; +// Every task source; the compiler fails here until a new one is listed. +const SOURCES = Object.keys({ api: true, linear: true } satisfies Record); export function render( tasks: ApiTask[], @@ -73,7 +75,7 @@ export const ls: CommandModule = { .command("ls") .description("List tasks, newest first") .addOption( - new Option("--source ", "only tasks from this source").choices(["api", "linear"]), + new Option("--source ", "only tasks from this source").choices(SOURCES), ) .option("--mine", "only tasks created with your token") .addOption( From e8b440ed096942e6f064d4b3f59cb647e0734ead Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Fri, 30 Oct 2026 20:12:11 +0530 Subject: [PATCH 13/22] feat(control-plane): integration registry with webhooks, status and Slack opt-in Integrations carry enabled, webhook and status. Core mounts each enabled integration's webhook at its existing path, serves /api/integrations/:name with the existing fields plus held sends, and skips a disabled integration's sends and hooks instead of holding them. Slack notifications are opted in per source with SLACK_NOTIFY_SOURCES (default linear), and mentions come from the integration's assignee lookup. Co-Authored-By: Claude Opus 5.5 --- .../src/__tests__/integrationsConfig.test.ts | 14 ++ .../src/__tests__/webhookRoutes.test.ts | 49 ++++++- control-plane/src/app.ts | 80 ++++++----- control-plane/src/config.ts | 6 +- .../__tests__/api.controller.stream.test.ts | 15 +- .../__tests__/api.controller.test.ts | 16 ++- .../src/integrations/__tests__/http.test.ts | 12 +- control-plane/src/integrations/api/index.ts | 23 +++- .../src/integrations/github/index.ts | 46 +++++-- .../src/integrations/linear/index.ts | 22 ++- control-plane/src/integrations/types.ts | 15 +- control-plane/src/integrationsConfig.ts | 19 ++- control-plane/src/routes/api.ts | 15 +- control-plane/src/routes/webhook.ts | 30 ++-- .../__tests__/eventDispatcher.test.ts | 80 ++++++----- .../services/__tests__/inbox.service.test.ts | 14 +- .../__tests__/integrationHooks.test.ts | 22 ++- .../__tests__/slackNotifier.service.test.ts | 130 ++++++++++-------- .../__tests__/taskProcessor.service.test.ts | 35 +++-- control-plane/src/services/eventDispatcher.ts | 72 +++++----- control-plane/src/services/inbox.service.ts | 16 ++- .../src/services/integrationHooks.ts | 14 +- control-plane/src/services/replyHolds.ts | 9 +- .../src/services/slackNotifier.service.ts | 103 +++++++------- .../src/services/taskProcessor.service.ts | 47 +++---- 25 files changed, 578 insertions(+), 326 deletions(-) diff --git a/control-plane/src/__tests__/integrationsConfig.test.ts b/control-plane/src/__tests__/integrationsConfig.test.ts index 69bf86b..07dfce3 100644 --- a/control-plane/src/__tests__/integrationsConfig.test.ts +++ b/control-plane/src/__tests__/integrationsConfig.test.ts @@ -107,9 +107,23 @@ describe("parseSlackConfig", () => { expect(parseSlackConfig({ ...SLACK_ENV, SLACK_ENABLED: "true" })).toEqual({ botToken: "xoxb-test", channel: "C123", + notifySources: ["linear"], }); }); + it("SLACK: SLACK_NOTIFY_SOURCES lists task sources; an unknown one fails", () => { + const on = { ...SLACK_ENV, SLACK_ENABLED: "true" }; + expect(parseSlackConfig({ ...on, SLACK_NOTIFY_SOURCES: "linear, api" })?.notifySources).toEqual( + ["linear", "api"], + ); + expect(parseSlackConfig({ ...on, SLACK_NOTIFY_SOURCES: " " })?.notifySources).toEqual([ + "linear", + ]); + expect(() => parseSlackConfig({ ...on, SLACK_NOTIFY_SOURCES: "linear,github" })).toThrow( + new Error('SLACK_NOTIFY_SOURCES: unknown task source "github" (allowed: linear, api)'), + ); + }); + it("SLACK: on with a setting missing", () => { expect(() => parseSlackConfig({ SLACK_ENABLED: "true" })).toThrow( new Error("SLACK_ENABLED is true but these are missing: SLACK_BOT_TOKEN, SLACK_CHANNEL_ID"), diff --git a/control-plane/src/__tests__/webhookRoutes.test.ts b/control-plane/src/__tests__/webhookRoutes.test.ts index 72b0560..ebace6e 100644 --- a/control-plane/src/__tests__/webhookRoutes.test.ts +++ b/control-plane/src/__tests__/webhookRoutes.test.ts @@ -1,6 +1,25 @@ import { describe, expect, it, spyOn } from "bun:test"; +import type { Context } from "hono"; +import type { TaskIntake } from "../integrations/intake"; +import type { IntegrationName } from "../integrations/types"; import { WEBHOOK_BODY_LIMIT_BYTES } from "../requestLimits"; -import { createWebhookRoutes } from "../routes/webhook"; +import { createWebhookRoutes as createRoutes } from "../routes/webhook"; + +const intake = {} as TaskIntake; + +// The routes for enabled integrations with these webhooks. +function createWebhookRoutes( + webhooks: Partial Promise }>>, +) { + return createRoutes( + Object.entries(webhooks).map(([name, w]) => ({ + name: name as IntegrationName, + enabled: true, + webhook: (c: Context) => w.handleWebhook(c), + })), + intake, + ); +} const OVERSIZED_BODY = "x".repeat(WEBHOOK_BODY_LIMIT_BYTES + 1); @@ -90,6 +109,34 @@ describe("createWebhookRoutes", () => { } }); + it("serves no webhook for a disabled integration, and hands the intake to an enabled one", async () => { + const seen: unknown[] = []; + const app = createRoutes( + [ + { + name: "linear", + enabled: false, + webhook: async () => new Response(null, { status: 200 }), + }, + { + name: "github", + enabled: true, + webhook: async (_c, i) => { + seen.push(i); + return new Response(null, { status: 200 }); + }, + }, + { name: "api", enabled: true }, + ], + intake, + ); + expect((await app.request("/webhook/linear", { method: "POST", body: "{}" })).status).toBe(404); + expect((await app.request("/webhook/api", { method: "POST", body: "{}" })).status).toBe(404); + expect((await app.request("/webhook/github", { method: "POST", body: "{}" })).status).toBe(200); + expect(seen).toHaveLength(1); + expect(seen[0]).toBe(intake); + }); + it("leaves disabled and unknown webhook routes as 404 even for oversized bodies", async () => { const app = createWebhookRoutes({ github: { diff --git a/control-plane/src/app.ts b/control-plane/src/app.ts index 489f87f..3419e99 100644 --- a/control-plane/src/app.ts +++ b/control-plane/src/app.ts @@ -8,8 +8,8 @@ import { HealthController } from "./controllers/health.controller"; import { LinearWebhookController } from "./controllers/linearWebhook.controller"; import { db } from "./db"; import { taskSource } from "./db/schema"; -import { apiIntegration } from "./integrations/api"; -import { githubIntegration } from "./integrations/github"; +import { ApiIntegration } from "./integrations/api"; +import { GithubIntegration } from "./integrations/github"; import { Intake } from "./integrations/intake"; import { LinearIntegration } from "./integrations/linear"; import type { Integration, IntegrationName } from "./integrations/types"; @@ -81,18 +81,31 @@ export function createApp(): AppBundle { trustedUsers: joinIfAny(config.github.trustedUsers), }); - // Every outbound message goes through integrations[round.source].deliver. + const linearStatus = new LinearStatusService( + config.linear ? linearTokenProvider : undefined, + logger, + ); + const githubStatus = new GithubStatusService(githubTokenState, { + enabled: config.github.webhooks !== undefined, + ciResults: config.github.ciResultEnabled, + lastWebhookAt: (): string | null => integrations.github.controller?.lastWebhookAt() ?? null, + }); + + // The registry. Every outbound message goes through + // integrations[round.source].deliver, and core skips a disabled + // integration's webhook, sends and hooks. const integrations = { - linear: new LinearIntegration(linearTokenProvider), - api: apiIntegration, - github: githubIntegration, + linear: new LinearIntegration(linearTokenProvider, config.linear !== undefined, linearStatus), + api: new ApiIntegration(config.api.enabled), + // GitHub rounds come only from its webhooks. + github: new GithubIntegration(config.github.webhooks !== undefined, githubStatus), } satisfies Record; // The registered integrations that create tasks (github creates none). Only // these sources are claimed and reconciled, so an older image never works a // task whose source it doesn't know (§10). const taskSources = taskSource.enumValues.filter((s) => s in integrations); - // Sends held because they failed before anything went out; the status - // routes will report them per integration (phase 2). + // Sends held because they failed before anything went out; each + // integration's status reports its own. const replyHolds = new ReplyHolds(); // Integrations' lifecycle hooks; TaskProcessor.stop() drains detached ones. const hooks = new IntegrationHooks(integrations, inboxRepository); @@ -111,19 +124,34 @@ export function createApp(): AppBundle { intake, ); - const agentSessionService = new AgentSessionService(intake, linearTokenProvider, inboxService); + // Today's webhook controllers enqueue through InboxService, which needs the + // registry, so each is attached to its module here, only when enabled. + if (config.linear) { + integrations.linear.controller = new LinearWebhookController( + new AgentSessionService(intake, linearTokenProvider, inboxService), + (at) => linearStatus.recordWebhook(at), + ); + } + if (config.github.webhooks) { + integrations.github.controller = new GitHubWebhookController( + inboxService, + intake, + config.github.webhooks, + ); + } const slackNotifier = new SlackNotifierService( taskRepository, config.slack?.botToken ?? "", config.slack?.channel ?? "", + config.slack?.notifySources ?? [], + hooks, ); const eventDispatcher = new EventDispatcher( taskRepository, inboxRepository, sandboxManager, - linearTokenProvider, slackNotifier, integrations, replyHolds, @@ -144,31 +172,11 @@ export function createApp(): AppBundle { hooks, ); - const linearStatus = new LinearStatusService( - config.linear ? linearTokenProvider : undefined, - logger, - ); - - // Each webhook exists only when its integration is configured. - const linearWebhookController = config.linear - ? new LinearWebhookController(agentSessionService, (at) => linearStatus.recordWebhook(at)) - : undefined; - const githubWebhookController = config.github.webhooks - ? new GitHubWebhookController(inboxService, intake, config.github.webhooks) - : undefined; - const githubStatus = new GithubStatusService(githubTokenState, { - enabled: githubWebhookController !== undefined, - ciResults: config.github.ciResultEnabled, - lastWebhookAt: () => githubWebhookController?.lastWebhookAt() ?? null, - }); const healthController = new HealthController(db); const app = new Hono(); app.route("/", createHealthRoutes(healthController)); - app.route( - "/", - createWebhookRoutes({ linear: linearWebhookController, github: githubWebhookController }), - ); + app.route("/", createWebhookRoutes(Object.values(integrations), intake)); // /api only mounts when explicitly enabled: a CP deployed before the // internal ingress exists must expose nothing. @@ -184,13 +192,19 @@ export function createApp(): AppBundle { sseMaxDurationMs: config.api.sseMaxDurationMs, clock: realClock, }); + // Each integration's own status body, plus its held sends (§4 I14). + const statuses = Object.fromEntries( + Object.values(integrations).map((i) => [ + i.name, + { status: async () => ({ ...(await i.status()), holds: replyHolds.of(i.name) }) }, + ]), + ) as Record>; app.route( "/", createApiRoutes(apiController, new ApiAuthService(apiTokenRepository), { authEnabled: config.api.authEnabled, allowedHosts: config.api.allowedHosts, - linearStatus: () => linearStatus.status(), - githubStatus: () => githubStatus.status(), + integrations: statuses, }), ); stopApiStreams = () => apiController.closeStreams(); diff --git a/control-plane/src/config.ts b/control-plane/src/config.ts index d2908cc..001855d 100644 --- a/control-plane/src/config.ts +++ b/control-plane/src/config.ts @@ -96,8 +96,10 @@ export const config = { // integrationsConfig.ts. linear: parseLinearConfig(process.env), // Undefined unless SLACK_ENABLED=true; then SLACK_BOT_TOKEN (xoxb-…, posts - // task-completion updates) and SLACK_CHANNEL_ID (C…) are required. The token - // matches the /token/i log-redaction pattern. Parsed by integrationsConfig.ts. + // task-completion updates) and SLACK_CHANNEL_ID (C…) are required, and + // SLACK_NOTIFY_SOURCES picks the task sources that post (default linear). + // The token matches the /token/i log-redaction pattern. Parsed by + // integrationsConfig.ts. slack: parseSlackConfig(process.env), github: { // The control plane's own GitHub token (it runs outside the sandbox diff --git a/control-plane/src/controllers/__tests__/api.controller.stream.test.ts b/control-plane/src/controllers/__tests__/api.controller.stream.test.ts index 5127b29..0d807bc 100644 --- a/control-plane/src/controllers/__tests__/api.controller.stream.test.ts +++ b/control-plane/src/controllers/__tests__/api.controller.stream.test.ts @@ -123,11 +123,16 @@ function setup(o: { { authEnabled: false, allowedHosts: [], - linearStatus: async () => ({ configured: false }), - githubStatus: () => ({ - token: { source: "env", expiresAt: null, readAt: null, lastError: null }, - webhooks: { enabled: false, lastWebhookAt: null, ciResults: false }, - }), + integrations: { + linear: { status: async () => ({ configured: false }) }, + github: { + status: async () => ({ + token: { source: "env", expiresAt: null, readAt: null, lastError: null }, + webhooks: { enabled: false, lastWebhookAt: null, ciResults: false }, + }), + }, + api: { status: async () => ({ enabled: true }) }, + }, }, ); return { app, service, controller, clock }; diff --git a/control-plane/src/controllers/__tests__/api.controller.test.ts b/control-plane/src/controllers/__tests__/api.controller.test.ts index fa444d1..bc69c36 100644 --- a/control-plane/src/controllers/__tests__/api.controller.test.ts +++ b/control-plane/src/controllers/__tests__/api.controller.test.ts @@ -57,8 +57,11 @@ function setup(opts: { authEnabled: boolean; allowedHosts?: string[] }) { const app = createApiRoutes(controller, auth, { authEnabled: opts.authEnabled, allowedHosts: opts.allowedHosts ?? [], - linearStatus: async () => LINEAR_STATUS, - githubStatus: () => GITHUB_STATUS, + integrations: { + linear: { status: async () => LINEAR_STATUS }, + github: { status: async () => GITHUB_STATUS }, + api: { status: async () => ({ enabled: true }) }, + }, }); const req = (path: string, init: RequestInit & { json?: unknown } = {}) => app.request(path, { @@ -101,6 +104,7 @@ const routes: Array<[string, string, unknown?]> = [ ["GET", "/api/me"], ["GET", "/api/integrations/linear"], ["GET", "/api/integrations/github"], + ["GET", "/api/integrations/api"], ]; describe("ApiController", () => { @@ -118,6 +122,14 @@ describe("ApiController", () => { expect(await res.json()).toEqual(GITHUB_STATUS); }); + it("GET /api/integrations/api returns the status body; an unknown name is 404", async () => { + const { req } = setup({ authEnabled: true }); + expect(await (await req("/api/integrations/api")).json()).toEqual({ enabled: true }); + for (const name of ["jira", "constructor"]) { + expect((await req(`/api/integrations/${name}`)).status).toBe(404); + } + }); + describe("auth on", () => { it.each(routes)( "%s %s without a token is 401 with the uniform body", diff --git a/control-plane/src/integrations/__tests__/http.test.ts b/control-plane/src/integrations/__tests__/http.test.ts index 34ad881..f9ddb12 100644 --- a/control-plane/src/integrations/__tests__/http.test.ts +++ b/control-plane/src/integrations/__tests__/http.test.ts @@ -193,10 +193,14 @@ describe("Linear deliver: SDK failures", () => { const target: ReplyTarget = { id: "sess-1", spec: { kind: "linear_session" } }; const reply = { type: "reply", body: "hi" } as const; const integrationFor = (apiUrl: string) => - new LinearIntegration({ - getAuthorizedLinearClient: (signal?: AbortSignal) => - Promise.resolve(new LinearClient({ accessToken: "t", apiUrl, signal })), - }); + new LinearIntegration( + { + getAuthorizedLinearClient: (signal?: AbortSignal) => + Promise.resolve(new LinearClient({ accessToken: "t", apiUrl, signal })), + }, + true, + { status: async () => ({ configured: false }) }, + ); const deliverTo = (apiUrl: string, signal = new AbortController().signal) => integrationFor(apiUrl) .deliver({ signal, log: logger }, target, reply) diff --git a/control-plane/src/integrations/api/index.ts b/control-plane/src/integrations/api/index.ts index 25d7f17..bfeef3a 100644 --- a/control-plane/src/integrations/api/index.ts +++ b/control-plane/src/integrations/api/index.ts @@ -1,10 +1,21 @@ import type { Integration } from "../types"; +export type ApiStatus = { enabled: boolean }; + // API tasks have no outbound channel: the task_events log carries replies and -// failures (failTaskInbox / markFailed write round_failed there). -export const apiIntegration: Integration = { - name: "api", - async deliver() { +// failures (failTaskInbox / markFailed write round_failed there). No webhook: +// the /api controller and ApiTaskService call TaskIntake. +export class ApiIntegration implements Integration { + readonly name = "api"; + + // API_ENABLED. + constructor(readonly enabled: boolean) {} + + async deliver(): Promise { // Nothing to post. - }, -}; + } + + async status(): Promise { + return { enabled: this.enabled }; + } +} diff --git a/control-plane/src/integrations/github/index.ts b/control-plane/src/integrations/github/index.ts index 4730635..2c74b07 100644 --- a/control-plane/src/integrations/github/index.ts +++ b/control-plane/src/integrations/github/index.ts @@ -1,3 +1,4 @@ +import type { Context } from "hono"; import { addIssueCommentReaction, addReviewCommentReaction, @@ -7,8 +8,10 @@ import { requestReviewers, } from "../../clients/github"; import { config } from "../../config"; +import type { GitHubWebhookController } from "../../controllers/githubWebhook.controller"; import type { Task } from "../../db/schema"; -import type { InboxRow, ReplyTargetSpec } from "../../repositories/inbox.repository"; +import type { InboxRow, ReplyTarget, ReplyTargetSpec } from "../../repositories/inbox.repository"; +import type { GithubStatus, GithubStatusService } from "../../services/githubStatus.service"; import { WORKSPACE_MISSING_MESSAGE } from "../../services/inbox.service"; import { retryable } from "../http"; import type { Outbound, SendCtx } from "../outbound"; @@ -21,9 +24,32 @@ type GithubSpec = Extract< // GitHub rounds (comments, reviews, CI results) post on the PR. A PR shows no // live progress, and it is public: a failure's raw detail never goes there. -export const githubIntegration: Integration = { - name: "github", - async deliver(ctx, target, message) { +// GitHub creates no tasks: its rounds come only from its webhooks, so it is +// enabled exactly when they are, and disabling them skips open rounds' replies. +export class GithubIntegration implements Integration { + readonly name = "github"; + // Today's webhook controller. It enqueues through InboxService, which needs + // this registry, so app.ts attaches it afterwards (only when enabled). + controller?: GitHubWebhookController; + + constructor( + // GITHUB_WEBHOOKS_ENABLED. + readonly enabled: boolean, + private statusService: Pick, + ) {} + + // Today's responses: 403 for untrusted actors, 503 when the trusted set + // can't be fetched. The controller already holds the intake. + async webhook(c: Context): Promise { + if (!this.controller) return c.notFound(); + return this.controller.handleWebhook(c); + } + + async status(): Promise { + return this.statusService.status(); + } + + async deliver(ctx: SendCtx, target: ReplyTarget, message: Outbound): Promise { const spec = target.spec; if (spec.kind === "linear_session" || spec.kind === "api_session") { throw new Error(`github cannot deliver to a ${spec.kind} target`); @@ -38,16 +64,20 @@ export const githubIntegration: Integration = { await deliverFailure(ctx, spec, message); return; } - }, + } // A finished review re-requests review from its reviewer, once (a replayed // `completed` doesn't). Best-effort: a failure is logged, nothing waits on it. - async onRoundFinished(ctx, task, round) { + async onRoundFinished( + ctx: HookCtx & { firstTransition: boolean }, + task: Task, + round: InboxRow, + ): Promise { if (ctx.firstTransition && round.kind === "review") { await requestReviewFromOriginalReviewer(ctx, round, task); } - }, -}; + } +} async function requestReviewFromOriginalReviewer( ctx: HookCtx, diff --git a/control-plane/src/integrations/linear/index.ts b/control-plane/src/integrations/linear/index.ts index d2ce11a..706f001 100644 --- a/control-plane/src/integrations/linear/index.ts +++ b/control-plane/src/integrations/linear/index.ts @@ -1,4 +1,5 @@ import { type LinearClient, LinearError, RatelimitedLinearError } from "@linear/sdk"; +import type { Context } from "hono"; import { getUserById } from "../../clients/github"; import { attachUrlToIssue, @@ -9,9 +10,11 @@ import { getIssueHumanAssignee, uploadIssueAttachment, } from "../../clients/linear"; +import type { LinearWebhookController } from "../../controllers/linearWebhook.controller"; import type { Task } from "../../db/schema"; import type { InboxRow, ReplyTarget } from "../../repositories/inbox.repository"; import { WORKSPACE_MISSING_MESSAGE } from "../../services/inbox.service"; +import type { LinearStatus, LinearStatusService } from "../../services/linearStatus.service"; import type { LinearTokenProvider } from "../../services/linearTokenProvider"; import { rateLimitWaitMs, retryable } from "../http"; import { type Outbound, RetryableError, type SendCtx } from "../outbound"; @@ -20,13 +23,30 @@ import type { HookCtx, Integration, Person, PrRef } from "../types"; // Linear rounds post to the agent session. target.id IS the agent session id // by construction (the enqueue paths stamp the live session id). The session // is private to the workspace, so a failure shows its raw detail. -export class LinearIntegration implements Integration { +export class LinearIntegration implements Integration { readonly name = "linear"; + // Today's webhook controller. It enqueues through InboxService, which needs + // this registry, so app.ts attaches it afterwards (only when enabled). + controller?: Pick; constructor( private linearTokenProvider: Pick, + // LINEAR_ENABLED. + readonly enabled: boolean, + private statusService: Pick, ) {} + // Today's responses, including 503 session_not_found when a prompt arrives + // before its session (Linear retries). The controller already holds the intake. + async webhook(c: Context): Promise { + if (!this.controller) return c.notFound(); + return this.controller.handleWebhook(c); + } + + status(): Promise { + return this.statusService.status(); + } + async deliver(ctx: SendCtx, target: ReplyTarget, message: Outbound): Promise { if (target.spec.kind !== "linear_session") { throw new Error(`linear cannot deliver to a ${target.spec.kind} target`); diff --git a/control-plane/src/integrations/types.ts b/control-plane/src/integrations/types.ts index 1a20acc..d04ffb8 100644 --- a/control-plane/src/integrations/types.ts +++ b/control-plane/src/integrations/types.ts @@ -1,8 +1,10 @@ +import type { Context } from "hono"; import type { Task, taskSource } from "../db/schema"; import type { logger } from "../logger"; import type { InboxRow, ReplyTarget } from "../repositories/inbox.repository"; import type { ApiTaskSource } from "../types/api"; import type { InboxSource } from "../types/inboxMessage"; +import type { TaskIntake } from "./intake"; import type { Outbound, SendCtx } from "./outbound"; type Logger = typeof logger; @@ -23,8 +25,15 @@ export type SourceChecks = [ Assert>, ]; -export interface Integration { +export interface Integration { name: IntegrationName; + // Core never calls webhook, deliver, hooks or assignee of a disabled module: + // it logs and skips (a skipped send counts as sent, never held). + enabled: boolean; + + // Incoming. Core mounts POST /webhook/ only when enabled (404 + // otherwise), behind the shared body limit and raw-body read. + webhook?(c: Context, intake: TaskIntake): Promise; // Outgoing. Core labels each message; the integration decides where (or // whether) it shows up. Throws RetryableError only before anything was sent. @@ -41,6 +50,10 @@ export interface Integration { outcome: Outcome | null, ): Promise; assignee?(ctx: HookCtx, task: Task): Promise; // throws when the lookup itself fails + + // Health. Core serves GET /api/integrations/ (behind API auth, only + // when the API is enabled) with this body plus the integration's held sends. + status(): Promise; } // isCurrent() is false once a restart rotated the task's assignment id. diff --git a/control-plane/src/integrationsConfig.ts b/control-plane/src/integrationsConfig.ts index 93d2048..9c2a3bd 100644 --- a/control-plane/src/integrationsConfig.ts +++ b/control-plane/src/integrationsConfig.ts @@ -2,6 +2,9 @@ // testable without mocking ../config. config.ts calls these at startup; a bad // combination fails boot with an error that names what is missing. +import { taskSource } from "./db/schema"; +import type { TaskSource } from "./integrations/types"; + export type LinearConfig = { clientId: string; clientSecret: string; @@ -13,6 +16,8 @@ export type SlackConfig = { botToken: string; // Channel id (C…) the updates are posted to. channel: string; + // The task sources whose tasks post (SLACK_NOTIFY_SOURCES; default linear). + notifySources: TaskSource[]; }; export type GithubWebhooksConfig = { @@ -80,7 +85,8 @@ export function parseLinearConfig(env: Env): LinearConfig | undefined { } // Slack is on only with SLACK_ENABLED=true; then the token and channel are -// required. +// required. SLACK_NOTIFY_SOURCES lists the task sources that post (comma- or +// whitespace-separated, default linear); an unknown source fails startup. export function parseSlackConfig(env: Env): SlackConfig | undefined { if (!parseSwitch(env, "SLACK_ENABLED")) return undefined; const botToken = present(env, "SLACK_BOT_TOKEN"); @@ -89,7 +95,16 @@ export function parseSlackConfig(env: Env): SlackConfig | undefined { if (!botToken) missing.push("SLACK_BOT_TOKEN"); if (!channel) missing.push("SLACK_CHANNEL_ID"); if (!botToken || !channel) throw missingError("SLACK_ENABLED", missing); - return { botToken, channel }; + const sources = parseTrustList(env.SLACK_NOTIFY_SOURCES); + const known: readonly string[] = taskSource.enumValues; + const unknown = sources.find((s) => !known.includes(s)); + if (unknown !== undefined) { + throw new Error( + `SLACK_NOTIFY_SOURCES: unknown task source "${unknown}" (allowed: ${known.join(", ")})`, + ); + } + const notifySources: TaskSource[] = sources.length > 0 ? (sources as TaskSource[]) : ["linear"]; + return { botToken, channel, notifySources }; } export function parseGithubConfig(env: Env): GithubIntegrationConfig { diff --git a/control-plane/src/routes/api.ts b/control-plane/src/routes/api.ts index 6abfefa..cdd0202 100644 --- a/control-plane/src/routes/api.ts +++ b/control-plane/src/routes/api.ts @@ -1,11 +1,10 @@ import { Hono } from "hono"; import { bodyLimit } from "hono/body-limit"; import { type ApiController, type ApiEnv, errorBody } from "../controllers/api.controller"; +import type { Integration, IntegrationName } from "../integrations/types"; import { logger } from "../logger"; import { API_BODY_LIMIT_BYTES } from "../requestLimits"; import type { ApiAuthService } from "../services/apiAuth.service"; -import type { GithubStatus } from "../services/githubStatus.service"; -import type { LinearStatus } from "../services/linearStatus.service"; // /api. Mounted by app.ts only when API_ENABLED=true. // Order matters: the host check runs first (a disallowed host learns nothing, @@ -16,8 +15,8 @@ export function createApiRoutes( opts: { authEnabled: boolean; allowedHosts: string[]; - linearStatus: () => Promise; - githubStatus: () => GithubStatus; + // GET /api/integrations/, for every integration, enabled or not. + integrations: Record>; }, ): Hono { const app = new Hono(); @@ -66,7 +65,11 @@ export function createApiRoutes( app.get("/api/tasks/:ref/events/stream", (c) => controller.streamEvents(c)); app.get("/api/tasks/:ref/events", (c) => controller.listEvents(c)); app.get("/api/me", (c) => controller.me(c)); - app.get("/api/integrations/linear", async (c) => c.json(await opts.linearStatus())); - app.get("/api/integrations/github", (c) => c.json(opts.githubStatus())); + app.get("/api/integrations/:name", async (c) => { + const name = c.req.param("name"); + // An unknown name is a 404, as an unregistered route. + if (!Object.hasOwn(opts.integrations, name)) return c.notFound(); + return c.json(await opts.integrations[name as IntegrationName].status()); + }); return app; } diff --git a/control-plane/src/routes/webhook.ts b/control-plane/src/routes/webhook.ts index 2a9e652..6f014b7 100644 --- a/control-plane/src/routes/webhook.ts +++ b/control-plane/src/routes/webhook.ts @@ -1,17 +1,16 @@ import { type Context, Hono, type MiddlewareHandler } from "hono"; import { bodyLimit } from "hono/body-limit"; -import type { GitHubWebhookController } from "../controllers/githubWebhook.controller"; -import type { LinearWebhookController } from "../controllers/linearWebhook.controller"; +import type { TaskIntake } from "../integrations/intake"; +import type { Integration } from "../integrations/types"; import { WEBHOOK_BODY_LIMIT_BYTES } from "../requestLimits"; -// Each webhook is registered only when its integration is configured; an -// unconfigured one answers 404. -export function createWebhookRoutes(controllers: { - linear?: LinearWebhookController; - github?: GitHubWebhookController; -}): Hono { +// POST /webhook/ is registered only for an enabled integration that has +// a webhook; any other name answers 404. +export function createWebhookRoutes( + integrations: Pick[], + intake: TaskIntake, +): Hono { const app = new Hono(); - const { linear, github } = controllers; const payloadTooLarge = (c: Context) => c.json({ error: "payload too large" }, 413); const webhookBodyLimit = bodyLimit({ @@ -32,13 +31,12 @@ export function createWebhookRoutes(controllers: { await next(); }; - if (linear) - app.post("/webhook/linear", webhookBodyLimit, readLimitedWebhookBody, (c) => - linear.handleWebhook(c), - ); - if (github) - app.post("/webhook/github", webhookBodyLimit, readLimitedWebhookBody, (c) => - github.handleWebhook(c), + for (const integration of integrations) { + const webhook = integration.webhook?.bind(integration); + if (!integration.enabled || !webhook) continue; + app.post(`/webhook/${integration.name}`, webhookBodyLimit, readLimitedWebhookBody, (c) => + webhook(c, intake), ); + } return app; } diff --git a/control-plane/src/services/__tests__/eventDispatcher.test.ts b/control-plane/src/services/__tests__/eventDispatcher.test.ts index 332091d..2636dda 100644 --- a/control-plane/src/services/__tests__/eventDispatcher.test.ts +++ b/control-plane/src/services/__tests__/eventDispatcher.test.ts @@ -79,8 +79,8 @@ mock.module("../../clients/github", () => ({ const { EventDispatcher } = await import("../eventDispatcher"); const { LinearIntegration } = await import("../../integrations/linear"); -const { githubIntegration } = await import("../../integrations/github"); -const { apiIntegration } = await import("../../integrations/api"); +const { GithubIntegration } = await import("../../integrations/github"); +const { ApiIntegration } = await import("../../integrations/api"); const { HttpError } = await import("../../integrations/http"); const { ReplyHolds } = await import("../replyHolds"); const { IntegrationHooks } = await import("../integrationHooks"); @@ -161,6 +161,15 @@ describe("EventDispatcher", () => { let holds: InstanceType; let dispatcher: InstanceType; + // The registry, with GitHub's webhooks (and so its integration) on or off. + const makeIntegrations = (githubEnabled: boolean) => ({ + linear: new LinearIntegration(linearTokenProvider, true, { + status: async () => ({ configured: false }), + }), + api: new ApiIntegration(true), + github: new GithubIntegration(githubEnabled, { status: () => ({}) as never }), + }); + beforeEach(() => { for (const m of [ emitThought, @@ -205,19 +214,18 @@ describe("EventDispatcher", () => { } as unknown as Pick; notifyTerminal = mock(() => Promise.resolve()); - slackNotifier = { notifyTerminal } as unknown as SlackNotifierService; + // Slack on with the default SLACK_NOTIFY_SOURCES (linear). + slackNotifier = { + notifyTerminal, + notifies: (source: string) => source === "linear", + } as unknown as SlackNotifierService; holds = new ReplyHolds(); - const integrations = { - linear: new LinearIntegration(linearTokenProvider), - api: apiIntegration, - github: githubIntegration, - }; + const integrations = makeIntegrations(true); dispatcher = new EventDispatcher( taskRepo, inboxRepo, sandboxManager, - linearTokenProvider, slackNotifier, integrations, holds, @@ -589,6 +597,8 @@ describe("EventDispatcher", () => { const delivered: unknown[] = []; const record = (name: "linear" | "api" | "github") => ({ name, + enabled: true, + status: async () => ({}), deliver: async (_ctx: unknown, target: unknown, message: unknown) => { delivered.push([name, target, message]); }, @@ -602,7 +612,6 @@ describe("EventDispatcher", () => { taskRepo, inboxRepo, sandboxManager, - linearTokenProvider, slackNotifier, integrations, new ReplyHolds(), @@ -986,23 +995,21 @@ describe("EventDispatcher", () => { expect(postIssueComment).toHaveBeenCalledTimes(2); }); - it("still posts GitHub replies when GitHub webhooks are off", async () => { - // characterization: changes in phase 2 (spec §13: a disabled integration's replies are skipped) - const webhooks = mockedConfig.github.webhooks; - mockedConfig.github.webhooks = undefined; - try { - getByIdForTask.mockResolvedValue(ghCommentRow()); - await dispatcher.handle(responseEvent("555"), makeTask()); - expect(postIssueComment).toHaveBeenCalledWith( - "o", - "r", - 1, - "good catch, fixing", - expect.any(AbortSignal), - ); - } finally { - mockedConfig.github.webhooks = webhooks; - } + it("skips GitHub replies when GitHub webhooks are off: not posted, not held (§13)", async () => { + const integrations = makeIntegrations(false); + const d = new EventDispatcher( + taskRepo, + inboxRepo, + sandboxManager, + slackNotifier, + integrations, + holds, + new IntegrationHooks(integrations, inboxRepo), + ); + getByIdForTask.mockResolvedValue(ghCommentRow()); + expect(await d.handle(responseEvent("555"), makeTask())).toBe("ok"); + expect(postIssueComment).not.toHaveBeenCalled(); + expect(holds.stats()).toEqual({}); }); it("a Linear token failure does not hold a GitHub round's reply or completion on a Linear task", async () => { @@ -1189,7 +1196,6 @@ describe("EventDispatcher", () => { await dispatcher.handle(completedEvent(), makeTask()); expect(notifyTerminal).toHaveBeenCalledTimes(1); expect(notifyTerminal).toHaveBeenCalledWith( - expect.anything(), expect.objectContaining({ id: "task-1" }), "completed", ); @@ -1197,11 +1203,7 @@ describe("EventDispatcher", () => { it("forwards the event status to the Slack notify (partial_failure)", async () => { await dispatcher.handle({ ...completedEvent(), status: "partial_failure" }, makeTask()); - expect(notifyTerminal).toHaveBeenCalledWith( - expect.anything(), - expect.anything(), - "partial_failure", - ); + expect(notifyTerminal).toHaveBeenCalledWith(expect.anything(), "partial_failure"); }); it("does NOT post to Slack on an idempotent re-fire (already responded)", async () => { @@ -1385,12 +1387,18 @@ describe("EventDispatcher", () => { expect(markResponded).toHaveBeenCalled(); }); - it("Linear assignment: a token fetch failure holds the event before markResponded", async () => { + it("Linear assignment: a token fetch failure no longer holds; it marks and notifies Slack (§13)", async () => { + // Slack's mention comes from assignee(), which fails (logged) and the + // post goes out without it; completed needs no Linear client. (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( new Error("token refresh failed"), ); - expect(await dispatcher.handle(completedEvent(), makeTask())).toBe("hold"); - expect(markResponded).not.toHaveBeenCalled(); + expect(await dispatcher.handle(completedEvent(), makeTask())).toBe("ok"); + expect(markResponded).toHaveBeenCalled(); + expect(notifyTerminal).toHaveBeenCalledWith( + expect.objectContaining({ id: "task-1" }), + "completed", + ); }); it("still marks responded when linearClient is null", async () => { diff --git a/control-plane/src/services/__tests__/inbox.service.test.ts b/control-plane/src/services/__tests__/inbox.service.test.ts index e68dad7..d582021 100644 --- a/control-plane/src/services/__tests__/inbox.service.test.ts +++ b/control-plane/src/services/__tests__/inbox.service.test.ts @@ -68,8 +68,8 @@ mock.module("../../clients/github", () => ({ const { InboxService, WORKSPACE_MISSING_MESSAGE } = await import("../inbox.service"); const { LinearIntegration } = await import("../../integrations/linear"); -const { githubIntegration } = await import("../../integrations/github"); -const { apiIntegration } = await import("../../integrations/api"); +const { GithubIntegration } = await import("../../integrations/github"); +const { ApiIntegration } = await import("../../integrations/api"); const { IntegrationHooks } = await import("../integrationHooks"); const makeTask = (overrides: Partial = {}): Task => ({ @@ -176,8 +176,14 @@ describe("InboxService", () => { getAuthorizedLinearClient: mock(() => Promise.resolve({})), } as unknown as Pick; - linear = new LinearIntegration(linearTokenProvider); - const integrations = { linear, api: apiIntegration, github: githubIntegration }; + linear = new LinearIntegration(linearTokenProvider, true, { + status: async () => ({ configured: false }), + }); + const integrations = { + linear, + api: new ApiIntegration(true), + github: new GithubIntegration(true, { status: () => ({}) as never }), + }; service = new InboxService( inboxRepo, sandboxManager, diff --git a/control-plane/src/services/__tests__/integrationHooks.test.ts b/control-plane/src/services/__tests__/integrationHooks.test.ts index a011612..3fcc12b 100644 --- a/control-plane/src/services/__tests__/integrationHooks.test.ts +++ b/control-plane/src/services/__tests__/integrationHooks.test.ts @@ -15,7 +15,7 @@ const row = (id: string, kind = "assignment", source = "linear") => source, reply_targets: [{ id: "sess-1", spec: { kind: "linear_session" } }], }) as unknown as InboxRow; -const noop: Integration = { name: "api", async deliver() {} }; +const noop: Integration = { name: "api", enabled: true, async deliver() {}, async status() {} }; const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); function hooksWith( @@ -57,6 +57,26 @@ describe("IntegrationHooks", () => { } }); + it("skips a disabled integration's hooks: logged, never called, never held", async () => { + const info = spyOn(logger, "info").mockImplementation(() => logger); + try { + const assignee = mock(() => Promise.resolve(null)); + const onPrOpened = mock(() => Promise.reject(new RetryableError("429"))); + const { hooks } = hooksWith({ enabled: false, assignee, onPrOpened }); + const pr = { url: "u", repo: "r", branch: "b" }; + expect(await hooks.assignee(task, row("a-1"), {})).toBeUndefined(); + expect(await hooks.prOpened(task, row("a-1"), pr, true)).toBeUndefined(); + expect(assignee).not.toHaveBeenCalled(); + expect(onPrOpened).not.toHaveBeenCalled(); + expect(info).toHaveBeenCalledWith( + "assignee skipped: integration disabled", + expect.objectContaining({ taskId: "task-1", integration: "linear" }), + ); + } finally { + info.mockRestore(); + } + }); + it("onPrOpened alone lets a RetryableError escape, so pr_opened holds", async () => { const error = spyOn(logger, "error").mockImplementation(() => logger); try { diff --git a/control-plane/src/services/__tests__/slackNotifier.service.test.ts b/control-plane/src/services/__tests__/slackNotifier.service.test.ts index 21ebf1d..5dcd44b 100644 --- a/control-plane/src/services/__tests__/slackNotifier.service.test.ts +++ b/control-plane/src/services/__tests__/slackNotifier.service.test.ts @@ -1,5 +1,4 @@ import { beforeEach, describe, expect, it, mock } from "bun:test"; -import type { LinearClient } from "@linear/sdk"; import type { Task } from "../../db/schema"; import type { TaskRepository } from "../../repositories/task.repository"; @@ -39,23 +38,22 @@ const makeTask = (overrides: Partial = {}): Task => ({ ...overrides, }); -const client = {} as unknown as LinearClient; - describe("SlackNotifierService", () => { let post: ReturnType; - let resolveAssignee: ReturnType; + let assignee: ReturnType; let lookupSlackUser: ReturnType; let getById: ReturnType; let taskRepo: TaskRepository; - const make = (botToken = "xoxb-test", channel = "C1") => + const make = (botToken = "xoxb-test", channel = "C1", sources: Ctor[3] = ["linear"]) => new SlackNotifierService( taskRepo, botToken, channel, - post as unknown as Ctor[3], - resolveAssignee as unknown as Ctor[4], - lookupSlackUser as unknown as Ctor[5], + sources, + { assignee } as unknown as Ctor[4], + post as unknown as Ctor[5], + lookupSlackUser as unknown as Ctor[6], ); // The text posted by the single post() call. @@ -63,14 +61,14 @@ describe("SlackNotifierService", () => { beforeEach(() => { post = mock(() => Promise.resolve({ ts: "1" })); - resolveAssignee = mock(() => Promise.resolve(null)); + assignee = mock(() => Promise.resolve(null)); lookupSlackUser = mock(() => Promise.resolve(null)); getById = mock((id: string) => Promise.resolve(makeTask({ id, prUrls: [PR] }))); taskRepo = { getById } as unknown as TaskRepository; }); it("posts a ✅ completion into the configured channel with the ticket + PR links", async () => { - await make().notifyTerminal(client, makeTask(), "completed"); + await make().notifyTerminal(makeTask(), "completed"); expect(post).toHaveBeenCalledTimes(1); const arg = (post.mock.calls[0] as unknown as [{ channel: string; threadTs?: string }])[0]; @@ -84,7 +82,7 @@ describe("SlackNotifierService", () => { it("uses singular 'PR' with no bullet for a single PR", async () => { // Default getById returns exactly one PR. - await make().notifyTerminal(client, makeTask(), "completed"); + await make().notifyTerminal(makeTask(), "completed"); const text = postedText(); expect(text).toContain( "Here is the PR for review: ", @@ -99,7 +97,7 @@ describe("SlackNotifierService", () => { prUrls: [PR, { repo: "acme/web", url: "https://github.com/acme/web/pull/2" }], }), ); - await make().notifyTerminal(client, makeTask(), "completed"); + await make().notifyTerminal(makeTask(), "completed"); const text = postedText(); expect(text).toContain("Here are the PRs for review:"); expect(text).toContain("• "); @@ -107,21 +105,21 @@ describe("SlackNotifierService", () => { }); it("uses ⚠️ for partial_failure and still lists PRs", async () => { - await make().notifyTerminal(client, makeTask(), "partial_failure"); + await make().notifyTerminal(makeTask(), "partial_failure"); const text = postedText(); expect(text).toContain("⚠️"); expect(text).toContain("acme/api"); }); it("uses 💤 for no_action and omits PR lines", async () => { - await make().notifyTerminal(client, makeTask(), "no_action"); + await make().notifyTerminal(makeTask(), "no_action"); const text = postedText(); expect(text).toContain("💤"); expect(text).not.toContain("github.com"); }); it("uses ❌ for failed and includes the failure message", async () => { - await make().notifyTerminal(client, makeTask(), "failed", "sandbox ran out of memory"); + await make().notifyTerminal(makeTask(), "failed", "sandbox ran out of memory"); const text = postedText(); expect(text).toContain("❌"); expect(text).toContain("sandbox ran out of memory"); @@ -129,13 +127,13 @@ describe("SlackNotifierService", () => { it("labels the ticket link 'Linear ticket' when the task has no title", async () => { getById.mockResolvedValue(makeTask({ title: "", prUrls: [PR] })); - await make().notifyTerminal(client, makeTask({ title: "" }), "completed"); + await make().notifyTerminal(makeTask({ title: "" }), "completed"); expect(postedText()).toContain(""); }); it("re-reads the task for the freshest pr_urls (stale in-memory task had none)", async () => { // The task handed in predates the pr_opened write; getById returns the PR. - await make().notifyTerminal(client, makeTask({ prUrls: [] }), "completed"); + await make().notifyTerminal(makeTask({ prUrls: [] }), "completed"); expect(getById).toHaveBeenCalledWith("task-1"); expect(postedText()).toContain("github.com/acme/api/pull/1"); }); @@ -144,44 +142,65 @@ describe("SlackNotifierService", () => { // A DB blip on the optional freshness re-read must NOT drop the whole post — // the thread is resolved and the in-memory task already carries the PR. getById.mockRejectedValue(new Error("db connection reset")); - await make().notifyTerminal(client, makeTask({ prUrls: [PR] }), "completed"); + await make().notifyTerminal(makeTask({ prUrls: [PR] }), "completed"); expect(post).toHaveBeenCalledTimes(1); expect(postedText()).toContain(""); }); it("no-ops when Slack is disabled (empty token) — never posts", async () => { - await make("").notifyTerminal(client, makeTask(), "completed"); + await make("").notifyTerminal(makeTask(), "completed"); expect(post).not.toHaveBeenCalled(); }); it("no-ops when no channel is configured", async () => { - await make("xoxb-test", "").notifyTerminal(client, makeTask(), "completed"); + await make("xoxb-test", "").notifyTerminal(makeTask(), "completed"); expect(post).not.toHaveBeenCalled(); }); it("skips a non-linear task", async () => { - await make().notifyTerminal( - client, - makeTask({ source: "github" as Task["source"] }), - "completed", - ); + await make().notifyTerminal(makeTask({ source: "github" as Task["source"] }), "completed"); + expect(post).not.toHaveBeenCalled(); + }); + + it("skips an API task unless SLACK_NOTIFY_SOURCES has api", async () => { + const notifier = make(); + expect(notifier.notifies("api")).toBe(false); + await notifier.notifyTerminal(makeTask({ source: "api" }), "completed"); expect(post).not.toHaveBeenCalled(); }); + it("posts for an opted-in API task: labelled by its key when untitled, never nudged", async () => { + const task = makeTask({ + source: "api", + key: "DBX-API-7", + title: "", + externalUrl: "https://devbox.example.com/api/tasks/DBX-API-7", + }); + getById.mockResolvedValue({ ...task, prUrls: [PR] }); + await make("xoxb-test", "C1", ["linear", "api"]).notifyTerminal(task, "completed"); + expect(postedText()).toContain(""); + expect(assignee).not.toHaveBeenCalled(); + }); + describe("connect-GitHub nudge", () => { + // Linear's assignee() for a member who hasn't connected GitHub in Linear. const unmapped = { id: "usr-1", - name: "Ada Lovelace", displayName: "ada", email: "ada@acme.com", - gitHubUserId: null, + codeHost: "unlinked", }; + const mapped = { ...unmapped, codeHost: { githubLogin: "ada" } }; it("appends the nudge when the human assignee has no GitHub connected and PRs exist", async () => { - resolveAssignee.mockResolvedValue(unmapped); - await make().notifyTerminal(client, makeTask(), "completed"); + assignee.mockResolvedValue(unmapped); + await make().notifyTerminal(makeTask(), "completed"); const text = postedText(); - expect(resolveAssignee).toHaveBeenCalledWith(client, "issue-1"); + expect(assignee).toHaveBeenCalledWith( + expect.objectContaining({ id: "task-1" }), + null, + expect.anything(), + ); expect(text).toContain("⚠️ ada: I couldn't assign the PR to you"); expect(text).toContain("*Linear → Settings → Connected accounts*"); // The nudge rides directly under the PR list. @@ -191,41 +210,42 @@ describe("SlackNotifierService", () => { }); it("omits the nudge when the assignee is already mapped", async () => { - resolveAssignee.mockResolvedValue({ ...unmapped, gitHubUserId: "583231" }); - await make().notifyTerminal(client, makeTask(), "completed"); + assignee.mockResolvedValue(mapped); + await make().notifyTerminal(makeTask(), "completed"); expect(postedText()).not.toContain("Connected accounts"); }); it("omits the nudge when there is no human assignee", async () => { - resolveAssignee.mockResolvedValue(null); - await make().notifyTerminal(client, makeTask(), "completed"); + assignee.mockResolvedValue(null); + await make().notifyTerminal(makeTask(), "completed"); expect(postedText()).not.toContain("Connected accounts"); }); it("never checks the assignee when there are no PRs (nothing to assign)", async () => { getById.mockResolvedValue(makeTask({ prUrls: [] })); - await make().notifyTerminal(client, makeTask(), "completed"); - expect(resolveAssignee).not.toHaveBeenCalled(); + await make().notifyTerminal(makeTask(), "completed"); + expect(assignee).not.toHaveBeenCalled(); expect(postedText()).not.toContain("Connected accounts"); }); it("still posts (without the nudge) when the assignee lookup fails", async () => { - resolveAssignee.mockRejectedValue(new Error("linear down")); - await make().notifyTerminal(client, makeTask(), "completed"); + // IntegrationHooks.assignee logs the failure and yields undefined. + assignee.mockResolvedValue(undefined); + await make().notifyTerminal(makeTask(), "completed"); expect(post).toHaveBeenCalledTimes(1); expect(postedText()).not.toContain("Connected accounts"); }); it("appends the nudge on partial_failure and failed too (PRs are listed there)", async () => { - resolveAssignee.mockResolvedValue(unmapped); - await make().notifyTerminal(client, makeTask(), "failed", "boom"); + assignee.mockResolvedValue(unmapped); + await make().notifyTerminal(makeTask(), "failed", "boom"); expect(postedText()).toContain("I couldn't assign the PR"); }); it("@-mentions the assignee when their Slack id resolves from the Linear email", async () => { - resolveAssignee.mockResolvedValue(unmapped); + assignee.mockResolvedValue(unmapped); lookupSlackUser.mockResolvedValue("U0ADA"); - await make().notifyTerminal(client, makeTask(), "completed"); + await make().notifyTerminal(makeTask(), "completed"); expect(lookupSlackUser).toHaveBeenCalledWith("ada@acme.com"); const text = postedText(); expect(text).toContain("⚠️ <@U0ADA>: I couldn't assign the PR"); @@ -233,36 +253,36 @@ describe("SlackNotifierService", () => { }); it("falls back to the display name when the Slack lookup finds nobody", async () => { - resolveAssignee.mockResolvedValue(unmapped); + assignee.mockResolvedValue(unmapped); lookupSlackUser.mockResolvedValue(null); - await make().notifyTerminal(client, makeTask(), "completed"); + await make().notifyTerminal(makeTask(), "completed"); expect(postedText()).toContain("⚠️ ada: I couldn't assign the PR"); }); it("falls back to the display name when the Slack lookup throws", async () => { - resolveAssignee.mockResolvedValue(unmapped); + assignee.mockResolvedValue(unmapped); lookupSlackUser.mockRejectedValue(new Error("missing_scope")); - await make().notifyTerminal(client, makeTask(), "completed"); + await make().notifyTerminal(makeTask(), "completed"); expect(postedText()).toContain("⚠️ ada: I couldn't assign the PR"); }); it("never looks up Slack when the assignee is already mapped", async () => { - resolveAssignee.mockResolvedValue({ ...unmapped, gitHubUserId: "583231" }); - await make().notifyTerminal(client, makeTask(), "completed"); + assignee.mockResolvedValue(mapped); + await make().notifyTerminal(makeTask(), "completed"); expect(lookupSlackUser).not.toHaveBeenCalled(); }); it("skips the Slack lookup when the assignee has no email (falls back to name)", async () => { - resolveAssignee.mockResolvedValue({ ...unmapped, email: "" }); - await make().notifyTerminal(client, makeTask(), "completed"); + assignee.mockResolvedValue({ ...unmapped, email: "" }); + await make().notifyTerminal(makeTask(), "completed"); expect(lookupSlackUser).not.toHaveBeenCalled(); expect(postedText()).toContain("⚠️ ada:"); }); it("escapes mrkdwn in the display-name fallback (no injection)", async () => { - resolveAssignee.mockResolvedValue({ ...unmapped, displayName: " <@U999>" }); + assignee.mockResolvedValue({ ...unmapped, displayName: " <@U999>" }); lookupSlackUser.mockResolvedValue(null); - await make().notifyTerminal(client, makeTask(), "completed"); + await make().notifyTerminal(makeTask(), "completed"); const text = postedText(); expect(text).toContain("<!channel> <@U999>"); expect(text).not.toContain(""); @@ -272,10 +292,10 @@ describe("SlackNotifierService", () => { it("never checks the assignee on no_action, even when the task carries PRs", async () => { // no_action renders no PR section, so there is nothing for the nudge to // ride on — the Linear/Slack lookups must be skipped entirely. - resolveAssignee.mockResolvedValue(unmapped); + assignee.mockResolvedValue(unmapped); getById.mockResolvedValue(makeTask({ prUrls: [PR] })); - await make().notifyTerminal(client, makeTask(), "no_action"); - expect(resolveAssignee).not.toHaveBeenCalled(); + await make().notifyTerminal(makeTask(), "no_action"); + expect(assignee).not.toHaveBeenCalled(); expect(postedText()).not.toContain("Connected accounts"); }); }); diff --git a/control-plane/src/services/__tests__/taskProcessor.service.test.ts b/control-plane/src/services/__tests__/taskProcessor.service.test.ts index 8e1abbf..f89852e 100644 --- a/control-plane/src/services/__tests__/taskProcessor.service.test.ts +++ b/control-plane/src/services/__tests__/taskProcessor.service.test.ts @@ -92,8 +92,8 @@ mock.module("../../clients/github", () => ({ const { TaskProcessorService } = await import("../taskProcessor.service"); const { LinearIntegration } = await import("../../integrations/linear"); -const { githubIntegration } = await import("../../integrations/github"); -const { apiIntegration } = await import("../../integrations/api"); +const { GithubIntegration } = await import("../../integrations/github"); +const { ApiIntegration } = await import("../../integrations/api"); const { logger } = await import("../../logger"); const { ReplyHolds } = await import("../replyHolds"); const { IntegrationHooks } = await import("../integrationHooks"); @@ -288,13 +288,19 @@ describe("TaskProcessorService", () => { inboxService = { sweepUndelivered, deadLetterWorkspaceMissing } as unknown as InboxService; notifyTerminal = mock(() => Promise.resolve()); - slackNotifier = { notifyTerminal } as unknown as SlackNotifierService; + // Slack on with the default SLACK_NOTIFY_SOURCES (linear). + slackNotifier = { + notifyTerminal, + notifies: (source: string) => source === "linear", + } as unknown as SlackNotifierService; holds = new ReplyHolds(); integrations = { - linear: new LinearIntegration(linearTokenProvider), - api: apiIntegration, - github: githubIntegration, + linear: new LinearIntegration(linearTokenProvider, true, { + status: async () => ({ configured: false }), + }), + api: new ApiIntegration(true), + github: new GithubIntegration(true, { status: () => ({}) as never }), }; hooks = new IntegrationHooks(integrations, inboxRepo); service = new TaskProcessorService( @@ -581,7 +587,6 @@ describe("TaskProcessorService", () => { await service.stop(); expect(notifyTerminal).toHaveBeenCalledWith( - expect.anything(), expect.objectContaining({ id: "task-1" }), "failed", expect.stringContaining("control-plane logs"), @@ -884,7 +889,7 @@ describe("TaskProcessorService", () => { }); }); - it("a Linear token failure skips only the Linear post; GitHub posts, dead-letter and release go on", async () => { + it("a Linear token failure skips only the Linear post; GitHub and Slack posts, dead-letter and release go on", async () => { getAuthorizedLinearClient.mockRejectedValue(new Error("token refresh failed")); (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValue([ row("a", "assignment", "linear", [{ id: "sess-1", spec: { kind: "linear_session" } }]), @@ -902,8 +907,12 @@ describe("TaskProcessorService", () => { ownerId: "pod-test", alsoClearSandbox: true, }); - // characterization: changes in phase 2 (Slack notifier sends without the assignee mention; spec §13) - expect(notifyTerminal).not.toHaveBeenCalled(); + // Sent without the assignee mention (its lookup fails; §13). + expect(notifyTerminal).toHaveBeenCalledWith( + expect.objectContaining({ id: "task-1" }), + "failed", + MSG, + ); }); }); @@ -1238,9 +1247,11 @@ describe("TaskProcessorService", () => { taskRepo, { getByIdForTask: mock(() => Promise.resolve(round)) } as unknown as InboxRepository, sandboxManager, - linearTokenProvider, slackNotifier, - { ...integrations, github: { name: "github", deliver } }, + { + ...integrations, + github: { name: "github", enabled: true, status: async () => ({}), deliver }, + }, holds, hooks, 20, // send timeout (ms) diff --git a/control-plane/src/services/eventDispatcher.ts b/control-plane/src/services/eventDispatcher.ts index aa1438c..0bcd7c8 100644 --- a/control-plane/src/services/eventDispatcher.ts +++ b/control-plane/src/services/eventDispatcher.ts @@ -1,4 +1,3 @@ -import type { LinearClient } from "@linear/sdk"; import { addPrAssignees, parsePrUrl, postIssueComment } from "../clients/github"; import { config } from "../config"; import type { Task } from "../db/schema"; @@ -10,7 +9,6 @@ import type { InboxRepository, InboxRow } from "../repositories/inbox.repository import type { TaskRepository } from "../repositories/task.repository"; import type { AgentEmit, HarnessEvent } from "../types/harnessEvent"; import type { IntegrationHooks } from "./integrationHooks"; -import type { LinearTokenProvider } from "./linearTokenProvider"; import type { ReplyHolds } from "./replyHolds"; import type { SandboxManager } from "./sandboxManager.service"; import type { SlackNotifierService } from "./slackNotifier.service"; @@ -62,8 +60,7 @@ export class EventDispatcher { private taskRepo: TaskRepository, private inboxRepo: InboxRepository, private sandboxManager: SandboxManager, - private linearTokenProvider: Pick, - private slackNotifier: SlackNotifierService, + private slackNotifier: Pick, private integrations: Record, private holds: ReplyHolds, private hooks: IntegrationHooks, @@ -74,8 +71,7 @@ export class EventDispatcher { try { return await this.apply(event, task); } catch (err) { - // A task-scoped call (the owner's onPrOpened, or the Linear client for - // Slack on completed) failed before anything was sent: hold on the + // The owner's onPrOpened failed before anything was sent: hold on the // task's own integration. deliver holds on the round's integration itself. if (!(err instanceof RetryableError)) throw err; this.holds.hold(task.id, event, task.source, err); @@ -171,12 +167,6 @@ export class EventDispatcher { } case "completed": { - // The Slack notify below needs a Linear client for a Linear task's - // assignment round (until Slack moves to assignee()). Fetched before - // marking, so a Linear token failure holds the event with the row - // still open. - const linearClient = - inboxRow?.kind === "assignment" ? await this.linearClientFor(task) : null; // Bare terminal marker — the user-facing message already went out via // the preceding agent_response / agent_error emit. Mark the inbox row // responded so the reaper guard clears and the sandbox can be freed. @@ -194,15 +184,18 @@ export class EventDispatcher { this.hooks.roundFinished(task, inboxRow, outcome, marked, async () => sandboxId ? this.sandboxManager.extractArchive(sandboxId) : null, ); - // Slack completion update to the configured channel. - // Assignment-only (the top-level task, not comment/review rounds) and - // gated on the first inbox transition so a cursor-replay can't double- - // post. Fire-and-forget: a Slack outage must not stall the dispatch - // loop. The notifier no-ops when Slack is disabled. - if (marked && linearClient) { - void this.slackNotifier - .notifyTerminal(linearClient, task, event.status ?? "completed") - .catch((err) => logErr("slack completion notify failed", err)); + // Slack completion update to the configured channel, for a task + // whose source is opted in (SLACK_NOTIFY_SOURCES). Assignment-only + // (the top-level task, not comment/review rounds) and gated on the + // first inbox transition so a cursor-replay can't double-post. + // Detached: a Slack outage (or the assignee lookup for its mention) + // must not stall the dispatch loop. + if (marked && inboxRow.kind === "assignment" && this.slackNotifier.notifies(task.source)) { + this.hooks.detach( + task.id, + "slack", + this.slackNotifier.notifyTerminal(task, event.status ?? "completed"), + ); } break; } @@ -219,24 +212,14 @@ export class EventDispatcher { return "ok"; } - // Nothing has been sent when the token fetch fails, so handle() holds. - private async linearClientFor(task: Task): Promise { - if (task.source !== "linear") return null; - try { - return await this.linearTokenProvider.getAuthorizedLinearClient(); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - throw new RetryableError(`Linear token fetch failed: ${msg}`); - } - } - // Delivers one agent emit to one thread: looks up the target on the inbox // row's reply_targets[] by target_id, labels the emit, and hands both to the // round's integration, which decides where (or whether) it shows up. Core - // never checks what an integration can show. The send runs under - // sendTimeoutMs (SEND_TIMEOUT_MS). A RetryableError (nothing was sent) holds - // the event on the round's integration; a timeout or any other failure is - // logged and the event dropped. + // never checks what an integration can show. A disabled integration's send + // is skipped and counts as sent. The send runs under sendTimeoutMs + // (SEND_TIMEOUT_MS). A RetryableError (nothing was sent) holds the event on + // the round's integration; a timeout or any other failure is logged and the + // event dropped. private async deliver( event: AgentEmit, inboxRow: InboxRow | null, @@ -259,13 +242,20 @@ export class EventDispatcher { return "ok"; // cursor still advances — drop } + // A source this build has no integration for (rows written by a newer + // build before a rollback) is skipped like a disabled one. + const integration: Integration | undefined = this.integrations[inboxRow.source]; + if (!integration?.enabled) { + logger.info(`${event.type} skipped: integration unknown or disabled`, { + taskId: task.id, + inboxId: inboxRow.id, + source: inboxRow.source, + }); + return "ok"; + } try { await withTimeout(this.sendTimeoutMs, (signal) => - this.integrations[inboxRow.source].deliver( - { signal, log: logger }, - target, - labelAgentEmit(event, inboxRow), - ), + integration.deliver({ signal, log: logger }, target, labelAgentEmit(event, inboxRow)), ); return "ok"; } catch (err) { diff --git a/control-plane/src/services/inbox.service.ts b/control-plane/src/services/inbox.service.ts index 02fb6d1..eb91d45 100644 --- a/control-plane/src/services/inbox.service.ts +++ b/control-plane/src/services/inbox.service.ts @@ -307,13 +307,17 @@ export class InboxService { const target = entry.reply_targets[0]; if (!target) return; // A source this build has no integration for (say, rows written by a newer - // build before a rollback) gets no notice; its row is still failed. + // build before a rollback), or a disabled one, gets no notice; its row is + // still failed. const integration: Integration | undefined = this.integrations[entry.source]; - if (!integration) { - logger.warn("workspace_missing: no integration for the row's source; notice skipped", { - inboxId: entry.id, - source: entry.source, - }); + if (!integration?.enabled) { + logger.warn( + "workspace_missing: the row's integration is unknown or disabled; notice skipped", + { + inboxId: entry.id, + source: entry.source, + }, + ); return; } await withTimeout(SEND_TIMEOUT_MS, (signal) => diff --git a/control-plane/src/services/integrationHooks.ts b/control-plane/src/services/integrationHooks.ts index cdaa166..7f98571 100644 --- a/control-plane/src/services/integrationHooks.ts +++ b/control-plane/src/services/integrationHooks.ts @@ -21,12 +21,13 @@ const HOOK_TIMEOUT_MS = { } as const; type HookName = keyof typeof HOOK_TIMEOUT_MS; // What a detached promise is doing, for the shutdown log. -type DetachedWork = HookName | "killSandbox"; +type DetachedWork = HookName | "killSandbox" | "slack"; // Runs integrations' lifecycle hooks (§4). onTaskStarted, onPrOpened and // assignee go to the task's own integration; onRoundFinished goes to the -// round's. Each call runs under its hook's timeout with an abort signal and -// never throws: a failure is logged and the caller goes on. The one exception +// round's. A disabled integration's hooks are skipped (logged). Each call runs +// under its hook's timeout with an abort signal and never throws: a failure is +// logged and the caller goes on. The one exception // is a RetryableError from onPrOpened, which escapes so the dispatcher holds // pr_opened as it holds a send (the hook must be idempotent). Detached work is // tracked so TaskProcessor.stop() can drain it. @@ -69,7 +70,8 @@ export class IntegrationHooks { } // The task's assignee, for detached work. undefined when the integration - // has no assignee or the lookup failed (logged here, with `logMeta`). + // has no assignee, is disabled, or the lookup failed (logged here, with + // `logMeta`). assignee( task: Task, round: InboxRow | null, @@ -141,6 +143,10 @@ export class IntegrationHooks { ): Promise { if (!owner[hook]) return undefined; const meta = { ...logMeta, taskId: task.id, integration: owner.name }; + if (!owner.enabled) { + logger.info(`${hook} skipped: integration disabled`, meta); + return undefined; + } try { return await withTimeout(this.timeoutsMs[hook], async (signal) => { // A failed lookup still runs the hook; isCurrent() then answers false. diff --git a/control-plane/src/services/replyHolds.ts b/control-plane/src/services/replyHolds.ts index a8e2bce..0d4e270 100644 --- a/control-plane/src/services/replyHolds.ts +++ b/control-plane/src/services/replyHolds.ts @@ -66,8 +66,13 @@ export class ReplyHolds { for (const id of this.byTask.keys()) if (!ownedIds.has(id)) this.byTask.delete(id); } - // Held tasks and the oldest hold's age, per integration (for phase 2's - // status routes). + // One integration's held tasks and oldest hold age, for its status body. + of(integration: IntegrationName): { heldTasks: number; oldestHoldAgeMs: number } { + const s = this.stats()[integration]; + return { heldTasks: s?.heldTasks ?? 0, oldestHoldAgeMs: s?.oldestHoldMs ?? 0 }; + } + + // Held tasks and the oldest hold's age, per integration. stats(): Partial> { const now = Date.now(); const out: Partial> = {}; diff --git a/control-plane/src/services/slackNotifier.service.ts b/control-plane/src/services/slackNotifier.service.ts index cc10016..19d242d 100644 --- a/control-plane/src/services/slackNotifier.service.ts +++ b/control-plane/src/services/slackNotifier.service.ts @@ -1,8 +1,8 @@ -import type { LinearClient } from "@linear/sdk"; -import { getIssueHumanAssignee } from "../clients/linear"; import { lookupSlackUserIdByEmail, postSlackMessage } from "../clients/slack"; import type { Task } from "../db/schema"; +import type { TaskSource } from "../integrations/types"; import type { TaskRepository } from "../repositories/task.repository"; +import type { IntegrationHooks } from "./integrationHooks"; // Terminal-state granularity. `completed | partial_failure | no_action` come // from the sandbox `completed` event's status field; `failed` is used by both @@ -13,11 +13,12 @@ export type TerminalStatus = "completed" | "partial_failure" | "no_action" | "fa type PrUrlEntry = { repo: string; url: string }; // Posts a single task-completion update to the configured Slack channel (a -// message per Linear-sourced task). Disabled unless both a bot token and a -// channel are configured. +// message per task whose source is opted in). Disabled unless both a bot token +// and a channel are configured. // -// Best-effort by contract: callers fire-and-forget with a `.catch`. A Slack -// outage or a missing thread must never affect task dispatch or teardown. +// Best-effort by contract: callers detach it (IntegrationHooks.detach logs a +// failure). A Slack outage or a missing thread must never affect task dispatch +// or teardown. export class SlackNotifierService { constructor( private readonly taskRepo: TaskRepository, @@ -27,19 +28,24 @@ export class SlackNotifierService { private readonly botToken: string, // config.slack.channel in production. Empty ⇒ Slack disabled (no-op). private readonly channel: string, + // config.slack.notifySources (SLACK_NOTIFY_SOURCES): the task sources + // whose tasks post. + private readonly notifySources: readonly TaskSource[], + // The owner integration's assignee(), for the connect-GitHub nudge. + private readonly hooks: Pick, // Injected for testability; default to the real implementations. private readonly post: typeof postSlackMessage = postSlackMessage, - private readonly resolveAssignee: typeof getIssueHumanAssignee = getIssueHumanAssignee, private readonly lookupSlackUser: typeof lookupSlackUserIdByEmail = lookupSlackUserIdByEmail, ) {} - async notifyTerminal( - linearClient: LinearClient, - task: Task, - status: TerminalStatus, - message?: string, - ): Promise { - if (!this.enabledFor(task)) return; + // Common gate for every Slack post: a token and a channel are provisioned + // and the source is opted in. Callers check it before notifyTerminal. + notifies(source: TaskSource): boolean { + return Boolean(this.botToken && this.channel) && this.notifySources.includes(source); + } + + async notifyTerminal(task: Task, status: TerminalStatus, message?: string): Promise { + if (!this.notifies(task.source)) return; // Re-read for the freshest pr_urls — the in-memory task handed in from the // reconcile loop can predate the pr_opened write that appended the PR. This @@ -49,52 +55,41 @@ export class SlackNotifierService { const fresh = await this.taskRepo.getById(task.id).catch(() => null); const t = fresh ?? task; const prUrls = parsePrUrls(t.prUrls); - // Nudge shown alongside PR links when the ticket's human assignee hasn't - // connected GitHub in Linear — the PR couldn't be assigned to them + // Nudge shown alongside PR links when a Linear ticket's human assignee + // hasn't connected GitHub in Linear — the PR couldn't be assigned to them // (eventDispatcher's pr_opened mirror skipped with no_github_mapping) and // self-service is the only fix. Re-checked lazily here so a user who // connected mid-run isn't nagged. Skipped for no_action, whose message // renders no PR section for the nudge to ride on. - const wantsNudge = status !== "no_action" && prUrls.length > 0; - const connectNote = wantsNudge ? await this.connectGitHubNote(linearClient, t) : ""; + const wantsNudge = t.source === "linear" && status !== "no_action" && prUrls.length > 0; + const connectNote = wantsNudge ? await this.connectGitHubNote(t) : ""; const text = buildMessage(status, t, prUrls, connectNote, message); await this.post({ channel: this.channel, text }); } // "" when there is nothing to nag about: no human assignee, already mapped, - // or the Linear lookup failed (best-effort — never blocks the post). - private async connectGitHubNote(linearClient: LinearClient, task: Task): Promise { - try { - const human = await this.resolveAssignee(linearClient, task.externalId); - if (!human || human.gitHubUserId) return ""; - // @-mention the assignee so the nudge notifies them (Slack only pings - // members of the channel — a non-member renders un-notified). Linear and - // Slack share the org email, so lookupByEmail is reliable here (it's only - // GitHub where personal emails diverge — the whole reason for this nudge). - // Degrades to the plain display name on any lookup miss (users_not_found, - // bot missing the users:read.email scope, transport error). displayName - // is member-editable — escape it so it can't inject mrkdwn (, - // forged <@mentions>, disguised links) into a trusted bot message. - const slackId = human.email - ? await this.lookupSlackUser(human.email).catch(() => null) - : null; - const mention = slackId ? `<@${slackId}>` : escapeMrkdwn(human.displayName); - return ( - `\n⚠️ ${mention}: I couldn't assign the PR to you — your GitHub account ` + - "isn't connected in Linear. Connect it under *Linear → Settings → Connected accounts* " + - "and future PRs will be assigned to you automatically." - ); - } catch { - return ""; - } - } - - // Common gate for every Slack post: a token and a channel are provisioned - // and the task is Linear-sourced. - private enabledFor(task: Task): boolean { - if (!this.botToken || !this.channel) return false; - return task.source === "linear"; + // or the lookup failed (IntegrationHooks logs it; never blocks the post). + private async connectGitHubNote(task: Task): Promise { + const person = await this.hooks.assignee(task, null, { purpose: "slack_nudge" }); + if (person?.codeHost !== "unlinked") return ""; + // @-mention the assignee so the nudge notifies them (Slack only pings + // members of the channel — a non-member renders un-notified). Linear and + // Slack share the org email, so lookupByEmail is reliable here (it's only + // GitHub where personal emails diverge — the whole reason for this nudge). + // Degrades to the plain display name on any lookup miss (users_not_found, + // bot missing the users:read.email scope, transport error). displayName + // is member-editable — escape it so it can't inject mrkdwn (, + // forged <@mentions>, disguised links) into a trusted bot message. + const slackId = person.email + ? await this.lookupSlackUser(person.email).catch(() => null) + : null; + const mention = slackId ? `<@${slackId}>` : escapeMrkdwn(person.displayName); + return ( + `\n⚠️ ${mention}: I couldn't assign the PR to you — your GitHub account ` + + "isn't connected in Linear. Connect it under *Linear → Settings → Connected accounts* " + + "and future PRs will be assigned to you automatically." + ); } } @@ -149,10 +144,12 @@ function prSection(prUrls: PrUrlEntry[]): string { return `\nHere are the PRs for review:\n${list}`; } -// The Linear-ticket link fragment shared by every message ("", or -// the bare title when the task has no external URL). +// The task's link fragment shared by every message ("", or the +// bare title when the task has no external URL). An untitled task is labelled +// "Linear ticket" on Linear and by its key elsewhere. function ticketLink(task: Task): string { - return task.externalUrl ? link(task.externalUrl, task.title || "Linear ticket") : task.title; + const label = task.title || (task.source === "linear" ? "Linear ticket" : task.key); + return task.externalUrl ? link(task.externalUrl, label) : task.title; } function buildMessage( diff --git a/control-plane/src/services/taskProcessor.service.ts b/control-plane/src/services/taskProcessor.service.ts index 01c62de..84a43ba 100644 --- a/control-plane/src/services/taskProcessor.service.ts +++ b/control-plane/src/services/taskProcessor.service.ts @@ -43,7 +43,7 @@ export class TaskProcessorService { private readonly sandboxManager: SandboxManager; private readonly eventDispatcher: EventDispatcher; private readonly inboxService: InboxService; - private readonly slackNotifier: SlackNotifierService; + private readonly slackNotifier: Pick; private readonly integrations: Record; private readonly holds: ReplyHolds; // Sources this build has an integration for; claimNext and @@ -58,7 +58,7 @@ export class TaskProcessorService { sandboxManager: SandboxManager, eventDispatcher: EventDispatcher, inboxService: InboxService, - slackNotifier: SlackNotifierService, + slackNotifier: Pick, integrations: Record, holds: ReplyHolds, taskSources: Task["source"][], @@ -817,10 +817,11 @@ export class TaskProcessorService { const seen = new Set(); for (const row of failedRows) { // A source this build has no integration for (say, rows written by a - // newer build before a rollback) gets no notice; its row is still failed. + // newer build before a rollback), or a disabled one, gets no notice; its + // row is still failed. const integration: Integration | undefined = this.integrations[row.source]; - if (!integration) { - logger.warn("failTask: no integration for the row's source; notice skipped", { + if (!integration?.enabled) { + logger.warn("failTask: the row's integration is unknown or disabled; notice skipped", { taskId: task.id, inboxId: row.id, source: row.source, @@ -853,26 +854,22 @@ export class TaskProcessorService { const current = await this.taskRepo.getByIdOwnedBy(task.id, config.podId); if (current?.sandboxId) await this.killSandboxQuietly(current.sandboxId); } - // Slack failure update to the configured channel. Gated on the ASSIGNMENT - // (top-level) round being among the rows just dead-lettered, mirroring the - // completion path's kind === "assignment" gate in eventDispatcher. Without - // this, a failing follow-up comment/review/ci_result round (a new inbox row - // on the SAME Linear task, appended after the assignment already completed - // and posted "✅ devbox completed") would post a contradictory "❌ devbox - // failed" into that same thread — only the top-level delegated task should - // ever notify Slack. Tying it to this failTaskInbox transition also keeps it - // at-most-once. Fire-and-forget: a Slack outage must not affect teardown; - // the notifier no-ops when Slack is disabled. - if (assignment && task.source === "linear") { - const client = await this.linearTokenProvider.getAuthorizedLinearClient().catch((e) => { - logErr("failTask: getAuthorizedLinearClient failed", e); - return null; - }); - if (client) { - void this.slackNotifier - .notifyTerminal(client, task, "failed", message) - .catch((e) => logErr(`slack failure notify failed for task ${task.id}`, e)); - } + // Slack failure update to the configured channel, for a task whose source + // is opted in (SLACK_NOTIFY_SOURCES). Gated on the ASSIGNMENT (top-level) + // round being among the rows just dead-lettered, mirroring the completion + // path's kind === "assignment" gate in eventDispatcher. Without this, a + // failing follow-up comment/review/ci_result round (a new inbox row on the + // SAME task, appended after the assignment already completed and posted + // "✅ devbox completed") would post a contradictory "❌ devbox failed" into + // that same thread — only the top-level delegated task should ever notify + // Slack. Tying it to this failTaskInbox transition also keeps it + // at-most-once. Detached: a Slack outage must not affect teardown. + if (assignment && this.slackNotifier.notifies(task.source)) { + this.hooks.detach( + task.id, + "slack", + this.slackNotifier.notifyTerminal(task, "failed", message), + ); } await this.taskRepo.releaseLease({ taskId: task.id, From 34c41921b029882f94cb0fb960be2f47d00c6981 Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Fri, 30 Oct 2026 20:12:20 +0530 Subject: [PATCH 14/22] feat(deploy): plumb SLACK_NOTIFY_SOURCES through the chart and docs Helm values, schema and template, the install target file and the configuration docs gain it. Unset keeps Linear-only notifications. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 ++-- control-plane/.env.example | 3 +++ .../scripts/lib/integrations.test.ts | 17 ++++++++++++- control-plane/scripts/lib/integrations.ts | 24 +++++++++++++++---- deploy/helm/devbox/README.md | 1 + .../helm/devbox/templates/control-plane.yaml | 3 +++ deploy/helm/devbox/tests/access.test.ts | 8 +++++++ deploy/helm/devbox/tests/validation.test.ts | 6 +++++ deploy/helm/devbox/values.schema.json | 6 ++++- deploy/helm/devbox/values.yaml | 2 ++ docs/configuration.md | 2 +- docs/github.md | 5 ++-- docs/kubernetes/customizing.md | 1 + docs/linear.md | 7 +++--- skills/devbox/SKILL.md | 3 ++- skills/devbox/references/integrations.md | 2 ++ 16 files changed, 79 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 1109bc4..9054634 100644 --- a/README.md +++ b/README.md @@ -266,8 +266,8 @@ To report a vulnerability, see [SECURITY.md](SECURITY.md). reasoning, on port 80 or 443 under a DNS name. - New tasks come from the CLI and Linear. On GitHub, DevBox only picks up follow-ups on its own pull requests; it doesn't start work from GitHub - issues. Slack gets only completion updates for Linear tasks. No Jira and no - web UI. + issues. Slack gets only completion updates, for Linear tasks unless + configured for API tasks too. No Jira and no web UI. - Stopping a task from Linear isn't supported yet. - No stable extension interface. [docs/extending.md](docs/extending.md) shows where hooks would go. diff --git a/control-plane/.env.example b/control-plane/.env.example index 89f18cc..cd89603 100644 --- a/control-plane/.env.example +++ b/control-plane/.env.example @@ -15,6 +15,9 @@ LINEAR_ENABLED=false SLACK_ENABLED=false # SLACK_BOT_TOKEN= # SLACK_CHANNEL_ID= +# Task sources whose finished tasks post, comma-separated: linear, api. +# Default: linear. +# SLACK_NOTIFY_SOURCES=linear # The CP's own GitHub token. The Docker Compose stack (`bun run up`) does not # read this file: it passes GITHUB_TOKEN from the shell, `gh auth token` or a diff --git a/control-plane/scripts/lib/integrations.test.ts b/control-plane/scripts/lib/integrations.test.ts index 9eac497..49de052 100644 --- a/control-plane/scripts/lib/integrations.test.ts +++ b/control-plane/scripts/lib/integrations.test.ts @@ -100,6 +100,21 @@ describe("parseIntegrations", () => { ); }); + it("slack notifySources: a non-empty list of task sources", () => { + expect( + parseIntegrations( + { slack: { enabled: true, channelId: "C1", notifySources: ["linear", "api"] } }, + fail, + K8S, + ), + ).toEqual({ slack: { enabled: true, channelId: "C1", notifySources: ["linear", "api"] } }); + for (const notifySources of [[], ["github"], "linear"]) { + expect(failure({ slack: { enabled: true, channelId: "C1", notifySources } })).toBe( + "integrations.slack.notifySources: must be a non-empty list of: linear, api", + ); + } + }); + it("slack refused on Compose", () => { expect(failure({ slack: { enabled: true, channelId: "C123" } }, COMPOSE)).toBe( "integrations.slack: supported on Kubernetes only (the target file)", @@ -133,7 +148,7 @@ describe("parseIntegrations", () => { 'integrations.linear: unknown key "clientSecret" (allowed: enabled, clientId)', ); expect(failure({ slack: { enabled: true, channelId: "C1", botToken: "t" } })).toBe( - 'integrations.slack: unknown key "botToken" (allowed: enabled, channelId)', + 'integrations.slack: unknown key "botToken" (allowed: enabled, channelId, notifySources)', ); expect(failure({ githubApp: { enabled: false, privateKey: "s" } })).toBe( 'integrations.githubApp: unknown key "privateKey" (allowed: enabled, appId, slug, installationId, botUserId, tokens, tokenScope, ciResults, tokenFile, org, pending)', diff --git a/control-plane/scripts/lib/integrations.ts b/control-plane/scripts/lib/integrations.ts index ff12b44..27cc667 100644 --- a/control-plane/scripts/lib/integrations.ts +++ b/control-plane/scripts/lib/integrations.ts @@ -7,10 +7,13 @@ import { homedir } from "node:os"; import { dirname, resolve } from "node:path"; +import { taskSource } from "../../src/db/schema"; import { parsePublicUrl } from "./publicUrl"; export type LinearIntegration = { enabled: boolean; clientId?: string }; -export type SlackIntegration = { enabled: boolean; channelId?: string }; +// notifySources: the task sources whose tasks post (SLACK_NOTIFY_SOURCES); +// unset means the control plane's default, linear. +export type SlackIntegration = { enabled: boolean; channelId?: string; notifySources?: string[] }; // The GitHub App and its webhooks, one switch. tokenScope and ciResults // are always filled with their defaults; the other keys are required only when // enabled is true. @@ -138,9 +141,22 @@ function parseLinear(raw: unknown, fail: Fail): LinearIntegration { } function parseSlack(raw: unknown, fail: Fail): SlackIntegration { - const b = block(raw, "slack", ["channelId"], fail); - const channelId = setting(b, "integrations.slack", "channelId", nonEmpty(fail), fail); - return { enabled: b.enabled, ...(channelId !== undefined ? { channelId } : {}) }; + const f = "integrations.slack"; + const b = block(raw, "slack", ["channelId", "notifySources"], fail); + const channelId = setting(b, f, "channelId", nonEmpty(fail), fail); + const sources = b.obj.notifySources; + const known: readonly unknown[] = taskSource.enumValues; + if ( + sources !== undefined && + !(Array.isArray(sources) && sources.length > 0 && sources.every((s) => known.includes(s))) + ) { + return fail(`${f}.notifySources: must be a non-empty list of: ${known.join(", ")}`); + } + return { + enabled: b.enabled, + ...(channelId !== undefined ? { channelId } : {}), + ...(sources !== undefined ? { notifySources: sources as string[] } : {}), + }; } function parseGithubApp( diff --git a/deploy/helm/devbox/README.md b/deploy/helm/devbox/README.md index 4946ed3..e386ef1 100644 --- a/deploy/helm/devbox/README.md +++ b/deploy/helm/devbox/README.md @@ -60,6 +60,7 @@ docker run --rm -v "$PWD/deploy/helm/devbox:/helm-docs" jnorwood/helm-docs:v1.14 | integrations.linear.enabled | bool | `false` | Turns Linear on (LINEAR_ENABLED). Its secrets go in the integrations Secret. | | integrations.slack.channelId | string | `""` | Required when enabled: the Slack channel ID. | | integrations.slack.enabled | bool | `false` | Turns Slack on (SLACK_ENABLED). SLACK_BOT_TOKEN goes in the integrations Secret. | +| integrations.slack.notifySources | list | `[]` | Task sources whose finished tasks post (SLACK_NOTIFY_SOURCES): `linear`, `api`. Empty: `linear`. | | ironProxy.clusterIP | string | `""` | Required. A free address in the cluster's Service range; sandboxes use it as their DNS server. `bun run k8s:install` picks one. | | ironProxy.extraAllowlist | list | `[]` | Extra allowlist rules ({host, methods, paths}), appended as-is. | | ironProxy.image.repository | string | `"ironsh/iron-proxy"` | | diff --git a/deploy/helm/devbox/templates/control-plane.yaml b/deploy/helm/devbox/templates/control-plane.yaml index 6def2ec..528d2ea 100644 --- a/deploy/helm/devbox/templates/control-plane.yaml +++ b/deploy/helm/devbox/templates/control-plane.yaml @@ -144,6 +144,9 @@ spec: {{- with $slack.channelId }} - { name: SLACK_CHANNEL_ID, value: {{ . | quote }} } {{- end }} + {{- with $slack.notifySources }} + - { name: SLACK_NOTIFY_SOURCES, value: {{ join "," . | quote }} } + {{- end }} {{- /* A disabled App block may hold an App not yet installed: none of it renders. */}} {{- if $app.enabled }} - { name: GITHUB_WEBHOOKS_ENABLED, value: "true" } diff --git a/deploy/helm/devbox/tests/access.test.ts b/deploy/helm/devbox/tests/access.test.ts index e5167b0..4d7f747 100644 --- a/deploy/helm/devbox/tests/access.test.ts +++ b/deploy/helm/devbox/tests/access.test.ts @@ -252,6 +252,14 @@ describe("integration env", () => { expect(await envOf(["slack.yaml"])).toEqual({ SLACK_ENABLED: "true", SLACK_CHANNEL_ID: "C0123456789" }); }); + it("slack notifySources renders SLACK_NOTIFY_SOURCES", async () => { + expect(await envOf(["slack.yaml"], [], ['integrations.slack.notifySources=["linear","api"]'])).toEqual({ + SLACK_ENABLED: "true", + SLACK_CHANNEL_ID: "C0123456789", + SLACK_NOTIFY_SOURCES: "linear,api", + }); + }); + it("the GitHub App on renders the webhook switch, bot identity and CI results", async () => { expect(await envOf(["github-app.yaml"])).toEqual({ GITHUB_WEBHOOKS_ENABLED: "true", diff --git a/deploy/helm/devbox/tests/validation.test.ts b/deploy/helm/devbox/tests/validation.test.ts index 230bbcd..d40f721 100644 --- a/deploy/helm/devbox/tests/validation.test.ts +++ b/deploy/helm/devbox/tests/validation.test.ts @@ -73,6 +73,12 @@ describe("chart validation", () => { } }); + it("refuses an unknown task source in integrations.slack.notifySources", async () => { + const stderr = await failure(["integrations.slack.notifySources={linear,github}"]); + expect(stderr).toContain("values don't meet the specifications of the schema"); + expect(stderr).toContain("notifySources"); + }); + it("refuses the removed integrations.githubWebhooks", async () => { const stderr = await failure(["integrations.githubWebhooks.enabled=true"]); expect(stderr).toContain("values don't meet the specifications of the schema"); diff --git a/deploy/helm/devbox/values.schema.json b/deploy/helm/devbox/values.schema.json index ee98c75..b7ce8c5 100644 --- a/deploy/helm/devbox/values.schema.json +++ b/deploy/helm/devbox/values.schema.json @@ -117,7 +117,11 @@ "slack": { "type": "object", "additionalProperties": false, - "properties": { "enabled": { "type": "boolean" }, "channelId": { "type": "string" } } + "properties": { + "enabled": { "type": "boolean" }, + "channelId": { "type": "string" }, + "notifySources": { "type": "array", "items": { "enum": ["linear", "api"] } } + } }, "githubApp": { "type": "object", diff --git a/deploy/helm/devbox/values.yaml b/deploy/helm/devbox/values.yaml index deb695e..8252398 100644 --- a/deploy/helm/devbox/values.yaml +++ b/deploy/helm/devbox/values.yaml @@ -112,6 +112,8 @@ integrations: enabled: false # -- Required when enabled: the Slack channel ID. channelId: "" + # -- Task sources whose finished tasks post (SLACK_NOTIFY_SOURCES): `linear`, `api`. Empty: `linear`. + notifySources: [] githubApp: # -- Turns the GitHub App on: webhooks at /webhook/github (GITHUB_WEBHOOKS_ENABLED) and, with # `tokens: devbox`, the token minter. GITHUB_WEBHOOK_SECRET goes in the integrations Secret. diff --git a/docs/configuration.md b/docs/configuration.md index 5d065ea..6e3e9af 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -307,7 +307,7 @@ kube context, so a command never acts on a cluster you did not name. | `context` | yes | The kube context of the cluster. | | `profile` | no, default `k8s-` | The CLI profile the installer writes. Lower-case letters, digits and `-`; the default is the context made lower-case with other characters replaced by `-`. | | `values` | no | Helm values for the chart, passed as they are. `storage.storageClass` and `storage.volumeSnapshotClass` are required by the chart. Sandbox sizing goes under `values.sandbox`; the Compose-only `devbox.config.json` block is ignored. `repos` and `llm` are refused: the installer copies the full `llm` block, including effort and stage overrides, from `devbox.config.json`. `integrations` is refused: it goes at the top level. | -| `integrations` | no | The deployment's integrations: `linear` (`enabled`, `clientId`), `slack` (`enabled`, `channelId`) and `githubApp` (the fields [above](#integrations) except `tokenFile`; `slug` is copied as `botLogin`), with the same `enabled` rule. The installer copies them into the chart values. The top-level `publicUrl` (the public HTTPS URL webhooks reach, shared by all integrations) is read only by the CLI, never copied ([customizing.md](kubernetes/customizing.md#integrations)). The trust lists go under `values.github`. | +| `integrations` | no | The deployment's integrations: `linear` (`enabled`, `clientId`), `slack` (`enabled`, `channelId`, `notifySources`) and `githubApp` (the fields [above](#integrations) except `tokenFile`; `slug` is copied as `botLogin`), with the same `enabled` rule. The installer copies them into the chart values. The top-level `publicUrl` (the public HTTPS URL webhooks reach, shared by all integrations) is read only by the CLI, never copied ([customizing.md](kubernetes/customizing.md#integrations)). The trust lists go under `values.github`. | Any other key is an error: chart values go under `values`. The installer adds `repos`, `llm`, `integrations`, `ironProxy.clusterIP`, diff --git a/docs/github.md b/docs/github.md index 9066c84..804c5d8 100644 --- a/docs/github.md +++ b/docs/github.md @@ -696,8 +696,9 @@ bun run github:disable [--target ] [--no-keychain] [--forget [--yes]] - **`github:disable`** sets `integrations.githubApp.enabled` to `false` and keeps the settings and the saved secrets. The App and its webhooks are off, and devbox falls back to a personal token, so the next apply needs - `GITHUB_TOKEN`. Turn the App back on with `bun run github:setup`, or by - setting `enabled` to `true`. It asks nothing. A new App waiting for its + `GITHUB_TOKEN`. Comments and reviews already queued still run, but their + replies are not posted. Turn the App back on with `bun run github:setup`, or + by setting `enabled` to `true`. It asks nothing. A new App waiting for its install stays, and it prints `The new App is still waiting for its install: bun run github:setup resumes it, bun run github:disable --forget removes it`. - **`github:disable --forget`** removes the `integrations.githubApp` block (a diff --git a/docs/kubernetes/customizing.md b/docs/kubernetes/customizing.md index eb9b14e..8515245 100644 --- a/docs/kubernetes/customizing.md +++ b/docs/kubernetes/customizing.md @@ -251,6 +251,7 @@ enabled from that block ([Check the Secrets](README.md#check-the-secrets)). | `integrations.linear.clientId` | The Linear app's client ID. Required when enabled. | | `integrations.slack.enabled` | Turns on Slack notifications when a task finishes. | | `integrations.slack.channelId` | The Slack channel ID. Required when enabled. | +| `integrations.slack.notifySources` | The task sources whose finished tasks post, from `linear` and `api`. Default: `linear`. | | `integrations.githubApp.enabled` | Turns on the GitHub App: webhooks at `/webhook/github` (`@` mentions, reviews and failed CI runs reach the agent) and, with `tokens: devbox`, the token minter. See [docs/github.md](../github.md). | | `integrations.githubApp.appId`, `integrations.githubApp.installationId`, `integrations.githubApp.botUserId` | The App's ID, its installation's ID and the bot's numeric GitHub user ID. Required when enabled. | | `integrations.githubApp.botLogin` | The App's slug. The installer copies it from `slug` in the target file's block. Required when enabled. | diff --git a/docs/linear.md b/docs/linear.md index ebbdbe8..e3e3d11 100644 --- a/docs/linear.md +++ b/docs/linear.md @@ -297,9 +297,10 @@ bun run linear:disable [--target ] [--no-keychain] [--forget [--yes]] ``` - **`linear:disable`** sets `integrations.linear.enabled` to `false` and keeps - the client ID, the public URL and the saved secrets. Turn Linear back on with - the same app with `bun run linear:setup`, or by setting `enabled` to `true`. - It asks nothing. + the client ID, the public URL and the saved secrets. Open Linear tasks keep + running, but nothing more is posted to Linear. Turn Linear back on with the + same app with `bun run linear:setup`, or by setting `enabled` to `true`. It + asks nothing. - **`linear:disable --forget`** removes the `integrations.linear` block and the deployment's saved `LINEAR_CLIENT_SECRET` and `LINEAR_WEBHOOK_SECRET`. It lists what it deletes and asks `[y/N]`; Enter or Ctrl-D answers no, and diff --git a/skills/devbox/SKILL.md b/skills/devbox/SKILL.md index 7136683..e636de8 100644 --- a/skills/devbox/SKILL.md +++ b/skills/devbox/SKILL.md @@ -91,6 +91,7 @@ Say so instead of looking for a way around it: self-hosted model needs a server that speaks the Responses API with reasoning, on port 80 or 443 under a DNS name. - Work comes in from the CLI, Linear and GitHub. Slack only gets completion - updates for Linear tasks. No Jira, no web UI. + updates, for Linear tasks unless `notifySources` adds API tasks. No Jira, no + web UI. - A task can't be stopped from Linear yet. - No stable extension interface ([docs/extending.md](../../docs/extending.md)). diff --git a/skills/devbox/references/integrations.md b/skills/devbox/references/integrations.md index b7e4b27..6ca9fb8 100644 --- a/skills/devbox/references/integrations.md +++ b/skills/devbox/references/integrations.md @@ -67,6 +67,8 @@ works with one Linear workspace. Everything: Kubernetes only: `integrations.slack` in the target file and `SLACK_BOT_TOKEN` ([docs/kubernetes/customizing.md](../../../docs/kubernetes/customizing.md#integrations)). +Finished Linear tasks post by default; `notifySources: ["linear", "api"]` adds +API tasks. ## Turning one off From 040b0c9a73fba5b0f0bc11ea2fd52bba26912d3a Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Fri, 30 Oct 2026 20:55:47 +0530 Subject: [PATCH 15/22] fix(control-plane): migration lock order and upgrade notes Create the inbox index first and drop the old task-key index last, so the boot migration never holds tasks exclusively while waiting on inbox. The upgrade docs cover the lock-timeout retry, the duplicate-key failure and how to find such rows, and what older pods may log during the overlap. Co-Authored-By: Claude Opus 5.5 --- ...ql => 0001_inbox_source_kind_external.sql} | 0 ...key_lower.sql => 0002_tasks_key_lower.sql} | 0 control-plane/drizzle/meta/0001_snapshot.json | 35 ++++++++++++++++--- control-plane/drizzle/meta/_journal.json | 4 +-- docs/kubernetes/README.md | 33 +++++++++++++++++ 5 files changed, 66 insertions(+), 6 deletions(-) rename control-plane/drizzle/{0002_inbox_source_kind_external.sql => 0001_inbox_source_kind_external.sql} (100%) rename control-plane/drizzle/{0001_tasks_key_lower.sql => 0002_tasks_key_lower.sql} (100%) diff --git a/control-plane/drizzle/0002_inbox_source_kind_external.sql b/control-plane/drizzle/0001_inbox_source_kind_external.sql similarity index 100% rename from control-plane/drizzle/0002_inbox_source_kind_external.sql rename to control-plane/drizzle/0001_inbox_source_kind_external.sql diff --git a/control-plane/drizzle/0001_tasks_key_lower.sql b/control-plane/drizzle/0002_tasks_key_lower.sql similarity index 100% rename from control-plane/drizzle/0001_tasks_key_lower.sql rename to control-plane/drizzle/0002_tasks_key_lower.sql diff --git a/control-plane/drizzle/meta/0001_snapshot.json b/control-plane/drizzle/meta/0001_snapshot.json index da73214..2ea9e04 100644 --- a/control-plane/drizzle/meta/0001_snapshot.json +++ b/control-plane/drizzle/meta/0001_snapshot.json @@ -152,6 +152,33 @@ } }, "indexes": { + "inbox_source_kind_external_uniq": { + "name": "inbox_source_kind_external_uniq", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, "inbox_source_external_uniq": { "name": "inbox_source_external_uniq", "columns": [ @@ -460,13 +487,13 @@ "method": "btree", "with": {} }, - "tasks_key_lower_uniq": { - "name": "tasks_key_lower_uniq", + "tasks_key_uniq": { + "name": "tasks_key_uniq", "columns": [ { - "expression": "lower(\"key\")", + "expression": "key", + "isExpression": false, "asc": true, - "isExpression": true, "nulls": "last" } ], diff --git a/control-plane/drizzle/meta/_journal.json b/control-plane/drizzle/meta/_journal.json index 9e21983..e466440 100644 --- a/control-plane/drizzle/meta/_journal.json +++ b/control-plane/drizzle/meta/_journal.json @@ -13,14 +13,14 @@ "idx": 1, "version": "7", "when": 1793368811788, - "tag": "0001_tasks_key_lower", + "tag": "0001_inbox_source_kind_external", "breakpoints": true }, { "idx": 2, "version": "7", "when": 1793368820991, - "tag": "0002_inbox_source_kind_external", + "tag": "0002_tasks_key_lower", "breakpoints": true } ] diff --git a/docs/kubernetes/README.md b/docs/kubernetes/README.md index 6a8feff..c4ff54c 100644 --- a/docs/kubernetes/README.md +++ b/docs/kubernetes/README.md @@ -428,6 +428,39 @@ kubectl -n devbox-sandboxes create job devbox-snapshot-builder-manual-$(date +%s The snapshot builder also runs on a schedule (`snapshotBuilder.schedule`, every 6 hours by default). +### Database migrations + +A new control plane pod migrates the database when it starts, while the old +pods keep serving. The migrations hold short locks on the `tasks` and `inbox` +tables and wait at most 5 seconds for each. If a lock times out, the new pod +fails, restarts and tries again. Postgres may instead break a deadlock between +the migration and a request: then either the pod restarts, or the request +fails with a 500 and, for a webhook, the platform retries it. + +Task keys are unique regardless of case. The migration that enforces this +fails with `could not create unique index "tasks_key_lower_uniq"` and +`Key (lower(key))=(…) is duplicated` only if tasks were inserted by hand with +keys that differ only in case: devbox upper-cases Linear keys, and +`API_TASK_KEY_PREFIX` must match `^[A-Z][A-Z0-9-]*-$`. To list them: + +```sql +SELECT lower(key) AS key, array_agg(id) AS task_ids + FROM tasks GROUP BY lower(key) HAVING count(*) > 1; +``` + +Rename all but one key in each group; the pod migrates on its next start. + +While old and new pods overlap, and after a rollback to a release without the +`inbox_source_kind_external_uniq` index, an old pod can answer a webhook that +races its own redelivery with a 500: `duplicate key value violates unique +constraint "inbox_source_kind_external_uniq"`. The other request stored the +round and the platform retries, so nothing is lost. After such a rollback, +dropping the index stops the errors: + +```sql +DROP INDEX inbox_source_kind_external_uniq; +``` + ## Uninstall `k8s:uninstall` acts on the cluster named in the `--target` file, and on From a81f0e9f0a310774042ad3bec21aabc04bdbebc3 Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Fri, 30 Oct 2026 20:55:47 +0530 Subject: [PATCH 16/22] fix(control-plane): test gaps and log fields for intake and the registry Add tests for concurrent re-delegation, disabled integrations on the failure paths, the app's enabled mapping and status body, and the new ON CONFLICT target. Restore log fields and messages that dashboards may key on. Co-Authored-By: Claude Opus 5.5 --- control-plane/src/__tests__/app.test.ts | 64 +++++++++++++++++ control-plane/src/app.ts | 49 ++++++++----- .../src/integrations/__tests__/intake.test.ts | 42 ++++++++++- control-plane/src/integrations/intake.ts | 19 +++-- .../__tests__/inbox.repository.test.ts | 35 ++++++++- .../src/repositories/inbox.repository.ts | 5 +- .../src/repositories/task.repository.ts | 11 ++- .../__tests__/taskProcessor.service.test.ts | 72 +++++++++++++++++++ .../src/services/agentSession.service.ts | 3 +- control-plane/src/services/eventDispatcher.ts | 4 +- .../src/services/taskProcessor.service.ts | 4 +- 11 files changed, 275 insertions(+), 33 deletions(-) create mode 100644 control-plane/src/__tests__/app.test.ts diff --git a/control-plane/src/__tests__/app.test.ts b/control-plane/src/__tests__/app.test.ts new file mode 100644 index 0000000..ba22712 --- /dev/null +++ b/control-plane/src/__tests__/app.test.ts @@ -0,0 +1,64 @@ +import { afterEach, describe, expect, it, mock, setSystemTime } from "bun:test"; + +// config.ts reads required env at import; nothing here reads it. +mock.module("../config", () => ({ config: {} })); + +const { createIntegrations, statusesWithHolds } = await import("../app"); +const { RetryableError } = await import("../integrations/outbound"); +const { ReplyHolds } = await import("../services/replyHolds"); + +type Cfg = Parameters[0]; +const statusOf = (body: object) => ({ status: async () => body }) as never; +const registry = (cfg: Cfg, linear = {}, github = {}) => + createIntegrations(cfg, {} as never, statusOf(linear), statusOf(github)); + +describe("createIntegrations", () => { + it("enables each integration from its own setting", () => { + const enabled = (cfg: Cfg) => + Object.fromEntries(Object.values(registry(cfg)).map((i) => [i.name, i.enabled])); + const off = { api: { enabled: false }, github: {} }; + expect(enabled(off)).toEqual({ linear: false, api: false, github: false }); + expect(enabled({ ...off, linear: {} })).toEqual({ linear: true, api: false, github: false }); + expect(enabled({ ...off, api: { enabled: true } })).toEqual({ + linear: false, + api: true, + github: false, + }); + expect(enabled({ ...off, github: { webhooks: {} } })).toEqual({ + linear: false, + api: false, + github: true, + }); + }); +}); + +describe("statusesWithHolds", () => { + afterEach(() => { + setSystemTime(); + }); + + it("adds each integration's held sends to its own status body", async () => { + const T0 = new Date("2026-10-30T12:00:00Z").getTime(); + const holds = new ReplyHolds(); + setSystemTime(new Date(T0)); + holds.hold("t1", { inbox_id: "i1", seq: 1 }, "github", new RetryableError("HTTP 429")); + setSystemTime(new Date(T0 + 4_000)); + holds.hold("t2", { inbox_id: "i2", seq: 3 }, "github", new RetryableError("HTTP 429")); + setSystemTime(new Date(T0 + 5_000)); + const linear = { configured: true, lastWebhookAt: null, lastTokenError: null }; + const github = { token: { ok: true }, webhooks: { enabled: true, lastWebhookAt: null } }; + + const statuses = statusesWithHolds( + registry({ linear: {}, api: { enabled: true }, github: { webhooks: {} } }, linear, github), + holds, + ); + + const none = { heldTasks: 0, oldestHoldAgeMs: 0 }; + expect(await statuses.linear.status()).toEqual({ ...linear, holds: none }); + expect(await statuses.github.status()).toEqual({ + ...github, + holds: { heldTasks: 2, oldestHoldAgeMs: 5_000 }, + }); + expect(await statuses.api.status()).toEqual({ enabled: true, holds: none }); + }); +}); diff --git a/control-plane/src/app.ts b/control-plane/src/app.ts index 3419e99..84d693e 100644 --- a/control-plane/src/app.ts +++ b/control-plane/src/app.ts @@ -91,15 +91,7 @@ export function createApp(): AppBundle { lastWebhookAt: (): string | null => integrations.github.controller?.lastWebhookAt() ?? null, }); - // The registry. Every outbound message goes through - // integrations[round.source].deliver, and core skips a disabled - // integration's webhook, sends and hooks. - const integrations = { - linear: new LinearIntegration(linearTokenProvider, config.linear !== undefined, linearStatus), - api: new ApiIntegration(config.api.enabled), - // GitHub rounds come only from its webhooks. - github: new GithubIntegration(config.github.webhooks !== undefined, githubStatus), - } satisfies Record; + const integrations = createIntegrations(config, linearTokenProvider, linearStatus, githubStatus); // The registered integrations that create tasks (github creates none). Only // these sources are claimed and reconciled, so an older image never works a // task whose source it doesn't know (§10). @@ -192,19 +184,12 @@ export function createApp(): AppBundle { sseMaxDurationMs: config.api.sseMaxDurationMs, clock: realClock, }); - // Each integration's own status body, plus its held sends (§4 I14). - const statuses = Object.fromEntries( - Object.values(integrations).map((i) => [ - i.name, - { status: async () => ({ ...(await i.status()), holds: replyHolds.of(i.name) }) }, - ]), - ) as Record>; app.route( "/", createApiRoutes(apiController, new ApiAuthService(apiTokenRepository), { authEnabled: config.api.authEnabled, allowedHosts: config.api.allowedHosts, - integrations: statuses, + integrations: statusesWithHolds(integrations, replyHolds), }), ); stopApiStreams = () => apiController.closeStreams(); @@ -212,3 +197,33 @@ export function createApp(): AppBundle { return { app, taskProcessor, stopApiStreams }; } + +// The registry. Every outbound message goes through +// integrations[round.source].deliver, and core skips a disabled +// integration's webhook, sends and hooks. +export function createIntegrations( + cfg: { linear?: unknown; api: { enabled: boolean }; github: { webhooks?: unknown } }, + linearTokenProvider: Pick, + linearStatus: Pick, + githubStatus: Pick, +) { + return { + linear: new LinearIntegration(linearTokenProvider, cfg.linear !== undefined, linearStatus), + api: new ApiIntegration(cfg.api.enabled), + // GitHub rounds come only from its webhooks. + github: new GithubIntegration(cfg.github.webhooks !== undefined, githubStatus), + } satisfies Record; +} + +// Each integration's own status body, plus its held sends (§4 I14). +export function statusesWithHolds( + integrations: ReturnType, + replyHolds: Pick, +): Record> { + return Object.fromEntries( + Object.values(integrations).map((i) => [ + i.name, + { status: async () => ({ ...(await i.status()), holds: replyHolds.of(i.name) }) }, + ]), + ) as Record>; +} diff --git a/control-plane/src/integrations/__tests__/intake.test.ts b/control-plane/src/integrations/__tests__/intake.test.ts index 6e4af0e..3210691 100644 --- a/control-plane/src/integrations/__tests__/intake.test.ts +++ b/control-plane/src/integrations/__tests__/intake.test.ts @@ -219,7 +219,12 @@ describe("Intake", () => { flags: ["NEW"], }); - expect(res).toEqual({ taskId, key: "ISS-1", result: "restarted" }); + expect(res).toEqual({ + taskId, + key: "ISS-1", + result: "restarted", + previousTargetId: "sess-1", + }); expect(killSandbox).toHaveBeenCalledTimes(1); expect(killSandbox).toHaveBeenCalledWith("sb-old"); // In the tracked detached set, so shutdown drains it. @@ -237,6 +242,41 @@ describe("Intake", () => { }); }); + it("concurrent deliveries of one re-delegation restart the task once", async () => { + // The task row lock serializes them: the first restarts, the rest read + // the new target after it commits. Another transaction holds the + // assignment row, so the first restart waits while the rest pile up. + const { taskId } = await assignLinear(session("S0")); + await db.execute(sql`UPDATE tasks SET sandbox_id = 'sb-old' WHERE id = ${taskId}`); + const other = new SQL(TEST_DATABASE_URL); + try { + let release!: () => void; + const gate = new Promise((r) => { + release = r; + }); + let locked!: () => void; + const ready = new Promise((r) => { + locked = r; + }); + const holder = other.begin(async (tx) => { + await tx`SELECT 1 FROM inbox WHERE task_id = ${taskId} FOR UPDATE`; + locked(); + await gate; + }); + await ready; + const pending = Promise.all(Array.from({ length: 6 }, () => assignLinear(session("S1")))); + await new Promise((r) => setTimeout(r, 150)); + release(); + await holder; + const results = (await pending).map((r) => r.result).sort(); + expect(results).toEqual([...Array(5).fill("duplicate"), "restarted"]); + } finally { + await other.close(); + } + expect(killSandbox).toHaveBeenCalledTimes(1); + expect(killSandbox).toHaveBeenCalledWith("sb-old"); + }); + it("rejects a target of another integration, and a hint without an external id", async () => { await expect(assignLinear({ id: "t", spec: { kind: "api_session" } })).rejects.toThrow( "a linear round can't carry a api_session reply target", diff --git a/control-plane/src/integrations/intake.ts b/control-plane/src/integrations/intake.ts index 7d4912c..4005c63 100644 --- a/control-plane/src/integrations/intake.ts +++ b/control-plane/src/integrations/intake.ts @@ -34,7 +34,12 @@ export interface TaskIntake { target: (t: TaskRef) => ReplyTarget; // target.id = delegation id; optional target.order (monotonic number) requester?: Person; // tasks.metadata.requester createdBy?: string; // API token id - }): Promise<{ taskId: string; key: string; result: "created" | "restarted" | "duplicate" }>; + }): Promise<{ + taskId: string; + key: string; + result: "created" | "restarted" | "duplicate"; + previousTargetId?: string; // a restart's replaced delegation id + }>; restart( taskId: string, @@ -73,7 +78,7 @@ export class Intake implements TaskIntake { async assign(p: Parameters[0]): ReturnType { if ("hint" in p.key && !p.externalId) throw new Error("assign: a hint key needs externalId"); - const { prevSandboxId, ...t } = await this.taskRepo.assign({ + const { prevSandboxId, prevTargetId, ...t } = await this.taskRepo.assign({ ...p, key: "hint" in p.key @@ -85,8 +90,8 @@ export class Intake implements TaskIntake { return target; }, }); - this.killPreempted(t.id, prevSandboxId); - return { taskId: t.id, key: t.key, result: t.result }; + this.killPreempted(t.id, prevSandboxId, "preempted re-delegation"); + return { taskId: t.id, key: t.key, result: t.result, previousTargetId: prevTargetId }; } // API only: today's API restart, with the definition replacing the current @@ -108,7 +113,7 @@ export class Intake implements TaskIntake { inboxReplyTargets: [apiSessionTarget(taskId)], api: { title: def.title, description: def.description, experimentalFlags: def.flags }, }); - this.killPreempted(taskId, prevSandboxId); + this.killPreempted(taskId, prevSandboxId, "API restart"); } async followUp(p: Parameters[0]): ReturnType { @@ -140,14 +145,14 @@ export class Intake implements TaskIntake { // A restart's preempted sandbox, killed after commit in the tracked detached // set so shutdown drains it. The old workspace volume is kept (§15 bug 6). - private killPreempted(taskId: string, sandboxId: string | null): void { + private killPreempted(taskId: string, sandboxId: string | null, why: string): void { if (!sandboxId) return; this.hooks.detach( taskId, "killSandbox", this.sandboxManager.killSandbox(sandboxId).catch((err) => { const e = err instanceof Error ? err : new Error(String(err)); - logger.error("killSandbox(restart) failed", { + logger.error(`killSandbox(${why}) failed`, { taskId, sandboxId, message: e.message, diff --git a/control-plane/src/repositories/__tests__/inbox.repository.test.ts b/control-plane/src/repositories/__tests__/inbox.repository.test.ts index a86c032..3675072 100644 --- a/control-plane/src/repositories/__tests__/inbox.repository.test.ts +++ b/control-plane/src/repositories/__tests__/inbox.repository.test.ts @@ -1,5 +1,5 @@ import { beforeAll, beforeEach, describe, expect, it, mock } from "bun:test"; -import { sql } from "drizzle-orm"; +import { sql, TransactionRollbackError } from "drizzle-orm"; const TEST_DATABASE_URL = process.env.TEST_DATABASE_URL ?? "postgres://devbox:devbox@localhost:5432/devbox"; @@ -178,6 +178,39 @@ describe("InboxRepository", () => { const kinds = await db.execute<{ kind: string }>(sql`SELECT kind FROM inbox`); expect([...kinds]).toEqual([{ kind: "comment" }]); }); + + it("dedupes on (source, kind, external_id) once the old index is gone", async () => { + // Phase 3 drops inbox_source_external_uniq; here it goes in a + // transaction that is rolled back, so the insert's conflict target + // must be the new index. + const task = await insertTask(); + const review = { + source: "github" as const, + kind: "review" as const, + external_id: "555", + payload: { + review_body: { target_id: "555", body: "@bot" }, + inline_comments: [], + task_id: "task-1", + task_key: "T-1", + pr: { repo: "r", pr_url: "https://github.com/o/r/pull/7" }, + created_at: "", + }, + }; + const results: unknown[] = []; + await expect( + db.transaction(async (tx) => { + await tx.execute(sql`DROP INDEX inbox_source_external_uniq`); + for (const message of [ghComment("555"), ghComment("555"), review]) { + results.push( + await repo.appendInbox({ taskId: task.id, message, replyTargets: [] }, tx), + ); + } + tx.rollback(); + }), + ).rejects.toThrow(TransactionRollbackError); + expect(results).toEqual([{ id: expect.any(String) }, null, { id: expect.any(String) }]); + }); }); describe("listPendingByTask", () => { diff --git a/control-plane/src/repositories/inbox.repository.ts b/control-plane/src/repositories/inbox.repository.ts index 036b2df..da90822 100644 --- a/control-plane/src/repositories/inbox.repository.ts +++ b/control-plane/src/repositories/inbox.repository.ts @@ -239,11 +239,14 @@ export class InboxRepository { ); const seen = count?.n ?? 0; if (seen >= p.ciResultMax) { + // external_id is workflow_run:::. + const [, runId, , prNumber] = message.external_id.split(":").map(Number); logger.warn("ci_result loop breaker tripped — dropping CI failure", { taskId: p.taskId, seen, max: p.ciResultMax, - externalId: message.external_id, + runId, + prNumber, }); return { dropped: "ci_limit" }; } diff --git a/control-plane/src/repositories/task.repository.ts b/control-plane/src/repositories/task.repository.ts index 22fbdab..37c70fc 100644 --- a/control-plane/src/repositories/task.repository.ts +++ b/control-plane/src/repositories/task.repository.ts @@ -30,7 +30,8 @@ export class TaskRepository { // an order, one not above the stored order is a duplicate; otherwise the // same id is. A duplicate changes nothing; anything else restarts the task. // A new task gets its assignment row and, for API tasks, task_created. - // Returns the preempted sandbox for the caller to kill after commit. + // Returns the preempted sandbox for the caller to kill after commit, and on + // a restart the replaced delegation's target id. async assign(p: { source: TaskSource; externalId?: string; // required with a hint key; a sequence key is its own @@ -43,7 +44,11 @@ export class TaskRepository { requester?: Person; createdBy?: string; }): Promise< - TaskRef & { result: "created" | "restarted" | "duplicate"; prevSandboxId: string | null } + TaskRef & { + result: "created" | "restarted" | "duplicate"; + prevSandboxId: string | null; + prevTargetId?: string; + } > { return db.transaction(async (tx) => { // In a savepoint so a key collision leaves the transaction usable: a @@ -121,7 +126,7 @@ export class TaskRepository { }, tx, ); - return { ...t, result: "restarted", prevSandboxId }; + return { ...t, result: "restarted", prevSandboxId, prevTargetId: prev?.id }; }); } diff --git a/control-plane/src/services/__tests__/taskProcessor.service.test.ts b/control-plane/src/services/__tests__/taskProcessor.service.test.ts index f89852e..70e95c2 100644 --- a/control-plane/src/services/__tests__/taskProcessor.service.test.ts +++ b/control-plane/src/services/__tests__/taskProcessor.service.test.ts @@ -98,6 +98,9 @@ const { logger } = await import("../../logger"); const { ReplyHolds } = await import("../replyHolds"); const { IntegrationHooks } = await import("../integrationHooks"); const { EventDispatcher: RealEventDispatcher } = await import("../eventDispatcher"); +const { InboxService: RealInboxService, WORKSPACE_MISSING_MESSAGE } = await import( + "../inbox.service" +); const { config: mockedConfig } = (await import("../../config")) as unknown as { config: { shutdownDrainMs: number }; }; @@ -723,6 +726,52 @@ describe("TaskProcessorService", () => { alsoClearSandbox: true, }); }); + + it("a row whose integration is disabled gets no notice; it is still failed and the lease released", async () => { + integrations.github = new GithubIntegration(false, { status: () => ({}) as never }); + const row = { + ...survivingNonTerminalRow(), + id: "gc", + source: "github", + kind: "comment", + reply_targets: [ + { + id: "555", + spec: { kind: "issue_comment", owner: "o", repo: "r", prNumber: 7, commentId: 555 }, + }, + ], + }; + const markFailed = mock(() => Promise.resolve([{ id: "gc" }])); + (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValue([row]); + Object.assign(inboxRepo, { getById: mock(() => Promise.resolve(row)), markFailed }); + // The real dead-letter path in place of the stub. + Object.assign(service, { + inboxService: new RealInboxService(inboxRepo, sandboxManager, integrations, hooks, { + followUp: mock(), + }), + }); + + await ( + service as unknown as { handleTaskError(t: Task, e: unknown): Promise } + ).handleTaskError( + makeTask({ sandboxId: null, pvcName: "pvc-1" }), + new PvcNotFoundError("pvc-1"), + ); + + expect(postIssueComment).not.toHaveBeenCalled(); + expect(holds.stats()).toEqual({}); + expect(markFailed).toHaveBeenCalledWith({ + inboxId: "gc", + ownerId: "pod-test", + reason: "workspace_missing", + userMessage: WORKSPACE_MISSING_MESSAGE, + }); + expect(releaseLease).toHaveBeenCalledWith({ + taskId: "task-1", + ownerId: "pod-test", + alsoClearSandbox: true, + }); + }); }); describe("dispatch", () => { @@ -889,6 +938,29 @@ describe("TaskProcessorService", () => { }); }); + it("a row whose integration is disabled gets no notice; it is still dead-lettered and the lease released", async () => { + integrations.github = new GithubIntegration(false, { status: () => ({}) as never }); + (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValue([ + row("gc", "comment", "github", [issueComment("555", 7, 555)]), + ]); + failTaskInbox.mockResolvedValue([{ id: "gc" }]); + + await failTask(makeTask()); + + expect(postIssueComment).not.toHaveBeenCalled(); + expect(holds.stats()).toEqual({}); + expect(failTaskInbox).toHaveBeenCalledWith({ + taskId: "task-1", + ownerId: "pod-test", + reason: MSG, + }); + expect(releaseLease).toHaveBeenCalledWith({ + taskId: "task-1", + ownerId: "pod-test", + alsoClearSandbox: true, + }); + }); + it("a Linear token failure skips only the Linear post; GitHub and Slack posts, dead-letter and release go on", async () => { getAuthorizedLinearClient.mockRejectedValue(new Error("token refresh failed")); (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValue([ diff --git a/control-plane/src/services/agentSession.service.ts b/control-plane/src/services/agentSession.service.ts index c1e5e49..d4fa768 100644 --- a/control-plane/src/services/agentSession.service.ts +++ b/control-plane/src/services/agentSession.service.ts @@ -44,7 +44,7 @@ export class AgentSessionService { // the freshly-provisioned sandbox); a new one is a re-delegation, which // preempts any in-flight run and restarts from a clean slate (intake // kills the preempted sandbox). - const { taskId, result } = await this.intake.assign({ + const { taskId, result, previousTargetId } = await this.intake.assign({ source: "linear", externalId: event.issue.id, key: { hint: requireLinearTicketId(event.issue.url) }, @@ -61,6 +61,7 @@ export class AgentSessionService { } else if (result === "restarted") { logger.info("re-delegation detected; restarting task from clean slate", { taskId, + previousSessionId: previousTargetId, currentSessionId: event.agentSessionId, }); this.fireAndForgetEmitResponse(event.agentSessionId, RESTARTED_BODY, "restart-ack"); diff --git a/control-plane/src/services/eventDispatcher.ts b/control-plane/src/services/eventDispatcher.ts index 0bcd7c8..1af7ced 100644 --- a/control-plane/src/services/eventDispatcher.ts +++ b/control-plane/src/services/eventDispatcher.ts @@ -194,7 +194,9 @@ export class EventDispatcher { this.hooks.detach( task.id, "slack", - this.slackNotifier.notifyTerminal(task, event.status ?? "completed"), + this.slackNotifier + .notifyTerminal(task, event.status ?? "completed") + .catch((err) => logErr("slack completion notify failed", err, { taskId: task.id })), ); } break; diff --git a/control-plane/src/services/taskProcessor.service.ts b/control-plane/src/services/taskProcessor.service.ts index 84a43ba..71a3f97 100644 --- a/control-plane/src/services/taskProcessor.service.ts +++ b/control-plane/src/services/taskProcessor.service.ts @@ -868,7 +868,9 @@ export class TaskProcessorService { this.hooks.detach( task.id, "slack", - this.slackNotifier.notifyTerminal(task, "failed", message), + this.slackNotifier + .notifyTerminal(task, "failed", message) + .catch((e) => logErr(`slack failure notify failed for task ${task.id}`, e)), ); } await this.taskRepo.releaseLease({ From 4541e09db7e0de15000321fb47ddab99f842d69a Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Fri, 30 Oct 2026 21:32:21 +0530 Subject: [PATCH 17/22] fix(control-plane): keep leases warm and bound shutdown Renew every lease the pod holds before each reconcile, including tasks still provisioning, and never dispatch a task already in flight, so a long tick can't provision a task twice. Shutdown stops between events, failure notices go to exactly the rows failTaskInbox marked, and the drain also waits for work detached while draining. The upgrade docs no longer suggest dropping the new inbox index after a rollback: the migration is already recorded, so the next upgrade would not rebuild it. Co-Authored-By: Claude Opus 5.5 --- control-plane/docker-compose.yml | 4 +- .../__tests__/inbox.repository.test.ts | 24 +++ .../__tests__/task.repository.test.ts | 19 +-- .../src/repositories/inbox.repository.ts | 34 ++-- .../src/repositories/task.repository.ts | 15 +- .../__tests__/integrationHooks.test.ts | 28 ++++ .../__tests__/taskProcessor.service.test.ts | 147 ++++++++++++------ .../src/services/integrationHooks.ts | 11 +- .../src/services/taskProcessor.service.ts | 57 ++++--- deploy/helm/devbox/README.md | 2 +- deploy/helm/devbox/values.yaml | 8 +- docs/kubernetes/README.md | 11 +- 12 files changed, 231 insertions(+), 129 deletions(-) diff --git a/control-plane/docker-compose.yml b/control-plane/docker-compose.yml index c7c4b39..9938a99 100644 --- a/control-plane/docker-compose.yml +++ b/control-plane/docker-compose.yml @@ -93,8 +93,8 @@ services: image: devbox-control-plane build: *control-plane-build restart: unless-stopped - # Shutdown finishes the task it is reconciling (up to 110s when platforms - # are slow, plus 50s per extra PR it opens), then drains for + # Shutdown finishes the event it is handling (the rest of a task's + # reconcile: up to 90s when platforms are slow), then drains for # SHUTDOWN_DRAIN_MS (10s); Compose's default 10s would cut both short. stop_grace_period: 150s # Explicit, with no env_file. Only allowlisted HARNESS_* settings cross diff --git a/control-plane/src/repositories/__tests__/inbox.repository.test.ts b/control-plane/src/repositories/__tests__/inbox.repository.test.ts index 3675072..b58dd7a 100644 --- a/control-plane/src/repositories/__tests__/inbox.repository.test.ts +++ b/control-plane/src/repositories/__tests__/inbox.repository.test.ts @@ -335,6 +335,30 @@ describe("InboxRepository", () => { expect(await repo.listPendingByTask(task.id)).toEqual([]); }); + it("returns the rows it failed, whole, in arrival order", async () => { + const task = await insertTask({ ownerId: "pod-a" }); + const ids: string[] = []; + for (const external_id of ["first", "second"]) { + const row = await repo.appendInbox({ + taskId: task.id, + message: { + source: "linear", + kind: "assignment", + external_id, + payload: assignmentPayload, + }, + replyTargets: [linearTarget(`s-${external_id}`)], + }); + if (!row) throw new Error("insert failed"); + ids.push(row.id); + } + + const failed = await repo.failTaskInbox({ taskId: task.id, ownerId: "pod-a", reason: "x" }); + + expect(failed as unknown[]).toEqual(await Promise.all(ids.map((id) => repo.getById(id)))); + expect(failed[0]?.reply_targets).toEqual([linearTarget("s-first")]); + }); + it("does not touch an already-responded row", async () => { const task = await insertTask({ ownerId: "pod-a" }); const row = await repo.appendInbox({ diff --git a/control-plane/src/repositories/__tests__/task.repository.test.ts b/control-plane/src/repositories/__tests__/task.repository.test.ts index 997650d..dd13d95 100644 --- a/control-plane/src/repositories/__tests__/task.repository.test.ts +++ b/control-plane/src/repositories/__tests__/task.repository.test.ts @@ -1053,22 +1053,6 @@ describe("TaskRepository", () => { }); }); - describe("heartbeatTask", () => { - it("renews one owned task and returns 0 rows once the lease is someone else's", async () => { - const mine = await insertTask({ ownerId: "pod-a", heartbeatAt: new Date("2020-01-01") }); - const other = await insertTask({ ownerId: "pod-a", heartbeatAt: new Date("2020-01-01") }); - - expect(await repo.heartbeatTask({ taskId: mine.id, ownerId: "pod-a" })).toEqual([ - { id: mine.id }, - ]); - expect(await repo.heartbeatTask({ taskId: mine.id, ownerId: "pod-b" })).toEqual([]); - - const rows = await db.select().from(schema.tasks); - expect(rows.find((r) => r.id === mine.id)?.heartbeatAt?.getFullYear()).toBeGreaterThan(2020); - expect(rows.find((r) => r.id === other.id)?.heartbeatAt?.getFullYear()).toBe(2020); - }); - }); - describe("heartbeatOwnedBy / backdateHeartbeatsOwnedBy", () => { it("heartbeat updates only rows owned by us", async () => { const mine = await insertTask({ @@ -1082,7 +1066,8 @@ describe("TaskRepository", () => { heartbeatAt: new Date("2020-01-01"), }); - await repo.heartbeatOwnedBy("pod-a"); + // The renewed tasks: a task taken over is missing. + expect(await repo.heartbeatOwnedBy("pod-a")).toEqual([{ id: mine.id }]); const rows = await db.select().from(schema.tasks); expect(rows.find((r) => r.id === mine.id)?.heartbeatAt?.getFullYear()).toBeGreaterThan(2020); diff --git a/control-plane/src/repositories/inbox.repository.ts b/control-plane/src/repositories/inbox.repository.ts index da90822..7b67fdc 100644 --- a/control-plane/src/repositories/inbox.repository.ts +++ b/control-plane/src/repositories/inbox.repository.ts @@ -279,9 +279,7 @@ export class InboxRepository { return asRows(result).map(mapRow); } - // Every non-terminal row of a task (pending or delivered). failTask reads - // these before dead-lettering, then sends the failure message to the reply - // targets of the rows failTaskInbox transitioned. + // Every non-terminal row of a task (pending or delivered). async listNonTerminalByTask(taskId: string): Promise { const result = await db.execute(sql` SELECT * FROM inbox @@ -360,29 +358,33 @@ export class InboxRepository { // task-failure paths (provision failure, sandbox crash/timeout) call this so // a failed task cannot leave a row stuck pre-terminal — a non-terminal row // pins the idle reaper, and on a no-sandbox provision failure a 'pending' - // row would otherwise stay claimable forever. Returns the affected row ids: - // the caller logs them and sends the failure message only to those rows. + // row would otherwise stay claimable forever. Returns the rows it failed, in + // arrival order: the caller logs them and sends the failure message only to + // those rows. async failTaskInbox(params: { taskId: string; ownerId: string; reason: string; - }): Promise<{ id: string }[]> { + }): Promise { return db.transaction(async (tx) => { // Lock the task row first so task_events ids are handed out in commit // order per task for stable event cursors. Tasks before inbox, // the same order as restartTaskWithInbox. await tx.execute(sql`SELECT 1 FROM tasks WHERE id = ${params.taskId} FOR NO KEY UPDATE`); - const result = await tx.execute<{ id: string }>(sql` - UPDATE inbox i - SET status = 'failed', failure_reason = ${params.reason}, updated_at = now() - FROM tasks t - WHERE i.task_id = ${params.taskId} - AND i.task_id = t.id - AND t.owner_id = ${params.ownerId} - AND i.status IN ('pending', 'delivered') - RETURNING i.id; + const result = await tx.execute(sql` + WITH failed AS ( + UPDATE inbox i + SET status = 'failed', failure_reason = ${params.reason}, updated_at = now() + FROM tasks t + WHERE i.task_id = ${params.taskId} + AND i.task_id = t.id + AND t.owner_id = ${params.ownerId} + AND i.status IN ('pending', 'delivered') + RETURNING i.* + ) + SELECT * FROM failed ORDER BY created_at ASC; `); - const failed = asRows<{ id: string }>(result); + const failed = asRows(result).map(mapRow); // Write one round_failed event per failed row in the same transaction. for (const row of failed) { await insertCpEventIfApi(tx, { diff --git a/control-plane/src/repositories/task.repository.ts b/control-plane/src/repositories/task.repository.ts index 37c70fc..2050bf3 100644 --- a/control-plane/src/repositories/task.repository.ts +++ b/control-plane/src/repositories/task.repository.ts @@ -471,21 +471,12 @@ export class TaskRepository { } // As long as a pod owns a task it heartbeats; if it loses interest it calls - // releaseLease. - async heartbeatOwnedBy(ownerId: string): Promise { - await db - .update(schema.tasks) - .set({ heartbeatAt: sql`now()` }) - .where(eq(schema.tasks.ownerId, ownerId)); - } - - // One task's heartbeat, renewed right before its reconcile. Owner-guarded: - // 0 rows means the lease was taken over. - async heartbeatTask(params: { taskId: string; ownerId: string }): Promise<{ id: string }[]> { + // releaseLease. Returns the tasks renewed: one missing was taken over. + async heartbeatOwnedBy(ownerId: string): Promise<{ id: string }[]> { return db .update(schema.tasks) .set({ heartbeatAt: sql`now()` }) - .where(this.ownedByUs(params.taskId, params.ownerId)) + .where(eq(schema.tasks.ownerId, ownerId)) .returning({ id: schema.tasks.id }); } diff --git a/control-plane/src/services/__tests__/integrationHooks.test.ts b/control-plane/src/services/__tests__/integrationHooks.test.ts index 3fcc12b..5c5f28c 100644 --- a/control-plane/src/services/__tests__/integrationHooks.test.ts +++ b/control-plane/src/services/__tests__/integrationHooks.test.ts @@ -156,6 +156,34 @@ describe("IntegrationHooks", () => { expect(onTaskStarted).toHaveBeenCalledTimes(2); }); + it("drain also waits for work detached while it drains", async () => { + const warn = spyOn(logger, "warn").mockImplementation(() => logger); + try { + const { hooks } = hooksWith({}); + let finished = false; + hooks.detach( + "task-1", + "onRoundFinished", + sleep(10).then(() => + // A webhook still being served restarts a task meanwhile. + hooks.detach( + "task-2", + "killSandbox", + sleep(20).then(() => { + finished = true; + }), + ), + ), + ); + await hooks.drain(sleep(500)); + + expect(finished).toBe(true); + expect(warn).not.toHaveBeenCalled(); + } finally { + warn.mockRestore(); + } + }); + it("onRoundFinished routes by round.source; drain logs hook_abandoned for what is still running", async () => { const warn = spyOn(logger, "warn").mockImplementation(() => logger); try { diff --git a/control-plane/src/services/__tests__/taskProcessor.service.test.ts b/control-plane/src/services/__tests__/taskProcessor.service.test.ts index 70e95c2..72adf0d 100644 --- a/control-plane/src/services/__tests__/taskProcessor.service.test.ts +++ b/control-plane/src/services/__tests__/taskProcessor.service.test.ts @@ -157,7 +157,6 @@ const survivingNonTerminalRow = (taskId = "task-1", externalId = "issue-1"): any describe("TaskProcessorService", () => { let heartbeatOwnedBy: ReturnType; - let heartbeatTask: ReturnType; let claimNext: ReturnType; let attachSandboxResources: ReturnType; let backdateHeartbeatsOwnedBy: ReturnType; @@ -213,8 +212,8 @@ describe("TaskProcessorService", () => { postReviewCommentReply.mockReset(); postReviewCommentReply.mockResolvedValue({ id: 1, url: "u" }); - heartbeatOwnedBy = mock(() => Promise.resolve()); - heartbeatTask = mock(() => Promise.resolve([{ id: "task-1" }])); + // Renews every task the tests reconcile. + heartbeatOwnedBy = mock(() => Promise.resolve([{ id: "task-1" }, { id: "task-2" }])); claimNext = mock(() => Promise.resolve(null)); attachSandboxResources = mock(() => Promise.resolve([{ id: "task-1" }])); backdateHeartbeatsOwnedBy = mock(() => Promise.resolve()); @@ -228,7 +227,6 @@ describe("TaskProcessorService", () => { getByIdOwnedBy = mock((id: string) => Promise.resolve(makeTask({ id }))); taskRepo = { heartbeatOwnedBy, - heartbeatTask, claimNext, attachSandboxResources, backdateHeartbeatsOwnedBy, @@ -321,12 +319,11 @@ describe("TaskProcessorService", () => { ); }); - // failTask broadcasts to every non-terminal row's reply targets — stub one - // Linear session target so the user-facing failure message reaches emitError. + // failTask broadcasts to the reply targets of the rows failTaskInbox + // dead-letters — stub one with a Linear session target so the user-facing + // failure message reaches emitError. const stubLinearTarget = (task: Task): void => { - (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValueOnce([ - survivingNonTerminalRow(task.id, task.externalId), - ]); + failTaskInbox.mockResolvedValueOnce([survivingNonTerminalRow(task.id, task.externalId)]); }; describe("API tasks", () => { @@ -353,7 +350,7 @@ describe("TaskProcessorService", () => { it("failTask posts nothing to an api_session target and still dead-letters", async () => { const t = apiTask(); - (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValue([ + failTaskInbox.mockResolvedValue([ { ...survivingNonTerminalRow(t.id, t.externalId), source: "api", @@ -541,9 +538,9 @@ describe("TaskProcessorService", () => { createSandboxWithPvc.mockImplementation(() => Promise.reject(new Error("docker daemon gone with token=abcd")), ); - // failTask broadcasts the message to every non-terminal row's reply - // targets — stub one row with the assignment row's Linear session target. - (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValueOnce([ + // failTask broadcasts the message to the reply targets of the rows it + // dead-letters — stub one row with the assignment row's Linear session target. + failTaskInbox.mockResolvedValue([ { id: "inbox-1", task_id: task.id, @@ -559,7 +556,6 @@ describe("TaskProcessorService", () => { updated_at: new Date(), }, ]); - failTaskInbox.mockResolvedValue([{ id: "inbox-1" }]); service.start(); await waitFor(() => releaseLease.mock.calls.length > 0); @@ -580,10 +576,9 @@ describe("TaskProcessorService", () => { const task = makeTask({ sandboxId: null }); claimNext.mockResolvedValueOnce(task).mockResolvedValue(null); createSandboxWithPvc.mockImplementation(() => Promise.reject(new Error("provision boom"))); - stubLinearTarget(task); // failTaskInbox returning a non-empty set = rows actually dead-lettered on // THIS call; that transition is the gate for the (at-most-once) Slack post. - failTaskInbox.mockResolvedValueOnce([{ id: "inbox-1" }]); + stubLinearTarget(task); service.start(); await waitFor(() => notifyTerminal.mock.calls.length > 0); @@ -618,10 +613,7 @@ describe("TaskProcessorService", () => { created_at: new Date(), updated_at: new Date(), }; - (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValueOnce([ - commentRow, - ]); - failTaskInbox.mockResolvedValueOnce([{ id: "inbox-c" }]); + failTaskInbox.mockResolvedValueOnce([commentRow]); service.start(); await waitFor(() => releaseLease.mock.calls.length > 0); @@ -635,7 +627,6 @@ describe("TaskProcessorService", () => { const task = makeTask({ sandboxId: null }); claimNext.mockResolvedValueOnce(task).mockResolvedValue(null); createSandboxWithPvc.mockImplementation(() => Promise.reject(new Error("provision boom"))); - stubLinearTarget(task); // Default failTaskInbox returns [] → no transition → no Slack post. service.start(); await waitFor(() => releaseLease.mock.calls.length > 0); @@ -824,7 +815,7 @@ describe("TaskProcessorService", () => { }); it("posts once per distinct target id across every open row, by target kind", async () => { - (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValue([ + failTaskInbox.mockResolvedValue([ row("a", "assignment", "linear", [{ id: "sess-1", spec: { kind: "linear_session" } }]), // Same session as the assignment → deduped. row("lc", "comment", "linear", [{ id: "sess-1", spec: { kind: "linear_session" } }]), @@ -840,7 +831,6 @@ describe("TaskProcessorService", () => { // CI target: commentId 0 = "post a new PR comment". row("ci", "ci_result", "github", [issueComment("9001", 9, 0)]), ]); - failTaskInbox.mockResolvedValue(["a", "lc", "gc", "rv", "ci"].map((id) => ({ id }))); await failTask(makeTask()); @@ -864,12 +854,29 @@ describe("TaskProcessorService", () => { }); }); - it("dead-letters before posting, and posts only to the rows it dead-lettered", async () => { - (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValue([ + it("notices every row failTaskInbox dead-letters, whatever a separate read of the open rows saw", async () => { + // A follow-up committed after such a read, or the read failing, must not + // cost a dead-lettered row its notice, hook or Slack post. + (inboxRepo.listNonTerminalByTask as ReturnType).mockRejectedValue( + new Error("connection reset"), + ); + failTaskInbox.mockResolvedValue([ row("a", "assignment", "linear", [{ id: "sess-1", spec: { kind: "linear_session" } }]), - // Responded by a late `completed` before failTaskInbox ran: not transitioned. row("gc", "comment", "github", [issueComment("555", 7, 555)]), ]); + + await failTask(makeTask()); + + expect(emitError).toHaveBeenCalledWith(expect.anything(), "sess-1", MSG); + expect(postIssueComment).toHaveBeenCalledWith("o", "r", 7, MSG, expect.any(AbortSignal)); + expect(notifyTerminal).toHaveBeenCalledWith( + expect.objectContaining({ id: "task-1" }), + "failed", + MSG, + ); + }); + + it("dead-letters before posting, and posts only to the rows it dead-lettered", async () => { const order: string[] = []; emitError.mockImplementation(() => { order.push("post"); @@ -882,7 +889,11 @@ describe("TaskProcessorService", () => { }); failTaskInbox.mockImplementation(() => { order.push("mark"); - return Promise.resolve([{ id: "a" }]); + // The GitHub round was responded by a late `completed` before + // failTaskInbox ran: not transitioned, so not returned. + return Promise.resolve([ + row("a", "assignment", "linear", [{ id: "sess-1", spec: { kind: "linear_session" } }]), + ]); }); releaseLease.mockImplementation(() => { order.push("release"); @@ -897,11 +908,10 @@ describe("TaskProcessorService", () => { it("finishes a dead-lettered assignment's round: outcome failed, first transition, no archive", async () => { const onRoundFinished = spyOn(integrations.linear, "onRoundFinished"); - (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValue([ + failTaskInbox.mockResolvedValue([ row("a", "assignment", "linear", [{ id: "sess-1", spec: { kind: "linear_session" } }]), row("gc", "comment", "github", [issueComment("555", 7, 555)]), ]); - failTaskInbox.mockResolvedValue([{ id: "a" }, { id: "gc" }]); await failTask(makeTask()); await waitFor(() => onRoundFinished.mock.calls.length > 0); @@ -920,11 +930,10 @@ describe("TaskProcessorService", () => { }); it("a row whose source has no integration is skipped; other rows post and the lease is released", async () => { - (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValue([ + failTaskInbox.mockResolvedValue([ row("x", "comment", "unknown-source", [issueComment("777", 7, 777)]), row("gc", "comment", "github", [issueComment("555", 7, 555)]), ]); - failTaskInbox.mockResolvedValue([{ id: "x" }, { id: "gc" }]); await failTask(makeTask()); @@ -940,10 +949,9 @@ describe("TaskProcessorService", () => { it("a row whose integration is disabled gets no notice; it is still dead-lettered and the lease released", async () => { integrations.github = new GithubIntegration(false, { status: () => ({}) as never }); - (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValue([ + failTaskInbox.mockResolvedValue([ row("gc", "comment", "github", [issueComment("555", 7, 555)]), ]); - failTaskInbox.mockResolvedValue([{ id: "gc" }]); await failTask(makeTask()); @@ -963,11 +971,10 @@ describe("TaskProcessorService", () => { it("a Linear token failure skips only the Linear post; GitHub and Slack posts, dead-letter and release go on", async () => { getAuthorizedLinearClient.mockRejectedValue(new Error("token refresh failed")); - (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValue([ + failTaskInbox.mockResolvedValue([ row("a", "assignment", "linear", [{ id: "sess-1", spec: { kind: "linear_session" } }]), row("gc", "comment", "github", [issueComment("555", 7, 555)]), ]); - failTaskInbox.mockResolvedValue([{ id: "a" }, { id: "gc" }]); await failTask(makeTask()); @@ -1489,14 +1496,15 @@ describe("TaskProcessorService", () => { expect(holds.stats()).toEqual({}); }); - it("renews each task's lease right before its reconcile; a lost lease skips that task", async () => { + it("renews every lease this pod holds right before each reconcile; a lost lease skips that task", async () => { const t1 = makeTask({ id: "task-1", sandboxId: "sbx-1" }); const t2 = makeTask({ id: "task-2", sandboxId: "sbx-2" }); listInFlightOwnedBy.mockResolvedValueOnce([t1, t2]).mockResolvedValue([]); const order: string[] = []; - heartbeatTask.mockImplementation((p: { taskId: string }) => { - order.push(`heartbeat ${p.taskId}`); - return Promise.resolve(p.taskId === "task-2" ? [] : [{ id: p.taskId }]); + // task-2's lease was taken over. + heartbeatOwnedBy.mockImplementation(() => { + order.push("heartbeat"); + return Promise.resolve([{ id: "task-1" }, { id: "in-flight-provision" }]); }); getSandboxInfo.mockImplementation((id: string) => { order.push(`reconcile ${id}`); @@ -1504,12 +1512,65 @@ describe("TaskProcessorService", () => { }); service.start(); - await waitFor(() => order.length >= 3); + await waitFor(() => order.length >= 4); await new Promise((r) => setTimeout(r, 30)); await service.stop(); - expect(order).toEqual(["heartbeat task-1", "reconcile sbx-1", "heartbeat task-2"]); - expect(heartbeatTask).toHaveBeenCalledWith({ taskId: "task-1", ownerId: "pod-test" }); + // Tick start, then before each reconcile. + expect(order.slice(0, 4)).toEqual(["heartbeat", "heartbeat", "reconcile sbx-1", "heartbeat"]); + expect(order).not.toContain("reconcile sbx-2"); + expect(heartbeatOwnedBy).toHaveBeenCalledWith("pod-test"); + }); + + it("never provisions a task twice on this pod: a re-claim while its provision runs is skipped", async () => { + let release!: () => void; + createSandboxWithPvc.mockImplementation( + () => + new Promise((resolve) => { + release = () => resolve({ sandboxId: "sbx-1", pvcName: "devbox-pvc-task-1-aaaaaa" }); + }), + ); + // The lease of the running provision expired, so claimNext hands it back. + claimNext + .mockResolvedValueOnce(makeTask()) + .mockResolvedValueOnce(makeTask()) + .mockResolvedValue(null); + + service.start(); + await waitFor(() => claimNext.mock.calls.length >= 3); + release(); + await waitFor(() => attachSandboxResources.mock.calls.length > 0); + await service.stop(); + + expect(createSandboxWithPvc).toHaveBeenCalledTimes(1); + }); + + it("once shutdown starts, dispatches no further event of the task in hand", async () => { + const task = makeTask({ sandboxId: "sbx-1" }); + listInFlightOwnedBy.mockResolvedValue([task]); + const thought = (seq: number) => + ({ + seq, + ts: "t", + type: "agent_thought", + inbox_id: "inbox-1", + target_id: "sess-1", + body: "x", + }) as HarnessEvent; + readEvents.mockResolvedValue([thought(1), thought(2), thought(3)]); + let stopping: Promise | undefined; + dispatcherHandle.mockImplementation(() => { + stopping ??= service.stop(); + return Promise.resolve("ok" as const); + }); + + service.start(); + await waitFor(() => stopping !== undefined); + await stopping; + + // The event in hand finishes and its cursor is kept; the rest wait. + expect(dispatcherHandle).toHaveBeenCalledTimes(1); + expect(recordEventCursor).toHaveBeenCalledTimes(1); }); it("claims and lists only the configured task sources", async () => { @@ -1609,7 +1670,6 @@ describe("TaskProcessorService", () => { getSandboxPodStatus.mockResolvedValue({ exists: true, phase: "Running", oomKilled: false }); getSandboxInfo.mockResolvedValue(makeSandboxInfo({ metadata: { task_id: "other-task" } })); stubLinearTarget(task); - failTaskInbox.mockResolvedValue([{ id: "inbox-1" }]); service.start(); await waitFor(() => releaseLease.mock.calls.length > 0); @@ -1747,7 +1807,6 @@ describe("TaskProcessorService", () => { it("retries exhausted → dead-letters + fails with the give-up message", async () => { const task = makeTask({ sandboxId: "sbx-oom" }); stubLinearTarget(task); - failTaskInbox.mockResolvedValue([{ id: "inbox-1" }]); getSandboxPodStatus.mockResolvedValue({ exists: true, phase: "Failed", oomKilled: true }); // The count is durable in tasks.metadata; requeueForRetry persists it. Mirror // that on the shared task object so the next assess() reads the bumped count diff --git a/control-plane/src/services/integrationHooks.ts b/control-plane/src/services/integrationHooks.ts index 7f98571..8fb4eee 100644 --- a/control-plane/src/services/integrationHooks.ts +++ b/control-plane/src/services/integrationHooks.ts @@ -125,9 +125,16 @@ export class IntegrationHooks { } // Waits for detached work until `deadline` settles, then logs what is still - // running; the process exits right after. + // running; the process exits right after. Work detached meanwhile (say, a + // restart's sandbox kill from a webhook still being served) is waited for too. async drain(deadline: Promise): Promise { - await Promise.race([Promise.allSettled(this.detached.keys()), deadline]); + let expired = false; + void deadline.then(() => { + expired = true; + }); + while (this.detached.size > 0 && !expired) { + await Promise.race([Promise.allSettled(this.detached.keys()), deadline]); + } for (const { taskId, hook } of this.detached.values()) { logger.warn("hook abandoned at shutdown", { event: "hook_abandoned", taskId, hook }); } diff --git a/control-plane/src/services/taskProcessor.service.ts b/control-plane/src/services/taskProcessor.service.ts index 71a3f97..32f3079 100644 --- a/control-plane/src/services/taskProcessor.service.ts +++ b/control-plane/src/services/taskProcessor.service.ts @@ -6,7 +6,7 @@ import { PvcNotFoundError, SandboxCreateError } from "../errors"; import { SEND_TIMEOUT_MS, withTimeout } from "../integrations/http"; import type { Integration, IntegrationName } from "../integrations/types"; import { logger } from "../logger"; -import type { InboxRepository } from "../repositories/inbox.repository"; +import type { InboxRepository, InboxRow } from "../repositories/inbox.repository"; import type { TaskRepository } from "../repositories/task.repository"; import type { EventDispatcher } from "./eventDispatcher"; import { extractExperimentalFlags } from "./experimentalFlags"; @@ -25,7 +25,8 @@ export class TaskProcessorService { private running = false; private loopPromise: Promise | null = null; private sleepAbort: AbortController | null = null; - private readonly inflight = new Set>(); + // Provisions running on this pod, by task id. + private readonly inflight = new Map>(); // First tick (per task) we saw the backing pod ABSENT (kubectl NotFound). A // pod can be briefly gone between create and scheduling, so absence gets a // grace window (config.sandboxAbsentGraceMs) before we fail. Cleared the @@ -94,7 +95,7 @@ export class TaskProcessorService { // In-flight provisions, then detached hooks (some started by those // provisions), share one shutdownDrainMs budget. - const drain = Promise.allSettled([...this.inflight]); + const drain = Promise.allSettled([...this.inflight.values()]); const timeout = new Promise((resolve) => setTimeout(resolve, config.shutdownDrainMs)); await Promise.race([drain, timeout]); await this.hooks.drain(timeout); @@ -126,17 +127,19 @@ export class TaskProcessorService { // 2. Idle reaper — kill sandboxes idle past the ceiling, release leases. await this.reapIdle().catch((err) => logErr("reapIdle failed", err)); - // 3. Reconcile owned tasks with live sandboxes. Each task's lease is renewed - // right before its reconcile (and with each cursor advance inside it), so - // slow sends can't let it outlive the lease mid-tick; 0 rows means a - // sibling took it over. + // 3. Reconcile owned tasks with live sandboxes. Every lease this pod holds + // (in-flight provisions included) is renewed right before each reconcile, + // and the task's own with each cursor advance inside it, so slow sends + // can't let a lease expire mid-tick; a task not renewed was taken over. + // ponytail: one reconcile of many slow but successful sends (>120 s) can + // still outlive an in-flight provision's lease; a lease timer if it matters. const owned = await this.taskRepo.listInFlightOwnedBy(config.podId, this.taskSources); for (const task of owned) { // Shutting down: leave the rest to siblings (stop() backdates our // heartbeats), so shutdown waits for one task's reconcile at most (§10). if (!this.running) break; - const renewed = await this.taskRepo.heartbeatTask({ taskId: task.id, ownerId: config.podId }); - if (renewed.length === 0) { + const renewed = await this.taskRepo.heartbeatOwnedBy(config.podId); + if (!renewed.some((r) => r.id === task.id)) { logger.info("lease lost before reconcile; skipping", { taskId: task.id }); continue; } @@ -202,13 +205,21 @@ export class TaskProcessorService { } private dispatch(task: Task): void { + // A provision still running here whose lease expired can be claimed back; + // a second one would orphan a sandbox and a PVC. + if (this.inflight.has(task.id)) { + logger.warn("claimed a task whose provision is still running here; skipping", { + taskId: task.id, + }); + return; + } const p = this.runTask(task) .catch((err) => this.handleTaskError(task, err)) // Nothing awaits p, so a throw from handleTaskError itself (e.g. a DB // error releasing the lease) would be an unhandled rejection. Log it. .catch((err) => logErr(`handleTaskError failed for task ${task.id}`, err)) - .finally(() => this.inflight.delete(p)); - this.inflight.add(p); + .finally(() => this.inflight.delete(task.id)); + this.inflight.set(task.id, p); } // Creates or attaches a sandbox. Branches on `sandbox_id` nullability: @@ -402,6 +413,9 @@ export class TaskProcessorService { const events = await this.sandboxManager.readEvents(task.sandboxId, task.lastObservedEventSeq); for (const event of events) { if (currentTask === null) break; // self-evict: lease lost or task gone + // Shutting down: stop after the event in hand (the cursor is persisted + // per event), so shutdown waits for one event, not the backlog (§10). + if (!this.running) break; const result = await this.eventDispatcher.handle(event, currentTask); switch (result) { @@ -783,25 +797,18 @@ export class TaskProcessorService { opts: { killSandbox?: boolean } = {}, ): Promise { const { killSandbox = true } = opts; - const nonTerminal = await this.inboxRepo - .listNonTerminalByTask(task.id) - .catch((e): Awaited> => { - logErr(`failTask: listNonTerminalByTask failed for ${task.id}`, e); - return []; - }); // Dead-letter the inbox rows FIRST, and BEFORE releaseLease — failTaskInbox // is owner-guarded and needs owner_id still set. Only the rows it - // transitions get a failure message, so a racing restart or a late - // `completed` can't draw a "Task failed" on a live or finished round. + // transitions (as it returns them) get a failure message, so a racing + // restart or a late `completed` can't draw a "Task failed" on a live or + // finished round, and every row it fails gets one. const failed = await this.inboxRepo .failTaskInbox({ taskId: task.id, ownerId: config.podId, reason: message }) - .catch((e) => { + .catch((e): InboxRow[] => { logErr(`failTaskInbox failed for task ${task.id}`, e); - return [] as { id: string }[]; + return []; }); - const failedIds = new Set(failed.map((r) => r.id)); - const failedRows = nonTerminal.filter((r) => failedIds.has(r.id)); - const assignment = failedRows.find((r) => r.kind === "assignment"); + const assignment = failed.find((r) => r.kind === "assignment"); if (failed.length > 0) { logger.warn("inbox rows dead-lettered on task failure", { taskId: task.id, @@ -815,7 +822,7 @@ export class TaskProcessorService { // seeing the same error N times. At most once: there is no cursor to hold, // so a delivery error (RetryableError included) is logged and dropped. const seen = new Set(); - for (const row of failedRows) { + for (const row of failed) { // A source this build has no integration for (say, rows written by a // newer build before a rollback), or a disabled one, gets no notice; its // row is still failed. diff --git a/deploy/helm/devbox/README.md b/deploy/helm/devbox/README.md index e386ef1..eab9b4d 100644 --- a/deploy/helm/devbox/README.md +++ b/deploy/helm/devbox/README.md @@ -25,7 +25,7 @@ docker run --rm -v "$PWD/deploy/helm/devbox:/helm-docs" jnorwood/helm-docs:v1.14 | controlPlane.resources.limits.memory | string | `"1Gi"` | | | controlPlane.resources.requests.cpu | string | `"100m"` | | | controlPlane.resources.requests.memory | string | `"256Mi"` | | -| controlPlane.terminationGracePeriodSeconds | int | `150` | Seconds between SIGTERM and SIGKILL. At least one task's reconcile (up to 110s when platforms are slow, plus 50s per extra PR opened in that reconcile) plus SHUTDOWN_DRAIN_MS (10s) for in-flight provisions and integration hooks; raise it with SHUTDOWN_DRAIN_MS. | +| controlPlane.terminationGracePeriodSeconds | int | `150` | Seconds between SIGTERM and SIGKILL. At least the rest of one task's reconcile (up to 90s when platforms are slow: shutdown stops after the event in hand) plus SHUTDOWN_DRAIN_MS (10s) for in-flight provisions and integration hooks; raise it with SHUTDOWN_DRAIN_MS. | | database.existingSecret | string | `""` | With postgresql.enabled false: a Secret with key `DATABASE_URL`. | | extraEnv | list | `[]` | Extra control-plane env. Only explicitly allowlisted HARNESS_* settings reach sandboxes; never put a real secret in a forwarded name. | | extraManifests | list | `[]` | Extra manifests, rendered with tpl. | diff --git a/deploy/helm/devbox/values.yaml b/deploy/helm/devbox/values.yaml index 8252398..745ce4f 100644 --- a/deploy/helm/devbox/values.yaml +++ b/deploy/helm/devbox/values.yaml @@ -151,10 +151,10 @@ controlPlane: resources: requests: { cpu: 100m, memory: 256Mi } limits: { memory: 1Gi } - # -- Seconds between SIGTERM and SIGKILL. At least one task's reconcile (up to - # 110s when platforms are slow, plus 50s per extra PR opened in that reconcile) - # plus SHUTDOWN_DRAIN_MS (10s) for in-flight provisions and integration hooks; - # raise it with SHUTDOWN_DRAIN_MS. + # -- Seconds between SIGTERM and SIGKILL. At least the rest of one task's + # reconcile (up to 90s when platforms are slow: shutdown stops after the event + # in hand) plus SHUTDOWN_DRAIN_MS (10s) for in-flight provisions and + # integration hooks; raise it with SHUTDOWN_DRAIN_MS. terminationGracePeriodSeconds: 150 postgresql: diff --git a/docs/kubernetes/README.md b/docs/kubernetes/README.md index c4ff54c..1196c04 100644 --- a/docs/kubernetes/README.md +++ b/docs/kubernetes/README.md @@ -454,12 +454,11 @@ While old and new pods overlap, and after a rollback to a release without the `inbox_source_kind_external_uniq` index, an old pod can answer a webhook that races its own redelivery with a 500: `duplicate key value violates unique constraint "inbox_source_kind_external_uniq"`. The other request stored the -round and the platform retries, so nothing is lost. After such a rollback, -dropping the index stops the errors: - -```sql -DROP INDEX inbox_source_kind_external_uniq; -``` +round and the platform retries, so nothing is lost. Leave the index in place +after a rollback: the database has recorded its migration, so a dropped index +is not rebuilt on the next upgrade, and the new release then fails to store +any follow-up (a comment, review, CI result or API message) with `there is no +unique or exclusion constraint matching the ON CONFLICT specification`. ## Uninstall From 757cb73daf4b9d4395db5dc6455398adea6b6b47 Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Fri, 30 Oct 2026 21:37:21 +0530 Subject: [PATCH 18/22] refactor(control-plane): keep integration specifics out of core Linear's issue-state change and Provisioning action move into its onTaskStarted hook, GitHub's deliver narrows to its own target kinds, one helper delivers for the dispatcher and both failure paths, failure notices dedupe per source and target, Slack calls get the send timeout, and tests share one fake-integration fixture, so a new integration needs no core edits. Co-Authored-By: Claude Opus 5.5 --- control-plane/src/app.ts | 1 - .../__tests__/api.controller.stream.test.ts | 5 +- .../__tests__/api.controller.test.ts | 5 +- .../__tests__/fakeIntegrations.ts | 23 +++++ .../src/integrations/github/index.ts | 11 ++- control-plane/src/integrations/http.ts | 38 +++++++- .../src/integrations/linear/index.ts | 20 ++++ .../src/repositories/inbox.repository.ts | 19 ---- .../__tests__/eventDispatcher.test.ts | 20 ++-- .../services/__tests__/inbox.service.test.ts | 5 +- .../__tests__/integrationHooks.test.ts | 6 +- .../__tests__/slackNotifier.service.test.ts | 8 +- .../__tests__/taskProcessor.service.test.ts | 27 ++++-- control-plane/src/services/eventDispatcher.ts | 33 +++---- control-plane/src/services/inbox.service.ts | 32 ++----- .../src/services/slackNotifier.service.ts | 12 ++- .../src/services/taskProcessor.service.ts | 95 ++++++------------- 17 files changed, 200 insertions(+), 160 deletions(-) create mode 100644 control-plane/src/integrations/__tests__/fakeIntegrations.ts diff --git a/control-plane/src/app.ts b/control-plane/src/app.ts index 84d693e..82939f0 100644 --- a/control-plane/src/app.ts +++ b/control-plane/src/app.ts @@ -153,7 +153,6 @@ export function createApp(): AppBundle { const taskProcessor = new TaskProcessorService( taskRepository, inboxRepository, - linearTokenProvider, sandboxManager, eventDispatcher, inboxService, diff --git a/control-plane/src/controllers/__tests__/api.controller.stream.test.ts b/control-plane/src/controllers/__tests__/api.controller.stream.test.ts index 0d807bc..9ef1afe 100644 --- a/control-plane/src/controllers/__tests__/api.controller.stream.test.ts +++ b/control-plane/src/controllers/__tests__/api.controller.stream.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, mock, spyOn } from "bun:test"; import type { Clock } from "../../clock"; import { ApiError } from "../../errors"; +import { fakeIntegrations } from "../../integrations/__tests__/fakeIntegrations"; import { logger } from "../../logger"; import { createApiRoutes } from "../../routes/api"; import type { ApiEventEnvelope, SseStateData } from "../../types/api"; @@ -123,7 +124,7 @@ function setup(o: { { authEnabled: false, allowedHosts: [], - integrations: { + integrations: fakeIntegrations({ linear: { status: async () => ({ configured: false }) }, github: { status: async () => ({ @@ -132,7 +133,7 @@ function setup(o: { }), }, api: { status: async () => ({ enabled: true }) }, - }, + }), }, ); return { app, service, controller, clock }; diff --git a/control-plane/src/controllers/__tests__/api.controller.test.ts b/control-plane/src/controllers/__tests__/api.controller.test.ts index bc69c36..0db2bb2 100644 --- a/control-plane/src/controllers/__tests__/api.controller.test.ts +++ b/control-plane/src/controllers/__tests__/api.controller.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, mock, spyOn } from "bun:test"; import { ApiError } from "../../errors"; +import { fakeIntegrations } from "../../integrations/__tests__/fakeIntegrations"; import { logger } from "../../logger"; import { createApiRoutes } from "../../routes/api"; import type { ApiTask } from "../../types/api"; @@ -57,11 +58,11 @@ function setup(opts: { authEnabled: boolean; allowedHosts?: string[] }) { const app = createApiRoutes(controller, auth, { authEnabled: opts.authEnabled, allowedHosts: opts.allowedHosts ?? [], - integrations: { + integrations: fakeIntegrations({ linear: { status: async () => LINEAR_STATUS }, github: { status: async () => GITHUB_STATUS }, api: { status: async () => ({ enabled: true }) }, - }, + }), }); const req = (path: string, init: RequestInit & { json?: unknown } = {}) => app.request(path, { diff --git a/control-plane/src/integrations/__tests__/fakeIntegrations.ts b/control-plane/src/integrations/__tests__/fakeIntegrations.ts new file mode 100644 index 0000000..42d5589 --- /dev/null +++ b/control-plane/src/integrations/__tests__/fakeIntegrations.ts @@ -0,0 +1,23 @@ +import type { Integration, IntegrationName } from "../types"; + +const noop = (name: IntegrationName): Integration => ({ + name, + enabled: true, + async deliver() {}, + async status() { + return {}; + }, +}); + +// A registry for tests: every integration a no-op (enabled, delivers nothing) +// unless given. A new integration gets its default here, not in every test. +export function fakeIntegrations< + O extends Partial>>, +>(overrides: O): Omit, keyof O> & O { + const defaults: Record = { + linear: noop("linear"), + api: noop("api"), + github: noop("github"), + }; + return { ...defaults, ...overrides }; +} diff --git a/control-plane/src/integrations/github/index.ts b/control-plane/src/integrations/github/index.ts index 2c74b07..059185c 100644 --- a/control-plane/src/integrations/github/index.ts +++ b/control-plane/src/integrations/github/index.ts @@ -50,8 +50,13 @@ export class GithubIntegration implements Integration { } async deliver(ctx: SendCtx, target: ReplyTarget, message: Outbound): Promise { + // Its own three kinds only (post() switches over them exhaustively). const spec = target.spec; - if (spec.kind === "linear_session" || spec.kind === "api_session") { + if ( + spec.kind !== "issue_comment" && + spec.kind !== "review_comment" && + spec.kind !== "review_body" + ) { throw new Error(`github cannot deliver to a ${spec.kind} target`); } switch (message.type) { @@ -63,6 +68,10 @@ export class GithubIntegration implements Integration { case "failure": await deliverFailure(ctx, spec, message); return; + default: { + const _exhaustive: never = message; + void _exhaustive; + } } } diff --git a/control-plane/src/integrations/http.ts b/control-plane/src/integrations/http.ts index f1bdd29..efb69af 100644 --- a/control-plane/src/integrations/http.ts +++ b/control-plane/src/integrations/http.ts @@ -1,4 +1,7 @@ -import { RetryableError } from "./outbound"; +import { logger } from "../logger"; +import type { InboxRow, ReplyTarget } from "../repositories/inbox.repository"; +import { type Outbound, RetryableError } from "./outbound"; +import type { Integration, IntegrationName } from "./types"; // Shared rules for integrations' outbound HTTP (I16): the typed errors clients // throw, which failures may be retried, and how long one send may take. @@ -98,3 +101,36 @@ export async function withTimeout( clearTimeout(timer); } } + +// Hands `message` for one of `round`'s targets to the round's integration +// (§4), under the send timeout. An integration this build doesn't know (rows +// written by a newer build before a rollback) or a disabled one is skipped and +// logged: that counts as sent. Any other failure is logged and the message +// dropped, except a RetryableError (nothing was sent), which is rethrown: each +// caller decides whether it holds or drops. +export async function deliverToRound( + integrations: Record, + round: InboxRow, + target: ReplyTarget, + message: Outbound, + meta: Record, + timeoutMs = SEND_TIMEOUT_MS, +): Promise<"ok" | "dropped"> { + const log = { ...meta, inboxId: round.id, source: round.source, targetId: target.id }; + const integration: Integration | undefined = integrations[round.source]; + if (!integration?.enabled) { + logger.info(`${message.type} skipped: integration unknown or disabled`, log); + return "ok"; + } + try { + await withTimeout(timeoutMs, (signal) => + integration.deliver({ signal, log: logger }, target, message), + ); + return "ok"; + } catch (err) { + if (err instanceof RetryableError) throw err; + const e = err instanceof Error ? err : new Error(String(err)); + logger.error(`deliver ${message.type} failed`, { ...log, message: e.message, stack: e.stack }); + return "dropped"; + } +} diff --git a/control-plane/src/integrations/linear/index.ts b/control-plane/src/integrations/linear/index.ts index 706f001..f70b3d4 100644 --- a/control-plane/src/integrations/linear/index.ts +++ b/control-plane/src/integrations/linear/index.ts @@ -8,6 +8,7 @@ import { emitResponse, emitThought, getIssueHumanAssignee, + setIssueStarted, uploadIssueAttachment, } from "../../clients/linear"; import type { LinearWebhookController } from "../../controllers/linearWebhook.controller"; @@ -79,12 +80,31 @@ export class LinearIntegration implements Integration { if (text) await emitError(client, target.id, text); return; } + default: { + const _exhaustive: never = message; + void _exhaustive; + } } } catch (err) { throw linearRetryable(err) ?? err; } } + // A fresh provision: moves the issue to its team's started state, then shows + // "Provisioning" on the session. Each is best-effort; a team without a + // started state still gets the action. + async onTaskStarted(ctx: HookCtx, task: Task, target: ReplyTarget): Promise { + const client = await this.client(ctx.signal); + await setIssueStarted(client, task.externalId).catch((err: unknown) => { + const e = err instanceof Error ? err : new Error(String(err)); + ctx.log.error(`setIssueStarted failed for task ${task.id}`, { + message: e.message, + stack: e.stack, + }); + }); + await emitAction(client, target.id, "Provisioning", "infra"); + } + // Links the PR on the issue. Idempotent (Linear updates an existing link), // so it runs on every pr_opened dispatch; a RetryableError holds the event. async onPrOpened(ctx: HookCtx, task: Task, pr: PrRef): Promise { diff --git a/control-plane/src/repositories/inbox.repository.ts b/control-plane/src/repositories/inbox.repository.ts index 7b67fdc..120816c 100644 --- a/control-plane/src/repositories/inbox.repository.ts +++ b/control-plane/src/repositories/inbox.repository.ts @@ -319,25 +319,6 @@ export class InboxRepository { return row ? mapRow(row) : null; } - // Reply target id for task-scoped CP-originated emits (provisioning - // lifecycle). Reads the first entry of the assignment row's reply_targets — - // the canonical store, written on `created` and overwritten on - // re-delegation. An assignment row carries exactly one reply target by - // construction. Returns null when the assignment row is missing or its - // reply_targets is empty. - async getAssignmentReplyTargetId(taskId: string): Promise { - const result = await db.execute<{ target_id: string | null }>(sql` - SELECT reply_targets->0->>'id' AS target_id - FROM inbox - WHERE task_id = ${taskId} AND kind = 'assignment' - LIMIT 1; - `); - const row = Array.isArray(result) - ? (result[0] as { target_id: string | null } | undefined) - : undefined; - return row?.target_id ?? null; - } - // pending → delivered. Owner-guarded (a stale-lease pod must not advance a // row it no longer owns) and status-guarded (a re-sweep can't regress a row // already past 'pending'). diff --git a/control-plane/src/services/__tests__/eventDispatcher.test.ts b/control-plane/src/services/__tests__/eventDispatcher.test.ts index 2636dda..dbd6572 100644 --- a/control-plane/src/services/__tests__/eventDispatcher.test.ts +++ b/control-plane/src/services/__tests__/eventDispatcher.test.ts @@ -4,6 +4,7 @@ import { type LinearClient, LinearError, RatelimitedLinearError } from "@linear/ // mock.module barrel factories below, and it pins the real contract. import type { LinearHumanUser } from "../../clients/linearApi"; import type { Task } from "../../db/schema"; +import { fakeIntegrations } from "../../integrations/__tests__/fakeIntegrations"; import type { InboxRepository, InboxRow, InboxStatus } from "../../repositories/inbox.repository"; import type { TaskRepository } from "../../repositories/task.repository"; import type { HarnessEvent } from "../../types/harnessEvent"; @@ -162,13 +163,14 @@ describe("EventDispatcher", () => { let dispatcher: InstanceType; // The registry, with GitHub's webhooks (and so its integration) on or off. - const makeIntegrations = (githubEnabled: boolean) => ({ - linear: new LinearIntegration(linearTokenProvider, true, { - status: async () => ({ configured: false }), - }), - api: new ApiIntegration(true), - github: new GithubIntegration(githubEnabled, { status: () => ({}) as never }), - }); + const makeIntegrations = (githubEnabled: boolean) => + fakeIntegrations({ + linear: new LinearIntegration(linearTokenProvider, true, { + status: async () => ({ configured: false }), + }), + api: new ApiIntegration(true), + github: new GithubIntegration(githubEnabled, { status: () => ({}) as never }), + }); beforeEach(() => { for (const m of [ @@ -603,11 +605,11 @@ describe("EventDispatcher", () => { delivered.push([name, target, message]); }, }); - const integrations = { + const integrations = fakeIntegrations({ linear: record("linear"), api: record("api"), github: record("github"), - }; + }); const d = new EventDispatcher( taskRepo, inboxRepo, diff --git a/control-plane/src/services/__tests__/inbox.service.test.ts b/control-plane/src/services/__tests__/inbox.service.test.ts index d582021..9526ae4 100644 --- a/control-plane/src/services/__tests__/inbox.service.test.ts +++ b/control-plane/src/services/__tests__/inbox.service.test.ts @@ -1,5 +1,6 @@ import { beforeEach, describe, expect, it, mock, spyOn } from "bun:test"; import type { Task } from "../../db/schema"; +import { fakeIntegrations } from "../../integrations/__tests__/fakeIntegrations"; import type { InboxRepository, InboxRow } from "../../repositories/inbox.repository"; import type { LinearTokenProvider } from "../linearTokenProvider"; import type { SandboxManager } from "../sandboxManager.service"; @@ -179,11 +180,11 @@ describe("InboxService", () => { linear = new LinearIntegration(linearTokenProvider, true, { status: async () => ({ configured: false }), }); - const integrations = { + const integrations = fakeIntegrations({ linear, api: new ApiIntegration(true), github: new GithubIntegration(true, { status: () => ({}) as never }), - }; + }); service = new InboxService( inboxRepo, sandboxManager, diff --git a/control-plane/src/services/__tests__/integrationHooks.test.ts b/control-plane/src/services/__tests__/integrationHooks.test.ts index 5c5f28c..e91d622 100644 --- a/control-plane/src/services/__tests__/integrationHooks.test.ts +++ b/control-plane/src/services/__tests__/integrationHooks.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, mock, spyOn } from "bun:test"; import type { Task } from "../../db/schema"; +import { fakeIntegrations } from "../../integrations/__tests__/fakeIntegrations"; import type { HookCtx, Integration } from "../../integrations/types"; import type { InboxRepository, InboxRow } from "../../repositories/inbox.repository"; @@ -24,11 +25,10 @@ function hooksWith( timeoutMs?: number, ) { const inboxRepo = { getAssignment: mock(() => Promise.resolve(row("a-1"))) }; - const integrations = { + const integrations = fakeIntegrations({ linear: { ...noop, name: "linear" as const, ...linear }, - api: noop, github: { ...noop, name: "github" as const, ...github }, - }; + }); const t = timeoutMs; const timeouts = t === undefined diff --git a/control-plane/src/services/__tests__/slackNotifier.service.test.ts b/control-plane/src/services/__tests__/slackNotifier.service.test.ts index 5dcd44b..2a228da 100644 --- a/control-plane/src/services/__tests__/slackNotifier.service.test.ts +++ b/control-plane/src/services/__tests__/slackNotifier.service.test.ts @@ -67,6 +67,12 @@ describe("SlackNotifierService", () => { taskRepo = { getById } as unknown as TaskRepository; }); + it("bounds the post with an abort signal", async () => { + await make().notifyTerminal(makeTask(), "completed"); + const arg = (post.mock.calls[0] as unknown as [{ signal?: unknown }])[0]; + expect(arg.signal).toBeInstanceOf(AbortSignal); + }); + it("posts a ✅ completion into the configured channel with the ticket + PR links", async () => { await make().notifyTerminal(makeTask(), "completed"); @@ -246,7 +252,7 @@ describe("SlackNotifierService", () => { assignee.mockResolvedValue(unmapped); lookupSlackUser.mockResolvedValue("U0ADA"); await make().notifyTerminal(makeTask(), "completed"); - expect(lookupSlackUser).toHaveBeenCalledWith("ada@acme.com"); + expect(lookupSlackUser).toHaveBeenCalledWith("ada@acme.com", expect.any(AbortSignal)); const text = postedText(); expect(text).toContain("⚠️ <@U0ADA>: I couldn't assign the PR"); expect(text).not.toContain("⚠️ ada:"); diff --git a/control-plane/src/services/__tests__/taskProcessor.service.test.ts b/control-plane/src/services/__tests__/taskProcessor.service.test.ts index 72adf0d..8a1c359 100644 --- a/control-plane/src/services/__tests__/taskProcessor.service.test.ts +++ b/control-plane/src/services/__tests__/taskProcessor.service.test.ts @@ -4,7 +4,8 @@ import { SandboxApiException } from "@alibaba-group/opensandbox"; import type { LinearClient } from "@linear/sdk"; import type { Task } from "../../db/schema"; import { PvcNotFoundError, SandboxCreateError } from "../../errors"; -import type { Integration } from "../../integrations/types"; +import { fakeIntegrations } from "../../integrations/__tests__/fakeIntegrations"; +import type { Integration, IntegrationName } from "../../integrations/types"; import type { InboxRepository } from "../../repositories/inbox.repository"; import type { TaskRepository } from "../../repositories/task.repository"; import type { HarnessEvent } from "../../types/harnessEvent"; @@ -190,10 +191,8 @@ describe("TaskProcessorService", () => { let notifyTerminal: ReturnType; let slackNotifier: SlackNotifierService; let holds: InstanceType; - let integrations: { + let integrations: Record & { linear: InstanceType; - api: Integration; - github: Integration; }; let hooks: InstanceType; let service: InstanceType; @@ -243,7 +242,7 @@ describe("TaskProcessorService", () => { failTaskInbox = mock(() => Promise.resolve([] as Array<{ id: string }>)); inboxRepo = { failTaskInbox, - getAssignmentReplyTargetId: mock(() => Promise.resolve("sess-1")), + getAssignment: mock(() => Promise.resolve(survivingNonTerminalRow())), // Default: one SURVIVING non-terminal row (with a Linear session reply // target). retryTask now treats "no surviving non-terminal row" as a clean // terminal (no requeue), so the retry-path tests need work left to resume; @@ -296,18 +295,17 @@ describe("TaskProcessorService", () => { } as unknown as SlackNotifierService; holds = new ReplyHolds(); - integrations = { + integrations = fakeIntegrations({ linear: new LinearIntegration(linearTokenProvider, true, { status: async () => ({ configured: false }), }), api: new ApiIntegration(true), github: new GithubIntegration(true, { status: () => ({}) as never }), - }; + }); hooks = new IntegrationHooks(integrations, inboxRepo); service = new TaskProcessorService( taskRepo, inboxRepo, - linearTokenProvider, sandboxManager, eventDispatcher, inboxService, @@ -854,6 +852,18 @@ describe("TaskProcessorService", () => { }); }); + it("dedupes per integration: equal target ids of two sources each get a notice", async () => { + failTaskInbox.mockResolvedValue([ + row("a", "assignment", "linear", [{ id: "555", spec: { kind: "linear_session" } }]), + row("gc", "comment", "github", [issueComment("555", 7, 555)]), + ]); + + await failTask(makeTask()); + + expect(emitError).toHaveBeenCalledWith(expect.anything(), "555", MSG); + expect(postIssueComment).toHaveBeenCalledWith("o", "r", 7, MSG, expect.any(AbortSignal)); + }); + it("notices every row failTaskInbox dead-letters, whatever a separate read of the open rows saw", async () => { // A follow-up committed after such a read, or the read failing, must not // cost a dead-lettered row its notice, hook or Slack post. @@ -1338,7 +1348,6 @@ describe("TaskProcessorService", () => { service = new TaskProcessorService( taskRepo, inboxRepo, - linearTokenProvider, sandboxManager, dispatcher, inboxService, diff --git a/control-plane/src/services/eventDispatcher.ts b/control-plane/src/services/eventDispatcher.ts index 1af7ced..1df0aa0 100644 --- a/control-plane/src/services/eventDispatcher.ts +++ b/control-plane/src/services/eventDispatcher.ts @@ -1,7 +1,7 @@ import { addPrAssignees, parsePrUrl, postIssueComment } from "../clients/github"; import { config } from "../config"; import type { Task } from "../db/schema"; -import { SEND_TIMEOUT_MS, withTimeout } from "../integrations/http"; +import { deliverToRound, SEND_TIMEOUT_MS, withTimeout } from "../integrations/http"; import { type Outbound, RetryableError } from "../integrations/outbound"; import type { Integration, IntegrationName } from "../integrations/types"; import { logger } from "../logger"; @@ -244,29 +244,20 @@ export class EventDispatcher { return "ok"; // cursor still advances — drop } - // A source this build has no integration for (rows written by a newer - // build before a rollback) is skipped like a disabled one. - const integration: Integration | undefined = this.integrations[inboxRow.source]; - if (!integration?.enabled) { - logger.info(`${event.type} skipped: integration unknown or disabled`, { - taskId: task.id, - inboxId: inboxRow.id, - source: inboxRow.source, - }); - return "ok"; - } try { - await withTimeout(this.sendTimeoutMs, (signal) => - integration.deliver({ signal, log: logger }, target, labelAgentEmit(event, inboxRow)), + return await deliverToRound( + this.integrations, + inboxRow, + target, + labelAgentEmit(event, inboxRow), + { taskId: task.id, eventType: event.type }, + this.sendTimeoutMs, ); - return "ok"; } catch (err) { - if (err instanceof RetryableError) { - this.holds.hold(task.id, event, inboxRow.source, err); - return "hold"; - } - logErr(`deliver ${event.type} failed for target ${target.id}`, err); - return "dropped"; + // Nothing was sent: the event holds on the round's integration. + if (!(err instanceof RetryableError)) throw err; + this.holds.hold(task.id, event, inboxRow.source, err); + return "hold"; } } diff --git a/control-plane/src/services/inbox.service.ts b/control-plane/src/services/inbox.service.ts index eb91d45..8f7e9c9 100644 --- a/control-plane/src/services/inbox.service.ts +++ b/control-plane/src/services/inbox.service.ts @@ -1,6 +1,6 @@ import { config } from "../config"; import type { Task } from "../db/schema"; -import { SEND_TIMEOUT_MS, withTimeout } from "../integrations/http"; +import { deliverToRound } from "../integrations/http"; import type { TaskIntake } from "../integrations/intake"; import type { Integration, IntegrationName } from "../integrations/types"; import { logger } from "../logger"; @@ -306,27 +306,15 @@ export class InboxService { if (entry.kind === "assignment") this.hooks.roundFinished(task, entry, "failed", true); const target = entry.reply_targets[0]; if (!target) return; - // A source this build has no integration for (say, rows written by a newer - // build before a rollback), or a disabled one, gets no notice; its row is - // still failed. - const integration: Integration | undefined = this.integrations[entry.source]; - if (!integration?.enabled) { - logger.warn( - "workspace_missing: the row's integration is unknown or disabled; notice skipped", - { - inboxId: entry.id, - source: entry.source, - }, - ); - return; - } - await withTimeout(SEND_TIMEOUT_MS, (signal) => - integration.deliver({ signal, log: logger }, target, { - type: "failure", - cause: "workspace_missing", - round: entry, - }), - ).catch((err) => logErr(`workspace_missing notice failed for inbox ${entry.id}`, err)); + // A row whose integration is unknown or disabled gets no notice; it is + // still failed. No cursor to hold, so a RetryableError is dropped too. + await deliverToRound( + this.integrations, + entry, + target, + { type: "failure", cause: "workspace_missing", round: entry }, + { taskId: task.id }, + ).catch((err) => logErr(`workspace_missing notice for inbox ${entry.id} dropped`, err)); } } diff --git a/control-plane/src/services/slackNotifier.service.ts b/control-plane/src/services/slackNotifier.service.ts index 19d242d..f7cfe46 100644 --- a/control-plane/src/services/slackNotifier.service.ts +++ b/control-plane/src/services/slackNotifier.service.ts @@ -1,5 +1,6 @@ import { lookupSlackUserIdByEmail, postSlackMessage } from "../clients/slack"; import type { Task } from "../db/schema"; +import { SEND_TIMEOUT_MS, withTimeout } from "../integrations/http"; import type { TaskSource } from "../integrations/types"; import type { TaskRepository } from "../repositories/task.repository"; import type { IntegrationHooks } from "./integrationHooks"; @@ -65,7 +66,9 @@ export class SlackNotifierService { const connectNote = wantsNudge ? await this.connectGitHubNote(t) : ""; const text = buildMessage(status, t, prUrls, connectNote, message); - await this.post({ channel: this.channel, text }); + await withTimeout(SEND_TIMEOUT_MS, (signal) => + this.post({ channel: this.channel, text, signal }), + ); } // "" when there is nothing to nag about: no human assignee, already mapped, @@ -81,8 +84,11 @@ export class SlackNotifierService { // bot missing the users:read.email scope, transport error). displayName // is member-editable — escape it so it can't inject mrkdwn (, // forged <@mentions>, disguised links) into a trusted bot message. - const slackId = person.email - ? await this.lookupSlackUser(person.email).catch(() => null) + const email = person.email; + const slackId = email + ? await withTimeout(SEND_TIMEOUT_MS, (signal) => this.lookupSlackUser(email, signal)).catch( + () => null, + ) : null; const mention = slackId ? `<@${slackId}>` : escapeMrkdwn(person.displayName); return ( diff --git a/control-plane/src/services/taskProcessor.service.ts b/control-plane/src/services/taskProcessor.service.ts index 32f3079..eac5643 100644 --- a/control-plane/src/services/taskProcessor.service.ts +++ b/control-plane/src/services/taskProcessor.service.ts @@ -1,9 +1,9 @@ import type { SandboxInfo } from "@alibaba-group/opensandbox"; -import { emitAction, setIssueStarted } from "../clients/linear"; import { config } from "../config"; import type { Task } from "../db/schema"; import { PvcNotFoundError, SandboxCreateError } from "../errors"; -import { SEND_TIMEOUT_MS, withTimeout } from "../integrations/http"; +import { deliverToRound } from "../integrations/http"; +import type { Outbound } from "../integrations/outbound"; import type { Integration, IntegrationName } from "../integrations/types"; import { logger } from "../logger"; import type { InboxRepository, InboxRow } from "../repositories/inbox.repository"; @@ -12,7 +12,6 @@ import type { EventDispatcher } from "./eventDispatcher"; import { extractExperimentalFlags } from "./experimentalFlags"; import type { InboxService } from "./inbox.service"; import type { IntegrationHooks } from "./integrationHooks"; -import type { LinearTokenProvider } from "./linearTokenProvider"; import type { ReplyHolds } from "./replyHolds"; import { type PodStatus, SandboxManager } from "./sandboxManager.service"; import type { SlackNotifierService } from "./slackNotifier.service"; @@ -40,7 +39,6 @@ export class TaskProcessorService { private readonly reconcileFilesUnavailableSince = new Map(); private readonly taskRepo: TaskRepository; private readonly inboxRepo: InboxRepository; - private readonly linearTokenProvider: Pick; private readonly sandboxManager: SandboxManager; private readonly eventDispatcher: EventDispatcher; private readonly inboxService: InboxService; @@ -55,7 +53,6 @@ export class TaskProcessorService { constructor( taskRepo: TaskRepository, inboxRepo: InboxRepository, - linearTokenProvider: Pick, sandboxManager: SandboxManager, eventDispatcher: EventDispatcher, inboxService: InboxService, @@ -67,7 +64,6 @@ export class TaskProcessorService { ) { this.taskRepo = taskRepo; this.inboxRepo = inboxRepo; - this.linearTokenProvider = linearTokenProvider; this.sandboxManager = sandboxManager; this.eventDispatcher = eventDispatcher; this.inboxService = inboxService; @@ -224,8 +220,6 @@ export class TaskProcessorService { // Creates or attaches a sandbox. Branches on `sandbox_id` nullability: // (sandbox_id=null) is a fresh provision; (sandbox_id set) is an adoption. - // - // Linear and API tasks share this path; Linear-only emits are gated below. private async runTask(task: Task): Promise { if (task.sandboxId) { // Adoption — resume the existing sandbox. Attach is best-effort: a failure @@ -284,39 +278,19 @@ export class TaskProcessorService { return; } - // Fresh provision. Linear tasks: assert issue state + lifecycle emits on - // the agent session. API tasks have no Linear surface; their log gets - // sandbox_started from attachSandboxResources instead. - const isLinear = task.source === "linear"; - if (isLinear) { - try { - const client = await this.linearTokenProvider.getAuthorizedLinearClient(); - await setIssueStarted(client, task.externalId); - } catch (err) { - logErr(`setIssueStarted failed for task ${task.id}`, err); - } - } - - // Reply target id on the assignment inbox row is the canonical destination - // for task-scoped lifecycle emits. A claimed task must have an assignment - // row (claimNext requires at least one non-terminal inbox row, and the - // `created` path always inserts one) — a missing target id means the row - // was deleted out from under us, which is a hard invariant break. - const replyTargetId = await this.inboxRepo.getAssignmentReplyTargetId(task.id); - if (!replyTargetId) { + // Fresh provision. The assignment row's reply target is the canonical + // destination for task-scoped lifecycle emits. A claimed task must have an + // assignment row (claimNext requires at least one non-terminal inbox row, + // and the `created` path always inserts one) — a missing target means the + // row was deleted out from under us, which is a hard invariant break. + const assignment = await this.inboxRepo.getAssignment(task.id); + if (!assignment?.reply_targets[0]) { throw new Error(`runTask: assignment inbox row missing reply target for task ${task.id}`); } - if (isLinear) { - try { - const client = await this.linearTokenProvider.getAuthorizedLinearClient(); - await emitAction(client, replyTargetId, "Provisioning", "infra"); - } catch (err) { - logErr(`emitAction(Provisioning) failed for task ${task.id}`, err); - } - } - // The owner integration's onTaskStarted (none today), at most once per - // assignment on this pod. Never throws. + // The owner integration's onTaskStarted (Linear: issue state and the + // Provisioning action), at most once per assignment on this pod. Never + // throws. API tasks' logs get sandbox_started from attachSandboxResources. await this.hooks.taskStarted(task); // WORKSPACE is the only task-specific input passed at creation; the daemon @@ -817,35 +791,28 @@ export class TaskProcessorService { }); } - // Dedupe by target id — a Linear-source task has many rows sharing the - // same linear_session target id; posting once per id keeps the user from - // seeing the same error N times. At most once: there is no cursor to hold, - // so a delivery error (RetryableError included) is logged and dropped. + // Dedupe by target — a Linear-source task has many rows sharing the + // same linear_session target id; posting once per target keeps the user + // from seeing the same error N times. Keyed by source too: two platforms' + // ids can collide. A row whose integration is unknown or disabled gets no + // notice; it is still failed. const seen = new Set(); for (const row of failed) { - // A source this build has no integration for (say, rows written by a - // newer build before a rollback), or a disabled one, gets no notice; its - // row is still failed. - const integration: Integration | undefined = this.integrations[row.source]; - if (!integration?.enabled) { - logger.warn("failTask: the row's integration is unknown or disabled; notice skipped", { - taskId: task.id, - inboxId: row.id, - source: row.source, - }); - continue; - } for (const t of row.reply_targets) { - if (seen.has(t.id)) continue; - seen.add(t.id); - await withTimeout(SEND_TIMEOUT_MS, (signal) => - integration.deliver({ signal, log: logger }, t, { - type: "failure", - cause: "task_failed", - round: row, - summary: message, - }), - ).catch((e) => logErr(`failTask: post to ${t.spec.kind} target ${t.id} failed`, e)); + const key = `${row.source}:${t.id}`; + if (seen.has(key)) continue; + seen.add(key); + const notice: Outbound = { + type: "failure", + cause: "task_failed", + round: row, + summary: message, + }; + // At most once: there is no cursor to hold, so a RetryableError is + // dropped too. + await deliverToRound(this.integrations, row, t, notice, { taskId: task.id }).catch((e) => + logErr(`failTask: notice to target ${t.id} dropped`, e), + ); } } From 2149936a7248e54dd6638a523f730e7facc7e792 Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Fri, 30 Oct 2026 21:39:31 +0530 Subject: [PATCH 19/22] fix(control-plane): validate delegation order; API-only restart TaskIntake rejects a delegation order that isn't a non-negative safe integer, since NaN would make every redelivery restart the task, and restart refuses non-API tasks. Also fixes stale test names and comments. Co-Authored-By: Claude Opus 5.5 --- .../src/integrations/__tests__/intake.test.ts | 24 ++++++++++++++++++- control-plane/src/integrations/intake.ts | 7 ++++++ .../__tests__/task.repository.test.ts | 4 ++-- .../__tests__/eventDispatcher.test.ts | 9 +++---- .../src/services/__tests__/replyHolds.test.ts | 11 ++++----- .../__tests__/taskProcessor.service.test.ts | 10 ++++---- control-plane/src/services/replyHolds.ts | 15 ++++-------- control-plane/src/types/harnessEvent.ts | 3 ++- control-plane/src/types/inboxMessage.ts | 4 ++-- 9 files changed, 55 insertions(+), 32 deletions(-) diff --git a/control-plane/src/integrations/__tests__/intake.test.ts b/control-plane/src/integrations/__tests__/intake.test.ts index 3210691..6cf53c2 100644 --- a/control-plane/src/integrations/__tests__/intake.test.ts +++ b/control-plane/src/integrations/__tests__/intake.test.ts @@ -196,6 +196,20 @@ describe("Intake", () => { expect(round?.reply_targets).toEqual([session("d:7", 7)]); }); + it("rejects an order that isn't a non-negative integer; a valid order still dedupes", async () => { + await assignLinear(session("d:5", 5)); + for (const bad of [Number.NaN, -1, 1.5]) { + await expect(assignLinear(session("d:5", bad))).rejects.toThrow( + "target.order must be a non-negative integer", + ); + } + expect((await assignLinear(session("d:5", 5))).result).toBe("duplicate"); + const [round] = await db.execute<{ reply_targets: ReplyTarget[] }>( + sql`SELECT reply_targets FROM inbox`, + ); + expect(round?.reply_targets).toEqual([session("d:5", 5)]); + }); + it("a duplicate kills nothing", async () => { const { taskId } = await assignLinear(session("sess-1")); await db.execute(sql`UPDATE tasks SET sandbox_id = 'sb-1' WHERE id = ${taskId}`); @@ -314,7 +328,7 @@ describe("Intake", () => { }); it("a review whose id equals a stored comment id is a duplicate while the old index exists", async () => { - // changes in phase 3 (spec I9: the old (source, external_id) index goes) + // characterization: changes in phase 3 (spec I9: the old (source, external_id) index goes) const task = await insertTask({ key: "DBX-API-1" }); await intake.followUp({ task, message: ghComment("555"), targets: [ghTarget("555")] }); const review = await intake.followUp({ @@ -425,6 +439,14 @@ describe("Intake", () => { }); }); + it("restart is API-only", async () => { + const task = await insertTask({ source: "linear", externalId: "issue-1", key: "ISS-1" }); + const def = { title: "T2", description: null, flags: [] }; + await expect(intake.restart(task.id, def)).rejects.toThrow("is not an API task"); + const [row] = await db.execute(sql`SELECT title FROM tasks`); + expect(row).toEqual({ title: "t" }); + }); + it("findTask matches on source and external id", async () => { const t = await insertTask({ source: "linear", externalId: "issue-9", key: "ISS-9" }); expect((await intake.findTask("linear", "issue-9"))?.id).toBe(t.id); diff --git a/control-plane/src/integrations/intake.ts b/control-plane/src/integrations/intake.ts index 4005c63..e1535fa 100644 --- a/control-plane/src/integrations/intake.ts +++ b/control-plane/src/integrations/intake.ts @@ -87,6 +87,12 @@ export class Intake implements TaskIntake { target: (ref) => { const target = p.target(ref); checkTargets(p.source, [target]); + // A NaN order is never "not above" the stored one, so every + // redelivery would restart the task. + const { order } = target; + if (order !== undefined && !(Number.isSafeInteger(order) && order >= 0)) { + throw new Error(`assign: target.order must be a non-negative integer, not ${order}`); + } return target; }, }); @@ -102,6 +108,7 @@ export class Intake implements TaskIntake { ): Promise { const task = await this.taskRepo.getById(taskId); if (!task) throw new Error(`restart: task ${taskId} not found`); + if (task.source !== "api") throw new Error(`restart: task ${taskId} is not an API task`); const { prevSandboxId } = await this.taskRepo.restartTaskWithInbox({ taskId, inboxPayload: { diff --git a/control-plane/src/repositories/__tests__/task.repository.test.ts b/control-plane/src/repositories/__tests__/task.repository.test.ts index dd13d95..bf4c577 100644 --- a/control-plane/src/repositories/__tests__/task.repository.test.ts +++ b/control-plane/src/repositories/__tests__/task.repository.test.ts @@ -119,7 +119,7 @@ describe("TaskRepository", () => { }); describe("assign (Linear created)", () => { - it("inserts task + inbox row in one tx, created=true on fresh insert", async () => { + it("inserts task + inbox row in one tx; a fresh insert is created", async () => { const result = await assignLinear({ session: "sess-1", flags: ["FAST_PATH"] }); expect(result.result).toBe("created"); @@ -152,7 +152,7 @@ describe("TaskRepository", () => { }); }); - it("created=false on conflict; inbox INSERT is a no-op; existing reply_targets persist", async () => { + it("a redelivery is a duplicate; inbox INSERT is a no-op; existing reply_targets persist", async () => { await assignLinear({ session: "sess-1" }); // Seed a durable retry budget on the existing row. A duplicate `created` diff --git a/control-plane/src/services/__tests__/eventDispatcher.test.ts b/control-plane/src/services/__tests__/eventDispatcher.test.ts index dbd6572..642cecd 100644 --- a/control-plane/src/services/__tests__/eventDispatcher.test.ts +++ b/control-plane/src/services/__tests__/eventDispatcher.test.ts @@ -720,7 +720,7 @@ describe("EventDispatcher", () => { expect(await dispatcher.handle(prEvent(), makeTask())).toBe("hold"); expect(await dispatcher.handle(prEvent(), makeTask())).toBe("hold"); expect(attachUrlToIssue).toHaveBeenCalledTimes(2); - expect(holds.stats().linear?.heldTasks).toBe(1); + expect(holds.of("linear").heldTasks).toBe(1); }); it("self-evicts when appendPrUrl returns 0 rows (lease lost); skips attachUrlToIssue", async () => { @@ -1011,7 +1011,7 @@ describe("EventDispatcher", () => { getByIdForTask.mockResolvedValue(ghCommentRow()); expect(await d.handle(responseEvent("555"), makeTask())).toBe("ok"); expect(postIssueComment).not.toHaveBeenCalled(); - expect(holds.stats()).toEqual({}); + expect(holds.of("github").heldTasks).toBe(0); }); it("a Linear token failure does not hold a GitHub round's reply or completion on a Linear task", async () => { @@ -1053,13 +1053,14 @@ describe("EventDispatcher", () => { getByIdForTask.mockResolvedValue(ghCommentRow()); postIssueComment.mockRejectedValueOnce(new HttpError("HTTP 429", 429)); expect(await dispatcher.handle(responseEvent("555"), makeTask())).toBe("hold"); - expect(Object.keys(holds.stats())).toEqual(["github"]); + expect(holds.of("github").heldTasks).toBe(1); + expect(holds.of("linear").heldTasks).toBe(0); // pr_opened's Linear client fetch is the task's (Linear's) hold. (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( new Error("token refresh failed"), ); expect(await dispatcher.handle(prEvent(), makeTask({ id: "task-2" }))).toBe("hold"); - expect(holds.stats().linear?.heldTasks).toBe(1); + expect(holds.of("linear").heldTasks).toBe(1); }); it("a Linear send gets a per-call client bound to the send's signal", async () => { diff --git a/control-plane/src/services/__tests__/replyHolds.test.ts b/control-plane/src/services/__tests__/replyHolds.test.ts index 6f6b99b..1f227bc 100644 --- a/control-plane/src/services/__tests__/replyHolds.test.ts +++ b/control-plane/src/services/__tests__/replyHolds.test.ts @@ -58,7 +58,7 @@ describe("ReplyHolds", () => { at(15 * 60_000); holds.hold("t1", event(6), "github", err); expect(warn).toHaveBeenCalledTimes(3); - expect(holds.stats()).toEqual({ github: { heldTasks: 1, oldestHoldMs: 0 } }); + expect(holds.of("github")).toEqual({ heldTasks: 1, oldestHoldAgeMs: 0 }); } finally { warn.mockRestore(); } @@ -73,13 +73,12 @@ describe("ReplyHolds", () => { holds.hold("b", event(1), "linear", err); holds.hold("c", event(1), "github", err); at(120_000); - expect(holds.stats()).toEqual({ - linear: { heldTasks: 2, oldestHoldMs: 120_000 }, - github: { heldTasks: 1, oldestHoldMs: 60_000 }, - }); + expect(holds.of("linear")).toEqual({ heldTasks: 2, oldestHoldAgeMs: 120_000 }); + expect(holds.of("github")).toEqual({ heldTasks: 1, oldestHoldAgeMs: 60_000 }); holds.release("a"); holds.retain(new Set(["a", "b"])); - expect(holds.stats()).toEqual({ linear: { heldTasks: 1, oldestHoldMs: 60_000 } }); + expect(holds.of("linear")).toEqual({ heldTasks: 1, oldestHoldAgeMs: 60_000 }); + expect(holds.of("github")).toEqual({ heldTasks: 0, oldestHoldAgeMs: 0 }); }); }); diff --git a/control-plane/src/services/__tests__/taskProcessor.service.test.ts b/control-plane/src/services/__tests__/taskProcessor.service.test.ts index 8a1c359..9367d7a 100644 --- a/control-plane/src/services/__tests__/taskProcessor.service.test.ts +++ b/control-plane/src/services/__tests__/taskProcessor.service.test.ts @@ -748,7 +748,7 @@ describe("TaskProcessorService", () => { ); expect(postIssueComment).not.toHaveBeenCalled(); - expect(holds.stats()).toEqual({}); + expect(holds.of("github").heldTasks).toBe(0); expect(markFailed).toHaveBeenCalledWith({ inboxId: "gc", ownerId: "pod-test", @@ -966,7 +966,7 @@ describe("TaskProcessorService", () => { await failTask(makeTask()); expect(postIssueComment).not.toHaveBeenCalled(); - expect(holds.stats()).toEqual({}); + expect(holds.of("github").heldTasks).toBe(0); expect(failTaskInbox).toHaveBeenCalledWith({ taskId: "task-1", ownerId: "pod-test", @@ -1405,7 +1405,7 @@ describe("TaskProcessorService", () => { await waitFor(() => attachSandbox.mock.calls.length > 0); await service.stop(); - expect(holds.stats()).toEqual({}); + expect(holds.of("github").heldTasks).toBe(0); }); it("once shutdown starts, reconciles no further owned task and claims nothing", async () => { @@ -1458,7 +1458,7 @@ describe("TaskProcessorService", () => { await service.stop(); expect(readEvents).not.toHaveBeenCalled(); - expect(holds.stats().github?.heldTasks).toBe(1); + expect(holds.of("github").heldTasks).toBe(1); }); it("a dropped send advances the cursor, then ends this task's events for the tick", async () => { @@ -1502,7 +1502,7 @@ describe("TaskProcessorService", () => { await new Promise((r) => setTimeout(r, 30)); await service.stop(); - expect(holds.stats()).toEqual({}); + expect(holds.of("linear").heldTasks).toBe(0); }); it("renews every lease this pod holds right before each reconcile; a lost lease skips that task", async () => { diff --git a/control-plane/src/services/replyHolds.ts b/control-plane/src/services/replyHolds.ts index 0d4e270..eb9c29a 100644 --- a/control-plane/src/services/replyHolds.ts +++ b/control-plane/src/services/replyHolds.ts @@ -68,19 +68,12 @@ export class ReplyHolds { // One integration's held tasks and oldest hold age, for its status body. of(integration: IntegrationName): { heldTasks: number; oldestHoldAgeMs: number } { - const s = this.stats()[integration]; - return { heldTasks: s?.heldTasks ?? 0, oldestHoldAgeMs: s?.oldestHoldMs ?? 0 }; - } - - // Held tasks and the oldest hold's age, per integration. - stats(): Partial> { const now = Date.now(); - const out: Partial> = {}; + const out = { heldTasks: 0, oldestHoldAgeMs: 0 }; for (const h of this.byTask.values()) { - const s = out[h.integration] ?? { heldTasks: 0, oldestHoldMs: 0 }; - s.heldTasks += 1; - s.oldestHoldMs = Math.max(s.oldestHoldMs, now - h.since); - out[h.integration] = s; + if (h.integration !== integration) continue; + out.heldTasks += 1; + out.oldestHoldAgeMs = Math.max(out.oldestHoldAgeMs, now - h.since); } return out; } diff --git a/control-plane/src/types/harnessEvent.ts b/control-plane/src/types/harnessEvent.ts index 30567ac..20842cb 100644 --- a/control-plane/src/types/harnessEvent.ts +++ b/control-plane/src/types/harnessEvent.ts @@ -13,7 +13,8 @@ // // `target_id` is always present and matches an inbox row's reply_targets[] // entry: -// - assignment: the Linear agent session id (AssignmentPayload.target_id) +// - assignment: the delegation id (AssignmentPayload.target_id; Linear's +// is the agent session id) // - comment: the GH comment id (CommentPayload.target_id) // - review: a review_body or inline-comment target_id (the agent picks one // per item for response; for thought/action it conventionally targets diff --git a/control-plane/src/types/inboxMessage.ts b/control-plane/src/types/inboxMessage.ts index 64e96a4..10f605f 100644 --- a/control-plane/src/types/inboxMessage.ts +++ b/control-plane/src/types/inboxMessage.ts @@ -33,8 +33,8 @@ export type AssignmentPayload = TaskIdentity & { title: string; description: string | null; external_url: string; - // Linear agent session id. Matches the row's reply_targets[0].id (a - // linear_session target); daemon stamps it as the per-round `--target` for + // The delegation id (Linear: the agent session id). Matches the row's + // reply_targets[0].id; daemon stamps it as the per-round `--target` for // emits. target_id: string; }; From b7f72f1426358220fc0ce973cd7ec467839e50d8 Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Fri, 30 Oct 2026 21:43:47 +0530 Subject: [PATCH 20/22] fix(control-plane): renew all leases during long reconciles Within a tick, renew every lease the pod holds once a quarter of the lease has passed, checked before each reconcile and each event, and stop a task's events if its own lease was lost. One long reconcile no longer lets a sibling pod take over the pod's other tasks. Co-Authored-By: Claude Opus 5.5 --- .../__tests__/taskProcessor.service.test.ts | 75 +++++++++++++++++-- .../src/services/taskProcessor.service.ts | 44 ++++++++--- 2 files changed, 104 insertions(+), 15 deletions(-) diff --git a/control-plane/src/services/__tests__/taskProcessor.service.test.ts b/control-plane/src/services/__tests__/taskProcessor.service.test.ts index 9367d7a..53b6213 100644 --- a/control-plane/src/services/__tests__/taskProcessor.service.test.ts +++ b/control-plane/src/services/__tests__/taskProcessor.service.test.ts @@ -1,4 +1,4 @@ -import { beforeEach, describe, expect, it, mock, spyOn } from "bun:test"; +import { afterEach, beforeEach, describe, expect, it, mock, spyOn } from "bun:test"; import type { SandboxInfo } from "@alibaba-group/opensandbox"; import { SandboxApiException } from "@alibaba-group/opensandbox"; import type { LinearClient } from "@linear/sdk"; @@ -103,7 +103,7 @@ const { InboxService: RealInboxService, WORKSPACE_MISSING_MESSAGE } = await impo "../inbox.service" ); const { config: mockedConfig } = (await import("../../config")) as unknown as { - config: { shutdownDrainMs: number }; + config: { shutdownDrainMs: number; leaseTimeoutMs: number }; }; const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); const { RetryableError } = await import("../../integrations/outbound"); @@ -1505,7 +1505,7 @@ describe("TaskProcessorService", () => { expect(holds.of("linear").heldTasks).toBe(0); }); - it("renews every lease this pod holds right before each reconcile; a lost lease skips that task", async () => { + it("renews every lease this pod holds at tick start; a task it didn't renew is skipped", async () => { const t1 = makeTask({ id: "task-1", sandboxId: "sbx-1" }); const t2 = makeTask({ id: "task-2", sandboxId: "sbx-2" }); listInFlightOwnedBy.mockResolvedValueOnce([t1, t2]).mockResolvedValue([]); @@ -1521,16 +1521,79 @@ describe("TaskProcessorService", () => { }); service.start(); - await waitFor(() => order.length >= 4); + await waitFor(() => order.length >= 2); await new Promise((r) => setTimeout(r, 30)); await service.stop(); - // Tick start, then before each reconcile. - expect(order.slice(0, 4)).toEqual(["heartbeat", "heartbeat", "reconcile sbx-1", "heartbeat"]); + expect(order.slice(0, 2)).toEqual(["heartbeat", "reconcile sbx-1"]); expect(order).not.toContain("reconcile sbx-2"); expect(heartbeatOwnedBy).toHaveBeenCalledWith("pod-test"); }); + describe("a reconcile longer than a quarter of the lease", () => { + const thought = (seq: number) => + ({ + seq, + ts: "t", + type: "agent_thought", + inbox_id: "inbox-1", + target_id: "sess-1", + body: "x", + }) as HarnessEvent; + let leaseMs: number; + beforeEach(() => { + leaseMs = mockedConfig.leaseTimeoutMs; + mockedConfig.leaseTimeoutMs = 40; // renew after 10 ms + listInFlightOwnedBy + .mockResolvedValueOnce([makeTask({ sandboxId: "sbx-1" })]) + .mockResolvedValue([]); + readEvents.mockResolvedValueOnce([thought(1), thought(2), thought(3)]); + }); + afterEach(() => { + mockedConfig.leaseTimeoutMs = leaseMs; + }); + + it("renews every lease again before a later event", async () => { + const order: string[] = []; + heartbeatOwnedBy.mockImplementation(() => { + order.push("heartbeat"); + return Promise.resolve([{ id: "task-1" }]); + }); + // Each send outlasts a quarter of the lease. + dispatcherHandle.mockImplementation(async (event: HarnessEvent) => { + order.push(`handle ${event.seq}`); + await sleep(15); + return "ok" as const; + }); + + service.start(); + await waitFor(() => recordEventCursor.mock.calls.length >= 3); + await service.stop(); + + expect(order.slice(order.indexOf("handle 1"), order.indexOf("handle 2"))).toContain( + "heartbeat", + ); + }); + + it("stops the task's events once a renewal shows its lease taken over", async () => { + // Ours until the first send; taken over by the renewal after it. + heartbeatOwnedBy.mockImplementation(() => + Promise.resolve(dispatcherHandle.mock.calls.length > 0 ? [] : [{ id: "task-1" }]), + ); + dispatcherHandle.mockImplementation(async () => { + await sleep(15); + return "ok" as const; + }); + + service.start(); + await waitFor(() => recordEventCursor.mock.calls.length >= 1); + await sleep(50); + await service.stop(); + + expect(dispatcherHandle).toHaveBeenCalledTimes(1); + }); + }); + it("never provisions a task twice on this pod: a re-claim while its provision runs is skipped", async () => { let release!: () => void; createSandboxWithPvc.mockImplementation( diff --git a/control-plane/src/services/taskProcessor.service.ts b/control-plane/src/services/taskProcessor.service.ts index eac5643..1688fca 100644 --- a/control-plane/src/services/taskProcessor.service.ts +++ b/control-plane/src/services/taskProcessor.service.ts @@ -26,6 +26,9 @@ export class TaskProcessorService { private sleepAbort: AbortController | null = null; // Provisions running on this pod, by task id. private readonly inflight = new Map>(); + // The last renewal of this pod's leases: when it was sent and what it renewed. + private leasesRenewedAt = 0; + private renewedTaskIds = new Set(); // First tick (per task) we saw the backing pod ABSENT (kubectl NotFound). A // pod can be briefly gone between create and scheduling, so absence gets a // grace window (config.sandboxAbsentGraceMs) before we fail. Cleared the @@ -118,24 +121,22 @@ export class TaskProcessorService { private async tick(): Promise { // 1. Heartbeat refresh — keep owned-task leases warm. - await this.taskRepo.heartbeatOwnedBy(config.podId); + await this.renewLeases(); // 2. Idle reaper — kill sandboxes idle past the ceiling, release leases. await this.reapIdle().catch((err) => logErr("reapIdle failed", err)); - // 3. Reconcile owned tasks with live sandboxes. Every lease this pod holds - // (in-flight provisions included) is renewed right before each reconcile, - // and the task's own with each cursor advance inside it, so slow sends - // can't let a lease expire mid-tick; a task not renewed was taken over. - // ponytail: one reconcile of many slow but successful sends (>120 s) can - // still outlive an in-flight provision's lease; a lease timer if it matters. + // 3. Reconcile owned tasks with live sandboxes. Before each reconcile and + // each event inside it, every lease this pod holds (in-flight provisions + // included) is renewed once a quarter of the lease has passed, so slow + // sends can't let a lease expire mid-tick; a task not renewed was taken + // over. const owned = await this.taskRepo.listInFlightOwnedBy(config.podId, this.taskSources); for (const task of owned) { // Shutting down: leave the rest to siblings (stop() backdates our // heartbeats), so shutdown waits for one task's reconcile at most (§10). if (!this.running) break; - const renewed = await this.taskRepo.heartbeatOwnedBy(config.podId); - if (!renewed.some((r) => r.id === task.id)) { + if (!(await this.leaseHeld(task.id))) { logger.info("lease lost before reconcile; skipping", { taskId: task.id }); continue; } @@ -183,6 +184,22 @@ export class TaskProcessorService { this.dispatch(claimed); } + // Renews every lease this pod holds and notes which it renewed. + private async renewLeases(): Promise { + const at = Date.now(); + const rows = await this.taskRepo.heartbeatOwnedBy(config.podId); + this.leasesRenewedAt = at; + this.renewedTaskIds = new Set(rows.map((r) => r.id)); + } + + // Renews every lease once a quarter of the lease has passed since the last + // renewal (each step between checks is bounded well under a lease), then says + // whether `taskId` is still ours: one missing from the renewal was taken over. + private async leaseHeld(taskId: string): Promise { + if (Date.now() - this.leasesRenewedAt >= config.leaseTimeoutMs / 4) await this.renewLeases(); + return this.renewedTaskIds.has(taskId); + } + private async reapIdle(): Promise { const reaped = await this.taskRepo.reapIdleTasks({ ownerId: config.podId, @@ -390,6 +407,15 @@ export class TaskProcessorService { // Shutting down: stop after the event in hand (the cursor is persisted // per event), so shutdown waits for one event, not the backlog (§10). if (!this.running) break; + // A long run of slow sends must not let any lease expire; ours lost + // means a sibling took this task over. + if (!(await this.leaseHeld(currentTask.id))) { + logger.info("lease lost mid-reconcile; self-evicting", { + taskId: currentTask.id, + seq: event.seq, + }); + return; + } const result = await this.eventDispatcher.handle(event, currentTask); switch (result) { From 9ef33fd5914849a1b1a78f55282538ee59282fe7 Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Fri, 30 Oct 2026 22:07:59 +0530 Subject: [PATCH 21/22] chore(control-plane): make comments stand on their own Comments and test names say what the code does and why, without citing documents outside the repo. Co-Authored-By: Claude Opus 5.5 --- control-plane/src/app.ts | 4 ++-- control-plane/src/db/schema.ts | 5 +++-- .../src/integrations/__tests__/intake.test.ts | 4 ++-- control-plane/src/integrations/http.ts | 16 ++++++++-------- control-plane/src/integrations/intake.ts | 4 ++-- .../__tests__/inbox.repository.test.ts | 8 ++++---- .../__tests__/task.repository.test.ts | 6 +++--- .../src/repositories/inbox.repository.ts | 8 ++++---- .../src/repositories/task.repository.ts | 4 ++-- .../services/__tests__/apiTask.service.test.ts | 2 +- .../services/__tests__/eventDispatcher.test.ts | 4 ++-- .../__tests__/taskProcessor.service.test.ts | 4 ++-- control-plane/src/services/eventDispatcher.ts | 2 +- control-plane/src/services/integrationHooks.ts | 4 ++-- control-plane/src/services/replyHolds.ts | 2 +- .../src/services/taskProcessor.service.ts | 6 +++--- 16 files changed, 42 insertions(+), 41 deletions(-) diff --git a/control-plane/src/app.ts b/control-plane/src/app.ts index 82939f0..39ef078 100644 --- a/control-plane/src/app.ts +++ b/control-plane/src/app.ts @@ -94,7 +94,7 @@ export function createApp(): AppBundle { const integrations = createIntegrations(config, linearTokenProvider, linearStatus, githubStatus); // The registered integrations that create tasks (github creates none). Only // these sources are claimed and reconciled, so an older image never works a - // task whose source it doesn't know (§10). + // task whose source it doesn't know. const taskSources = taskSource.enumValues.filter((s) => s in integrations); // Sends held because they failed before anything went out; each // integration's status reports its own. @@ -214,7 +214,7 @@ export function createIntegrations( } satisfies Record; } -// Each integration's own status body, plus its held sends (§4 I14). +// Each integration's own status body, plus its held sends. export function statusesWithHolds( integrations: ReturnType, replyHolds: Pick, diff --git a/control-plane/src/db/schema.ts b/control-plane/src/db/schema.ts index a51c4ed..947e052 100644 --- a/control-plane/src/db/schema.ts +++ b/control-plane/src/db/schema.ts @@ -127,8 +127,9 @@ export const inbox = pgTable( // Idempotency anchor: webhook retries / LB reroutes collapse via this. // Every ON CONFLICT targets (source, kind, external_id). uniqueIndex("inbox_source_kind_external_uniq").on(t.source, t.kind, t.externalId), - // The stricter pre-phase-2 index, kept for one release so older pods' - // ON CONFLICT (source, external_id) still has its arbiter. Phase 3 drops it. + // The older, stricter index, kept for one release so older pods' + // ON CONFLICT (source, external_id) still has its arbiter. A later release + // drops it. uniqueIndex("inbox_source_external_uniq").on(t.source, t.externalId), // Pending = not yet delivered or dead-lettered. Sweep + claimNext filter on this. index("inbox_pending_idx").on(t.taskId).where(sql`${t.status} = 'pending'`), diff --git a/control-plane/src/integrations/__tests__/intake.test.ts b/control-plane/src/integrations/__tests__/intake.test.ts index 6cf53c2..3ab2a0c 100644 --- a/control-plane/src/integrations/__tests__/intake.test.ts +++ b/control-plane/src/integrations/__tests__/intake.test.ts @@ -246,7 +246,7 @@ describe("Intake", () => { const [task] = await db.execute( sql`SELECT title, external_url, metadata, sandbox_id, pvc_name FROM tasks`, ); - // The old workspace volume is only unlinked, not deleted (§15 bug 6). + // The old workspace volume is only unlinked, not deleted (a known leak). expect(task).toEqual({ title: "Fix auth v2", external_url: "https://linear.app/x/issue/ISS-1/v2", @@ -328,7 +328,7 @@ describe("Intake", () => { }); it("a review whose id equals a stored comment id is a duplicate while the old index exists", async () => { - // characterization: changes in phase 3 (spec I9: the old (source, external_id) index goes) + // characterization: changes once the old (source, external_id) index is dropped const task = await insertTask({ key: "DBX-API-1" }); await intake.followUp({ task, message: ghComment("555"), targets: [ghTarget("555")] }); const review = await intake.followUp({ diff --git a/control-plane/src/integrations/http.ts b/control-plane/src/integrations/http.ts index efb69af..9d7e081 100644 --- a/control-plane/src/integrations/http.ts +++ b/control-plane/src/integrations/http.ts @@ -6,7 +6,7 @@ import type { Integration, IntegrationName } from "./types"; // Shared rules for integrations' outbound HTTP (I16): the typed errors clients // throw, which failures may be retried, and how long one send may take. -// Upper bound on one deliver() call (§4 I14). +// Upper bound on one deliver() call. export const SEND_TIMEOUT_MS = 20_000; // A non-2xx answer. The message keeps the client's own wording (tests and logs @@ -26,7 +26,7 @@ export class HttpError extends Error { // How long a rate limit without Retry-After waits: until the platform's reset // time (epoch ms), and at least a minute. GitHub may ban integrations that -// retry sooner (§4). +// retry sooner. export function rateLimitWaitMs(resetAtMs?: number): number { const untilReset = resetAtMs !== undefined && Number.isFinite(resetAtMs) ? resetAtMs - Date.now() : 0; @@ -53,10 +53,10 @@ const NOTHING_SENT = new Set([ "ERR_TLS_CERT_ALTNAME_INVALID", ]); -// The failure as a RetryableError when it is one of the exact retryable cases -// (§4): a rate limit (429), connection refused, DNS, TLS handshake, token -// fetch. Anything else (timeouts, 5xx, other 4xx) is undefined: the caller -// logs and drops it. +// The failure as a RetryableError when it is one of the exact retryable +// cases, where nothing was sent yet: a rate limit (429), connection refused, +// DNS, TLS handshake, token fetch. Anything else (timeouts, 5xx, other 4xx) +// is undefined: the caller logs and drops it. export function retryable(err: unknown): RetryableError | undefined { if (err instanceof RetryableError) return err; if (err instanceof TokenFetchError) return new RetryableError(err.message); @@ -102,8 +102,8 @@ export async function withTimeout( } } -// Hands `message` for one of `round`'s targets to the round's integration -// (§4), under the send timeout. An integration this build doesn't know (rows +// Hands `message` for one of `round`'s targets to the round's integration, +// under the send timeout. An integration this build doesn't know (rows // written by a newer build before a rollback) or a disabled one is skipped and // logged: that counts as sent. Any other failure is logged and the message // dropped, except a RetryableError (nothing was sent), which is rethrown: each diff --git a/control-plane/src/integrations/intake.ts b/control-plane/src/integrations/intake.ts index e1535fa..f869bc7 100644 --- a/control-plane/src/integrations/intake.ts +++ b/control-plane/src/integrations/intake.ts @@ -14,7 +14,7 @@ import type { SandboxManager } from "../services/sandboxManager.service"; import type { InboxMessage, TaskIdentity } from "../types/inboxMessage"; import type { IntegrationName, Person, TaskSource } from "./types"; -// The only way modules create work (§3, §5). +// The only way modules create work. export type TaskRef = { id: string; key: string }; type Draft = M extends InboxMessage @@ -151,7 +151,7 @@ export class Intake implements TaskIntake { } // A restart's preempted sandbox, killed after commit in the tracked detached - // set so shutdown drains it. The old workspace volume is kept (§15 bug 6). + // set so shutdown drains it. The old workspace volume is kept (a known leak). private killPreempted(taskId: string, sandboxId: string | null, why: string): void { if (!sandboxId) return; this.hooks.detach( diff --git a/control-plane/src/repositories/__tests__/inbox.repository.test.ts b/control-plane/src/repositories/__tests__/inbox.repository.test.ts index b58dd7a..c659f8c 100644 --- a/control-plane/src/repositories/__tests__/inbox.repository.test.ts +++ b/control-plane/src/repositories/__tests__/inbox.repository.test.ts @@ -150,9 +150,9 @@ describe("InboxRepository", () => { }); it("dedupes across kinds: a review whose id equals a stored comment id is dropped", async () => { - // characterization: changes in phase 3 (spec I9: phase 2 adds the - // (source, kind, external_id) index, but the old (source, external_id) - // one still rejects this row until phase 3 drops it) + // characterization: changes once the old index is dropped. The + // (source, kind, external_id) index allows this row, but the old + // (source, external_id) one still rejects it. const task = await insertTask(); await repo.appendInbox({ taskId: task.id, message: ghComment("555"), replyTargets: [] }); @@ -180,7 +180,7 @@ describe("InboxRepository", () => { }); it("dedupes on (source, kind, external_id) once the old index is gone", async () => { - // Phase 3 drops inbox_source_external_uniq; here it goes in a + // A later release drops inbox_source_external_uniq; here it goes in a // transaction that is rolled back, so the insert's conflict target // must be the new index. const task = await insertTask(); diff --git a/control-plane/src/repositories/__tests__/task.repository.test.ts b/control-plane/src/repositories/__tests__/task.repository.test.ts index bf4c577..c79e44e 100644 --- a/control-plane/src/repositories/__tests__/task.repository.test.ts +++ b/control-plane/src/repositories/__tests__/task.repository.test.ts @@ -168,7 +168,7 @@ describe("TaskRepository", () => { expect(result.result).toBe("duplicate"); const [task] = await db.select().from(schema.tasks); - // A duplicate changes nothing, not even the title (spec §13). + // A duplicate changes nothing, not even the title. expect(task?.title).toBe("Fix auth"); // retryCount survives the duplicate redelivery. expect((task?.metadata as { retryCount?: number })?.retryCount).toBe(2); @@ -184,7 +184,7 @@ describe("TaskRepository", () => { }); it("a duplicate created (same session) changes nothing: not the title, URL or flags, nor the run", async () => { - // Flipped in phase 2 (spec §13): it used to refresh title, URL and flags. + // It used to refresh title, URL and flags. const { id: taskId } = await assignLinear({ session: "sess-1", flags: ["OLD"] }); await db.execute(sql` UPDATE tasks @@ -1018,7 +1018,7 @@ describe("TaskRepository", () => { }); }); - describe("source filter (§10: an image never works a source it doesn't know)", () => { + describe("source filter (an image never works a source it doesn't know)", () => { it("claimNext and listInFlightOwnedBy skip a task whose source is not in the list", async () => { const api = await insertTask({ source: "api", externalId: "api-unclaimed" }); await insertInboxRow(api.id); diff --git a/control-plane/src/repositories/inbox.repository.ts b/control-plane/src/repositories/inbox.repository.ts index 120816c..cc9c953 100644 --- a/control-plane/src/repositories/inbox.repository.ts +++ b/control-plane/src/repositories/inbox.repository.ts @@ -183,10 +183,10 @@ export class InboxRepository { const result = await (tx ? tx.transaction(insert) : insert(db)); return asRows<{ id: string }>(result)[0] ?? null; } catch (err) { - // Until phase 3 drops it, the old (source, external_id) index also - // rejects a row whose id another kind of the same source already uses, - // with a 23505 instead of DO NOTHING. Dedupe it as before; phase 3 - // removes this with the index. + // Until a later release drops it, the old (source, external_id) index + // also rejects a row whose id another kind of the same source already + // uses, with a 23505 instead of DO NOTHING. Dedupe it as before; this + // goes with the index. if (isUniqueViolation(err, "inbox_source_external_uniq")) return null; throw err; } diff --git a/control-plane/src/repositories/task.repository.ts b/control-plane/src/repositories/task.repository.ts index 2050bf3..f7cbc5a 100644 --- a/control-plane/src/repositories/task.repository.ts +++ b/control-plane/src/repositories/task.repository.ts @@ -24,7 +24,7 @@ function asRows(result: unknown): T[] { } export class TaskRepository { - // TaskIntake.assign's transaction (§5). Inserts the task, or finds it by + // TaskIntake.assign's transaction. Inserts the task, or finds it by // (source, external_id) and locks it, then compares the stored delegation // (the assignment row's reply target) with the incoming one: when both carry // an order, one not above the stored order is a duplicate; otherwise the @@ -391,7 +391,7 @@ export class TaskRepository { // // Only tasks of `sources` (the sources this build has an integration for) are // claimed, so an older image never takes a task of a source it doesn't know: - // rollbacks and rolling overlaps stay safe without draining (§10). + // rollbacks and rolling overlaps stay safe without draining. async claimNext(params: { ownerId: string; leaseTimeoutMs: number; diff --git a/control-plane/src/services/__tests__/apiTask.service.test.ts b/control-plane/src/services/__tests__/apiTask.service.test.ts index 6da9930..c9d8c13 100644 --- a/control-plane/src/services/__tests__/apiTask.service.test.ts +++ b/control-plane/src/services/__tests__/apiTask.service.test.ts @@ -252,7 +252,7 @@ describe("ApiTaskService", () => { { id: expect.any(String), status: "pending", failure_reason: null, outcome: null }, ]); expect(rows[0]?.id).not.toBe((before as { id: string }).id); - // The old workspace volume is not deleted, only unlinked (spec §15 bug 6, out of scope). + // The old workspace volume is not deleted, only unlinked (a known leak). expect(killSandbox).toHaveBeenCalledTimes(1); expect(killSandbox).toHaveBeenCalledWith("sbx-old"); }); diff --git a/control-plane/src/services/__tests__/eventDispatcher.test.ts b/control-plane/src/services/__tests__/eventDispatcher.test.ts index 642cecd..47a5049 100644 --- a/control-plane/src/services/__tests__/eventDispatcher.test.ts +++ b/control-plane/src/services/__tests__/eventDispatcher.test.ts @@ -997,7 +997,7 @@ describe("EventDispatcher", () => { expect(postIssueComment).toHaveBeenCalledTimes(2); }); - it("skips GitHub replies when GitHub webhooks are off: not posted, not held (§13)", async () => { + it("skips GitHub replies when GitHub webhooks are off: not posted, not held", async () => { const integrations = makeIntegrations(false); const d = new EventDispatcher( taskRepo, @@ -1390,7 +1390,7 @@ describe("EventDispatcher", () => { expect(markResponded).toHaveBeenCalled(); }); - it("Linear assignment: a token fetch failure no longer holds; it marks and notifies Slack (§13)", async () => { + it("Linear assignment: a token fetch failure no longer holds; it marks and notifies Slack", async () => { // Slack's mention comes from assignee(), which fails (logged) and the // post goes out without it; completed needs no Linear client. (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( diff --git a/control-plane/src/services/__tests__/taskProcessor.service.test.ts b/control-plane/src/services/__tests__/taskProcessor.service.test.ts index 53b6213..2e22275 100644 --- a/control-plane/src/services/__tests__/taskProcessor.service.test.ts +++ b/control-plane/src/services/__tests__/taskProcessor.service.test.ts @@ -390,7 +390,7 @@ describe("TaskProcessorService", () => { }); expect(setIssueStarted).toHaveBeenCalledWith(expect.anything(), "issue-1"); expect(emitAction).toHaveBeenCalledWith(expect.anything(), "sess-1", "Provisioning", "infra"); - // The post-create "Sandbox provisioned" thought is gone (spec §13). + // The post-create "Sandbox provisioned" thought is gone. expect(emitThought).not.toHaveBeenCalled(); }); @@ -996,7 +996,7 @@ describe("TaskProcessorService", () => { ownerId: "pod-test", alsoClearSandbox: true, }); - // Sent without the assignee mention (its lookup fails; §13). + // Sent without the assignee mention (its lookup fails). expect(notifyTerminal).toHaveBeenCalledWith( expect.objectContaining({ id: "task-1" }), "failed", diff --git a/control-plane/src/services/eventDispatcher.ts b/control-plane/src/services/eventDispatcher.ts index 1df0aa0..bbc8391 100644 --- a/control-plane/src/services/eventDispatcher.ts +++ b/control-plane/src/services/eventDispatcher.ts @@ -17,7 +17,7 @@ import type { SlackNotifierService } from "./slackNotifier.service"; // - `ok`: handled; reconcileOne advances the cursor and goes on. // - `dropped`: the event's send timed out or failed for good. reconcileOne // advances past it, then stops this task's events for the tick, so a slow -// or failing platform can't eat the tick (§4 I14). +// or failing platform can't eat the tick. // - `hold`: a send (or the owner's onPrOpened) failed before anything went out // (RetryableError); the hold is recorded in ReplyHolds. reconcileOne stops // this task's events BEFORE advancing the cursor, so the event is retried diff --git a/control-plane/src/services/integrationHooks.ts b/control-plane/src/services/integrationHooks.ts index 8fb4eee..298b676 100644 --- a/control-plane/src/services/integrationHooks.ts +++ b/control-plane/src/services/integrationHooks.ts @@ -12,7 +12,7 @@ import type { import { logger } from "../logger"; import type { InboxRepository, InboxRow } from "../repositories/inbox.repository"; -// Upper bound on one hook call (§4). +// Upper bound on one hook call. const HOOK_TIMEOUT_MS = { onTaskStarted: 20_000, onPrOpened: 30_000, @@ -23,7 +23,7 @@ type HookName = keyof typeof HOOK_TIMEOUT_MS; // What a detached promise is doing, for the shutdown log. type DetachedWork = HookName | "killSandbox" | "slack"; -// Runs integrations' lifecycle hooks (§4). onTaskStarted, onPrOpened and +// Runs integrations' lifecycle hooks. onTaskStarted, onPrOpened and // assignee go to the task's own integration; onRoundFinished goes to the // round's. A disabled integration's hooks are skipped (logged). Each call runs // under its hook's timeout with an abort signal and never throws: a failure is diff --git a/control-plane/src/services/replyHolds.ts b/control-plane/src/services/replyHolds.ts index eb9c29a..d2453d5 100644 --- a/control-plane/src/services/replyHolds.ts +++ b/control-plane/src/services/replyHolds.ts @@ -14,7 +14,7 @@ type Hold = { warnedAt: number; }; -// Events whose send failed before anything went out (§4 I14). A held task +// Events whose send failed before anything went out. A held task // stops at that event, so it holds at most one: keyed by task, the hold is // (taskId, inboxId, seq) and a different event starts a new one. In memory and // with no time limit: the reconcile loop drops a hold when the cursor moves diff --git a/control-plane/src/services/taskProcessor.service.ts b/control-plane/src/services/taskProcessor.service.ts index 1688fca..711ec46 100644 --- a/control-plane/src/services/taskProcessor.service.ts +++ b/control-plane/src/services/taskProcessor.service.ts @@ -49,7 +49,7 @@ export class TaskProcessorService { private readonly integrations: Record; private readonly holds: ReplyHolds; // Sources this build has an integration for; claimNext and - // listInFlightOwnedBy see only these tasks (§10). + // listInFlightOwnedBy see only these tasks. private readonly taskSources: Task["source"][]; private readonly hooks: IntegrationHooks; @@ -134,7 +134,7 @@ export class TaskProcessorService { const owned = await this.taskRepo.listInFlightOwnedBy(config.podId, this.taskSources); for (const task of owned) { // Shutting down: leave the rest to siblings (stop() backdates our - // heartbeats), so shutdown waits for one task's reconcile at most (§10). + // heartbeats), so shutdown waits for one task's reconcile at most. if (!this.running) break; if (!(await this.leaseHeld(task.id))) { logger.info("lease lost before reconcile; skipping", { taskId: task.id }); @@ -405,7 +405,7 @@ export class TaskProcessorService { for (const event of events) { if (currentTask === null) break; // self-evict: lease lost or task gone // Shutting down: stop after the event in hand (the cursor is persisted - // per event), so shutdown waits for one event, not the backlog (§10). + // per event), so shutdown waits for one event, not the backlog. if (!this.running) break; // A long run of slow sends must not let any lease expire; ours lost // means a sibling took this task over. From 1827f6dcd70b56a5177204c42a6cdc37a7128709 Mon Sep 17 00:00:00 2001 From: Mihir Khambhati Date: Sat, 31 Oct 2026 12:24:03 +0530 Subject: [PATCH 22/22] chore(control-plane): drop a leftover design reference from a comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 5.5 🔒 Scanned for secrets using gitleaks 8.28.0 --- control-plane/src/integrations/http.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/control-plane/src/integrations/http.ts b/control-plane/src/integrations/http.ts index 9d7e081..dc54079 100644 --- a/control-plane/src/integrations/http.ts +++ b/control-plane/src/integrations/http.ts @@ -3,7 +3,7 @@ import type { InboxRow, ReplyTarget } from "../repositories/inbox.repository"; import { type Outbound, RetryableError } from "./outbound"; import type { Integration, IntegrationName } from "./types"; -// Shared rules for integrations' outbound HTTP (I16): the typed errors clients +// Shared rules for integrations' outbound HTTP: the typed errors clients // throw, which failures may be retried, and how long one send may take. // Upper bound on one deliver() call.