diff --git a/CLAUDE.md b/CLAUDE.md index a8026a6..fbfdf3a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -47,6 +47,8 @@ socket in the background, so a daemon that comes up later is picked up with no restart. A hangar unusable for 300 s — unreachable, or answering another protocol — sends one `hangar_offline` push per episode, none on recovery; the cockpit never pushes for being offline. +How many agents the hangar runs at once is capped by `CC_MOBILE_HANGAR_CAP`; see **Launch API**. + ## Commands ```bash @@ -502,7 +504,7 @@ that can refuse a keystroke is a worse failure than a missing line. `POST /api/launch` (`server/launch.ts`) dispatches an Obsidian task card onto the **hangar** without a phone: `{cwd, cardPath, vault, project, profileId?}` → -`201 {sessionId, claudeUuid}`. `cardPath` is relative to the vault, `vault` must +`201 {sessionId, claudeUuid}`, or `202 {queued: true, position}` when the hangar is at its cap (below). `cardPath` is relative to the vault, `vault` must equal the basename of `CC_MOBILE_VAULT_ROOT`, and `project` matches `[A-Za-z0-9._-]+` without a leading dot. It is the same `terminal_create` + `terminal_send` path the WS uses, so `CC_MOBILE_ALLOWED_ROOTS` and the profile @@ -525,7 +527,9 @@ path that does not resolve is `404 card_not_found`; one whose resolved path root that then cannot be read (a directory, mode 000) is `404 card_not_found`; a profile of a non-claude kind is `400 unsupported_kind`. The request's own `cwd` is checked next, with the terminal codes (`400 invalid_cwd`, -`403 path_not_allowed`), then its profile (`400 unknown_profile`). When `cwd` is +`403 path_not_allowed`), then its profile (`400 unknown_profile`). `409 already_queued` and +`503 hangar_offline` come after `unknown_profile` and before the worktree checks. +When `cwd` is inside a git repo the worktree checks follow, all before git writes anything: a repo git refuses to read (dubious ownership, a broken `.git`) is `500 worktree_failed`; a repo top level outside every allowed root, or a @@ -635,6 +639,22 @@ on the new pane before sending, writing into the same replay buffer the WS uses: a phone that connects later replays the turn. Its audit line has `device: "launch-api"`. +The hangar runs a bounded number of agents at once. The cap is +`CC_MOBILE_HANGAR_CAP` when set (blank counts as unset; a value that is not a +positive integer stops the server at startup), else performance cores − 2 with a +minimum of 1, else 4 when the core count is unknown. Occupancy is the hangar's +own `agent.list` plus launches admitted but not yet finished, so an agent the +user started by hand counts. Past the cap a launch is not refused but queued, +first in first out: the reply is `202 {queued: true, position}` and the +request's checks have already passed. A card already waiting, or admitted and still starting, answers +`409 already_queued`, and a hangar that cannot be asked how full it is answers +`503 hangar_offline` rather than guess it empty — never over-admit. When a slot +frees, the next launch is checked again against the live occupancy before it +starts; a failure at that point has no caller left to tell and is a log line +only, except that a dequeued launch whose prompt failed still writes today's +`prompt_send` audit line, `failed`, or `rejected` when the pane was busy (only failures before the prompt are log-only). The queue lives in memory, so a restart empties the queue and the +caller dispatches again. + ## Cards API `GET /api/cards` (`server/cards.ts`) lists the obw task cards an inbox can @@ -664,6 +684,12 @@ archiving a card is obw's, whose dependency check lives in its pm skill. The route needs no token, since it starts nothing; the root gate still runs in front of it. +Each card also carries `queued`, true when its launch waits for a hangar slot or was admitted and is still starting, +and the reply carries `hangar: {cap, occupied, queued}`. `hangar.queued` counts only +launches still waiting for a slot; one that is starting is counted in `occupied`. `occupied` is `null`, +not 0, when the hangar cannot be asked: offline is unknown, not empty. The GET +reads this state and starts nothing, queued launches included. + ## Security Constraints - cc-mobile generates no agent settings of its own: the argv it builds carries no diff --git a/server/__tests__/cards-api.test.ts b/server/__tests__/cards-api.test.ts index 49fb163..81695d8 100644 --- a/server/__tests__/cards-api.test.ts +++ b/server/__tests__/cards-api.test.ts @@ -12,7 +12,8 @@ import { } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { createCardsPlugin } from "../cards"; +import { createCardsPlugin, type HangarView } from "../cards"; +import { cardKey } from "../launch-queue"; import { testServerConfig } from "./ws-harness"; let tmp: string; @@ -47,9 +48,10 @@ afterEach(() => { rmSync(tmp, { recursive: true, force: true }); }); -async function get() { +async function get(hangar?: HangarView | null) { const app = createCardsPlugin({ config: testServerConfig, + ...(hangar !== undefined ? { hangar } : {}), workspaceRoot: workspace, obsidianConfigPath: obsidianConfig, launchesDir, @@ -69,10 +71,12 @@ interface CardsBody { status: string | null; priority: string | null; blockedBy: string[]; + queued: boolean; dispatchable: boolean; }[]; }[]; skipped: { repo: string; reason: string }[]; + hangar: { cap: number; occupied: number | null; queued: number } | null; } describe("GET /api/cards", () => { @@ -99,6 +103,7 @@ describe("GET /api/cards", () => { status: "todo", priority: "high", blockedBy: ["[[b]]"], + queued: false, dispatchable: false, }, ], @@ -239,6 +244,78 @@ describe("GET /api/cards", () => { expect(body.skipped).toEqual([{ repo: join(workspace, "p"), reason: "vault_not_found" }]); }); + test("no hangar view reports hangar: null", async () => { + const { body } = await get(); + expect(body.hangar).toBeNull(); + }); + + test("a hangar view reports its cap, occupancy and queue length", async () => { + const hangar = { has: () => false, status: async () => ({ cap: 4, occupied: 2, queued: 0 }) }; + const { body } = await get(hangar); + expect(body.hangar).toEqual({ cap: 4, occupied: 2, queued: 0 }); + }); + + describe("queued cards", () => { + const viewOf = (...keys: string[]): HangarView => ({ + has: (key) => keys.includes(key), + status: async () => ({ cap: 1, occupied: 1, queued: keys.length }), + }); + const verdicts = (body: CardsBody) => + Object.fromEntries( + body.projects[0].cards.map((c) => [ + c.cardPath.split("/").pop(), + { queued: c.queued, dispatchable: c.dispatchable }, + ]), + ); + + test("a waiting card is queued and not dispatchable; its sibling is unaffected", async () => { + repo("p", "vault: obsidian\npm:\n project: p\n"); + card("p", "a.md", "type: task\nstatus: todo"); + card("p", "b.md", "type: task\nstatus: todo"); + const { body } = await get(viewOf(cardKey("obsidian", "pm/p/tasks/a.md"))); + expect(verdicts(body)).toEqual({ + "a.md": { queued: true, dispatchable: false }, + "b.md": { queued: false, dispatchable: true }, + }); + }); + + test("a key spelled with a doubled slash names the same card", async () => { + repo("p", "vault: obsidian\npm:\n project: p\n"); + card("p", "a.md", "type: task\nstatus: todo"); + const { body } = await get(viewOf(cardKey("obsidian", "pm//p/tasks/a.md"))); + expect(verdicts(body)["a.md"].queued).toBe(true); + }); + + test("a binding written with a doubled slash still binds its card", async () => { + repo("p", "vault: obsidian\npm:\n project: p\n"); + card("p", "a.md", "type: task\nstatus: todo"); + write( + join(launchesDir, "u.json"), + JSON.stringify({ vault: "obsidian", cardPath: "pm//p/tasks/a.md" }), + ); + const { body } = await get(viewOf()); + expect(verdicts(body)["a.md"].dispatchable).toBe(false); + }); + + test("a blocked card that is queued is queued and not dispatchable", async () => { + repo("p", "vault: obsidian\npm:\n project: p\n"); + card("p", "a.md", 'type: task\nstatus: todo\nblocked_by: ["[[x]]"]'); + const { body } = await get(viewOf(cardKey("obsidian", "pm/p/tasks/a.md"))); + expect(verdicts(body)["a.md"]).toEqual({ queued: true, dispatchable: false }); + }); + + test("a bound card that left the queue is bound, not queued, not dispatchable", async () => { + repo("p", "vault: obsidian\npm:\n project: p\n"); + card("p", "a.md", "type: task\nstatus: todo"); + write( + join(launchesDir, "u.json"), + JSON.stringify({ vault: "obsidian", cardPath: "pm/p/tasks/a.md" }), + ); + const { body } = await get(viewOf()); + expect(verdicts(body)["a.md"]).toEqual({ queued: false, dispatchable: false }); + }); + }); + test("reading writes nothing anywhere under the workspace, the vault or the launches dir", async () => { repo("p", "vault: obsidian\npm:\n project: p\n"); card("p", "a.md", "type: task\nstatus: todo\nblocked_by: []"); diff --git a/server/__tests__/config.test.ts b/server/__tests__/config.test.ts index 7a6f284..703608d 100644 --- a/server/__tests__/config.test.ts +++ b/server/__tests__/config.test.ts @@ -19,6 +19,7 @@ describe("parseServerConfig", () => { delete process.env.CC_MOBILE_ALLOWED_ROOTS; delete process.env.BASE_PATH; delete process.env.CC_MOBILE_VAULT_ROOT; + delete process.env.CC_MOBILE_HANGAR_CAP; const result = parseServerConfig(["node", "index.ts"]); expect(result).toEqual({ port: 3001, @@ -30,6 +31,7 @@ describe("parseServerConfig", () => { hangarSession: null, launchToken: null, vaultRoot: null, + hangarCap: 4, }); cleanup(); }); @@ -38,6 +40,7 @@ describe("parseServerConfig", () => { delete process.env.CC_MOBILE_ALLOWED_ROOTS; delete process.env.BASE_PATH; delete process.env.CC_MOBILE_VAULT_ROOT; + delete process.env.CC_MOBILE_HANGAR_CAP; const result = parseServerConfig([ "node", "index.ts", @@ -58,6 +61,7 @@ describe("parseServerConfig", () => { hangarSession: null, launchToken: null, vaultRoot: null, + hangarCap: 4, }); cleanup(); }); @@ -168,4 +172,81 @@ describe("parseServerConfig", () => { expect(() => parseServerConfig(["node", "index.ts"])).toThrow("CC_MOBILE_PUSH_SCOPE"); }); }); + + describe("hangar cap from performance cores", () => { + const original = process.env.CC_MOBILE_HANGAR_CAP; + afterEach(() => { + if (original === undefined) delete process.env.CC_MOBILE_HANGAR_CAP; + else process.env.CC_MOBILE_HANGAR_CAP = original; + }); + const capFor = (readPerfCores?: () => number | null) => { + delete process.env.CC_MOBILE_HANGAR_CAP; + return parseServerConfig(["node", "index.ts"], readPerfCores ? { readPerfCores } : undefined) + .hangarCap; + }; + + test("6 cores -> 4", () => expect(capFor(() => 6)).toBe(4)); + test("10 cores -> 8", () => expect(capFor(() => 10)).toBe(8)); + test("3, 2 or 1 cores -> 1", () => { + for (const n of [3, 2, 1]) expect(capFor(() => n)).toBe(1); + }); + test("null -> 4", () => expect(capFor(() => null)).toBe(4)); + test("a throwing reader -> 4", () => + expect( + capFor(() => { + throw new Error("x"); + }), + ).toBe(4)); + test("no options -> 4", () => expect(capFor()).toBe(4)); + test("reader is called exactly once", () => { + let calls = 0; + capFor(() => { + calls++; + return 6; + }); + expect(calls).toBe(1); + }); + }); + + describe("CC_MOBILE_HANGAR_CAP", () => { + const original = process.env.CC_MOBILE_HANGAR_CAP; + afterEach(() => { + if (original === undefined) delete process.env.CC_MOBILE_HANGAR_CAP; + else process.env.CC_MOBILE_HANGAR_CAP = original; + }); + const parse = (v: string, readPerfCores: () => number | null = () => 6) => { + process.env.CC_MOBILE_HANGAR_CAP = v; + return parseServerConfig(["node", "index.ts"], { readPerfCores }); + }; + + test("pins the cap and never reads the cores", () => { + let calls = 0; + const cfg = parse("6", () => { + calls++; + return 2; + }); + expect(cfg.hangarCap).toBe(6); + expect(calls).toBe(0); + }); + + test("trims surrounding whitespace", () => expect(parse(" 3 ").hangarCap).toBe(3)); + test("accepts two digits", () => expect(parse("12").hangarCap).toBe(12)); + + test("a malformed value throws naming the variable and the value", () => { + for (const v of ["0", "04", "+4", "4.0", "-1", "abc", "4 4"]) { + expect(() => parse(v)).toThrow(/CC_MOBILE_HANGAR_CAP/); + expect(() => parse(v)).toThrow(v); + } + }); + + test("blank counts as unset", () => { + let calls = 0; + const cfg = parse(" ", () => { + calls++; + return 6; + }); + expect(cfg.hangarCap).toBe(4); + expect(calls).toBe(1); + }); + }); }); diff --git a/server/__tests__/docs-launch-claims.test.ts b/server/__tests__/docs-launch-claims.test.ts index f8bda87..dbf98c7 100644 --- a/server/__tests__/docs-launch-claims.test.ts +++ b/server/__tests__/docs-launch-claims.test.ts @@ -40,4 +40,31 @@ describe("CLAUDE.md launch docs", () => { ); expect(flat).toContain("except that `POST /api/launch` answers `503 hangar_unavailable`"); }); + + test("documents the hangar cap and its queue", () => { + for (const term of [ + "CC_MOBILE_HANGAR_CAP", + "202 {queued: true, position}", + "409 already_queued", + "503 hangar_offline", + "hangar: {cap, occupied, queued}", + "agent.list", + "restart empties the queue", + ]) { + expect(doc.replace(/\s+/g, " ")).toContain(term); + } + }); + + test("states how queued, admitted and dequeued launches answer", () => { + const flat = doc.replace(/\s+/g, " "); + expect(flat).toContain("waiting, or admitted and still starting, answers `409 already_queued`"); + expect(flat).toContain("waits for a hangar slot or was admitted and is still starting"); + expect(flat).toContain("come after `unknown_profile` and before the worktree checks"); + expect(flat).toContain( + "still writes today's `prompt_send` audit line, `failed`, or `rejected` when the pane was busy", + ); + expect(flat).toContain( + "`hangar.queued` counts only launches still waiting for a slot; one that is starting is counted in `occupied`", + ); + }); }); diff --git a/server/__tests__/hangar-assembled.test.ts b/server/__tests__/hangar-assembled.test.ts index 3396f72..38bd2e2 100644 --- a/server/__tests__/hangar-assembled.test.ts +++ b/server/__tests__/hangar-assembled.test.ts @@ -6,12 +6,21 @@ */ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { WebSocket as WsClient } from "ws"; import type { AgentProfileSource } from "../agents/profiles"; import { type AppBackend, createApp } from "../app"; +import { parseServerConfig, type ServerConfig } from "../config"; import { HerdrTransportError } from "../herdr/errors"; import { composeLaunchPrompt } from "../launch-prompt"; import { createSubscriptionStore } from "../push/subscription-store"; @@ -45,7 +54,7 @@ function fakeDaemon(cwd: string) { const calls: Array<{ method: string; params: Record }> = []; const typed: Array<{ pane: string; text?: string; keys?: string[] }> = []; let shellOnly = false; - const behaviour = { listFails: false, compatibleFails: false }; + const behaviour = { listFails: false, compatibleFails: false, empty: false }; const agent = { terminal_id: "t1", agent_status: "idle", @@ -67,7 +76,7 @@ function fakeDaemon(cwd: string) { agentList: async () => { counts.agentList++; if (behaviour.listFails) throw new HerdrTransportError("connect ENOENT"); - return [agent]; + return behaviour.empty ? [] : [agent]; }, agentGet: async () => ({ ...agent, interactive_ready: true }), sessionSnapshot: async () => ({ @@ -83,6 +92,8 @@ function fakeDaemon(cwd: string) { call: async (method: string, params: unknown) => { calls.push({ method, params: params as Record }); if (method === "workspace.create") { + // A pane that now exists is listed, whatever the test emptied before. + behaviour.empty = false; shellOnly = true; return { type: "workspace_created", @@ -189,6 +200,8 @@ function rig(options: { launchToken?: string; launchesDir?: string; agentProfiles?: AgentProfileSource; + config?: ServerConfig; + hangarCap?: number; }) { const cockpit = options.cockpit ?? fakeDaemon("/tmp/cockpitproj"); const hangar = options.hangar ?? fakeDaemon("/tmp/hangarproj"); @@ -196,19 +209,33 @@ function rig(options: { const sends: Array<{ payload: Record; opts: Record }> = []; const backendRef: { current: AppBackend | null } = { current: null }; const handle: { fire?: () => void } = {}; + // The queue's poll, driven by hand: nothing here waits on a real interval. + const queueTimer: { fn?: () => void } = {}; const paths = { auditLogPath: join(tmp, "audit.jsonl"), pushAttemptLogPath: join(tmp, "attempts.jsonl"), }; const app = createApp( { - ...testServerConfig, + ...(options.config ?? testServerConfig), hangarSession: options.hangarSession, + ...(options.hangarCap ? { hangarCap: options.hangarCap } : {}), ...(options.launchToken ? { launchToken: options.launchToken } : {}), ...(options.vaultRoot ? { vaultRoot: options.vaultRoot } : {}), }, { - ...(options.launchesDir ? { launchesDir: options.launchesDir } : {}), + launchesDir: options.launchesDir ?? join(tmp, "launches"), + workspaceRoot: join(tmp, "workspace"), + obsidianConfigPath: join(tmp, "obsidian.json"), + queueTimers: { + setIntervalFn: (fn) => { + queueTimer.fn = fn; + return 1; + }, + clearIntervalFn: () => { + queueTimer.fn = undefined; + }, + }, cardWorktrees: { create: async () => ({ kind: "not_a_repo" }), remove: async () => {} }, ...(options.agentProfiles ? { agentProfiles: options.agentProfiles } : {}), pushStore: createSubscriptionStore({ path: join(tmp, "subs.json") }), @@ -241,6 +268,12 @@ function rig(options: { body: JSON.stringify(body), }), ); + const cards = async () => { + const res = await app.handle(new Request("http://localhost/api/cards")); + return (await res.json()) as { + hangar: { cap: number; occupied: number | null; queued: number } | null; + }; + }; const subscribe = () => app.handle( new Request("http://localhost/api/push/subscribe", { @@ -298,8 +331,10 @@ function rig(options: { sends, backendRef, handle, + queueTimer, paths, launch, + cards, subscribe, phone, }; @@ -671,3 +706,103 @@ describe("AssembledLaunchOnHangarDaemon", () => { expect(methods(r.hangar)).not.toContain("workspace.create"); }); }); + +describe("HangarCap", () => { + const CARD = "# Task\nCARD-BODY do the thing\n"; + const body = { + cwd: "/tmp", + cardPath: "pm/cc-mobile/tasks/card.md", + vault: "obsidian", + project: "cc-mobile", + }; + + function capRig(options: { hangarCap?: number } = {}) { + const vault = join(tmp, "obsidian"); + mkdirSync(join(vault, "pm", "cc-mobile", "tasks"), { recursive: true }); + writeFileSync(join(vault, body.cardPath), CARD); + const r = rig({ hangarSession: "fleet", vaultRoot: vault, launchToken: "tok", ...options }); + const sendKeys = r.hangar.client.paneSendKeys; + r.hangar.client.paneSendKeys = async (pane, keys) => { + await sendKeys(pane, keys); + if (keys.includes("Enter")) r.hangar.status("working"); + }; + return r; + } + const created = (r: ReturnType) => + r.hangar.calls.filter((c) => c.method === "workspace.create").length; + + test("CardsReportHangar T6: GET /api/cards reads the hangar with one agent.list", async () => { + const r = rig({ hangarSession: "fleet" }); + expect((await r.cards()).hangar).toEqual({ cap: 4, occupied: 1, queued: 0 }); + expect(r.hangar.counts.agentList).toBe(1); + }); + + test("CardsReportHangar T8: with no hangar session the cards reply says hangar: null", async () => { + const r = rig({ hangarSession: null }); + expect((await r.cards()).hangar).toBeNull(); + }); + + test("CardsReportHangar T7: the cap is the pinned env, then the derived cores, then the fallback", async () => { + const saved = process.env.CC_MOBILE_HANGAR_CAP; + try { + process.env.CC_MOBILE_HANGAR_CAP = "7"; + const pinned = rig({ + hangarSession: "fleet", + config: parseServerConfig([], { readPerfCores: () => 10 }), + }); + expect((await pinned.cards()).hangar?.cap).toBe(7); + delete process.env.CC_MOBILE_HANGAR_CAP; + const derived = rig({ + hangarSession: "fleet", + config: parseServerConfig([], { readPerfCores: () => 10 }), + }); + expect((await derived.cards()).hangar?.cap).toBe(8); + } finally { + if (saved === undefined) delete process.env.CC_MOBILE_HANGAR_CAP; + else process.env.CC_MOBILE_HANGAR_CAP = saved; + } + expect((await rig({ hangarSession: "fleet" }).cards()).hangar?.cap).toBe(4); + }); + + test("LaunchRefusedWhileHangarOffline T4: a hangar that cannot list is 503 and creates nothing", async () => { + const r = capRig(); + r.hangar.behaviour.listFails = true; + const res = await r.launch(body); + expect(res.status).toBe(503); + expect(await res.json()).toEqual({ error: "hangar_offline" }); + expect(created(r)).toBe(0); + }); + + test("QueuedLaunchStartsOnPoll T4: a full hangar queues, and the poll starts the launch once a slot frees", async () => { + const r = capRig({ hangarCap: 1 }); + const res = await r.launch(body); + expect(res.status).toBe(202); + expect(await res.json()).toEqual({ queued: true, position: 1 }); + expect(created(r)).toBe(0); + expect(r.queueTimer.fn).toBeDefined(); + + r.hangar.behaviour.empty = true; + r.queueTimer.fn?.(); + await until(() => r.hangar.typed.length >= 2, "the prompt typed into the hangar pane"); + + expect(created(r)).toBe(1); + const files = readdirSync(join(tmp, "launches")); + expect(files).toHaveLength(1); + expect(JSON.parse(readFileSync(join(tmp, "launches", files[0]), "utf8")).paneId).toBe( + HANGAR_KEY, + ); + expect(r.hangar.typed[0]).toEqual({ pane: PANE, text: composeLaunchPrompt(CARD) }); + }); + + test("QueuedLaunchStartsAfterHangarTeardown T4: closing a hangar pane starts the head without a tick", async () => { + const r = capRig({ hangarCap: 1 }); + expect((await r.launch(body)).status).toBe(202); + expect(created(r)).toBe(0); + const phone = await r.phone(); + await phone.list(); + + r.hangar.behaviour.empty = true; + phone.socket.send(JSON.stringify({ type: "terminal_teardown", sessionId: HANGAR_KEY })); + await until(() => created(r) === 1, "the queued launch starting"); + }); +}); diff --git a/server/__tests__/hangar-cap.test.ts b/server/__tests__/hangar-cap.test.ts new file mode 100644 index 0000000..9138249 --- /dev/null +++ b/server/__tests__/hangar-cap.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, test } from "bun:test"; +import { + describeHangarCap, + readPerfCoresViaSysctl, + resolveHangarCapWithSource, +} from "../hangar-cap"; + +describe("readPerfCoresViaSysctl", () => { + test("reads the count via the absolute sysctl path", () => { + const calls: string[][] = []; + const n = readPerfCoresViaSysctl((argv) => { + calls.push(argv); + return { exitCode: 0, stdout: "6\n" }; + }); + expect(n).toBe(6); + expect(calls).toEqual([["/usr/sbin/sysctl", "-n", "hw.perflevel0.physicalcpu"]]); + }); + + test("non-zero exit -> null", () => { + expect(readPerfCoresViaSysctl(() => ({ exitCode: 1, stdout: "" }))).toBeNull(); + }); + + test("non-numeric output -> null", () => { + expect(readPerfCoresViaSysctl(() => ({ exitCode: 0, stdout: "abc\n" }))).toBeNull(); + }); + + test("zero -> null", () => { + expect(readPerfCoresViaSysctl(() => ({ exitCode: 0, stdout: "0\n" }))).toBeNull(); + }); + + test("a throwing runner -> null", () => { + expect( + readPerfCoresViaSysctl(() => { + throw new Error("ENOENT"); + }), + ).toBeNull(); + }); +}); + +describe("resolveHangarCapWithSource", () => { + test("source: blank override counts as unset and names the core count", () => { + expect(resolveHangarCapWithSource(" ", () => 6)).toEqual({ + cap: 4, + source: "perf-cores", + cores: 6, + }); + }); + + test("source: a pinned value names the override and skips the reader", () => { + let calls = 0; + const r = resolveHangarCapWithSource("3", () => { + calls++; + return 6; + }); + expect(r).toEqual({ cap: 3, source: "override" }); + expect(calls).toBe(0); + }); + + test("source: an unreadable count names the fallback", () => { + expect(resolveHangarCapWithSource(undefined, () => null)).toEqual({ + cap: 4, + source: "unreadable", + }); + }); + + test("source: the startup line reads each source", () => { + expect(describeHangarCap({ cap: 4, source: "perf-cores", cores: 6 })).toBe( + "hangar cap 4 (6 performance cores)", + ); + expect(describeHangarCap({ cap: 2, source: "override" })).toBe( + "hangar cap 2 (CC_MOBILE_HANGAR_CAP)", + ); + expect(describeHangarCap({ cap: 4, source: "unreadable" })).toBe( + "hangar cap 4 (performance cores unreadable)", + ); + }); +}); diff --git a/server/__tests__/launch-api-routing.test.ts b/server/__tests__/launch-api-routing.test.ts index 3bc3b29..5c40edf 100644 --- a/server/__tests__/launch-api-routing.test.ts +++ b/server/__tests__/launch-api-routing.test.ts @@ -99,7 +99,7 @@ function fakeHerdr(claude: Claude = {}) { return { client, typed, hooks }; } -function launchApp(claude?: Claude) { +function launchApp(claude?: Claude, hangarAgents = 0) { const { client, typed, hooks } = fakeHerdr(claude); const backend = createHerdrBackend({ client: client as never, @@ -112,6 +112,7 @@ function launchApp(claude?: Claude) { created.push(input.claudeUuid); return { name: "claude", paneRef: PANE }; }; + backend.hangarAgentCount = async () => hangarAgents; const eventBuffer = new EventBuffer(500); const dir = mkdtempSync(join(tmpdir(), "launch-audit-")); const vault = join(dir, "obsidian"); @@ -126,6 +127,7 @@ function launchApp(claude?: Claude) { gateEnv: {}, auditLogPath: join(dir, "a.jsonl"), launchesDir, + queueTimers: { setIntervalFn: () => 0, clearIntervalFn: () => {} }, cardWorktrees: { create: async () => ({ kind: "not_a_repo" }), remove: async () => {} }, }, ); @@ -163,6 +165,15 @@ test("an HTTP launch with no WebSocket client types the template and card, press expect(existsSync(join(launchesDir, `${claudeUuid}.json`))).toBe(true); }); +test("a launch beyond the default cap is held with its place in line, and nothing is typed", async () => { + const { post, typed, created } = launchApp(undefined, 4); + const r = await post(); + expect(r.status).toBe(202); + expect(await r.json()).toEqual({ queued: true, position: 1 }); + expect(typed).toEqual([]); + expect(created).toEqual([]); +}); + test("a busy pane answers 502 with the routing's code, and nothing is typed", async () => { const { post, typed, eventBuffer, created } = launchApp({ status: "working" }); const r = await post(); diff --git a/server/__tests__/launch-api.test.ts b/server/__tests__/launch-api.test.ts index b1953ac..d9e7ee6 100644 --- a/server/__tests__/launch-api.test.ts +++ b/server/__tests__/launch-api.test.ts @@ -17,6 +17,7 @@ import { type AgentProfile, emptyAgentProfileSource } from "../agents/profiles"; import { EventBuffer } from "../event-buffer"; import { createLaunchPlugin } from "../launch"; import { composeLaunchPrompt } from "../launch-prompt"; +import { cardKey, createLaunchQueue } from "../launch-queue"; import type { CardWorktreeRequest, CardWorktreeResult } from "../launch-worktree"; import type { CreateSessionInput } from "../terminal-backend"; import { testServerConfig } from "./ws-harness"; @@ -65,10 +66,49 @@ interface SetupOptions { launchesDir?: string; profiles?: AgentProfile[]; worktree?: CardWorktreeResult; + cap?: number; + /** What the hangar reports; change `s.occupancy.value` to move it. */ + occupancy?: number | null | Error; + /** `send` waits for this before answering. */ + holdSend?: Promise; } function setup(opts: SetupOptions = {}) { const calls: string[] = []; + // `calls` is writes only; `events` also records each occupancy read. + const events: string[] = []; + const record = (event: string) => { + calls.push(event); + events.push(event); + }; + const occupancy = { + value: opts.occupancy === undefined ? (0 as number | null | Error) : opts.occupancy, + reads: 0, + }; + const logs: string[] = []; + const timers = { set: [] as (() => void)[], cleared: 0 }; + const queue = createLaunchQueue({ + cap: opts.cap ?? 4, + occupancy: async () => { + occupancy.reads += 1; + events.push("occupancy"); + if (occupancy.value instanceof Error) throw occupancy.value; + return occupancy.value; + }, + setIntervalFn: (fn) => timers.set.push(fn), + clearIntervalFn: () => { + timers.cleared += 1; + }, + log: (line) => logs.push(line), + }); + /** Fires the queue's poll and lets the launch it starts run to its end. */ + const tick = async (cardPath: string = good.cardPath) => { + timers.set[timers.set.length - 1](); + const key = cardKey(good.vault, cardPath); + for (let i = 0; i < 400 && queue.has(key); i++) { + await new Promise((resolve) => setTimeout(resolve, 5)); + } + }; const worktreeRequests: CardWorktreeRequest[] = []; const creates: CreateSessionInput[] = []; const sends: { claudeUuid: string; content: string; confirmStart?: boolean }[] = []; @@ -77,20 +117,21 @@ function setup(opts: SetupOptions = {}) { const bindingAtSend: { text: string | null }[] = []; const backend = { createSession: async (input: CreateSessionInput) => { - calls.push(`create:${input.cwd}`); + record(`create:${input.cwd}`); creates.push(input); if (opts.createError) throw new Error(opts.createError); return { name: "t", paneRef: "fleet@w1:p2" }; }, teardown: async (key: string) => { - calls.push(`teardown:${key}`); + record(`teardown:${key}`); if (opts.teardownFails) throw new Error("teardown boom"); return { killed: true }; }, registerClient: () => {}, send: async (params: { claudeUuid: string; content: string; confirmStart?: boolean }) => { - calls.push(`send:${params.claudeUuid}`); + record(`send:${params.claudeUuid}`); sends.push(params); + await opts.holdSend; const file = join(launchesDir, `${creates[0]?.claudeUuid}.json`); bindingAtSend.push({ text: existsSync(file) ? readFileSync(file, "utf8") : null }); if (opts.sendFails) throw new Error("boom"); @@ -114,22 +155,28 @@ function setup(opts: SetupOptions = {}) { worktrees: { create: async (request: CardWorktreeRequest) => { worktreeRequests.push(request); - calls.push(`worktree:${request.cwd}:${request.cardName}`); + record(`worktree:${request.cwd}:${request.cardName}`); return opts.worktree ?? { kind: "not_a_repo" }; }, remove: async (worktree: { path: string }) => { - calls.push(`remove:${worktree.path}`); + record(`remove:${worktree.path}`); }, }, + queue, auditLog: { append: async (r: Record) => void audits.push(r) } as never, }); - const post = (body: unknown, auth: string | null = "Bearer s3cret") => + const post = ( + body: unknown, + auth: string | null = "Bearer s3cret", + headers: Record = {}, + ) => app.handle( new Request("http://localhost/api/launch", { method: "POST", headers: { "content-type": "application/json", "x-forwarded-for": "100.1.2.3", + ...headers, ...(auth ? { authorization: auth } : {}), }, body: JSON.stringify(body), @@ -137,6 +184,7 @@ function setup(opts: SetupOptions = {}) { ); const launches = () => (existsSync(launchesDir) ? readdirSync(launchesDir) : []); return { + opts, calls, worktreeRequests, creates, @@ -146,6 +194,12 @@ function setup(opts: SetupOptions = {}) { launchesDir, launches, bindingAtSend, + queue, + occupancy, + logs, + events, + timers, + tick, }; } @@ -720,3 +774,390 @@ describe("LaunchCardWorktree", () => { expect(s.calls[0]).toBe("worktree:/tmp:x.y"); }); }); + +function card(name: string, text = CARD) { + writeFileSync(join(vault, "pm", "cc-mobile", "tasks", name), text); + return `pm/cc-mobile/tasks/${name}`; +} + +/** A queue that is full: one slot, taken. */ +function full(opts: SetupOptions = {}) { + return setup({ cap: 1, occupancy: 1, ...opts }); +} + +const WRITES = ["worktree:/tmp:card", "create:/tmp", "send:fleet@w1:p2"]; + +describe("LaunchAdmittedWhenSlotFree", () => { + test("T1 a free slot starts at once, reading occupancy once before the worktree", async () => { + const s = setup({ cap: 4, occupancy: 3 }); + const r = await s.post(good); + expect(r.status).toBe(201); + const body = (await r.json()) as { sessionId: string; claudeUuid: string }; + expect(body.sessionId).toBe("fleet@w1:p2"); + expect(body.claudeUuid).toMatch(UUID_V4); + expect(s.events).toEqual(["occupancy", ...WRITES]); + }); + test("T4 a refused session creation answers today's 500", async () => { + const s = setup({ cap: 4, occupancy: 3, createError: "boom" }); + const r = await s.post(good); + expect(r.status).toBe(500); + expect(await r.json()).toEqual({ error: "terminal_error", message: "boom" }); + }); +}); + +describe("LaunchChecksPrecedeQueue", () => { + test("T1 a missing token is refused before the queue or occupancy is touched", async () => { + const s = setup({ cap: 4, occupancy: 4 }); + const r = await s.post(good, null); + expect(r.status).toBe(401); + expect(await r.json()).toEqual({ error: "unauthorized" }); + expect(s.occupancy.reads).toBe(0); + nothingHappened(s); + expect((await s.queue.status()).queued).toBe(0); + }); + test("T2 a wrong token is 401 even when the card already waits", async () => { + const s = full(); + expect((await s.post(good)).status).toBe(202); + expect((await s.post(good, "Bearer nope")).status).toBe(401); + }); + test("T3 an invalid body reads no occupancy", async () => { + const s = setup({ occupancy: null }); + const r = await s.post({}); + expect(r.status).toBe(400); + expect(await r.json()).toEqual({ error: "invalid_body" }); + expect(s.occupancy.reads).toBe(0); + }); + test("T4 a bad card path is refused on a full hangar", async () => { + const s = setup({ cap: 4, occupancy: 4 }); + const r = await s.post({ ...good, cardPath: "pm/../x.md" }); + expect(r.status).toBe(400); + expect(await r.json()).toEqual({ error: "invalid_card_path" }); + }); + test("T5 an unknown profile is refused on a full hangar", async () => { + const s = setup({ cap: 4, occupancy: 4 }); + const r = await s.post({ ...good, profileId: "nope" }); + expect(r.status).toBe(400); + expect(await r.json()).toEqual({ + error: "unknown_profile", + message: "Unknown agent profile: nope", + }); + }); + test("T6 a deleted card is 404 on a full hangar", async () => { + const s = setup({ cap: 4, occupancy: 4 }); + rmSync(join(vault, good.cardPath)); + const r = await s.post(good); + expect(r.status).toBe(404); + expect(await r.json()).toEqual({ error: "card_not_found" }); + expect(s.occupancy.reads).toBe(0); + }); +}); + +describe("LaunchQueuedWhenFull", () => { + test("T1-T2 a full hangar answers 202 with a growing position and writes nothing", async () => { + const s = setup({ cap: 4, occupancy: 4 }); + const r = await s.post(good); + expect(r.status).toBe(202); + expect(await r.json()).toEqual({ queued: true, position: 1 }); + nothingHappened(s); + const positions: unknown[] = []; + for (const name of ["b.md", "c.md", "d.md"]) { + const res = await s.post({ ...good, cardPath: card(name) }); + positions.push(((await res.json()) as { position: number }).position); + } + expect(positions).toEqual([2, 3, 4]); + nothingHappened(s); + }); + test("T3 a free slot goes to the head of the line, not to a newcomer", async () => { + let release!: () => void; + const s = setup({ + cap: 2, + occupancy: 2, + holdSend: new Promise((resolve) => (release = () => resolve(null))), + }); + const a = card("a.md"); + expect((await s.post({ ...good, cardPath: a })).status).toBe(202); + s.occupancy.value = 1; + const r = await s.post({ ...good, cardPath: card("d.md") }); + expect(r.status).toBe(202); + expect(await r.json()).toEqual({ queued: true, position: 1 }); + while (s.sends.length === 0) await new Promise((resolve) => setTimeout(resolve, 5)); + expect(s.worktreeRequests.map((w) => w.cardName)).toEqual(["a"]); + release(); + }); +}); + +describe("LaunchRefusedWhileHangarOffline", () => { + test("T1 an unknown occupancy is 503 and nothing is queued", async () => { + const s = setup({ occupancy: null }); + const r = await s.post(good); + expect(r.status).toBe(503); + expect(await r.json()).toEqual({ error: "hangar_offline" }); + nothingHappened(s); + expect((await s.queue.status()).queued).toBe(0); + }); + test("T2 a rejecting occupancy source is 503", async () => { + const s = setup({ occupancy: new Error("x") }); + const r = await s.post(good); + expect(r.status).toBe(503); + expect(await r.json()).toEqual({ error: "hangar_offline" }); + }); + test("T3 a poll while offline starts nothing", async () => { + const s = full(); + expect((await s.post(good)).status).toBe(202); + s.occupancy.value = null; + const b = card("b.md"); + expect((await s.post({ ...good, cardPath: b })).status).toBe(503); + s.timers.set[0](); + await new Promise((resolve) => setTimeout(resolve, 20)); + nothingHappened(s); + }); +}); + +describe("DuplicateCardRefused", () => { + test("T1 the same card twice is 409 and the queue is unchanged", async () => { + const s = full(); + expect((await s.post(good)).status).toBe(202); + const r = await s.post(good); + expect(r.status).toBe(409); + expect(await r.json()).toEqual({ error: "already_queued" }); + expect((await s.queue.status()).queued).toBe(1); + }); + test("T2 two spellings of one path are one card", async () => { + const s = full(); + expect((await s.post({ ...good, cardPath: "pm//cc-mobile/tasks/card.md" })).status).toBe(202); + const r = await s.post({ ...good, cardPath: "pm/cc-mobile/tasks/./card.md" }); + expect(r.status).toBe(409); + expect(await r.json()).toEqual({ error: "already_queued" }); + }); + test("T3 a card whose launch is still starting is 409", async () => { + let release!: () => void; + const s = setup({ holdSend: new Promise((resolve) => (release = () => resolve(null))) }); + const first = s.post(good); + while (s.sends.length === 0) await new Promise((resolve) => setTimeout(resolve, 5)); + const second = await s.post(good); + expect(second.status).toBe(409); + expect(await second.json()).toEqual({ error: "already_queued" }); + release(); + expect((await first).status).toBe(201); + }); + test("T4 a launch that failed outright is not remembered", async () => { + const s = setup({ createError: "boom" }); + expect((await s.post(good)).status).toBe(500); + expect((await s.post(good)).status).toBe(500); + }); + test("T5 the duplicate check precedes the occupancy read", async () => { + const s = full(); + expect((await s.post(good)).status).toBe(202); + s.occupancy.value = null; + expect((await s.post(good)).status).toBe(409); + }); +}); + +describe("DequeuedLaunchRunsTodaysWrites", () => { + test("T1 a dequeued launch runs the same writes, attributed to the enqueuing request", async () => { + const s = full(); + expect((await s.post(good, "Bearer s3cret", { "x-forwarded-for": "100.1.2.3" })).status).toBe( + 202, + ); + nothingHappened(s); + s.occupancy.value = 0; + await s.tick(); + expect(s.calls).toEqual(WRITES); + const files = s.launches(); + expect(files).toHaveLength(1); + expect(JSON.parse(readFileSync(join(s.launchesDir, files[0]), "utf8"))).toEqual({ + cardPath: "pm/cc-mobile/tasks/card.md", + vault: "obsidian", + project: "cc-mobile", + paneId: "fleet@w1:p2", + createdAt: expect.stringMatching(/^\d{4}-\d\d-\d\dT/), + }); + expect(s.audits).toEqual([ + { + action: "prompt_send", + paneId: "fleet@w1:p2", + ip: "100.1.2.3", + device: "launch-api", + outcome: "dispatched", + }, + ]); + expect(s.sends[0].confirmStart).toBe(true); + }); + test("T2 the pane starts in the card's worktree on the hangar", async () => { + const wt = join(tmp, "wt"); + mkdirSync(wt); + const s = full({ + worktree: { + kind: "created", + repo: "/repo", + path: "/repo/.claude/worktrees/card", + branch: "fleet/card", + cwd: wt, + }, + }); + await s.post(good); + s.occupancy.value = 0; + await s.tick(); + expect(s.creates[0].cwd).toBe(wt); + expect(s.creates[0].side).toBe("hangar"); + }); + test("T3 the binding exists before the prompt is typed", async () => { + const s = full(); + await s.post(good); + s.occupancy.value = 0; + await s.tick(); + expect(s.bindingAtSend[0].text).not.toBeNull(); + }); +}); + +describe("DequeuedLaunchReadsCardAtDequeue", () => { + test("T1 the prompt typed is the card as it stands at dequeue", async () => { + const s = full(); + expect((await s.post(good)).status).toBe(202); + card("card.md", "# Task\nCARD-BODY v2\n"); + s.occupancy.value = 0; + await s.tick(); + expect(s.sends[0].content).toBe(composeLaunchPrompt("# Task\nCARD-BODY v2\n")); + }); + test("T2 the profile's argv is the one listed at dequeue", async () => { + const s = full({ profiles: [claudeAuto] }); + expect((await s.post({ ...good, profileId: "claude-auto" })).status).toBe(202); + s.opts.profiles = [{ ...claudeAuto, args: ["--permission-mode", "plan"] }]; + s.occupancy.value = 0; + await s.tick(); + expect(s.creates[0].profileArgs).toEqual(["--permission-mode", "plan"]); + }); +}); + +describe("DequeueFailureFreesCard", () => { + const created: CardWorktreeResult = { + kind: "created", + repo: "/repo", + path: "/repo/.claude/worktrees/card", + branch: "fleet/card", + cwd: "/tmp", + }; + const key = cardKey("obsidian", good.cardPath); + + test("T1 a refused worktree frees the card and removes nothing", async () => { + const s = full(); + await s.post(good); + s.opts.worktree = { kind: "refused", code: "worktree_exists", message: "taken" }; + s.occupancy.value = 0; + await s.tick(); + expect(s.queue.has(key)).toBe(false); + expect(s.calls.some((c) => c.startsWith("remove:"))).toBe(false); + s.occupancy.value = 1; + expect((await s.post(good)).status).not.toBe(409); + }); + test("T2 a failed session creation removes the worktree it made and frees the card", async () => { + const s = full({ worktree: created }); + await s.post(good); + s.opts.createError = "boom"; + s.occupancy.value = 0; + await s.tick(); + expect(s.calls).toContain(`remove:${created.path}`); + expect(s.queue.has(key)).toBe(false); + s.occupancy.value = 1; + expect((await s.post(good)).status).not.toBe(409); + }); + test("T3 a failed binding write tears the pane down, removes the worktree and frees the card", async () => { + writeFileSync(join(tmp, "afile"), "x"); + const s = full({ worktree: created, launchesDir: join(tmp, "afile", "sub") }); + await s.post(good); + s.occupancy.value = 0; + await s.tick(); + expect(s.calls).toContain("teardown:fleet@w1:p2"); + expect(s.calls).toContain(`remove:${created.path}`); + expect(s.queue.has(key)).toBe(false); + s.occupancy.value = 1; + expect((await s.post(good)).status).not.toBe(409); + }); + test("T4 a card deleted while it waited frees the card, and a restored card can be dispatched", async () => { + const s = full(); + await s.post(good); + const path = join(vault, good.cardPath); + rmSync(path); + s.occupancy.value = 0; + await s.tick(); + expect(s.queue.has(key)).toBe(false); + writeFileSync(path, CARD); + s.occupancy.value = 1; + expect((await s.post(good)).status).not.toBe(409); + }); +}); + +describe("DequeueFailureLogsOnly", () => { + test("T1 a card deleted while waiting leaves one log line and no write", async () => { + const s = full(); + await s.post(good); + rmSync(join(vault, good.cardPath)); + s.occupancy.value = 0; + await s.tick(); + expect(s.logs).toHaveLength(1); + expect(s.logs[0]).toMatch(/pm\/cc-mobile\/tasks\/card\.md/); + expect(s.logs[0]).toMatch(/card_not_found/); + expect(s.audits).toEqual([]); + expect(s.calls).toEqual([]); + }); + test("T2 a refused session creation logs terminal_error", async () => { + const s = full(); + await s.post(good); + s.opts.createError = "boom"; + s.occupancy.value = 0; + await s.tick(); + expect(s.logs).toHaveLength(1); + expect(s.logs[0]).toMatch(/terminal_error/); + expect(s.audits).toEqual([]); + }); + test("T3 a log line carries neither the token nor the card text", async () => { + const lines: string[] = []; + const first = full(); + await first.post(good); + rmSync(join(vault, good.cardPath)); + first.occupancy.value = 0; + await first.tick(); + lines.push(...first.logs); + writeFileSync(join(vault, good.cardPath), CARD); + const second = full(); + await second.post(good); + second.opts.createError = "boom"; + second.occupancy.value = 0; + await second.tick(); + lines.push(...second.logs); + expect(lines).toHaveLength(2); + for (const line of lines) { + expect(line).not.toContain("s3cret"); + expect(line).not.toContain("CARD-BODY"); + } + }); + test("T4 a launch admitted at once logs nothing when it fails", async () => { + const s = setup({ createError: "boom" }); + expect((await s.post(good)).status).toBe(500); + expect(s.logs).toHaveLength(0); + }); +}); + +describe("DequeuedPromptFailureKeepsBinding", () => { + test("T1 an unconfirmed prompt keeps the pane and binding, audits failed and frees the card", async () => { + const s = full(); + await s.post(good); + s.opts.sendFails = true; + s.occupancy.value = 0; + await s.tick(); + expect(s.launches()).toHaveLength(1); + expect(s.calls.some((c) => c.startsWith("remove:") || c.startsWith("teardown:"))).toBe(false); + expect(s.audits).toEqual([ + { + action: "prompt_send", + paneId: "fleet@w1:p2", + ip: "100.1.2.3", + device: "launch-api", + outcome: "failed", + }, + ]); + expect(s.logs).toHaveLength(1); + expect(s.logs[0]).toMatch(/prompt_failed/); + expect(s.queue.has(cardKey("obsidian", good.cardPath))).toBe(false); + }); +}); diff --git a/server/__tests__/launch-queue.test.ts b/server/__tests__/launch-queue.test.ts new file mode 100644 index 0000000..34c50ea --- /dev/null +++ b/server/__tests__/launch-queue.test.ts @@ -0,0 +1,347 @@ +import { describe, expect, test } from "bun:test"; +import { + cardKey, + createLaunchQueue, + type LaunchQueueOptions, + type StartResult, +} from "../launch-queue"; + +interface Deferred { + promise: Promise; + resolve(value: T): void; + reject(error: unknown): void; +} + +function deferred(): Deferred { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); + +function rig(opts: { cap: number; deferredReads?: boolean; intervalMs?: number }) { + const source = { + value: 0 as number | null | Error, + deferredReads: opts.deferredReads ?? false, + pending: [] as Deferred[], + calls: 0, + }; + const timers = { + set: [] as { fn: () => void; ms: number }[], + cleared: 0, + }; + const logs: string[] = []; + const options: LaunchQueueOptions = { + cap: opts.cap, + occupancy: () => { + source.calls += 1; + if (source.deferredReads) { + const d = deferred(); + source.pending.push(d); + return d.promise; + } + if (source.value instanceof Error) return Promise.reject(source.value); + return Promise.resolve(source.value); + }, + setIntervalFn: (fn, ms) => { + timers.set.push({ fn, ms }); + return timers.set.length; + }, + clearIntervalFn: () => { + timers.cleared += 1; + }, + log: (line) => logs.push(line), + ...(opts.intervalMs !== undefined ? { intervalMs: opts.intervalMs } : {}), + }; + const queue = createLaunchQueue(options); + const tick = async () => { + timers.set[timers.set.length - 1].fn(); + await flush(); + }; + return { queue, source, timers, logs, tick }; +} + +/** A start that records its invocation and finishes when told. */ +function entry(name: string) { + const d = deferred(); + const probe = { started: 0 }; + return { + key: cardKey("v", `pm/p/tasks/${name}.md`), + label: `v/pm/p/tasks/${name}.md`, + start: () => { + probe.started += 1; + return d.promise; + }, + finish: d, + probe, + }; +} + +describe("cardKey", () => { + test("one card, whichever way its path is spelled", () => { + expect(cardKey("v", "pm//p/tasks/a.md")).toBe(cardKey("v", "pm/p/tasks/./a.md")); + expect(cardKey("v", "a.md")).not.toBe(cardKey("w", "a.md")); + }); +}); + +describe("OccupancyNeverExceedsCap", () => { + test("T1 five concurrent submits against cap 4 start four", async () => { + const { queue } = rig({ cap: 4 }); + const results = await Promise.all(["a", "b", "c", "d", "e"].map((n) => queue.submit(entry(n)))); + expect(results.filter((r) => r.kind === "started").length).toBe(4); + expect(results[4]).toEqual({ kind: "queued", position: 1 }); + }); + + test("T2 serialized reads start one of two on a cap of one", async () => { + const { queue, source } = rig({ cap: 1, deferredReads: true }); + const a = queue.submit(entry("a")); + const b = queue.submit(entry("b")); + await flush(); + source.pending[0].resolve(0); + await flush(); + source.pending[1]?.resolve(0); + const results = await Promise.all([a, b]); + expect(results.map((r) => r.kind).sort()).toEqual(["queued", "started"]); + }); + + test("T3 a reading issued before a finish does not release its slot", async () => { + const { queue, source, tick } = rig({ cap: 1 }); + const a = entry("a"); + const b = entry("b"); + expect((await queue.submit(a)).kind).toBe("started"); + expect((await queue.submit(b)).kind).toBe("queued"); + source.deferredReads = true; + await tick(); + expect(source.pending.length).toBe(1); + a.finish.resolve({ ok: false, reason: "prompt_failed" }); + await flush(); + source.pending[0].resolve(0); + await flush(); + expect(b.probe.started).toBe(0); + source.pending[1].resolve(0); + await flush(); + expect(b.probe.started).toBe(1); + }); + + test("T4 a reading after the finish that still lists the pane keeps the head waiting", async () => { + const { queue, source } = rig({ cap: 1 }); + const a = entry("a"); + const b = entry("b"); + await queue.submit(a); + await queue.submit(b); + source.value = 1; + a.finish.resolve({ ok: true }); + await flush(); + expect(b.probe.started).toBe(0); + source.value = 0; + await queue.kick(); + expect(b.probe.started).toBe(1); + }); + + test("T5 status counts a running launch on top of the reading", async () => { + const { queue, source } = rig({ cap: 2 }); + await queue.submit(entry("a")); + source.value = 1; + expect((await queue.status()).occupied).toBe(2); + }); + + test("T6 an occupancy rejection is offline and starts nothing", async () => { + const { queue, source } = rig({ cap: 1 }); + source.value = new Error("x"); + const e = entry("a"); + expect(await queue.submit(e)).toEqual({ kind: "hangar_offline" }); + expect(e.probe.started).toBe(0); + }); +}); + +describe("QueuedLaunchRechecksOnFinish", () => { + async function full() { + const r = rig({ cap: 1 }); + const a = entry("a"); + const b = entry("b"); + await r.queue.submit(a); + await r.queue.submit(b); + return { ...r, a, b }; + } + + test("T1 a failed finish starts the head with no tick", async () => { + const { a, b } = await full(); + a.finish.resolve({ ok: false, reason: "terminal_error" }); + await flush(); + expect(b.probe.started).toBe(1); + }); + + test("T2 a rejected start starts the head", async () => { + const { a, b } = await full(); + a.finish.reject(new Error("x")); + await flush(); + expect(b.probe.started).toBe(1); + }); + + test("T3 a successful finish with a free reading starts the head", async () => { + const { a, b } = await full(); + a.finish.resolve({ ok: true }); + await flush(); + expect(b.probe.started).toBe(1); + }); + + test("T4 a successful finish whose reading is still full re-reads and keeps waiting", async () => { + const { a, b, source } = await full(); + source.value = 1; + const before = source.calls; + a.finish.resolve({ ok: true }); + await flush(); + expect(b.probe.started).toBe(0); + expect(source.calls).toBe(before + 1); + }); +}); + +describe("QueuedLaunchStartsOnPoll", () => { + test("T1 a free tick starts the waiting launches in order", async () => { + const { queue, source, tick } = rig({ cap: 3 }); + source.value = 3; + const order: string[] = []; + const mk = (n: string) => ({ + ...entry(n), + start: () => { + order.push(n); + return new Promise(() => {}); + }, + }); + for (const n of ["a", "b", "c"]) await queue.submit(mk(n)); + source.value = 0; + await tick(); + expect(order).toEqual(["a", "b", "c"]); + }); + + test("T2 only the free slots start", async () => { + const { queue, source, tick } = rig({ cap: 2 }); + source.value = 2; + const es = ["a", "b", "c"].map(entry); + for (const e of es) await queue.submit(e); + source.value = 0; + await tick(); + expect(es.map((e) => e.probe.started)).toEqual([1, 1, 0]); + expect((await queue.status()).queued).toBe(1); + }); + + test("T3 an unknown reading starts nothing and keeps every position", async () => { + const { queue, source, tick } = rig({ cap: 1 }); + source.value = 1; + const a = entry("a"); + await queue.submit(a); + source.value = null; + await tick(); + expect(a.probe.started).toBe(0); + expect((await queue.status()).queued).toBe(1); + source.value = 0; + await tick(); + expect(a.probe.started).toBe(1); + }); +}); + +describe("QueuePollRunsOnlyWhileWaiting", () => { + test("T1 an idle queue holds no timer and reads nothing", () => { + const { timers, source } = rig({ cap: 1 }); + expect(timers.set.length).toBe(0); + expect(source.calls).toBe(0); + }); + + test("T2 the timer is armed once at 10 000 ms", async () => { + const { queue, source, timers } = rig({ cap: 1 }); + source.value = 1; + await queue.submit(entry("a")); + await queue.submit(entry("b")); + expect(timers.set.length).toBe(1); + expect(timers.set[0].ms).toBe(10_000); + }); + + test("T3 the timer is cleared once when nothing waits", async () => { + const { queue, source, timers, tick } = rig({ cap: 2 }); + source.value = 2; + await queue.submit(entry("a")); + await queue.submit(entry("b")); + source.value = 0; + await tick(); + expect(timers.cleared).toBe(1); + }); + + test("T4 intervalMs is passed through", async () => { + const { queue, source, timers } = rig({ cap: 1, intervalMs: 50 }); + source.value = 1; + await queue.submit(entry("a")); + expect(timers.set[0].ms).toBe(50); + }); +}); + +describe("HangarStatus", () => { + test("T3 cap 1, reading 1, two waiting", async () => { + const { queue, source } = rig({ cap: 1 }); + source.value = 1; + await queue.submit(entry("a")); + await queue.submit(entry("b")); + expect(await queue.status()).toEqual({ cap: 1, occupied: 1, queued: 2 }); + }); + + test("T4 an unknown reading is null, not zero", async () => { + const { queue, source } = rig({ cap: 4 }); + source.value = null; + expect(await queue.status()).toEqual({ cap: 4, occupied: null, queued: 0 }); + }); + + test("T5 status never starts a waiting launch", async () => { + const { queue, source } = rig({ cap: 1 }); + source.value = 1; + const a = entry("a"); + await queue.submit(a); + source.value = 0; + const status = await queue.status(); + expect(a.probe.started).toBe(0); + expect(status.queued).toBe(1); + }); +}); + +describe("DequeueFailureLogsOnly queue", () => { + test("T5 a rejected dequeued start logs its label and unexpected_error once", async () => { + const { queue, source, logs, tick } = rig({ cap: 1 }); + source.value = 1; + const a = entry("a"); + await queue.submit(a); + source.value = 0; + a.finish.reject(new Error("secret detail")); + await tick(); + expect(logs.length).toBe(1); + expect(logs[0]).toContain(a.label); + expect(logs[0]).toContain("unexpected_error"); + expect(logs[0]).not.toContain("secret detail"); + }); + + test("a launch admitted at once logs nothing when it fails", async () => { + const { queue, logs } = rig({ cap: 1 }); + const a = entry("a"); + const result = await queue.submit(a); + a.finish.resolve({ ok: false, reason: "terminal_error" }); + if (result.kind === "started") await result.outcome; + await flush(); + expect(logs).toEqual([]); + }); +}); + +describe("DuplicateCardRefused queue", () => { + test("a key waiting or running is refused and the queue is unchanged", async () => { + const { queue, source } = rig({ cap: 1 }); + const a = entry("a"); + expect((await queue.submit(a)).kind).toBe("started"); + expect(await queue.submit(entry("a"))).toEqual({ kind: "already_queued" }); + source.value = 1; + await queue.submit(entry("b")); + source.value = null; + expect(await queue.submit(entry("b"))).toEqual({ kind: "already_queued" }); + expect((await queue.status()).queued).toBe(1); + }); +}); diff --git a/server/app.ts b/server/app.ts index edf0c80..7a64e49 100644 --- a/server/app.ts +++ b/server/app.ts @@ -29,8 +29,10 @@ import { createHerdrClient, type HerdrClient } from "./herdr/client"; import { createSidedBackend } from "./herdr/sided-backend"; import { hangarKey, resolveHerdrSides, routeSessionKey, type Side } from "./herdr/sides"; import { createSocketWatch, type SocketWatchOptions } from "./herdr/socket-watch"; +import { FALLBACK_HANGAR_CAP } from "./hangar-cap"; import { createLaunchPlugin } from "./launch"; import { defaultLaunchesDir } from "./launch-binding"; +import { createLaunchQueue, type LaunchQueueOptions } from "./launch-queue"; import { type CardWorktrees, gitCardWorktrees } from "./launch-worktree"; import { stripBasePath } from "./path-utils"; import { createAttemptLog } from "./push/attempt-log"; @@ -78,6 +80,8 @@ export interface AppBackend extends WsBackend { pushSubscriberCount?(): number; /** A pane's working directory for push copy. Optional for the same reason. */ paneCwd?(paneId: string): Promise; + /** How many agents the hangar holds, `null` when it cannot say. Optional like `paneCwd`. */ + hangarAgentCount?(): Promise; } /** @@ -152,6 +156,8 @@ export interface AppTestDeps { * was open — the one moment nothing else can observe it. */ eventBuffer?: EventBuffer; + /** Timing seams for the launch queue's poll; production passes nothing. */ + queueTimers?: Pick; } /** Builds the whole server. The returned app has not been listened on. */ @@ -256,6 +262,15 @@ export function createApp(serverConfig: ServerConfig, deps: AppTestDeps = {}) { }) : undefined; + // One queue for the process: the launch route submits to it, the cards route + // reads it, and a hangar teardown kicks it. The occupancy source is + // late-bound like `cwdOf`, since the backend is built below. + const launchQueue = createLaunchQueue({ + cap: serverConfig.hangarCap ?? FALLBACK_HANGAR_CAP, + occupancy: () => backend.hangarAgentCount?.() ?? Promise.resolve(null), + ...deps.queueTimers, + }); + function buildSidedBackend() { const cockpitClient: WatchableClient = deps.herdrClient ?? createHerdrClient({ socketPath: cockpitSocket.socketPath }); @@ -273,6 +288,7 @@ export function createApp(serverConfig: ServerConfig, deps: AppTestDeps = {}) { deps.hangarHerdrClient ?? createHerdrClient({ socketPath: hangarSocket.socketPath }); return createSidedBackend({ cockpit, + onHangarTeardown: () => void launchQueue.kick(), hangar: { name, backend: createHerdrBackend({ @@ -334,6 +350,7 @@ export function createApp(serverConfig: ServerConfig, deps: AppTestDeps = {}) { eventBuffer, launchesDir: deps.launchesDir ?? defaultLaunchesDir(), worktrees: deps.cardWorktrees ?? gitCardWorktrees(), + queue: launchQueue, }), ) .use( @@ -342,6 +359,7 @@ export function createApp(serverConfig: ServerConfig, deps: AppTestDeps = {}) { workspaceRoot: deps.workspaceRoot ?? defaultWorkspaceRoot(), obsidianConfigPath: deps.obsidianConfigPath ?? defaultObsidianConfigPath(), launchesDir: deps.launchesDir ?? defaultLaunchesDir(), + hangar: hangarSession === null ? null : launchQueue, }), ) .get("*", async ({ request }) => { diff --git a/server/cards.ts b/server/cards.ts index 178066a..7432ec3 100644 --- a/server/cards.ts +++ b/server/cards.ts @@ -4,6 +4,7 @@ import { basename, join } from "node:path"; import { Elysia } from "elysia"; import type { ServerConfig } from "./config"; import { resolveByLstat } from "./launch"; +import { cardKey, type HangarStatus } from "./launch-queue"; import { isWithinRoot } from "./path-utils"; export interface Card { @@ -13,6 +14,8 @@ export interface Card { status: string | null; priority: string | null; blockedBy: string[]; + /** Waiting for a hangar slot, or admitted and not yet finished starting. */ + queued: boolean; dispatchable: boolean; } @@ -29,6 +32,12 @@ export interface CardsReply { skipped: { repo: string; reason: "invalid_config" | "vault_not_found" | "tasks_outside_vault" }[]; } +/** What `createCardsPlugin` reads of the launch queue. */ +export interface HangarView { + has(key: string): boolean; + status(): Promise; +} + export interface CardSources { /** Its direct children are the repos scanned for `.obsidian.yaml`. */ workspaceRoot: string; @@ -89,7 +98,7 @@ async function vaultPaths(configPath: string): Promise> { return byName; } -/** `vault\0cardPath` of every card a launch has already bound. */ +/** The set of `cardKey` values of every card a launch has already bound. */ async function boundCards(dir: string): Promise> { const bound = new Set(); const names = await readdir(dir).catch(() => [] as string[]); @@ -101,7 +110,7 @@ async function boundCards(dir: string): Promise> { } catch {} const vault = text(binding?.vault); const cardPath = text(binding?.cardPath); - if (vault && cardPath) bound.add(`${vault}\0${cardPath}`); + if (vault && cardPath) bound.add(cardKey(vault, cardPath)); } return bound; } @@ -132,6 +141,7 @@ async function readProjectCards( tasksDir: string, project: string, bound: Set, + isQueued: (vault: string, cardPath: string) => boolean, ): Promise { const dir = join("pm", project, "tasks"); const entries = await readdir(tasksDir, { withFileTypes: true }).catch(() => []); @@ -142,12 +152,14 @@ async function readProjectCards( const fm = frontmatter((await readOrNull(join(tasksDir, entry.name))) ?? ""); const blockedBy = blockers(fm.blocked_by); const status = text(fm.status); + const queued = isQueued(vault, cardPath); cards.push({ cardPath, title: text(fm.title), status, priority: text(fm.priority), blockedBy, + queued, // obw's frontier requires `type: task` and counts a blocker only by a // `[[` in its value. The template writes an empty `session:` on every // card, so only a value counts as bound. @@ -156,14 +168,18 @@ async function readProjectCards( status === "todo" && !blockedBy.some((b) => b.includes("[[")) && text(fm.session) === null && - !bound.has(`${vault}\0${cardPath}`), + !bound.has(cardKey(vault, cardPath)) && + !queued, }); } return cards.sort((a, b) => a.cardPath.localeCompare(b.cardPath)); } /** Reads `.obsidian.yaml` files and the cards they point at. Writes nothing. */ -export async function readCards(sources: CardSources): Promise { +export async function readCards( + sources: CardSources, + isQueued: (vault: string, cardPath: string) => boolean = () => false, +): Promise { const vaults = await vaultPaths(sources.obsidianConfigPath).catch( () => new Map(), ); @@ -225,7 +241,8 @@ export async function readCards(sources: CardSources): Promise { vault, project, repos, - cards: tasksDir === null ? [] : await readProjectCards(vault, tasksDir, project, bound), + cards: + tasksDir === null ? [] : await readProjectCards(vault, tasksDir, project, bound, isQueued), }); } projects.sort((a, b) => a.project.localeCompare(b.project) || a.vault.localeCompare(b.vault)); @@ -233,6 +250,15 @@ export async function readCards(sources: CardSources): Promise { return { projects, skipped }; } -export function createCardsPlugin(opts: CardSources & { config: ServerConfig }) { - return new Elysia().get(`${opts.config.basePath}/api/cards`, () => readCards(opts)); +export function createCardsPlugin( + opts: CardSources & { config: ServerConfig; hangar?: HangarView | null }, +) { + const { hangar } = opts; + return new Elysia().get(`${opts.config.basePath}/api/cards`, async () => { + const reply = await readCards( + opts, + hangar ? (vault, cardPath) => hangar.has(cardKey(vault, cardPath)) : undefined, + ); + return { ...reply, hangar: hangar ? await hangar.status() : null }; + }); } diff --git a/server/config.ts b/server/config.ts index abfd7f5..1c1a4ca 100644 --- a/server/config.ts +++ b/server/config.ts @@ -1,5 +1,6 @@ import { homedir } from "node:os"; import { resolve } from "node:path"; +import { type HangarCapResolution, resolveHangarCapWithSource } from "./hangar-cap"; export interface ServerConfig { port: number; @@ -15,6 +16,8 @@ export interface ServerConfig { launchToken?: string | null; /** Absolute Obsidian vault root for `/api/launch` cards; null (unset/blank) means none. */ vaultRoot?: string | null; + /** Max concurrent hangar agents. See `resolveHangarCapWithSource`. */ + hangarCap?: number; } /** @@ -41,7 +44,21 @@ export function parseBasePath(basePath: string | undefined): string { return basePath; } -export function parseServerConfig(argv: string[]): ServerConfig { +export interface ParseServerConfigOptions { + readPerfCores?: () => number | null; + /** Receives how the cap was resolved, for the startup line. */ + onHangarCap?: (resolution: HangarCapResolution) => void; +} + +export function parseServerConfig( + argv: string[], + options: ParseServerConfigOptions = {}, +): ServerConfig { + const capResolution = resolveHangarCapWithSource( + process.env.CC_MOBILE_HANGAR_CAP, + options.readPerfCores ?? (() => null), + ); + options.onHangarCap?.(capResolution); const config: ServerConfig = { port: 3001, hostname: "0.0.0.0", @@ -52,6 +69,7 @@ export function parseServerConfig(argv: string[]): ServerConfig { hangarSession: parseHangarSession(), launchToken: process.env.CC_MOBILE_LAUNCH_TOKEN?.trim() || null, vaultRoot: parseVaultRoot(), + hangarCap: capResolution.cap, }; for (let i = 0; i < argv.length; i++) { diff --git a/server/hangar-cap.ts b/server/hangar-cap.ts new file mode 100644 index 0000000..ac2fa1c --- /dev/null +++ b/server/hangar-cap.ts @@ -0,0 +1,69 @@ +export const FALLBACK_HANGAR_CAP = 4; + +export type HangarCapResolution = { + cap: number; + source: "override" | "perf-cores" | "unreadable"; + /** Set when `source` is "perf-cores". */ + cores?: number; +}; + +/** + * A non-blank `env` pins the cap and must be a plain positive integer: a typo + * throws rather than becoming a different cap than the operator wrote. Unset or + * blank, the cap is performance cores minus two (floor one), or the fallback + * when the count cannot be read. + */ +export function resolveHangarCapWithSource( + env: string | undefined, + readPerfCores: () => number | null, +): HangarCapResolution { + const pinned = env?.trim(); + if (pinned) { + if (!/^[1-9][0-9]*$/.test(pinned)) { + throw new Error(`CC_MOBILE_HANGAR_CAP must be a positive integer, got "${env}"`); + } + return { cap: Number.parseInt(pinned, 10), source: "override" }; + } + let n: number | null; + try { + n = readPerfCores(); + } catch { + return { cap: FALLBACK_HANGAR_CAP, source: "unreadable" }; + } + if (n === null || !Number.isInteger(n) || n < 1) { + return { cap: FALLBACK_HANGAR_CAP, source: "unreadable" }; + } + return { cap: Math.max(1, n - 2), source: "perf-cores", cores: n }; +} + +export function describeHangarCap({ cap, source, cores }: HangarCapResolution): string { + const from = + source === "override" + ? "CC_MOBILE_HANGAR_CAP" + : source === "perf-cores" + ? `${cores} performance cores` + : "performance cores unreadable"; + return `hangar cap ${cap} (${from})`; +} + +type RunSysctl = (argv: string[]) => { exitCode: number; stdout: string }; + +const defaultRun: RunSysctl = (argv) => { + const proc = Bun.spawnSync(argv, { stdout: "pipe", stderr: "ignore" }); + return { exitCode: proc.exitCode ?? 1, stdout: proc.stdout.toString() }; +}; + +/** + * Absolute path on purpose: pm2's PATH lacks /usr/sbin, and a bare `sysctl` + * would fail there and silently land on the fallback cap. + */ +export function readPerfCoresViaSysctl(run: RunSysctl = defaultRun): number | null { + try { + const { exitCode, stdout } = run(["/usr/sbin/sysctl", "-n", "hw.perflevel0.physicalcpu"]); + const text = stdout.trim(); + if (exitCode !== 0 || !/^[1-9][0-9]*$/.test(text)) return null; + return Number.parseInt(text, 10); + } catch { + return null; + } +} diff --git a/server/herdr/backend.test.ts b/server/herdr/backend.test.ts index c2d0316..88a08e6 100644 --- a/server/herdr/backend.test.ts +++ b/server/herdr/backend.test.ts @@ -1535,3 +1535,85 @@ describe("ClearRotationNotice wiring", () => { } }); }); + +describe("HangarAgentCount backend", () => { + const row = (pane: string, workspace: string) => ({ + pane_id: pane, + workspace_id: workspace, + agent: "claude", + agent_status: "idle", + cwd: "/tmp", + agent_session: { kind: "id", value: pane }, + }); + + function countingClient(agentList?: () => Promise) { + const counts = { sessionSnapshot: 0, agentGet: 0 }; + const client = { + agentGet: async () => { + counts.agentGet += 1; + return {}; + }, + ...(agentList ? { agentList } : {}), + sessionSnapshot: async () => { + counts.sessionSnapshot += 1; + return { workspaces: [], panes: [], agents: [] }; + }, + subscribeEvents: async () => ({ stop: () => {} }), + } as unknown as NonNullable; + return { client, counts }; + } + + test("T5 counts every entry with one agent.list and nothing else", async () => { + const { client, counts } = countingClient(async () => [ + row("w1:p1", "w1"), + { ...row("w2:p1", "w2"), workspace_label: "ccme2e-x" }, + ]); + const backend = createHerdrBackend({ client }); + try { + expect(await backend.agentCount()).toBe(2); + expect(counts.sessionSnapshot).toBe(0); + expect(counts.agentGet).toBe(0); + } finally { + await backend.teardownAll(); + } + }); + + test("T6 a transport failure is null", async () => { + const { client } = countingClient(async () => { + throw new HerdrTransportError("connect failed"); + }); + const backend = createHerdrBackend({ client }); + try { + expect(await backend.agentCount()).toBeNull(); + } finally { + await backend.teardownAll(); + } + }); + + test("unanswered agent.list: the occupancy read is null within about 2 s", async () => { + // A client honouring the per-call deadline the way the transport does. + const { client } = countingClient((async (options?: { timeoutMs?: number }) => { + await new Promise((_, reject) => + setTimeout(() => reject(new HerdrTransportError("timeout")), options?.timeoutMs ?? 10_000), + ); + }) as never); + const backend = createHerdrBackend({ client }); + try { + const started = Date.now(); + expect(await backend.agentCount()).toBeNull(); + expect(Date.now() - started).toBeLessThan(3_000); + } finally { + await backend.teardownAll(); + } + }); + + test("T7 a client without agentList is null", async () => { + const { client } = countingClient(); + const backend = createHerdrBackend({ client }); + try { + expect(await backend.agentCount()).toBeNull(); + } finally { + await backend.teardownAll(); + } + }); +}); diff --git a/server/herdr/backend.ts b/server/herdr/backend.ts index f23e8f9..4b45b2f 100644 --- a/server/herdr/backend.ts +++ b/server/herdr/backend.ts @@ -22,7 +22,12 @@ import type { ClientSink, TerminalBackend, TerminalSessionInfo } from "../termin import { createTranscriptDelivery } from "../transcript/delivery"; import { type PageCursor, readTranscriptPage, type TranscriptPage } from "../transcript/page"; import { type AgentState, statesFromSnapshot } from "./agent-state"; -import { createHerdrClient, type HerdrClient, SUPPORTED_PROTOCOL } from "./client"; +import { + AGENT_COUNT_TIMEOUT_MS, + createHerdrClient, + type HerdrClient, + SUPPORTED_PROTOCOL, +} from "./client"; import { classifyHerdrFailure } from "./errors"; import { createAgentNotice } from "./notice/agent-notice"; import { createHerdrPaneEvents } from "./pane-events"; @@ -197,6 +202,13 @@ export interface HerdrTerminalBackend extends TerminalBackend { * rejects: whatever herdr cannot answer is null. */ paneCwd(paneId: string): Promise; + /** + * How many entries the daemon's `agent.list` returns — every one, with no + * label suppression, in a single call. `null` when the daemon cannot say + * (client without `agentList`, or the call fails); never rejects, and never + * opens the event stream. + */ + agentCount(): Promise; } /** @@ -487,6 +499,14 @@ export function createHerdrBackend(options: HerdrBackendOptions = {}): HerdrTerm }, hasSession: (claudeUuid) => registry.hasSession(claudeUuid), listLive: () => registry.listSessions(), + async agentCount() { + if (typeof client.agentList !== "function") return null; + try { + return (await client.agentList({ timeoutMs: AGENT_COUNT_TIMEOUT_MS })).length; + } catch { + return null; + } + }, async paneCwd(paneId) { try { const info = await client.agentGet(paneId); diff --git a/server/herdr/client.test.ts b/server/herdr/client.test.ts index 982e731..4687b77 100644 --- a/server/herdr/client.test.ts +++ b/server/herdr/client.test.ts @@ -289,3 +289,16 @@ describe("herdr client: AgentWait", () => { expect(calls[0]?.options?.timeoutMs).toBe(65000); }); }); + +describe("herdr client: unanswered agent.list", () => { + it("unanswered agent.list is bounded by the caller's short deadline, the default stays the transport's", async () => { + const { transport, calls } = fakeTransport(() => resultOf(AGENT_LIST_LINE)); + const client = createHerdrClient({ transport }); + + await client.agentList({ timeoutMs: 2_000 }); + await client.agentList(); + + expect(calls[0]?.options?.timeoutMs).toBe(2_000); + expect(calls[1]?.options?.timeoutMs).toBeUndefined(); + }); +}); diff --git a/server/herdr/client.ts b/server/herdr/client.ts index 829b11f..9fa7364 100644 --- a/server/herdr/client.ts +++ b/server/herdr/client.ts @@ -73,6 +73,8 @@ const AGENT_WAIT_DEADLINE_MARGIN_MS = 5_000; * seconds, not the transport's 10 s default. */ const INTEGRATION_LIST_TIMEOUT_MS = 2_000; +/** Same reason for the launch queue's occupancy read: unknown must come back fast. */ +export const AGENT_COUNT_TIMEOUT_MS = 2_000; export interface HerdrClientOptions { /** Explicit socket path; falls back to $HERDR_SOCKET_PATH, then ~/.config/herdr/herdr.sock. */ @@ -135,8 +137,13 @@ export function createHerdrClient(options: HerdrClientOptions = {}) { } /** Lists all detected agents across the daemon. */ - async function agentList(): Promise { - const result = await call("agent.list", {}, AgentListResultSchema); + async function agentList(options?: { timeoutMs?: number }): Promise { + const result = await call( + "agent.list", + {}, + AgentListResultSchema, + options?.timeoutMs === undefined ? undefined : { timeoutMs: options.timeoutMs }, + ); return result.agents; } diff --git a/server/herdr/sessions.ts b/server/herdr/sessions.ts index de0acac..eca8a0a 100644 --- a/server/herdr/sessions.ts +++ b/server/herdr/sessions.ts @@ -99,7 +99,7 @@ export interface SessionDescriptor { /** The client slice the listing needs; the real HerdrClient satisfies it. */ export interface SessionListingClient { - agentList(): Promise; + agentList(options?: { timeoutMs?: number }): Promise; agentGet(target: string): Promise; sessionSnapshot(): Promise; call(method: string, params: unknown, schema?: unknown): Promise; diff --git a/server/herdr/sided-backend.test.ts b/server/herdr/sided-backend.test.ts index 3d970a9..38b523d 100644 --- a/server/herdr/sided-backend.test.ts +++ b/server/herdr/sided-backend.test.ts @@ -40,6 +40,7 @@ function fakeBackend(overrides: Record = {}) { pushSubscriberCount: 0, readCapabilities: Promise.resolve({ ok: false, reason: "failed" }), paneCwd: Promise.resolve(null), + agentCount: Promise.resolve(0), ...overrides, }; const backend: Record = {}; @@ -457,3 +458,59 @@ describe("SidesStartInBackground", () => { expect(warns.some((w) => /boom/.test(w))).toBe(true); }); }); + +describe("HangarAgentCount", () => { + test("T1 delegates to the hangar only", async () => { + const { sided, cockpit, hangar } = make({}, { agentCount: Promise.resolve(3) }); + expect(await sided.hangarAgentCount()).toBe(3); + expect(hangar.calls.agentCount.length).toBe(1); + expect(cockpit.calls.agentCount).toBeUndefined(); + }); + test("T2 unknown stays null", async () => { + const { sided } = make({}, { agentCount: Promise.resolve(null) }); + expect(await sided.hangarAgentCount()).toBeNull(); + }); + test("T3 no hangar is null", async () => { + const { sided } = make({}, null); + expect(await sided.hangarAgentCount()).toBeNull(); + }); + test("T4 incompatible watch issues no call", async () => { + const hw = fakeWatch("incompatible"); + const { sided, hangar } = make({}, { agentCount: Promise.resolve(5) }, { hw: hw.watch }); + expect(await sided.hangarAgentCount()).toBeNull(); + expect(hangar.calls.agentCount).toBeUndefined(); + }); +}); + +describe("QueuedLaunchStartsAfterHangarTeardown", () => { + function withSpy(h: Record = {}) { + const cockpit = fakeBackend(); + const hangar = fakeBackend(h); + let calls = 0; + const sided = createSidedBackend({ + cockpit: { backend: cockpit.backend }, + hangar: { name: "fleet", backend: hangar.backend }, + onHangarTeardown: () => { + calls += 1; + }, + }); + return { sided, hangar, calls: () => calls }; + } + test("T1 hangar teardown notifies once", async () => { + const { sided, hangar, calls } = withSpy(); + await sided.teardown("fleet@w1:p1"); + expect(hangar.calls.teardown[0][0]).toBe("w1:p1"); + expect(calls()).toBe(1); + }); + test("T2 cockpit teardown does not", async () => { + const { sided, calls } = withSpy(); + await sided.teardown("w1:p1"); + expect(calls()).toBe(0); + }); + test("T3 rejected teardown still notifies and rethrows", async () => { + const boom = new Error("boom"); + const { sided, calls } = withSpy({ teardown: () => Promise.reject(boom) }); + await expect(sided.teardown("fleet@w1:p1")).rejects.toBe(boom); + expect(calls()).toBe(1); + }); +}); diff --git a/server/herdr/sided-backend.ts b/server/herdr/sided-backend.ts index 3493186..eb6ddf5 100644 --- a/server/herdr/sided-backend.ts +++ b/server/herdr/sided-backend.ts @@ -30,6 +30,8 @@ export interface SidedBackendOptions { cockpit: { backend: HerdrTerminalBackend; watch?: SocketWatch }; hangar?: { name: string; backend: HerdrTerminalBackend; watch?: SocketWatch }; warn?: (message: string) => void; + /** Called once after a hangar-routed teardown settles, whether it resolved or rejected. */ + onHangarTeardown?: () => void; } export function createSidedBackend(options: SidedBackendOptions) { @@ -154,9 +156,18 @@ export function createSidedBackend(options: SidedBackendOptions) { const { backend, paneId } = route(params.claudeUuid); return backend.send({ ...params, claudeUuid: paneId }); }, - teardown(key: string): Promise { - const { backend, paneId } = route(key); - return backend.teardown(paneId); + async teardown(key: string): Promise { + const { backend, paneId, side } = route(key); + try { + return await backend.teardown(paneId); + } finally { + if (side === "hangar") options.onHangarTeardown?.(); + } + }, + /** `null` is unknown, never zero: no hangar, an incompatible socket, or a daemon that did not answer. */ + async hangarAgentCount(): Promise { + if (!hangar || hangar.watch?.status() === "incompatible") return null; + return hangar.backend.agentCount(); }, readTranscriptPage( key: string, diff --git a/server/index.ts b/server/index.ts index 6f99a06..2738f17 100644 --- a/server/index.ts +++ b/server/index.ts @@ -1,11 +1,15 @@ import { existsSync } from "node:fs"; import { type AppBackend, createApp, DIST_DIR } from "./app"; import { parseServerConfig } from "./config"; +import { describeHangarCap, readPerfCoresViaSysctl } from "./hangar-cap"; import { verifyHerdrSides } from "./herdr/backend"; import { resolveHerdrSides } from "./herdr/sides"; const isProd = process.env.NODE_ENV === "production"; -const serverConfig = parseServerConfig(process.argv); +const serverConfig = parseServerConfig(process.argv, { + readPerfCores: readPerfCoresViaSysctl, + onHangarCap: (resolution) => console.log(describeHangarCap(resolution)), +}); // Wrapped rather than top-level await: pm2's bun fork container loads this file // with require(), which rejects an async module. diff --git a/server/integration/launch-worktree.e2e.test.ts b/server/integration/launch-worktree.e2e.test.ts index 4f1cf16..3a9c06f 100644 --- a/server/integration/launch-worktree.e2e.test.ts +++ b/server/integration/launch-worktree.e2e.test.ts @@ -18,6 +18,7 @@ import { join } from "node:path"; import { emptyAgentProfileSource } from "../agents/profiles"; import { EventBuffer } from "../event-buffer"; import { createLaunchPlugin } from "../launch"; +import { createLaunchQueue } from "../launch-queue"; import { gitCardWorktrees } from "../launch-worktree"; import type { CreateSessionInput } from "../terminal-backend"; @@ -95,6 +96,7 @@ function setup(opts: { createError?: string; allowedRoots?: string[] } = {}) { eventBuffer: new EventBuffer(10), launchesDir: join(tmp, "launches"), worktrees: gitCardWorktrees(), + queue: createLaunchQueue({ cap: 4, occupancy: async () => 0 }), }); const post = (card: string, cwd = repo) => { writeFileSync(join(vault, "pm", "demo", "tasks", `${card}.md`), `# ${card}\n`); diff --git a/server/launch-queue.ts b/server/launch-queue.ts new file mode 100644 index 0000000..f4b22f1 --- /dev/null +++ b/server/launch-queue.ts @@ -0,0 +1,190 @@ +import { posix } from "node:path"; + +export type StartResult = { ok: true } | { ok: false; reason: string }; + +export interface QueueEntry { + key: string; + /** What a log line calls this launch. Never the card's text or a credential. */ + label: string; + start: () => Promise; +} + +export type SubmitResult = + | { kind: "started"; outcome: Promise } + | { kind: "queued"; position: number } + | { kind: "already_queued" } + | { kind: "hangar_offline" }; + +export interface HangarStatus { + cap: number; + /** `null` is unknown (the hangar did not answer), never zero. */ + occupied: number | null; + queued: number; +} + +export interface LaunchQueueOptions { + cap: number; + occupancy: () => Promise; + intervalMs?: number; + setIntervalFn?: (fn: () => void, ms: number) => unknown; + clearIntervalFn?: (handle: unknown) => void; + log?: (line: string) => void; +} + +export interface LaunchQueue { + submit(entry: QueueEntry): Promise; + has(key: string): boolean; + status(): Promise; + /** Re-checks the waiting head now. Resolves once that check has run. */ + kick(): Promise; +} + +const DEFAULT_INTERVAL_MS = 10_000; + +/** One card, whichever way its path was spelled. */ +export function cardKey(vault: string, cardPath: string): string { + return `${vault}\0${posix.normalize(cardPath)}`; +} + +interface Reservation { + key: string; + /** Stamp of the finish; `null` while the launch is still running. */ + finishedAt: number | null; +} + +/** + * Admits launches while the hangar has a free slot and holds the rest in order. + * + * A launch the hangar has not listed yet would be invisible to a reading, so + * each admitted launch holds a slot (a reservation) until a reading issued + * after it finished. A reading issued at stamp t therefore counts the + * reservations that were unfinished at t on top of what the hangar reported. + * Decisions run one at a time, and a reservation is taken in the same + * synchronous step as the decision to start. + */ +export function createLaunchQueue(options: LaunchQueueOptions): LaunchQueue { + const log = options.log ?? ((line: string) => console.warn(line)); + const setIntervalFn = + options.setIntervalFn ?? + ((fn: () => void, ms: number) => { + const handle = setInterval(fn, ms); + handle.unref?.(); + return handle; + }); + const clearIntervalFn = + options.clearIntervalFn ?? ((handle: unknown) => clearInterval(handle as Timer)); + + const waiting: QueueEntry[] = []; + const reservations = new Set(); + let counter = 0; + let readingsInFlight = 0; + let timer: unknown = null; + let timerArmed = false; + let chain: Promise = Promise.resolve(); + + const serialize = (fn: () => Promise): Promise => { + const run = chain.then(fn); + chain = run.then( + () => {}, + () => {}, + ); + return run; + }; + + const syncTimer = () => { + if (waiting.length > 0 && !timerArmed) { + timerArmed = true; + timer = setIntervalFn(() => { + void serialize(() => decide()); + }, options.intervalMs ?? DEFAULT_INTERVAL_MS); + } else if (waiting.length === 0 && timerArmed) { + timerArmed = false; + clearIntervalFn(timer); + } + }; + + const heldAt = (stamp: number): number => { + let held = 0; + for (const r of reservations) if (r.finishedAt === null || r.finishedAt > stamp) held += 1; + return held; + }; + + /** + * What the hangar reports plus the launches that reading cannot have seen + * finish; `null` when the hangar cannot say. Counted before the reading is + * retired, since retiring one may prune the reservations it still needs. + */ + async function read(): Promise { + const stamp = ++counter; + readingsInFlight += 1; + try { + const count = await Promise.resolve() + .then(options.occupancy) + .catch(() => null); + return count === null ? null : count + heldAt(stamp); + } finally { + readingsInFlight -= 1; + if (readingsInFlight === 0) { + for (const r of reservations) if (r.finishedAt !== null) reservations.delete(r); + } + } + } + + function launch(entry: QueueEntry, dequeued: boolean): Promise { + const reservation: Reservation = { key: entry.key, finishedAt: null }; + reservations.add(reservation); + return (async (): Promise => { + try { + return await entry.start(); + } catch { + return { ok: false, reason: "unexpected_error" }; + } + })().then((result) => { + reservation.finishedAt = ++counter; + if (readingsInFlight === 0) reservations.delete(reservation); + if (dequeued && !result.ok) + log(`[launch-queue] ${entry.label} not started: ${result.reason}`); + void serialize(() => decide()); + return result; + }); + } + + /** Starts waiting launches head-first, then decides on `fresh`, on one reading. */ + async function decide(fresh?: QueueEntry): Promise { + if (fresh && has(fresh.key)) return { kind: "already_queued" }; + if (waiting.length === 0 && !fresh) return undefined; + let occupied = await read(); + if (occupied === null) return fresh ? { kind: "hangar_offline" } : undefined; + while (waiting.length > 0 && occupied < options.cap) { + const head = waiting.shift() as QueueEntry; + occupied += 1; + void launch(head, true); + } + syncTimer(); + if (!fresh) return undefined; + if (waiting.length === 0 && occupied < options.cap) { + return { kind: "started", outcome: launch(fresh, false) }; + } + waiting.push(fresh); + syncTimer(); + return { kind: "queued", position: waiting.length }; + } + + function has(key: string): boolean { + for (const r of reservations) if (r.finishedAt === null && r.key === key) return true; + return waiting.some((e) => e.key === key); + } + + return { + async submit(entry) { + return (await serialize(() => decide(entry))) as SubmitResult; + }, + has, + async status() { + return { cap: options.cap, occupied: await read(), queued: waiting.length }; + }, + async kick() { + await serialize(() => decide()); + }, + }; +} diff --git a/server/launch.ts b/server/launch.ts index 1545539..61fbc1f 100644 --- a/server/launch.ts +++ b/server/launch.ts @@ -4,12 +4,13 @@ import { lstat, readFile, readlink } from "node:fs/promises"; import { basename, dirname, isAbsolute, join } from "node:path"; import { Elysia } from "elysia"; import { z } from "zod"; -import type { AgentProfileSource } from "./agents/profiles"; +import type { AgentProfile, AgentProfileSource } from "./agents/profiles"; import type { AuditLog } from "./audit/audit-log"; import { captureClientIdentity } from "./audit/client-identity"; import type { ServerConfig } from "./config"; import type { EventBuffer } from "./event-buffer"; import { writeLaunchBinding } from "./launch-binding"; +import { cardKey, type LaunchQueue } from "./launch-queue"; import { composeLaunchPrompt } from "./launch-prompt"; import type { CardWorktrees } from "./launch-worktree"; import { expandPath, isWithinRoot, validateAllowedPath, validateCwd } from "./path-utils"; @@ -85,6 +86,23 @@ function tokenMatches(presented: string, expected: string): boolean { return timingSafeEqual(digest(presented), digest(expected)); } +type Refusal = { status: number; body: Record }; + +interface Checked { + cwd: string; + cardPath: string; + vault: string; + project: string; + profileId: string | undefined; + cardText: string; + /** The profile list as it stood when the checks ran, handed to session creation unchanged. */ + snapshot: AgentProfile[]; +} + +type LaunchOutcome = + | { ok: true; status: 201; body: Record } + | { ok: false; reason: string; status: number; body: Record }; + export function createLaunchPlugin(opts: { config: ServerConfig; backend: TerminalControlBackend & { @@ -106,6 +124,8 @@ export function createLaunchPlugin(opts: { launchesDir: string; /** Each card gets its own worktree of the repo its cwd is in. */ worktrees: CardWorktrees; + /** Holds the hangar to its cap; a launch beyond it waits here. */ + queue: LaunchQueue; auditLog?: AuditLog; }) { const audit = async (record: Parameters[0]) => { @@ -113,6 +133,146 @@ export function createLaunchPlugin(opts: { await opts.auditLog?.append(record); } catch {} }; + + /** Every check that writes nothing, in the order they have always run. */ + async function checkLaunch( + body: unknown, + vaultRoot: string, + ): Promise<{ ok: true; checked: Checked } | ({ ok: false } & Refusal)> { + const refuse = (status: number, error: string, message?: string) => ({ + ok: false as const, + status, + body: message === undefined ? { error } : { error, message }, + }); + const parsed = LaunchBody.safeParse(body); + if (!parsed.success) return refuse(400, "invalid_body"); + const { cwd, cardPath, vault, project, profileId } = parsed.data; + if (vault !== basename(vaultRoot)) return refuse(400, "vault_mismatch"); + if (isAbsolute(cardPath) || cardPath.split("/").includes("..") || !cardPath.endsWith(".md")) { + return refuse(400, "invalid_card_path"); + } + let card: { path: string; stats: Stats }; + let root: string; + try { + root = (await resolveByLstat(vaultRoot)).path; + card = await resolveByLstat(join(vaultRoot, cardPath)); + } catch { + return refuse(404, "card_not_found"); + } + if (!isWithinRoot(card.path, root)) return refuse(403, "card_not_allowed"); + let cardText: string; + try { + if (!card.stats.isFile()) throw new Error("not a regular file"); + cardText = await readFile(card.path, "utf8"); + } catch { + return refuse(404, "card_not_found"); + } + // A binding is keyed by claude's session id, which no other kind reports. + const snapshot = opts.agentProfiles.list(); + const profile = profileId ? snapshot.find(({ id }) => id === profileId) : undefined; + if (profile && profile.kind !== "claude") return refuse(400, "unsupported_kind"); + // The terminal checks run again in handleTerminalCreate, but only after + // the worktree exists; a refused cwd or profile must not leave one behind. + const expandedCwd = expandPath(cwd); + const cwdError = validateCwd(expandedCwd); + if (cwdError) return refuse(400, "invalid_cwd", cwdError); + if (!validateAllowedPath(expandedCwd, opts.config.allowedRoots)) { + return refuse(403, "path_not_allowed", "Project path is not in the allowed roots"); + } + if (profileId && !profile) { + return refuse(400, "unknown_profile", `Unknown agent profile: ${profileId}`); + } + return { ok: true, checked: { cwd, cardPath, vault, project, profileId, cardText, snapshot } }; + } + + /** Everything a launch writes: worktree, pane, binding, sink and prompt. */ + async function runLaunch(checked: Checked, ip: string | null): Promise { + const { cwd, cardPath, vault, project, profileId, cardText, snapshot } = checked; + const fail = ( + reason: string, + status: number, + body: Record, + ): LaunchOutcome => ({ ok: false, reason, status, body }); + const worktree = await opts.worktrees.create({ + cwd, + cardName: basename(cardPath, ".md"), + allowedRoots: opts.config.allowedRoots, + }); + if (worktree.kind === "refused") { + return fail(worktree.code, STATUS[worktree.code], { + error: worktree.code, + message: worktree.message, + }); + } + const discardWorktree = async () => { + if (worktree.kind === "created") await opts.worktrees.remove(worktree); + }; + const claudeUuid = randomUUID(); + let reply: Record = {}; + await handleTerminalCreate( + { claudeUuid, cwd: worktree.kind === "created" ? worktree.cwd : cwd, profileId }, + { + backend: { + createSession: (input) => opts.backend.createSession({ ...input, side: "hangar" }), + teardown: (key) => opts.backend.teardown(key), + }, + allowedRoots: opts.config.allowedRoots, + agentProfiles: { list: () => snapshot }, + send: (msg) => { + reply = msg; + }, + }, + ); + if (reply.type !== "terminal_created") { + await discardWorktree(); + const code = String(reply.code); + return fail(code, STATUS[code] ?? 500, { error: code, message: reply.message }); + } + const sessionId = String(reply.sessionId); + try { + await writeLaunchBinding(opts.launchesDir, claudeUuid, { + cardPath, + vault, + project, + paneId: sessionId, + createdAt: new Date().toISOString(), + }); + } catch { + await opts.backend.teardown(sessionId).catch(() => {}); + await discardWorktree(); + return fail("binding_failed", 500, { error: "binding_failed", sessionId, claudeUuid }); + } + // Routing types nothing into a pane with no sink — so bind one, owned by + // this launch alone. What it hears is for a phone's replay only: the + // launch's answer is the send's own outcome. + opts.backend.registerClient( + sessionId, + (event) => bufferSessionEvent(opts.eventBuffer, sessionId, event), + {}, + ); + let outcome: TerminalSendOutcome | undefined; + try { + outcome = await sendPrompt(opts.backend, audit, { + sessionId, + content: composeLaunchPrompt(cardText), + ip, + device: "launch-api", + confirmStart: true, + }); + } catch { + return fail("prompt_failed", 502, { error: "prompt_failed", sessionId, claudeUuid }); + } + if (outcome?.ok !== true) { + return fail("prompt_failed", 502, { + error: "prompt_failed", + code: outcome?.code, + sessionId, + claudeUuid, + }); + } + return { ok: true, status: 201, body: { sessionId, claudeUuid } }; + } + return new Elysia().post( `${opts.config.basePath}/api/launch`, async ({ request, body, set, server }) => { @@ -135,144 +295,55 @@ export function createLaunchPlugin(opts: { set.status = 503; return { error: "vault_unconfigured" }; } - const parsed = LaunchBody.safeParse(body); - if (!parsed.success) { - set.status = 400; - return { error: "invalid_body" }; - } - const { cwd, cardPath, vault, project, profileId } = parsed.data; - if (vault !== basename(vaultRoot)) { - set.status = 400; - return { error: "vault_mismatch" }; - } - if (isAbsolute(cardPath) || cardPath.split("/").includes("..") || !cardPath.endsWith(".md")) { - set.status = 400; - return { error: "invalid_card_path" }; - } - let card: { path: string; stats: Stats }; - let root: string; - try { - root = (await resolveByLstat(vaultRoot)).path; - card = await resolveByLstat(join(vaultRoot, cardPath)); - } catch { - set.status = 404; - return { error: "card_not_found" }; - } - if (!isWithinRoot(card.path, root)) { - set.status = 403; - return { error: "card_not_allowed" }; - } - let cardText: string; - try { - if (!card.stats.isFile()) throw new Error("not a regular file"); - cardText = await readFile(card.path, "utf8"); - } catch { - set.status = 404; - return { error: "card_not_found" }; - } - // A binding is keyed by claude's session id, which no other kind reports. - const snapshot = opts.agentProfiles.list(); - const profile = profileId ? snapshot.find(({ id }) => id === profileId) : undefined; - if (profile && profile.kind !== "claude") { - set.status = 400; - return { error: "unsupported_kind" }; - } - // The terminal checks run again in handleTerminalCreate, but only after - // the worktree exists; a refused cwd or profile must not leave one behind. - const expandedCwd = expandPath(cwd); - const cwdError = validateCwd(expandedCwd); - if (cwdError) { - set.status = 400; - return { error: "invalid_cwd", message: cwdError }; - } - if (!validateAllowedPath(expandedCwd, opts.config.allowedRoots)) { - set.status = 403; - return { error: "path_not_allowed", message: "Project path is not in the allowed roots" }; - } - if (profileId && !profile) { - set.status = 400; - return { error: "unknown_profile", message: `Unknown agent profile: ${profileId}` }; - } - const worktree = await opts.worktrees.create({ - cwd, - cardName: basename(cardPath, ".md"), - allowedRoots: opts.config.allowedRoots, - }); - if (worktree.kind === "refused") { - set.status = STATUS[worktree.code]; - return { error: worktree.code, message: worktree.message }; - } - const discardWorktree = async () => { - if (worktree.kind === "created") await opts.worktrees.remove(worktree); - }; - const claudeUuid = randomUUID(); - let reply: Record = {}; - await handleTerminalCreate( - { claudeUuid, cwd: worktree.kind === "created" ? worktree.cwd : cwd, profileId }, - { - backend: { - createSession: (input) => opts.backend.createSession({ ...input, side: "hangar" }), - teardown: (key) => opts.backend.teardown(key), - }, - allowedRoots: opts.config.allowedRoots, - agentProfiles: { list: () => snapshot }, - send: (msg) => { - reply = msg; - }, - }, - ); - if (reply.type !== "terminal_created") { - await discardWorktree(); - const code = String(reply.code); - set.status = STATUS[code] ?? 500; - return { error: code, message: reply.message }; - } - const sessionId = String(reply.sessionId); - try { - await writeLaunchBinding(opts.launchesDir, claudeUuid, { - cardPath, - vault, - project, - paneId: sessionId, - createdAt: new Date().toISOString(), - }); - } catch { - await opts.backend.teardown(sessionId).catch(() => {}); - await discardWorktree(); - set.status = 500; - return { error: "binding_failed", sessionId, claudeUuid }; + const first = await checkLaunch(body, vaultRoot); + if (!first.ok) { + set.status = first.status; + return first.body; } + const { cardPath, vault } = first.checked; const { ip } = captureClientIdentity({ headers: request.headers, remoteAddress: server?.requestIP(request)?.address, }); - // Routing types nothing into a pane with no sink — so bind one, owned by - // this launch alone. What it hears is for a phone's replay only: the - // launch's answer is the send's own outcome. - opts.backend.registerClient( - sessionId, - (event) => bufferSessionEvent(opts.eventBuffer, sessionId, event), - {}, - ); - let outcome: TerminalSendOutcome | undefined; - try { - outcome = await sendPrompt(opts.backend, audit, { - sessionId, - content: composeLaunchPrompt(cardText), - ip, - device: "launch-api", - confirmStart: true, - }); - } catch { - set.status = 502; - return { error: "prompt_failed", sessionId, claudeUuid }; - } - if (outcome?.ok !== true) { - set.status = 502; - return { error: "prompt_failed", code: outcome?.code, sessionId, claudeUuid }; + // The reply of a launch admitted at once is its own outcome; one that + // waits answers 202 now and reports a failure only in the server log. + let outcome: LaunchOutcome | undefined; + const submitted = await opts.queue.submit({ + key: cardKey(vault, cardPath), + label: `${vault}/${cardPath}`, + start: async () => { + const again = await checkLaunch(body, vaultRoot); + outcome = again.ok + ? await runLaunch(again.checked, ip) + : { + ok: false, + reason: String(again.body.error), + status: again.status, + body: again.body, + }; + return outcome.ok ? { ok: true } : { ok: false, reason: outcome.reason }; + }, + }); + switch (submitted.kind) { + case "already_queued": + set.status = 409; + return { error: "already_queued" }; + case "hangar_offline": + set.status = 503; + return { error: "hangar_offline" }; + case "queued": + set.status = 202; + return { queued: true, position: submitted.position }; + case "started": { + await submitted.outcome; + if (!outcome) { + set.status = 500; + return { error: "unexpected_error" }; + } + set.status = outcome.status; + return outcome.body; + } } - set.status = 201; - return { sessionId, claudeUuid }; }, ); }