diff --git a/README.md b/README.md index 1109bc4..9054634 100644 --- a/README.md +++ b/README.md @@ -266,8 +266,8 @@ To report a vulnerability, see [SECURITY.md](SECURITY.md). reasoning, on port 80 or 443 under a DNS name. - New tasks come from the CLI and Linear. On GitHub, DevBox only picks up follow-ups on its own pull requests; it doesn't start work from GitHub - issues. Slack gets only completion updates for Linear tasks. No Jira and no - web UI. + issues. Slack gets only completion updates, for Linear tasks unless + configured for API tasks too. No Jira and no web UI. - Stopping a task from Linear isn't supported yet. - No stable extension interface. [docs/extending.md](docs/extending.md) shows where hooks would go. diff --git a/cli/interface/commands/ls.ts b/cli/interface/commands/ls.ts index e9a71f5..af6cf80 100644 --- a/cli/interface/commands/ls.ts +++ b/cli/interface/commands/ls.ts @@ -15,6 +15,8 @@ export type LsArgs = CommonArgs & { }; const HEADER = ["KEY", "STATE", "TITLE", "DESCRIPTION", "CREATOR", "AGE"]; +// Every task source; the compiler fails here until a new one is listed. +const SOURCES = Object.keys({ api: true, linear: true } satisfies Record); export function render( tasks: ApiTask[], @@ -73,7 +75,7 @@ export const ls: CommandModule = { .command("ls") .description("List tasks, newest first") .addOption( - new Option("--source ", "only tasks from this source").choices(["api", "linear"]), + new Option("--source ", "only tasks from this source").choices(SOURCES), ) .option("--mine", "only tasks created with your token") .addOption( diff --git a/control-plane/.env.example b/control-plane/.env.example index 89f18cc..cd89603 100644 --- a/control-plane/.env.example +++ b/control-plane/.env.example @@ -15,6 +15,9 @@ LINEAR_ENABLED=false SLACK_ENABLED=false # SLACK_BOT_TOKEN= # SLACK_CHANNEL_ID= +# Task sources whose finished tasks post, comma-separated: linear, api. +# Default: linear. +# SLACK_NOTIFY_SOURCES=linear # The CP's own GitHub token. The Docker Compose stack (`bun run up`) does not # read this file: it passes GITHUB_TOKEN from the shell, `gh auth token` or a diff --git a/control-plane/docker-compose.yml b/control-plane/docker-compose.yml index 6fdf7f9..9938a99 100644 --- a/control-plane/docker-compose.yml +++ b/control-plane/docker-compose.yml @@ -93,6 +93,10 @@ services: image: devbox-control-plane build: *control-plane-build restart: unless-stopped + # Shutdown finishes the event it is handling (the rest of a task's + # reconcile: up to 90s when platforms are slow), then drains for + # SHUTDOWN_DRAIN_MS (10s); Compose's default 10s would cut both short. + stop_grace_period: 150s # Explicit, with no env_file. Only allowlisted HARNESS_* settings cross # into sandboxes, and a forwarded name must never hold a real secret. environment: diff --git a/control-plane/drizzle/0001_inbox_source_kind_external.sql b/control-plane/drizzle/0001_inbox_source_kind_external.sql new file mode 100644 index 0000000..ad5cb6b --- /dev/null +++ b/control-plane/drizzle/0001_inbox_source_kind_external.sql @@ -0,0 +1,2 @@ +SET LOCAL lock_timeout = '5s';--> statement-breakpoint +CREATE UNIQUE INDEX "inbox_source_kind_external_uniq" ON "inbox" USING btree ("source","kind","external_id"); \ No newline at end of file diff --git a/control-plane/drizzle/0002_tasks_key_lower.sql b/control-plane/drizzle/0002_tasks_key_lower.sql new file mode 100644 index 0000000..049627c --- /dev/null +++ b/control-plane/drizzle/0002_tasks_key_lower.sql @@ -0,0 +1,3 @@ +SET LOCAL lock_timeout = '5s';--> statement-breakpoint +CREATE UNIQUE INDEX "tasks_key_lower_uniq" ON "tasks" USING btree (lower("key"));--> statement-breakpoint +DROP INDEX "tasks_key_uniq"; \ No newline at end of file diff --git a/control-plane/drizzle/meta/0001_snapshot.json b/control-plane/drizzle/meta/0001_snapshot.json new file mode 100644 index 0000000..2ea9e04 --- /dev/null +++ b/control-plane/drizzle/meta/0001_snapshot.json @@ -0,0 +1,607 @@ +{ + "id": "f6737fb5-1099-49c5-be13-f3c095f0156c", + "prevId": "08e0c675-47da-4dc6-bb2e-1329f1a39eb5", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.api_tokens": { + "name": "api_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuidv7()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_hash": { + "name": "secret_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "github_login": { + "name": "github_login", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_tokens_name_unique": { + "name": "api_tokens_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.inbox": { + "name": "inbox", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuidv7()" + }, + "task_id": { + "name": "task_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "reply_targets": { + "name": "reply_targets", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "inbox_source_kind_external_uniq": { + "name": "inbox_source_kind_external_uniq", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_source_external_uniq": { + "name": "inbox_source_external_uniq", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_pending_idx": { + "name": "inbox_pending_idx", + "columns": [ + { + "expression": "task_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"inbox\".\"status\" = 'pending'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_task_created_idx": { + "name": "inbox_task_created_idx", + "columns": [ + { + "expression": "task_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "inbox_task_id_tasks_id_fk": { + "name": "inbox_task_id_tasks_id_fk", + "tableFrom": "inbox", + "tableTo": "tasks", + "columnsFrom": [ + "task_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.task_events": { + "name": "task_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "task_id": { + "name": "task_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "inbox_id": { + "name": "inbox_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "seq": { + "name": "seq", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "task_events_task_id_id_idx": { + "name": "task_events_task_id_id_idx", + "columns": [ + { + "expression": "task_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "task_events_task_id_tasks_id_fk": { + "name": "task_events_task_id_tasks_id_fk", + "tableFrom": "task_events", + "tableTo": "tasks", + "columnsFrom": [ + "task_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tasks": { + "name": "tasks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuidv7()" + }, + "source": { + "name": "source", + "type": "task_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_url": { + "name": "external_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "heartbeat_at": { + "name": "heartbeat_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "sandbox_id": { + "name": "sandbox_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pvc_name": { + "name": "pvc_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pr_urls": { + "name": "pr_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "last_observed_event_seq": { + "name": "last_observed_event_seq", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "tasks_source_external_id_uniq": { + "name": "tasks_source_external_id_uniq", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tasks_key_uniq": { + "name": "tasks_key_uniq", + "columns": [ + { + "expression": "key", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tasks_lease_idx": { + "name": "tasks_lease_idx", + "columns": [ + { + "expression": "heartbeat_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"tasks\".\"owner_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "tasks_pr_urls_gin_idx": { + "name": "tasks_pr_urls_gin_idx", + "columns": [ + { + "expression": "pr_urls", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "jsonb_path_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "tasks_owner_sandbox_idx": { + "name": "tasks_owner_sandbox_idx", + "columns": [ + { + "expression": "owner_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"tasks\".\"sandbox_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tasks_created_by_api_tokens_id_fk": { + "name": "tasks_created_by_api_tokens_id_fk", + "tableFrom": "tasks", + "tableTo": "api_tokens", + "columnsFrom": [ + "created_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.task_source": { + "name": "task_source", + "schema": "public", + "values": [ + "linear", + "api" + ] + } + }, + "schemas": {}, + "sequences": { + "public.api_task_key_seq": { + "name": "api_task_key_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/control-plane/drizzle/meta/0002_snapshot.json b/control-plane/drizzle/meta/0002_snapshot.json new file mode 100644 index 0000000..e8c4e1b --- /dev/null +++ b/control-plane/drizzle/meta/0002_snapshot.json @@ -0,0 +1,607 @@ +{ + "id": "4f733928-3a33-4bef-890a-703ae5d73d88", + "prevId": "f6737fb5-1099-49c5-be13-f3c095f0156c", + "version": "7", + "dialect": "postgresql", + "tables": { + "public.api_tokens": { + "name": "api_tokens", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuidv7()" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "secret_hash": { + "name": "secret_hash", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "github_login": { + "name": "github_login", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "last_used_at": { + "name": "last_used_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "revoked_at": { + "name": "revoked_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "api_tokens_name_unique": { + "name": "api_tokens_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.inbox": { + "name": "inbox", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuidv7()" + }, + "task_id": { + "name": "task_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "payload": { + "name": "payload", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "reply_targets": { + "name": "reply_targets", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "default": "'pending'" + }, + "failure_reason": { + "name": "failure_reason", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "inbox_source_kind_external_uniq": { + "name": "inbox_source_kind_external_uniq", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "kind", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_source_external_uniq": { + "name": "inbox_source_external_uniq", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_pending_idx": { + "name": "inbox_pending_idx", + "columns": [ + { + "expression": "task_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"inbox\".\"status\" = 'pending'", + "concurrently": false, + "method": "btree", + "with": {} + }, + "inbox_task_created_idx": { + "name": "inbox_task_created_idx", + "columns": [ + { + "expression": "task_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "created_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "inbox_task_id_tasks_id_fk": { + "name": "inbox_task_id_tasks_id_fk", + "tableFrom": "inbox", + "tableTo": "tasks", + "columnsFrom": [ + "task_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.task_events": { + "name": "task_events", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "bigserial", + "primaryKey": true, + "notNull": true + }, + "task_id": { + "name": "task_id", + "type": "uuid", + "primaryKey": false, + "notNull": true + }, + "origin": { + "name": "origin", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "inbox_id": { + "name": "inbox_id", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "seq": { + "name": "seq", + "type": "bigint", + "primaryKey": false, + "notNull": false + }, + "data": { + "name": "data", + "type": "jsonb", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "task_events_task_id_id_idx": { + "name": "task_events_task_id_id_idx", + "columns": [ + { + "expression": "task_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "task_events_task_id_tasks_id_fk": { + "name": "task_events_task_id_tasks_id_fk", + "tableFrom": "task_events", + "tableTo": "tasks", + "columnsFrom": [ + "task_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.tasks": { + "name": "tasks", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "uuid", + "primaryKey": true, + "notNull": true, + "default": "uuidv7()" + }, + "source": { + "name": "source", + "type": "task_source", + "typeSchema": "public", + "primaryKey": false, + "notNull": true + }, + "external_id": { + "name": "external_id", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_by": { + "name": "created_by", + "type": "uuid", + "primaryKey": false, + "notNull": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "external_url": { + "name": "external_url", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "metadata": { + "name": "metadata", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'{}'::jsonb" + }, + "owner_id": { + "name": "owner_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "heartbeat_at": { + "name": "heartbeat_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": false + }, + "sandbox_id": { + "name": "sandbox_id", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pvc_name": { + "name": "pvc_name", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "pr_urls": { + "name": "pr_urls", + "type": "jsonb", + "primaryKey": false, + "notNull": true, + "default": "'[]'::jsonb" + }, + "last_observed_event_seq": { + "name": "last_observed_event_seq", + "type": "bigint", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + }, + "updated_at": { + "name": "updated_at", + "type": "timestamp with time zone", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": { + "tasks_source_external_id_uniq": { + "name": "tasks_source_external_id_uniq", + "columns": [ + { + "expression": "source", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "external_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tasks_key_lower_uniq": { + "name": "tasks_key_lower_uniq", + "columns": [ + { + "expression": "lower(\"key\")", + "asc": true, + "isExpression": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + }, + "tasks_lease_idx": { + "name": "tasks_lease_idx", + "columns": [ + { + "expression": "heartbeat_at", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"tasks\".\"owner_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + }, + "tasks_pr_urls_gin_idx": { + "name": "tasks_pr_urls_gin_idx", + "columns": [ + { + "expression": "pr_urls", + "isExpression": false, + "asc": true, + "nulls": "last", + "opclass": "jsonb_path_ops" + } + ], + "isUnique": false, + "concurrently": false, + "method": "gin", + "with": {} + }, + "tasks_owner_sandbox_idx": { + "name": "tasks_owner_sandbox_idx", + "columns": [ + { + "expression": "owner_id", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": false, + "where": "\"tasks\".\"sandbox_id\" IS NOT NULL", + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "tasks_created_by_api_tokens_id_fk": { + "name": "tasks_created_by_api_tokens_id_fk", + "tableFrom": "tasks", + "tableTo": "api_tokens", + "columnsFrom": [ + "created_by" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + } + }, + "enums": { + "public.task_source": { + "name": "task_source", + "schema": "public", + "values": [ + "linear", + "api" + ] + } + }, + "schemas": {}, + "sequences": { + "public.api_task_key_seq": { + "name": "api_task_key_seq", + "schema": "public", + "increment": "1", + "startWith": "1", + "minValue": "1", + "maxValue": "9223372036854775807", + "cache": "1", + "cycle": false + } + }, + "roles": {}, + "policies": {}, + "views": {}, + "_meta": { + "columns": {}, + "schemas": {}, + "tables": {} + } +} \ No newline at end of file diff --git a/control-plane/drizzle/meta/_journal.json b/control-plane/drizzle/meta/_journal.json index a637ac5..e466440 100644 --- a/control-plane/drizzle/meta/_journal.json +++ b/control-plane/drizzle/meta/_journal.json @@ -8,6 +8,20 @@ "when": 1792183649849, "tag": "0000_init", "breakpoints": true + }, + { + "idx": 1, + "version": "7", + "when": 1793368811788, + "tag": "0001_inbox_source_kind_external", + "breakpoints": true + }, + { + "idx": 2, + "version": "7", + "when": 1793368820991, + "tag": "0002_tasks_key_lower", + "breakpoints": true } ] -} +} \ No newline at end of file diff --git a/control-plane/scripts/lib/integrations.test.ts b/control-plane/scripts/lib/integrations.test.ts index 9eac497..49de052 100644 --- a/control-plane/scripts/lib/integrations.test.ts +++ b/control-plane/scripts/lib/integrations.test.ts @@ -100,6 +100,21 @@ describe("parseIntegrations", () => { ); }); + it("slack notifySources: a non-empty list of task sources", () => { + expect( + parseIntegrations( + { slack: { enabled: true, channelId: "C1", notifySources: ["linear", "api"] } }, + fail, + K8S, + ), + ).toEqual({ slack: { enabled: true, channelId: "C1", notifySources: ["linear", "api"] } }); + for (const notifySources of [[], ["github"], "linear"]) { + expect(failure({ slack: { enabled: true, channelId: "C1", notifySources } })).toBe( + "integrations.slack.notifySources: must be a non-empty list of: linear, api", + ); + } + }); + it("slack refused on Compose", () => { expect(failure({ slack: { enabled: true, channelId: "C123" } }, COMPOSE)).toBe( "integrations.slack: supported on Kubernetes only (the target file)", @@ -133,7 +148,7 @@ describe("parseIntegrations", () => { 'integrations.linear: unknown key "clientSecret" (allowed: enabled, clientId)', ); expect(failure({ slack: { enabled: true, channelId: "C1", botToken: "t" } })).toBe( - 'integrations.slack: unknown key "botToken" (allowed: enabled, channelId)', + 'integrations.slack: unknown key "botToken" (allowed: enabled, channelId, notifySources)', ); expect(failure({ githubApp: { enabled: false, privateKey: "s" } })).toBe( 'integrations.githubApp: unknown key "privateKey" (allowed: enabled, appId, slug, installationId, botUserId, tokens, tokenScope, ciResults, tokenFile, org, pending)', diff --git a/control-plane/scripts/lib/integrations.ts b/control-plane/scripts/lib/integrations.ts index ff12b44..27cc667 100644 --- a/control-plane/scripts/lib/integrations.ts +++ b/control-plane/scripts/lib/integrations.ts @@ -7,10 +7,13 @@ import { homedir } from "node:os"; import { dirname, resolve } from "node:path"; +import { taskSource } from "../../src/db/schema"; import { parsePublicUrl } from "./publicUrl"; export type LinearIntegration = { enabled: boolean; clientId?: string }; -export type SlackIntegration = { enabled: boolean; channelId?: string }; +// notifySources: the task sources whose tasks post (SLACK_NOTIFY_SOURCES); +// unset means the control plane's default, linear. +export type SlackIntegration = { enabled: boolean; channelId?: string; notifySources?: string[] }; // The GitHub App and its webhooks, one switch. tokenScope and ciResults // are always filled with their defaults; the other keys are required only when // enabled is true. @@ -138,9 +141,22 @@ function parseLinear(raw: unknown, fail: Fail): LinearIntegration { } function parseSlack(raw: unknown, fail: Fail): SlackIntegration { - const b = block(raw, "slack", ["channelId"], fail); - const channelId = setting(b, "integrations.slack", "channelId", nonEmpty(fail), fail); - return { enabled: b.enabled, ...(channelId !== undefined ? { channelId } : {}) }; + const f = "integrations.slack"; + const b = block(raw, "slack", ["channelId", "notifySources"], fail); + const channelId = setting(b, f, "channelId", nonEmpty(fail), fail); + const sources = b.obj.notifySources; + const known: readonly unknown[] = taskSource.enumValues; + if ( + sources !== undefined && + !(Array.isArray(sources) && sources.length > 0 && sources.every((s) => known.includes(s))) + ) { + return fail(`${f}.notifySources: must be a non-empty list of: ${known.join(", ")}`); + } + return { + enabled: b.enabled, + ...(channelId !== undefined ? { channelId } : {}), + ...(sources !== undefined ? { notifySources: sources as string[] } : {}), + }; } function parseGithubApp( diff --git a/control-plane/src/__tests__/app.test.ts b/control-plane/src/__tests__/app.test.ts new file mode 100644 index 0000000..ba22712 --- /dev/null +++ b/control-plane/src/__tests__/app.test.ts @@ -0,0 +1,64 @@ +import { afterEach, describe, expect, it, mock, setSystemTime } from "bun:test"; + +// config.ts reads required env at import; nothing here reads it. +mock.module("../config", () => ({ config: {} })); + +const { createIntegrations, statusesWithHolds } = await import("../app"); +const { RetryableError } = await import("../integrations/outbound"); +const { ReplyHolds } = await import("../services/replyHolds"); + +type Cfg = Parameters[0]; +const statusOf = (body: object) => ({ status: async () => body }) as never; +const registry = (cfg: Cfg, linear = {}, github = {}) => + createIntegrations(cfg, {} as never, statusOf(linear), statusOf(github)); + +describe("createIntegrations", () => { + it("enables each integration from its own setting", () => { + const enabled = (cfg: Cfg) => + Object.fromEntries(Object.values(registry(cfg)).map((i) => [i.name, i.enabled])); + const off = { api: { enabled: false }, github: {} }; + expect(enabled(off)).toEqual({ linear: false, api: false, github: false }); + expect(enabled({ ...off, linear: {} })).toEqual({ linear: true, api: false, github: false }); + expect(enabled({ ...off, api: { enabled: true } })).toEqual({ + linear: false, + api: true, + github: false, + }); + expect(enabled({ ...off, github: { webhooks: {} } })).toEqual({ + linear: false, + api: false, + github: true, + }); + }); +}); + +describe("statusesWithHolds", () => { + afterEach(() => { + setSystemTime(); + }); + + it("adds each integration's held sends to its own status body", async () => { + const T0 = new Date("2026-10-30T12:00:00Z").getTime(); + const holds = new ReplyHolds(); + setSystemTime(new Date(T0)); + holds.hold("t1", { inbox_id: "i1", seq: 1 }, "github", new RetryableError("HTTP 429")); + setSystemTime(new Date(T0 + 4_000)); + holds.hold("t2", { inbox_id: "i2", seq: 3 }, "github", new RetryableError("HTTP 429")); + setSystemTime(new Date(T0 + 5_000)); + const linear = { configured: true, lastWebhookAt: null, lastTokenError: null }; + const github = { token: { ok: true }, webhooks: { enabled: true, lastWebhookAt: null } }; + + const statuses = statusesWithHolds( + registry({ linear: {}, api: { enabled: true }, github: { webhooks: {} } }, linear, github), + holds, + ); + + const none = { heldTasks: 0, oldestHoldAgeMs: 0 }; + expect(await statuses.linear.status()).toEqual({ ...linear, holds: none }); + expect(await statuses.github.status()).toEqual({ + ...github, + holds: { heldTasks: 2, oldestHoldAgeMs: 5_000 }, + }); + expect(await statuses.api.status()).toEqual({ enabled: true, holds: none }); + }); +}); diff --git a/control-plane/src/__tests__/integrationsConfig.test.ts b/control-plane/src/__tests__/integrationsConfig.test.ts index 69bf86b..07dfce3 100644 --- a/control-plane/src/__tests__/integrationsConfig.test.ts +++ b/control-plane/src/__tests__/integrationsConfig.test.ts @@ -107,9 +107,23 @@ describe("parseSlackConfig", () => { expect(parseSlackConfig({ ...SLACK_ENV, SLACK_ENABLED: "true" })).toEqual({ botToken: "xoxb-test", channel: "C123", + notifySources: ["linear"], }); }); + it("SLACK: SLACK_NOTIFY_SOURCES lists task sources; an unknown one fails", () => { + const on = { ...SLACK_ENV, SLACK_ENABLED: "true" }; + expect(parseSlackConfig({ ...on, SLACK_NOTIFY_SOURCES: "linear, api" })?.notifySources).toEqual( + ["linear", "api"], + ); + expect(parseSlackConfig({ ...on, SLACK_NOTIFY_SOURCES: " " })?.notifySources).toEqual([ + "linear", + ]); + expect(() => parseSlackConfig({ ...on, SLACK_NOTIFY_SOURCES: "linear,github" })).toThrow( + new Error('SLACK_NOTIFY_SOURCES: unknown task source "github" (allowed: linear, api)'), + ); + }); + it("SLACK: on with a setting missing", () => { expect(() => parseSlackConfig({ SLACK_ENABLED: "true" })).toThrow( new Error("SLACK_ENABLED is true but these are missing: SLACK_BOT_TOKEN, SLACK_CHANNEL_ID"), diff --git a/control-plane/src/__tests__/webhookRoutes.test.ts b/control-plane/src/__tests__/webhookRoutes.test.ts index 72b0560..ebace6e 100644 --- a/control-plane/src/__tests__/webhookRoutes.test.ts +++ b/control-plane/src/__tests__/webhookRoutes.test.ts @@ -1,6 +1,25 @@ import { describe, expect, it, spyOn } from "bun:test"; +import type { Context } from "hono"; +import type { TaskIntake } from "../integrations/intake"; +import type { IntegrationName } from "../integrations/types"; import { WEBHOOK_BODY_LIMIT_BYTES } from "../requestLimits"; -import { createWebhookRoutes } from "../routes/webhook"; +import { createWebhookRoutes as createRoutes } from "../routes/webhook"; + +const intake = {} as TaskIntake; + +// The routes for enabled integrations with these webhooks. +function createWebhookRoutes( + webhooks: Partial Promise }>>, +) { + return createRoutes( + Object.entries(webhooks).map(([name, w]) => ({ + name: name as IntegrationName, + enabled: true, + webhook: (c: Context) => w.handleWebhook(c), + })), + intake, + ); +} const OVERSIZED_BODY = "x".repeat(WEBHOOK_BODY_LIMIT_BYTES + 1); @@ -90,6 +109,34 @@ describe("createWebhookRoutes", () => { } }); + it("serves no webhook for a disabled integration, and hands the intake to an enabled one", async () => { + const seen: unknown[] = []; + const app = createRoutes( + [ + { + name: "linear", + enabled: false, + webhook: async () => new Response(null, { status: 200 }), + }, + { + name: "github", + enabled: true, + webhook: async (_c, i) => { + seen.push(i); + return new Response(null, { status: 200 }); + }, + }, + { name: "api", enabled: true }, + ], + intake, + ); + expect((await app.request("/webhook/linear", { method: "POST", body: "{}" })).status).toBe(404); + expect((await app.request("/webhook/api", { method: "POST", body: "{}" })).status).toBe(404); + expect((await app.request("/webhook/github", { method: "POST", body: "{}" })).status).toBe(200); + expect(seen).toHaveLength(1); + expect(seen[0]).toBe(intake); + }); + it("leaves disabled and unknown webhook routes as 404 even for oversized bodies", async () => { const app = createWebhookRoutes({ github: { diff --git a/control-plane/src/app.ts b/control-plane/src/app.ts index d8bda75..39ef078 100644 --- a/control-plane/src/app.ts +++ b/control-plane/src/app.ts @@ -7,6 +7,12 @@ import { GitHubWebhookController } from "./controllers/githubWebhook.controller" import { HealthController } from "./controllers/health.controller"; import { LinearWebhookController } from "./controllers/linearWebhook.controller"; import { db } from "./db"; +import { taskSource } from "./db/schema"; +import { ApiIntegration } from "./integrations/api"; +import { GithubIntegration } from "./integrations/github"; +import { Intake } from "./integrations/intake"; +import { LinearIntegration } from "./integrations/linear"; +import type { Integration, IntegrationName } from "./integrations/types"; import { logger } from "./logger"; import { apiTaskRepository } from "./repositories/apiTask.repository"; import { apiTokenRepository } from "./repositories/apiToken.repository"; @@ -22,8 +28,10 @@ import { ApiTaskService } from "./services/apiTask.service"; import { EventDispatcher } from "./services/eventDispatcher"; import { GithubStatusService } from "./services/githubStatus.service"; import { InboxService } from "./services/inbox.service"; +import { IntegrationHooks } from "./services/integrationHooks"; import { LinearStatusService } from "./services/linearStatus.service"; import { LinearTokenProvider } from "./services/linearTokenProvider"; +import { ReplyHolds } from "./services/replyHolds"; import { SandboxManager } from "./services/sandboxManager.service"; import { SlackNotifierService } from "./services/slackNotifier.service"; import { TaskProcessorService } from "./services/taskProcessor.service"; @@ -73,81 +81,102 @@ export function createApp(): AppBundle { trustedUsers: joinIfAny(config.github.trustedUsers), }); - const inboxService = new InboxService(inboxRepository, sandboxManager, linearTokenProvider); + const linearStatus = new LinearStatusService( + config.linear ? linearTokenProvider : undefined, + logger, + ); + const githubStatus = new GithubStatusService(githubTokenState, { + enabled: config.github.webhooks !== undefined, + ciResults: config.github.ciResultEnabled, + lastWebhookAt: (): string | null => integrations.github.controller?.lastWebhookAt() ?? null, + }); - const agentSessionService = new AgentSessionService( - taskRepository, + const integrations = createIntegrations(config, linearTokenProvider, linearStatus, githubStatus); + // The registered integrations that create tasks (github creates none). Only + // these sources are claimed and reconciled, so an older image never works a + // task whose source it doesn't know. + const taskSources = taskSource.enumValues.filter((s) => s in integrations); + // Sends held because they failed before anything went out; each + // integration's status reports its own. + const replyHolds = new ReplyHolds(); + // Integrations' lifecycle hooks; TaskProcessor.stop() drains detached ones. + const hooks = new IntegrationHooks(integrations, inboxRepository); + // The only way tasks and rounds are created; a restart's preempted sandbox + // is killed in the hooks' tracked detached set. + const intake = new Intake(taskRepository, inboxRepository, sandboxManager, hooks, { + apiKeyPrefix: config.api.taskKeyPrefix, + ciResultMaxPerTask: config.github.ciResultMaxPerTask, + }); + + const inboxService = new InboxService( inboxRepository, - linearTokenProvider, sandboxManager, - inboxService, + integrations, + hooks, + intake, ); + // Today's webhook controllers enqueue through InboxService, which needs the + // registry, so each is attached to its module here, only when enabled. + if (config.linear) { + integrations.linear.controller = new LinearWebhookController( + new AgentSessionService(intake, linearTokenProvider, inboxService), + (at) => linearStatus.recordWebhook(at), + ); + } + if (config.github.webhooks) { + integrations.github.controller = new GitHubWebhookController( + inboxService, + intake, + config.github.webhooks, + ); + } + const slackNotifier = new SlackNotifierService( taskRepository, config.slack?.botToken ?? "", config.slack?.channel ?? "", + config.slack?.notifySources ?? [], + hooks, ); const eventDispatcher = new EventDispatcher( taskRepository, inboxRepository, sandboxManager, - linearTokenProvider, slackNotifier, + integrations, + replyHolds, + hooks, ); const taskProcessor = new TaskProcessorService( taskRepository, inboxRepository, - linearTokenProvider, sandboxManager, eventDispatcher, inboxService, slackNotifier, + integrations, + replyHolds, + taskSources, + hooks, ); - const linearStatus = new LinearStatusService( - config.linear ? linearTokenProvider : undefined, - logger, - ); - - // Each webhook exists only when its integration is configured. - const linearWebhookController = config.linear - ? new LinearWebhookController(agentSessionService, (at) => linearStatus.recordWebhook(at)) - : undefined; - const githubWebhookController = config.github.webhooks - ? new GitHubWebhookController(inboxService, taskRepository, config.github.webhooks) - : undefined; - const githubStatus = new GithubStatusService(githubTokenState, { - enabled: githubWebhookController !== undefined, - ciResults: config.github.ciResultEnabled, - lastWebhookAt: () => githubWebhookController?.lastWebhookAt() ?? null, - }); const healthController = new HealthController(db); const app = new Hono(); app.route("/", createHealthRoutes(healthController)); - app.route( - "/", - createWebhookRoutes({ linear: linearWebhookController, github: githubWebhookController }), - ); + app.route("/", createWebhookRoutes(Object.values(integrations), intake)); // /api only mounts when explicitly enabled: a CP deployed before the // internal ingress exists must expose nothing. let stopApiStreams = () => {}; if (config.api.enabled) { - const apiTaskService = new ApiTaskService( - apiTaskRepository, - taskRepository, - taskEventRepository, - sandboxManager, - { - keyPrefix: config.api.taskKeyPrefix, - publicBaseUrl: config.api.publicBaseUrl, - now: () => new Date(), - }, - ); + const apiTaskService = new ApiTaskService(apiTaskRepository, intake, taskEventRepository, { + publicBaseUrl: config.api.publicBaseUrl, + now: () => new Date(), + }); const apiController = new ApiController(apiTaskService, { authEnabled: config.api.authEnabled, ssePollMs: config.api.ssePollMs, @@ -159,8 +188,7 @@ export function createApp(): AppBundle { createApiRoutes(apiController, new ApiAuthService(apiTokenRepository), { authEnabled: config.api.authEnabled, allowedHosts: config.api.allowedHosts, - linearStatus: () => linearStatus.status(), - githubStatus: () => githubStatus.status(), + integrations: statusesWithHolds(integrations, replyHolds), }), ); stopApiStreams = () => apiController.closeStreams(); @@ -168,3 +196,33 @@ export function createApp(): AppBundle { return { app, taskProcessor, stopApiStreams }; } + +// The registry. Every outbound message goes through +// integrations[round.source].deliver, and core skips a disabled +// integration's webhook, sends and hooks. +export function createIntegrations( + cfg: { linear?: unknown; api: { enabled: boolean }; github: { webhooks?: unknown } }, + linearTokenProvider: Pick, + linearStatus: Pick, + githubStatus: Pick, +) { + return { + linear: new LinearIntegration(linearTokenProvider, cfg.linear !== undefined, linearStatus), + api: new ApiIntegration(cfg.api.enabled), + // GitHub rounds come only from its webhooks. + github: new GithubIntegration(cfg.github.webhooks !== undefined, githubStatus), + } satisfies Record; +} + +// Each integration's own status body, plus its held sends. +export function statusesWithHolds( + integrations: ReturnType, + replyHolds: Pick, +): Record> { + return Object.fromEntries( + Object.values(integrations).map((i) => [ + i.name, + { status: async () => ({ ...(await i.status()), holds: replyHolds.of(i.name) }) }, + ]), + ) as Record>; +} diff --git a/control-plane/src/clients/__tests__/github.test.ts b/control-plane/src/clients/__tests__/github.test.ts index 385caa7..e0cd43d 100644 --- a/control-plane/src/clients/__tests__/github.test.ts +++ b/control-plane/src/clients/__tests__/github.test.ts @@ -25,8 +25,11 @@ const { getUserById, getUserIdByLogin, parsePrUrl, + postIssueComment, requestReviewers, } = (await import("../githubApi.ts" + "?github-impl-test")) as typeof import("../githubApi"); +const { HttpError, retryable } = await import("../../integrations/http"); +const { RetryableError } = await import("../../integrations/outbound"); // Build a fetch stub that returns each page in `pages` as a 200 JSON response, // recording the requested URLs. githubFetch does resp.text() → JSON.parse, so @@ -352,6 +355,63 @@ describe("requestReviewers", () => { }); }); +describe("githubFetch errors and signal", () => { + const originalFetch = globalThis.fetch; + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + it("a non-2xx is an HttpError with status and Retry-After; the message is unchanged", async () => { + globalThis.fetch = (() => + Promise.resolve( + new Response('{"message":"API rate limit exceeded"}', { + status: 429, + headers: { "retry-after": "30" }, + }), + )) as unknown as typeof fetch; + + const err = await postIssueComment("o", "r", 1, "hi").catch((e: unknown) => e); + expect(err).toBeInstanceOf(HttpError); + expect((err as InstanceType).status).toBe(429); + expect((err as InstanceType).retryAfterMs).toBe(30_000); + expect((err as Error).message).toBe( + 'GitHub POST /repos/o/r/issues/1/comments failed: HTTP 429 {"message":"API rate limit exceeded"}', + ); + }); + + it("a rate-limit 403 holds like a 429, waiting for x-ratelimit-reset and at least a minute; any other 403 drops", async () => { + const classify = async (status: number, headers: Record) => { + globalThis.fetch = (() => + Promise.resolve(new Response("{}", { status, headers }))) as unknown as typeof fetch; + return retryable(await postIssueComment("o", "r", 1, "hi").catch((e: unknown) => e)); + }; + const inFiveMinutes = Math.floor(Date.now() / 1000) + 300; // epoch seconds + const exhausted = await classify(403, { + "x-ratelimit-remaining": "0", + "x-ratelimit-reset": String(inFiveMinutes), + }); + expect(exhausted).toBeInstanceOf(RetryableError); + expect(exhausted?.retryAfterMs).toBeGreaterThan(290_000); + expect(exhausted?.retryAfterMs).toBeLessThanOrEqual(300_000); + expect((await classify(403, { "retry-after": "30" }))?.retryAfterMs).toBe(30_000); + // No Retry-After and no reset ahead: at least a minute, never the next tick. + expect((await classify(429, {}))?.retryAfterMs).toBe(60_000); + expect(await classify(403, { "x-ratelimit-remaining": "12" })).toBeUndefined(); + }); + + it("passes the caller's signal to fetch", async () => { + let seen: AbortSignal | null | undefined; + globalThis.fetch = ((_url: string, init?: RequestInit) => { + seen = init?.signal; + return Promise.resolve(Response.json({ id: 1, html_url: "u" })); + }) as unknown as typeof fetch; + + const ac = new AbortController(); + await postIssueComment("o", "r", 1, "hi", ac.signal); + expect(seen).toBe(ac.signal); + }); +}); + describe("addPrAssignees", () => { const originalFetch = globalThis.fetch; afterEach(() => { diff --git a/control-plane/src/clients/githubApi.ts b/control-plane/src/clients/githubApi.ts index 1eb2e0b..71e66d3 100644 --- a/control-plane/src/clients/githubApi.ts +++ b/control-plane/src/clients/githubApi.ts @@ -13,6 +13,7 @@ import { readFileSync } from "node:fs"; import { config } from "../config"; import { githubApiBaseUrl } from "../githubToken/mint"; +import { HttpError, parseRetryAfter, rateLimitWaitMs, TokenFetchError } from "../integrations/http"; import { logger } from "../logger"; import { type GithubTokenProvider, @@ -42,8 +43,13 @@ function tokenProvider(): GithubTokenProvider { return githubToken; } +// No token means nothing was sent, so a send may be retried (TokenFetchError). function token(): string { - return tokenProvider()(); + try { + return tokenProvider()(); + } catch (err) { + throw new TokenFetchError(err instanceof Error ? err.message : String(err)); + } } // The token provider's state, for GET /api/integrations/github. @@ -60,22 +66,39 @@ function githubHeaders(): Record { }; } -// One request. Throws on non-2xx with status + a truncated body snippet — the -// token is never included in the message. +// One request. Throws an HttpError on non-2xx with status + a truncated body +// snippet (and Retry-After) — the token is never included in the message. async function githubFetch( method: string, path: string, body?: unknown, + signal?: AbortSignal, ): Promise<{ status: number; json: unknown }> { const resp = await fetch(`${API_BASE}${path}`, { method, headers: githubHeaders(), body: body === undefined ? undefined : JSON.stringify(body), + signal, }); const text = await resp.text(); if (!resp.ok) { const snippet = text.slice(0, 500); - throw new Error(`GitHub ${method} ${path} failed: HTTP ${resp.status} ${snippet}`); + // `?.`: the client tests stub fetch with header-less partial responses. + const header = (name: string) => resp.headers?.get(name) ?? null; + // GitHub's rate limit is a 429, or a 403 carrying x-ratelimit-remaining: 0 + // or Retry-After; any other 403 is a refusal. Without Retry-After, a rate + // limit waits for x-ratelimit-reset (epoch seconds), and at least a minute. + const rateLimited = + resp.status === 429 || + (resp.status === 403 && + (header("x-ratelimit-remaining") === "0" || header("retry-after") !== null)); + throw new HttpError( + `GitHub ${method} ${path} failed: HTTP ${resp.status} ${snippet}`, + resp.status, + parseRetryAfter(header("retry-after")) ?? + (rateLimited ? rateLimitWaitMs(Number(header("x-ratelimit-reset")) * 1000) : undefined), + rateLimited, + ); } let json: unknown = null; if (text.length > 0) { @@ -99,11 +122,13 @@ export async function postIssueComment( repo: string, prNumber: number, body: string, + signal?: AbortSignal, ): Promise<{ id: number; url: string }> { const { json } = await githubFetch( "POST", `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/issues/${prNumber}/comments`, { body }, + signal, ); const r = asRecord(json); return { id: Number(r.id), url: String(r.html_url ?? "") }; @@ -117,11 +142,13 @@ export async function requestReviewers( repo: string, prNumber: number, reviewers: string[], + signal?: AbortSignal, ): Promise { await githubFetch( "POST", `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/pulls/${prNumber}/requested_reviewers`, { reviewers }, + signal, ); } @@ -133,11 +160,13 @@ export async function postReviewCommentReply( prNumber: number, inReplyTo: number, body: string, + signal?: AbortSignal, ): Promise<{ id: number; url: string }> { const { json } = await githubFetch( "POST", `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/pulls/${prNumber}/comments`, { body, in_reply_to: inReplyTo }, + signal, ); const r = asRecord(json); return { id: Number(r.id), url: String(r.html_url ?? "") }; @@ -150,11 +179,13 @@ export async function addIssueCommentReaction( repo: string, commentId: number, content: ReactionContent, + signal?: AbortSignal, ): Promise { await githubFetch( "POST", `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/issues/comments/${commentId}/reactions`, { content }, + signal, ); } @@ -165,11 +196,13 @@ export async function addReviewCommentReaction( repo: string, commentId: number, content: ReactionContent, + signal?: AbortSignal, ): Promise { await githubFetch( "POST", `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/pulls/comments/${commentId}/reactions`, { content }, + signal, ); } @@ -294,8 +327,16 @@ export async function addPrAssignees( // GET /user/{id} — resolve an immutable numeric GitHub account id to the // current login. Ids survive renames and are never re-issued, so this is the // rename-proof lookup. Throws on 404 (githubFetch); callers catch. -export async function getUserById(id: string | number): Promise<{ login: string; id: number }> { - const { json } = await githubFetch("GET", `/user/${encodeURIComponent(String(id))}`); +export async function getUserById( + id: string | number, + signal?: AbortSignal, +): Promise<{ login: string; id: number }> { + const { json } = await githubFetch( + "GET", + `/user/${encodeURIComponent(String(id))}`, + undefined, + signal, + ); const r = asRecord(json); return { login: String(r.login ?? ""), id: Number(r.id) }; } diff --git a/control-plane/src/clients/linearApi.ts b/control-plane/src/clients/linearApi.ts index eb8bcfa..b6ccbab 100644 --- a/control-plane/src/clients/linearApi.ts +++ b/control-plane/src/clients/linearApi.ts @@ -72,6 +72,8 @@ type UploadIssueAttachmentParams = { contentType: string; subtitle?: string; metadata?: Record; + // The SDK calls take the client's own signal; this one bounds the raw upload. + signal?: AbortSignal; }; // Uploads bytes to Linear-managed object storage, then attaches the uploaded @@ -104,6 +106,7 @@ export async function uploadIssueAttachment( method: "PUT", headers: uploadHeaders, body: params.bytes, + signal: params.signal, }); if (!uploadResp.ok) { const responseBody = (await uploadResp.text().catch(() => "")).slice(0, 500); diff --git a/control-plane/src/clients/linearOauth.ts b/control-plane/src/clients/linearOauth.ts index fb459ca..802c821 100644 --- a/control-plane/src/clients/linearOauth.ts +++ b/control-plane/src/clients/linearOauth.ts @@ -1,3 +1,5 @@ +import { TokenFetchError } from "../integrations/http"; + // ── Client-credentials token ───────────────────────────────────────────────── // The exports in this section are pure: they never read `config`. @@ -12,8 +14,9 @@ export type LinearCredentials = { clientId: string; clientSecret: string }; export type ClientCredentialsToken = { accessToken: string; expiresAt: Date; scope: string }; // A failed token request. `code` and `description` are the OAuth `error` and -// `error_description` fields; the response body is never kept otherwise. -export class LinearTokenError extends Error { +// `error_description` fields; the response body is never kept otherwise. A +// TokenFetchError: nothing was sent with the missing token, so a send may retry. +export class LinearTokenError extends TokenFetchError { status?: number; code?: string; description?: string; diff --git a/control-plane/src/clients/slackApi.ts b/control-plane/src/clients/slackApi.ts index 274ccdc..aa0bd64 100644 --- a/control-plane/src/clients/slackApi.ts +++ b/control-plane/src/clients/slackApi.ts @@ -27,6 +27,7 @@ export type PostSlackMessageParams = { text: string; // Optional Block Kit blocks; `text` stays the fallback/notification string. blocks?: unknown[]; + signal?: AbortSignal; }; // POST /chat.postMessage into a channel, or into a thread when threadTs is set. IMPORTANT: Slack returns HTTP 200 even @@ -46,6 +47,7 @@ export async function postSlackMessage(params: PostSlackMessageParams): Promise< text: params.text, ...(params.blocks ? { blocks: params.blocks } : {}), }), + signal: params.signal, }); const text = await resp.text(); if (!resp.ok) { @@ -68,10 +70,14 @@ export async function postSlackMessage(params: PostSlackMessageParams): Promise< // or missing_scope when the bot lacks `users:read.email`) — the caller's mention // degrades to a plain name, never worth failing a message over. Throws only on // a transport-level error, matching postSlackMessage. -export async function lookupSlackUserIdByEmail(email: string): Promise { +export async function lookupSlackUserIdByEmail( + email: string, + signal?: AbortSignal, +): Promise { const resp = await fetch(`${API_BASE}/users.lookupByEmail?email=${encodeURIComponent(email)}`, { method: "GET", headers: { Authorization: `Bearer ${slackToken()}` }, + signal, }); const text = await resp.text(); if (!resp.ok) { diff --git a/control-plane/src/config.ts b/control-plane/src/config.ts index d2908cc..001855d 100644 --- a/control-plane/src/config.ts +++ b/control-plane/src/config.ts @@ -96,8 +96,10 @@ export const config = { // integrationsConfig.ts. linear: parseLinearConfig(process.env), // Undefined unless SLACK_ENABLED=true; then SLACK_BOT_TOKEN (xoxb-…, posts - // task-completion updates) and SLACK_CHANNEL_ID (C…) are required. The token - // matches the /token/i log-redaction pattern. Parsed by integrationsConfig.ts. + // task-completion updates) and SLACK_CHANNEL_ID (C…) are required, and + // SLACK_NOTIFY_SOURCES picks the task sources that post (default linear). + // The token matches the /token/i log-redaction pattern. Parsed by + // integrationsConfig.ts. slack: parseSlackConfig(process.env), github: { // The control plane's own GitHub token (it runs outside the sandbox diff --git a/control-plane/src/controllers/__tests__/api.controller.stream.test.ts b/control-plane/src/controllers/__tests__/api.controller.stream.test.ts index 5127b29..9ef1afe 100644 --- a/control-plane/src/controllers/__tests__/api.controller.stream.test.ts +++ b/control-plane/src/controllers/__tests__/api.controller.stream.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, mock, spyOn } from "bun:test"; import type { Clock } from "../../clock"; import { ApiError } from "../../errors"; +import { fakeIntegrations } from "../../integrations/__tests__/fakeIntegrations"; import { logger } from "../../logger"; import { createApiRoutes } from "../../routes/api"; import type { ApiEventEnvelope, SseStateData } from "../../types/api"; @@ -123,10 +124,15 @@ function setup(o: { { authEnabled: false, allowedHosts: [], - linearStatus: async () => ({ configured: false }), - githubStatus: () => ({ - token: { source: "env", expiresAt: null, readAt: null, lastError: null }, - webhooks: { enabled: false, lastWebhookAt: null, ciResults: false }, + integrations: fakeIntegrations({ + linear: { status: async () => ({ configured: false }) }, + github: { + status: async () => ({ + token: { source: "env", expiresAt: null, readAt: null, lastError: null }, + webhooks: { enabled: false, lastWebhookAt: null, ciResults: false }, + }), + }, + api: { status: async () => ({ enabled: true }) }, }), }, ); diff --git a/control-plane/src/controllers/__tests__/api.controller.test.ts b/control-plane/src/controllers/__tests__/api.controller.test.ts index fa444d1..0db2bb2 100644 --- a/control-plane/src/controllers/__tests__/api.controller.test.ts +++ b/control-plane/src/controllers/__tests__/api.controller.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it, mock, spyOn } from "bun:test"; import { ApiError } from "../../errors"; +import { fakeIntegrations } from "../../integrations/__tests__/fakeIntegrations"; import { logger } from "../../logger"; import { createApiRoutes } from "../../routes/api"; import type { ApiTask } from "../../types/api"; @@ -57,8 +58,11 @@ function setup(opts: { authEnabled: boolean; allowedHosts?: string[] }) { const app = createApiRoutes(controller, auth, { authEnabled: opts.authEnabled, allowedHosts: opts.allowedHosts ?? [], - linearStatus: async () => LINEAR_STATUS, - githubStatus: () => GITHUB_STATUS, + integrations: fakeIntegrations({ + linear: { status: async () => LINEAR_STATUS }, + github: { status: async () => GITHUB_STATUS }, + api: { status: async () => ({ enabled: true }) }, + }), }); const req = (path: string, init: RequestInit & { json?: unknown } = {}) => app.request(path, { @@ -101,6 +105,7 @@ const routes: Array<[string, string, unknown?]> = [ ["GET", "/api/me"], ["GET", "/api/integrations/linear"], ["GET", "/api/integrations/github"], + ["GET", "/api/integrations/api"], ]; describe("ApiController", () => { @@ -118,6 +123,14 @@ describe("ApiController", () => { expect(await res.json()).toEqual(GITHUB_STATUS); }); + it("GET /api/integrations/api returns the status body; an unknown name is 404", async () => { + const { req } = setup({ authEnabled: true }); + expect(await (await req("/api/integrations/api")).json()).toEqual({ enabled: true }); + for (const name of ["jira", "constructor"]) { + expect((await req(`/api/integrations/${name}`)).status).toBe(404); + } + }); + describe("auth on", () => { it.each(routes)( "%s %s without a token is 401 with the uniform body", diff --git a/control-plane/src/controllers/__tests__/githubWebhook.controller.test.ts b/control-plane/src/controllers/__tests__/githubWebhook.controller.test.ts index 36c831b..8ab5956 100644 --- a/control-plane/src/controllers/__tests__/githubWebhook.controller.test.ts +++ b/control-plane/src/controllers/__tests__/githubWebhook.controller.test.ts @@ -2,8 +2,8 @@ import { beforeEach, describe, expect, it, mock, spyOn } from "bun:test"; import { createHmac } from "node:crypto"; import { Hono } from "hono"; import type { Task } from "../../db/schema"; +import type { TaskIntake } from "../../integrations/intake"; import { logger } from "../../logger"; -import type { TaskRepository } from "../../repositories/task.repository"; import type { InboxService } from "../../services/inbox.service"; const WEBHOOK_SECRET = "gh-test-secret"; @@ -373,7 +373,7 @@ describe("GitHubWebhookController.handleWebhook", () => { enqueueReview, enqueueCiResult, } as unknown as InboxServiceLike as InboxService; - const taskRepo = { findByPrUrl } as unknown as TaskRepository; + const intake = { findTaskByPrUrl: findByPrUrl } as unknown as TaskIntake; // Reset the module-mocked fetchOrgMembers so per-test mockImplementationOnce // overrides don't leak across cases. Default: resolves to the trusted-set // logins used across positive-path bodies. The bot login `acme-bot[bot]` is @@ -384,7 +384,7 @@ describe("GitHubWebhookController.handleWebhook", () => { getUserIdByLogin.mockReset(); getUserIdByLogin.mockResolvedValue(null); clock = { now: 1_000_000 }; - controller = new GitHubWebhookController(inboxService, taskRepo, SETTINGS, () => clock.now); + controller = new GitHubWebhookController(inboxService, intake, SETTINGS, () => clock.now); const a = new Hono(); a.post("/webhook/github", (c) => controller.handleWebhook(c)); return a; @@ -462,6 +462,17 @@ describe("GitHubWebhookController.handleWebhook", () => { expect(addIssueCommentReaction).toHaveBeenCalledWith("o", "r", 555, "eyes"); }); + // No delivery-id dedupe here: the inbox's ON CONFLICT drops the duplicate row. + it("a redelivered comment enqueues and reacts again", async () => { + const body = commentBody("alice"); + expect((await post(body, "issue_comment", sign(body))).status).toBe(200); + expect((await post(body, "issue_comment", sign(body))).status).toBe(200); + await flushMicrotasks(); + expect(enqueueComment).toHaveBeenCalledTimes(2); + expect(enqueueComment.mock.calls[1]).toEqual(enqueueComment.mock.calls[0]); + expect(addIssueCommentReaction).toHaveBeenCalledTimes(2); + }); + it("does not react to a shortened alias of the bot login", async () => { const body = commentBody("alice", "acme-bot[bot]", "@bot fix"); const res = await post(body, "issue_comment", sign(body)); @@ -689,6 +700,29 @@ describe("GitHubWebhookController.handleWebhook", () => { expect(addReviewCommentReaction).toHaveBeenCalledWith("o", "r", 12, "eyes"); }); + it("a redelivered review refetches its inline comments, enqueues and reacts again", async () => { + getReviewComments.mockResolvedValue([ + { id: 11, path: "a.ts", line: 3, body: "nit", user: { login: "reviewer" } }, + ]); + const body = JSON.stringify({ + action: "submitted", + repository: { name: "r", owner: { login: "o" } }, + pull_request: { number: 7, user: { login: "acme-bot[bot]" } }, + review: { + id: 88, + body: "@acme-bot please handle this review", + user: { login: "a", id: 2 }, + submitted_at: "2026-04-23T12:00:00Z", + }, + }); + expect((await post(body, "pull_request_review", sign(body))).status).toBe(200); + expect((await post(body, "pull_request_review", sign(body))).status).toBe(200); + await flushMicrotasks(); + expect(getReviewComments).toHaveBeenCalledTimes(2); + expect(enqueueReview).toHaveBeenCalledTimes(2); + expect(addReviewCommentReaction).toHaveBeenCalledTimes(2); + }); + it("enqueues a review on an API task's PR without parsing its URL", async () => { findByPrUrl.mockResolvedValue(apiTask); const body = JSON.stringify({ @@ -934,11 +968,11 @@ describe("GitHubWebhookController.handleWebhook", () => { enqueueReview: mock(() => Promise.resolve()), enqueueCiResult: flagOffEnqueueCiResult, } as unknown as InboxService; - const flagOffRepo = { findByPrUrl: flagOffFindByPrUrl } as unknown as TaskRepository; + const flagOffIntake = { findTaskByPrUrl: flagOffFindByPrUrl } as unknown as TaskIntake; fetchOrgMembers.mockResolvedValue(new Set([1, 2, 3]) as ReadonlySet); const flagOffApp = new Hono(); flagOffApp.post("/webhook/github", (c) => - new Ctrl(flagOffService, flagOffRepo, SETTINGS).handleWebhook(c), + new Ctrl(flagOffService, flagOffIntake, SETTINGS).handleWebhook(c), ); const body = workflowRunBody(); @@ -1102,10 +1136,10 @@ describe("GitHubWebhookController.handleWebhook", () => { enqueueReview, enqueueCiResult, } as unknown as InboxService; - const taskRepo = { findByPrUrl } as unknown as TaskRepository; + const intake = { findTaskByPrUrl: findByPrUrl } as unknown as TaskIntake; controller = new GitHubWebhookController( inboxService, - taskRepo, + intake, { ...SETTINGS, ...settings }, () => clock.now, ); diff --git a/control-plane/src/controllers/api.controller.ts b/control-plane/src/controllers/api.controller.ts index 1ddd67a..0b92c9b 100644 --- a/control-plane/src/controllers/api.controller.ts +++ b/control-plane/src/controllers/api.controller.ts @@ -1,7 +1,9 @@ import type { Context } from "hono"; import { type SSEStreamingApi, streamSSE } from "hono/streaming"; import type { Clock } from "../clock"; +import { taskSource } from "../db/schema"; import { ApiError } from "../errors"; +import type { TaskSource } from "../integrations/types"; import { logger } from "../logger"; import type { ApiTaskService, @@ -37,6 +39,7 @@ const TEXT_MAX_BYTES = 64 * 1024; const FLAGS_MAX = 16; const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; const STATES: readonly ApiTaskState[] = ["queued", "running", "idle", "failed"]; +const SOURCES: readonly string[] = taskSource.enumValues; export function errorBody(code: ApiErrorCode, message: string): ApiErrorBody { return { error: { code, message } }; @@ -142,7 +145,7 @@ export class ApiController { // Last-Event-ID / ?after=. Errors before the first byte are JSON; errors // after are logged and close the stream (no message to the client). async streamEvents(c: Context): Promise { - let target: { id: string; source: "linear" | "api" }; + let target: { id: string; source: TaskSource }; let start: number; try { start = parseStreamStart(c.req.header("last-event-id"), c.req.query("after")); @@ -205,7 +208,7 @@ export class ApiController { private async runStream( stream: SSEStreamingApi, - target: { id: string; source: "linear" | "api" }, + target: { id: string; source: TaskSource }, start: number, signal: AbortSignal, ): Promise { @@ -369,8 +372,12 @@ function parseListQuery(c: Context): ListTasksInput { }; const source = c.req.query("source"); if (source !== undefined) { - if (source !== "linear" && source !== "api") throw invalid("source must be linear or api"); - q.source = source; + if (!SOURCES.includes(source)) { + throw invalid( + `source must be ${SOURCES.slice(0, -1).join(", ")} or ${SOURCES[SOURCES.length - 1]}`, + ); + } + q.source = source as TaskSource; } const state = c.req.query("state"); if (state !== undefined) { diff --git a/control-plane/src/controllers/githubWebhook.controller.ts b/control-plane/src/controllers/githubWebhook.controller.ts index 9dfa505..4d4cc81 100644 --- a/control-plane/src/controllers/githubWebhook.controller.ts +++ b/control-plane/src/controllers/githubWebhook.controller.ts @@ -8,10 +8,10 @@ import { getUserIdByLogin, } from "../clients/github"; import { config } from "../config"; +import type { TaskIntake } from "../integrations/intake"; import type { GithubWebhooksConfig } from "../integrationsConfig"; import { logger } from "../logger"; import type { ReplyTargetSpec } from "../repositories/inbox.repository"; -import type { TaskRepository } from "../repositories/task.repository"; import type { InboxService } from "../services/inbox.service"; // ──── Parsed event surface ──── @@ -159,7 +159,7 @@ export function extractPrOpenerLogin(eventHeader: string | undefined, payload: u } // Canonical PR URL — built deterministically so it is byte-identical to the -// html_url the harness stores via pr_opened (findByPrUrl does an exact jsonb +// html_url the harness stores via pr_opened (findTaskByPrUrl does an exact jsonb // containment match). Never trust the payload's html_url formatting. function prUrlOf(owner: string, repo: string, prNumber: number): string { return `https://github.com/${owner}/${repo}/pull/${prNumber}`; @@ -341,7 +341,7 @@ export class GitHubWebhookController { constructor( private inboxService: InboxService, - private taskRepo: TaskRepository, + private intake: Pick, private readonly settings: GithubWebhooksConfig, private readonly now: () => number = Date.now, ) { @@ -453,7 +453,7 @@ export class GitHubWebhookController { // trust gate and DB lookup so unrelated org activity doesn't fan out // into API calls. // workflow_run events have no single PR opener field. The task-lookup - // (findByPrUrl) already scopes to bot-opened PRs, so the pre-filter is + // (findTaskByPrUrl) already scopes to bot-opened PRs, so the pre-filter is // redundant for this event type. if (eventHeader !== "workflow_run" && prOpenerLogin !== this.botPayloadLogin) { this.reject(c, "not_bot_pr"); @@ -510,7 +510,7 @@ export class GitHubWebhookController { // and logs each trip — nothing to surface in the response here. for (const pr of parsed.pullRequests) { const prUrl = prUrlOf(parsed.owner, parsed.repo, pr.number); - const task = await this.taskRepo.findByPrUrl(prUrl); + const task = await this.intake.findTaskByPrUrl(prUrl); if (!task) continue; logger.debug("workflow_run matched a bot-opened task; enqueueing ci_result", { taskId: task.id, @@ -550,7 +550,7 @@ export class GitHubWebhookController { // comment / review — task has no status; every comment enqueues an inbox // row. The sweep recreates the sandbox from PVC if needed and surfaces a // PVC-missing error if the workspace has already been pruned. - const task = await this.taskRepo.findByPrUrl(parsed.prUrl); + const task = await this.intake.findTaskByPrUrl(parsed.prUrl); if (!task) return c.json({ status: "ignored" }, 200); if (parsed.kind === "comment") { diff --git a/control-plane/src/db/__tests__/migration.baseline.test.ts b/control-plane/src/db/__tests__/migration.baseline.test.ts index 06e9aa6..ecc78ae 100644 --- a/control-plane/src/db/__tests__/migration.baseline.test.ts +++ b/control-plane/src/db/__tests__/migration.baseline.test.ts @@ -15,6 +15,15 @@ const journal = JSON.parse(readFileSync("drizzle/meta/_journal.json", "utf8")) a entries: Array<{ tag: string; when: number }>; }; +describe("drizzle journal", () => { + it("has strictly increasing timestamps", () => { + // The migrator applies a migration only when its `when` is past the last + // applied one, so a file out of order would be skipped for good. + const whens = journal.entries.map((e) => e.when); + expect(whens.every((w, i) => i === 0 || w > (whens[i - 1] ?? Number.NaN))).toBe(true); + }); +}); + async function applyBaseline(client: SQL): Promise { const migration = readFileSync(`drizzle/${journal.entries[0]?.tag}.sql`, "utf8"); for (const statement of migration.split("--> statement-breakpoint")) { diff --git a/control-plane/src/db/schema.ts b/control-plane/src/db/schema.ts index 8a55e56..947e052 100644 --- a/control-plane/src/db/schema.ts +++ b/control-plane/src/db/schema.ts @@ -74,7 +74,8 @@ export const tasks = pgTable( }, (t) => [ uniqueIndex("tasks_source_external_id_uniq").on(t.source, t.externalId), - uniqueIndex("tasks_key_uniq").on(t.key), + // Keys are unique case-insensitively; readers match lower(key) = lower($1). + uniqueIndex("tasks_key_lower_uniq").on(sql`lower(${t.key})`), // Heartbeat scans only care whether a pod owns the task. index("tasks_lease_idx").on(t.heartbeatAt).where(sql`${t.ownerId} IS NOT NULL`), // GitHub webhook PR→task lookup. @@ -124,6 +125,11 @@ export const inbox = pgTable( }, (t) => [ // Idempotency anchor: webhook retries / LB reroutes collapse via this. + // Every ON CONFLICT targets (source, kind, external_id). + uniqueIndex("inbox_source_kind_external_uniq").on(t.source, t.kind, t.externalId), + // The older, stricter index, kept for one release so older pods' + // ON CONFLICT (source, external_id) still has its arbiter. A later release + // drops it. uniqueIndex("inbox_source_external_uniq").on(t.source, t.externalId), // Pending = not yet delivered or dead-lettered. Sweep + claimNext filter on this. index("inbox_pending_idx").on(t.taskId).where(sql`${t.status} = 'pending'`), diff --git a/control-plane/src/integrations/__tests__/fakeIntegrations.ts b/control-plane/src/integrations/__tests__/fakeIntegrations.ts new file mode 100644 index 0000000..42d5589 --- /dev/null +++ b/control-plane/src/integrations/__tests__/fakeIntegrations.ts @@ -0,0 +1,23 @@ +import type { Integration, IntegrationName } from "../types"; + +const noop = (name: IntegrationName): Integration => ({ + name, + enabled: true, + async deliver() {}, + async status() { + return {}; + }, +}); + +// A registry for tests: every integration a no-op (enabled, delivers nothing) +// unless given. A new integration gets its default here, not in every test. +export function fakeIntegrations< + O extends Partial>>, +>(overrides: O): Omit, keyof O> & O { + const defaults: Record = { + linear: noop("linear"), + api: noop("api"), + github: noop("github"), + }; + return { ...defaults, ...overrides }; +} diff --git a/control-plane/src/integrations/__tests__/http.test.ts b/control-plane/src/integrations/__tests__/http.test.ts new file mode 100644 index 0000000..f9ddb12 --- /dev/null +++ b/control-plane/src/integrations/__tests__/http.test.ts @@ -0,0 +1,304 @@ +import { afterEach, describe, expect, it, mock } from "bun:test"; +import { LinearClient } from "@linear/sdk"; +import type { ReplyTarget } from "../../repositories/inbox.repository"; + +// A token file that does not exist: the GitHub client has no token to send. +mock.module("../../config", () => ({ + config: { podId: "pod-test", github: { tokenSettings: { file: "/nonexistent/devbox/token" } } }, +})); + +const { HttpError, TokenFetchError, parseRetryAfter, retryable, withTimeout } = await import( + "../http" +); +const { RetryableError } = await import("../outbound"); +const { LinearIntegration } = await import("../linear"); +const { LinearTokenError } = await import("../../clients/linearOauth"); +// A fresh instance (see github.test.ts): never the possibly-mocked barrel. +const { postIssueComment } = (await import( + "../../clients/githubApi.ts" + "?http-test" +)) as typeof import("../../clients/githubApi"); +const { logger } = await import("../../logger"); + +// What fetch throws for `url`. +async function fetchError(url: string): Promise { + try { + await fetch(url); + } catch (err) { + return err; + } + throw new Error(`fetch ${url} succeeded`); +} + +const codeOf = (err: unknown) => (err as { code?: unknown }).code; + +// A local TCP server whose sockets do `onOpen` / `onData`. +function tcpServer(handlers: { + open?: (s: { write(d: string): void; end(): void }) => void; + data?: (s: { end(): void }) => void; +}) { + return Bun.listen({ + hostname: "127.0.0.1", + port: 0, + socket: { open: (s) => handlers.open?.(s), data: (s) => handlers.data?.(s) }, + }); +} + +// The exact shapes bun 1.4.4's fetch throws; retryable() is written from these. +describe("retryable: bun fetch failures", () => { + it("connection refused is a TypeError with code ECONNREFUSED: retryable", async () => { + const closed = tcpServer({}); + const port = closed.port; + closed.stop(true); + const err = await fetchError(`http://127.0.0.1:${port}/`); + expect(err).toBeInstanceOf(TypeError); + expect(codeOf(err)).toBe("ECONNREFUSED"); + expect(retryable(err)).toBeInstanceOf(RetryableError); + }); + + it("an unknown host is a TypeError with code ENOTFOUND (EAI_AGAIN without a resolver): retryable", async () => { + const err = await fetchError("https://devbox-http-test.invalid/"); + expect(err).toBeInstanceOf(TypeError); + expect(["ENOTFOUND", "EAI_AGAIN"]).toContain(codeOf(err) as string); + expect(retryable(err)).toBeInstanceOf(RetryableError); + }); + + it("a failed TLS handshake is a TypeError with code EPROTO: retryable", async () => { + const plain = tcpServer({ + open: (s) => { + s.write("HTTP/1.1 200 OK\r\n\r\nnot tls"); + s.end(); + }, + }); + try { + const err = await fetchError(`https://127.0.0.1:${plain.port}/`); + expect(err).toBeInstanceOf(TypeError); + expect(codeOf(err)).toBe("EPROTO"); + expect(retryable(err)).toBeInstanceOf(RetryableError); + } finally { + plain.stop(true); + } + }); + + it("a reset after the request went out (ECONNRESET) is not retryable", async () => { + const resetter = tcpServer({ data: (s) => s.end() }); + try { + const err = await fetchError(`http://127.0.0.1:${resetter.port}/`); + expect(codeOf(err)).toBe("ECONNRESET"); + expect(retryable(err)).toBeUndefined(); + } finally { + resetter.stop(true); + } + }); + + it("a timed-out (aborted) fetch is not retryable: it may have landed", async () => { + const hang = tcpServer({}); + try { + const err = await withTimeout(20, (signal) => + fetch(`http://127.0.0.1:${hang.port}/`, { signal }), + ).catch((e: unknown) => e); + expect((err as Error).message).toBe("timed out after 20 ms"); + expect(retryable(err)).toBeUndefined(); + } finally { + hang.stop(true); + } + }); +}); + +describe("retryable: typed errors", () => { + it("a 429 is retryable and keeps its Retry-After; 5xx and other 4xx are not", () => { + const limited = retryable(new HttpError("HTTP 429 slow down", 429, 7000)); + expect(limited).toBeInstanceOf(RetryableError); + expect(limited?.message).toBe("HTTP 429 slow down"); + expect(limited?.retryAfterMs).toBe(7000); + for (const status of [400, 403, 404, 422, 500, 502, 503]) { + expect(retryable(new HttpError(`HTTP ${status}`, status))).toBeUndefined(); + } + }); + + it("a certificate failure in the TLS handshake (DEPTH_ZERO_SELF_SIGNED_CERT) is retryable", () => { + const err = Object.assign(new TypeError("self signed certificate"), { + code: "DEPTH_ZERO_SELF_SIGNED_CERT", + }); + expect(retryable(err)).toBeInstanceOf(RetryableError); + }); + + it("a token-fetch failure is retryable, Linear's included", () => { + expect(retryable(new TokenFetchError("no token"))).toBeInstanceOf(RetryableError); + expect(retryable(new LinearTokenError("Linear token request failed"))).toBeInstanceOf( + RetryableError, + ); + }); + + it("anything else is not retryable", () => { + expect(retryable(new Error("boom"))).toBeUndefined(); + expect(retryable("boom")).toBeUndefined(); + expect(retryable(null)).toBeUndefined(); + }); +}); + +describe("parseRetryAfter", () => { + it("reads delay-seconds and HTTP dates; ignores anything else", () => { + expect(parseRetryAfter("7")).toBe(7000); + expect(parseRetryAfter("0")).toBe(0); + const inAMinute = parseRetryAfter(new Date(Date.now() + 60_000).toUTCString()) ?? 0; + expect(inAMinute).toBeGreaterThan(58_000); + expect(inAMinute).toBeLessThanOrEqual(60_000); + expect(parseRetryAfter(new Date(Date.now() - 60_000).toUTCString())).toBe(0); + expect(parseRetryAfter("soon")).toBeUndefined(); + expect(parseRetryAfter(null)).toBeUndefined(); + }); +}); + +describe("withTimeout", () => { + it("returns the send's result and leaves its signal alone", async () => { + let seen: AbortSignal | undefined; + const out = await withTimeout(1000, async (signal) => { + seen = signal; + return "sent"; + }); + expect(out).toBe("sent"); + expect(seen?.aborted).toBe(false); + }); + + it("gives up after the bound even when the send ignores its signal, and aborts the signal", async () => { + let seen: AbortSignal | undefined; + const err = await withTimeout(20, (signal) => { + seen = signal; + return new Promise(() => {}); + }).catch((e: unknown) => e); + expect((err as Error).message).toBe("timed out after 20 ms"); + expect(seen?.aborted).toBe(true); + }); +}); + +describe("GitHub client", () => { + const originalFetch = globalThis.fetch; + afterEach(() => { + globalThis.fetch = originalFetch; + }); + + it("a missing token is a TokenFetchError and sends nothing", async () => { + const fetchStub = mock(() => Promise.resolve(new Response("{}"))); + globalThis.fetch = fetchStub as unknown as typeof fetch; + const err = await postIssueComment("o", "r", 1, "hi").catch((e: unknown) => e); + expect(err).toBeInstanceOf(TokenFetchError); + expect(retryable(err)).toBeInstanceOf(RetryableError); + expect(fetchStub).not.toHaveBeenCalled(); + }); +}); + +// Linear's SDK wraps every failure in a LinearError; the integration maps the +// SDK's rate-limit error and the raw transport error underneath. +describe("Linear deliver: SDK failures", () => { + const target: ReplyTarget = { id: "sess-1", spec: { kind: "linear_session" } }; + const reply = { type: "reply", body: "hi" } as const; + const integrationFor = (apiUrl: string) => + new LinearIntegration( + { + getAuthorizedLinearClient: (signal?: AbortSignal) => + Promise.resolve(new LinearClient({ accessToken: "t", apiUrl, signal })), + }, + true, + { status: async () => ({ configured: false }) }, + ); + const deliverTo = (apiUrl: string, signal = new AbortController().signal) => + integrationFor(apiUrl) + .deliver({ signal, log: logger }, target, reply) + .then(() => "sent") + .catch((e: unknown) => e); + + const server = (respond: () => Response | Promise) => + Bun.serve({ hostname: "127.0.0.1", port: 0, fetch: respond }); + + it("an HTTP 429 is retryable with Retry-After", async () => { + const s = server( + () => new Response("Too many requests", { status: 429, headers: { "retry-after": "12" } }), + ); + try { + const err = await deliverTo(`http://127.0.0.1:${s.port}/graphql`); + expect(err).toBeInstanceOf(RetryableError); + expect((err as InstanceType).retryAfterMs).toBe(12_000); + } finally { + s.stop(true); + } + }); + + it("a ratelimited GraphQL error is retryable", async () => { + const s = server(() => + Response.json( + { + errors: [ + { + message: "Rate limit exceeded", + extensions: { type: "ratelimited", code: "RATELIMITED" }, + }, + ], + }, + { status: 400 }, + ), + ); + try { + expect(await deliverTo(`http://127.0.0.1:${s.port}/graphql`)).toBeInstanceOf(RetryableError); + } finally { + s.stop(true); + } + }); + + it("a rate limit without Retry-After waits for Linear's reset time, at least a minute", async () => { + const ratelimited = (headers: Record) => + server(() => + Response.json( + { errors: [{ message: "Rate limit exceeded", extensions: { type: "ratelimited" } }] }, + { status: 400, headers }, + ), + ); + const waitOf = async (headers: Record) => { + const s = ratelimited(headers); + try { + const err = await deliverTo(`http://127.0.0.1:${s.port}/graphql`); + expect(err).toBeInstanceOf(RetryableError); + return (err as InstanceType).retryAfterMs; + } finally { + s.stop(true); + } + }; + const inTwoMinutes = await waitOf({ + "x-ratelimit-requests-reset": String(Date.now() + 120_000), // epoch ms + }); + expect(inTwoMinutes).toBeGreaterThan(110_000); + expect(inTwoMinutes).toBeLessThanOrEqual(120_000); + expect(await waitOf({})).toBe(60_000); + }); + + it("a 500 is not retryable", async () => { + const s = server(() => new Response("boom", { status: 500 })); + try { + const err = await deliverTo(`http://127.0.0.1:${s.port}/graphql`); + expect(err).toBeInstanceOf(Error); + expect(err).not.toBeInstanceOf(RetryableError); + } finally { + s.stop(true); + } + }); + + it("connection refused is retryable", async () => { + const closed = tcpServer({}); + const port = closed.port; + closed.stop(true); + expect(await deliverTo(`http://127.0.0.1:${port}/graphql`)).toBeInstanceOf(RetryableError); + }); + + it("the per-call client honours the send's signal (aborted: not retryable)", async () => { + const s = server(() => new Promise(() => {})); + try { + const ac = new AbortController(); + const pending = deliverTo(`http://127.0.0.1:${s.port}/graphql`, ac.signal); + ac.abort(new Error("timed out after 20000 ms")); + const err = await pending; + expect(err).toBeInstanceOf(Error); + expect(err).not.toBeInstanceOf(RetryableError); + } finally { + s.stop(true); + } + }); +}); diff --git a/control-plane/src/integrations/__tests__/intake.test.ts b/control-plane/src/integrations/__tests__/intake.test.ts new file mode 100644 index 0000000..3ab2a0c --- /dev/null +++ b/control-plane/src/integrations/__tests__/intake.test.ts @@ -0,0 +1,455 @@ +import { beforeAll, beforeEach, describe, expect, it, mock } from "bun:test"; +import { SQL } from "bun"; +import { sql } from "drizzle-orm"; + +const TEST_DATABASE_URL = + process.env.TEST_DATABASE_URL ?? "postgres://devbox:devbox@localhost:5432/devbox"; +mock.module("../../config", () => ({ config: { databaseUrl: TEST_DATABASE_URL } })); + +const { db } = await import("../../db"); +const { TaskRepository } = await import("../../repositories/task.repository"); +const { InboxRepository } = await import("../../repositories/inbox.repository"); +const { Intake } = await import("../intake"); +const { migrate } = await import("drizzle-orm/bun-sql/migrator"); + +import type { ReplyTarget } from "../../repositories/inbox.repository"; +import type { InboxMessageDraft, TaskIntake } from "../intake"; + +const session = (id: string, order?: number): ReplyTarget => ({ + id, + spec: { kind: "linear_session" }, + ...(order === undefined ? {} : { order }), +}); + +const PR = "https://github.com/o/r/pull/7"; +const ghComment = (id: string): InboxMessageDraft => ({ + source: "github", + kind: "comment", + external_id: id, + payload: { target_id: id, body: "@bot fix", pr: { repo: "r", pr_url: PR }, created_at: "" }, +}); +const ghTarget = (id: string): ReplyTarget => ({ + id, + spec: { kind: "issue_comment", owner: "o", repo: "r", prNumber: 7, commentId: Number(id) }, +}); +const ciResult = (run: number): InboxMessageDraft => ({ + source: "github", + kind: "ci_result", + external_id: `workflow_run:${run}:1:7`, + payload: { + target_id: String(run), + workflow_name: "CI", + workflow_path: ".github/workflows/ci.yml", + conclusion: "failure", + run_url: "u", + head_sha: "abc", + head_branch: "b", + pr: { repo: "r", pr_url: PR }, + }, +}); + +async function insertTask(o: { + source?: string; + externalId?: string; + key: string; + prs?: string[]; +}) { + const prUrls = (o.prs ?? []).map((url) => ({ repo: "r", url, branch: "b", commit_sha: "c" })); + const [row] = await db.execute<{ id: string; key: string }>(sql` + INSERT INTO tasks (source, external_id, key, title, external_url, pr_urls) + VALUES (${o.source ?? "api"}, ${o.externalId ?? crypto.randomUUID()}, ${o.key}, 't', 'u', + ${JSON.stringify(prUrls)}::text::jsonb) + RETURNING id, key;`); + return row as { id: string; key: string }; +} + +describe("Intake", () => { + let killSandbox: ReturnType; + let detach: ReturnType; + let intake: TaskIntake; + + // A Linear `created`: issue-1 / ISS-1, delegated through `target`. + const assignLinear = ( + target: ReplyTarget, + o: Partial[0]> = {}, + ) => + intake.assign({ + source: "linear", + externalId: "issue-1", + key: { hint: "ISS-1" }, + title: "Fix auth", + description: "desc", + flags: [], + url: () => "https://linear.app/x/issue/ISS-1", + target: () => target, + ...o, + }); + + beforeAll(async () => { + await migrate(db, { migrationsFolder: "./drizzle" }); + }); + beforeEach(async () => { + await db.execute( + sql`TRUNCATE TABLE task_events, inbox, tasks, api_tokens RESTART IDENTITY CASCADE`, + ); + killSandbox = mock(() => Promise.resolve()); + detach = mock(() => {}); + intake = new Intake( + new TaskRepository(), + new InboxRepository(), + { killSandbox }, + { detach }, + { apiKeyPrefix: "DBX-API-", ciResultMaxPerTask: 20 }, + ); + }); + + describe("assign", () => { + it("creates the task and its assignment round; the platform id is the round's external id", async () => { + const requester = { id: "u-1", displayName: "Ann", codeHost: "unlinked" as const }; + const res = await assignLinear(session("sess-1"), { flags: ["FOO"], requester }); + expect(res).toEqual({ taskId: expect.any(String), key: "ISS-1", result: "created" }); + + const [task] = await db.execute( + sql`SELECT source, external_id, key, title, external_url, metadata FROM tasks`, + ); + expect(task).toEqual({ + source: "linear", + external_id: "issue-1", + key: "ISS-1", + title: "Fix auth", + external_url: "https://linear.app/x/issue/ISS-1", + metadata: { experimentalFlags: ["FOO"], requester }, + }); + const rounds = await db.execute( + sql`SELECT source, kind, external_id, payload, reply_targets FROM inbox`, + ); + expect([...rounds]).toEqual([ + { + source: "linear", + kind: "assignment", + external_id: "issue-1", + payload: { + title: "Fix auth", + description: "desc", + external_url: "https://linear.app/x/issue/ISS-1", + target_id: "sess-1", + task_id: res.taskId, + task_key: "ISS-1", + }, + reply_targets: [session("sess-1")], + }, + ]); + // task_events is API-only. + expect(await db.execute(sql`SELECT 1 FROM task_events`)).toHaveLength(0); + }); + + it("a hint another task holds, in any case, gets the source prefix", async () => { + await insertTask({ key: "iss-1" }); + expect((await assignLinear(session("sess-1"))).key).toBe("LINEAR-ISS-1"); + }); + + it("makes the hint branch-safe", async () => { + expect((await assignLinear(session("sess-1"), { key: { hint: "ISS 1/x" } })).key).toBe( + "ISS-1-x", + ); + }); + + it("retries once when a concurrent insert takes the key", async () => { + // Another transaction holds an uncommitted task with the key: the check + // doesn't see it, the insert waits on the index, then fails with 23505 + // once it commits, and the retry picks the prefixed key. + const other = new SQL(TEST_DATABASE_URL); + try { + let release!: () => void; + const gate = new Promise((r) => { + release = r; + }); + let inserted!: () => void; + const ready = new Promise((r) => { + inserted = r; + }); + const holder = other.begin(async (tx) => { + await tx`INSERT INTO tasks (source, external_id, key, title, external_url) + VALUES ('api', 'other', 'ISS-1', 't', 'u')`; + inserted(); + await gate; + }); + await ready; + const pending = assignLinear(session("sess-1")); + await new Promise((r) => setTimeout(r, 150)); + release(); + await holder; + expect((await pending).key).toBe("LINEAR-ISS-1"); + } finally { + await other.close(); + } + }); + + it("compares delegation order when both targets carry one", async () => { + await assignLinear(session("d:5", 5)); + expect((await assignLinear(session("d:3", 3))).result).toBe("duplicate"); + expect((await assignLinear(session("d:5", 5))).result).toBe("duplicate"); + expect((await assignLinear(session("d:7", 7))).result).toBe("restarted"); + const [round] = await db.execute<{ reply_targets: ReplyTarget[] }>( + sql`SELECT reply_targets FROM inbox`, + ); + expect(round?.reply_targets).toEqual([session("d:7", 7)]); + }); + + it("rejects an order that isn't a non-negative integer; a valid order still dedupes", async () => { + await assignLinear(session("d:5", 5)); + for (const bad of [Number.NaN, -1, 1.5]) { + await expect(assignLinear(session("d:5", bad))).rejects.toThrow( + "target.order must be a non-negative integer", + ); + } + expect((await assignLinear(session("d:5", 5))).result).toBe("duplicate"); + const [round] = await db.execute<{ reply_targets: ReplyTarget[] }>( + sql`SELECT reply_targets FROM inbox`, + ); + expect(round?.reply_targets).toEqual([session("d:5", 5)]); + }); + + it("a duplicate kills nothing", async () => { + const { taskId } = await assignLinear(session("sess-1")); + await db.execute(sql`UPDATE tasks SET sandbox_id = 'sb-1' WHERE id = ${taskId}`); + expect((await assignLinear(session("sess-1"))).result).toBe("duplicate"); + expect(killSandbox).not.toHaveBeenCalled(); + expect(detach).not.toHaveBeenCalled(); + }); + + // Moved from agentSession.service.test ("kills the preempted sandbox when a + // re-delegation restarts a live run"): intake now does the kill. + it("kills the preempted sandbox when a re-delegation restarts a live run", async () => { + const ann = { id: "u-1", displayName: "Ann", codeHost: "unlinked" as const }; + const { taskId } = await assignLinear(session("sess-1"), { flags: ["OLD"], requester: ann }); + await db.execute( + sql`UPDATE tasks SET sandbox_id = 'sb-old', pvc_name = 'pvc-old' WHERE id = ${taskId}`, + ); + + const res = await assignLinear(session("sess-2"), { + title: "Fix auth v2", + url: () => "https://linear.app/x/issue/ISS-1/v2", + flags: ["NEW"], + }); + + expect(res).toEqual({ + taskId, + key: "ISS-1", + result: "restarted", + previousTargetId: "sess-1", + }); + expect(killSandbox).toHaveBeenCalledTimes(1); + expect(killSandbox).toHaveBeenCalledWith("sb-old"); + // In the tracked detached set, so shutdown drains it. + expect(detach).toHaveBeenCalledWith(taskId, "killSandbox", expect.any(Promise)); + const [task] = await db.execute( + sql`SELECT title, external_url, metadata, sandbox_id, pvc_name FROM tasks`, + ); + // The old workspace volume is only unlinked, not deleted (a known leak). + expect(task).toEqual({ + title: "Fix auth v2", + external_url: "https://linear.app/x/issue/ISS-1/v2", + metadata: { experimentalFlags: ["NEW"] }, + sandbox_id: null, + pvc_name: null, + }); + }); + + it("concurrent deliveries of one re-delegation restart the task once", async () => { + // The task row lock serializes them: the first restarts, the rest read + // the new target after it commits. Another transaction holds the + // assignment row, so the first restart waits while the rest pile up. + const { taskId } = await assignLinear(session("S0")); + await db.execute(sql`UPDATE tasks SET sandbox_id = 'sb-old' WHERE id = ${taskId}`); + const other = new SQL(TEST_DATABASE_URL); + try { + let release!: () => void; + const gate = new Promise((r) => { + release = r; + }); + let locked!: () => void; + const ready = new Promise((r) => { + locked = r; + }); + const holder = other.begin(async (tx) => { + await tx`SELECT 1 FROM inbox WHERE task_id = ${taskId} FOR UPDATE`; + locked(); + await gate; + }); + await ready; + const pending = Promise.all(Array.from({ length: 6 }, () => assignLinear(session("S1")))); + await new Promise((r) => setTimeout(r, 150)); + release(); + await holder; + const results = (await pending).map((r) => r.result).sort(); + expect(results).toEqual([...Array(5).fill("duplicate"), "restarted"]); + } finally { + await other.close(); + } + expect(killSandbox).toHaveBeenCalledTimes(1); + expect(killSandbox).toHaveBeenCalledWith("sb-old"); + }); + + it("rejects a target of another integration, and a hint without an external id", async () => { + await expect(assignLinear({ id: "t", spec: { kind: "api_session" } })).rejects.toThrow( + "a linear round can't carry a api_session reply target", + ); + await expect(assignLinear(session("sess-1"), { externalId: undefined })).rejects.toThrow( + "a hint key needs externalId", + ); + expect(await db.execute(sql`SELECT 1 FROM tasks`)).toHaveLength(0); + }); + }); + + describe("followUp", () => { + it("inserts the round with the task's identity; a redelivery is a duplicate", async () => { + const task = await insertTask({ key: "DBX-API-1", prs: [PR] }); + const first = await intake.followUp({ + task, + message: ghComment("555"), + targets: [ghTarget("555")], + prUrl: PR, + }); + const [row] = await db.execute<{ id: string; payload: unknown }>( + sql`SELECT id, payload FROM inbox`, + ); + expect(first).toEqual({ roundId: (row as { id: string }).id }); + expect(row?.payload).toEqual({ + ...ghComment("555").payload, + task_id: task.id, + task_key: "DBX-API-1", + }); + expect( + await intake.followUp({ task, message: ghComment("555"), targets: [ghTarget("555")] }), + ).toEqual({ dropped: "duplicate" }); + // A GitHub round on an API task writes nothing to the log. + expect(await db.execute(sql`SELECT 1 FROM task_events`)).toHaveLength(0); + }); + + it("a review whose id equals a stored comment id is a duplicate while the old index exists", async () => { + // characterization: changes once the old (source, external_id) index is dropped + const task = await insertTask({ key: "DBX-API-1" }); + await intake.followUp({ task, message: ghComment("555"), targets: [ghTarget("555")] }); + const review = await intake.followUp({ + task, + message: { + source: "github", + kind: "review", + external_id: "555", + payload: { + review_body: { target_id: "555", body: "@bot" }, + inline_comments: [], + pr: { repo: "r", pr_url: PR }, + created_at: "", + }, + }, + targets: [ + { + id: "555", + spec: { kind: "review_body", owner: "o", repo: "r", prNumber: 7, reviewId: 555 }, + }, + ], + }); + expect(review).toEqual({ dropped: "duplicate" }); + }); + + it("drops a round whose PR is no longer on the task", async () => { + const task = await insertTask({ key: "DBX-API-1" }); + expect( + await intake.followUp({ + task, + message: ghComment("555"), + targets: [ghTarget("555")], + prUrl: PR, + }), + ).toEqual({ dropped: "stale" }); + expect(await db.execute(sql`SELECT 1 FROM inbox`)).toHaveLength(0); + }); + + it("writes message_queued for an inserted API message only", async () => { + const task = await insertTask({ key: "DBX-API-1" }); + const message: InboxMessageDraft = { + source: "api", + kind: "comment", + external_id: "m-1", + payload: { target_id: task.id, body: "more", pr: null, created_at: "" }, + }; + const targets: ReplyTarget[] = [{ id: task.id, spec: { kind: "api_session" } }]; + const first = await intake.followUp({ task, message, targets }); + expect(await intake.followUp({ task, message, targets })).toEqual({ dropped: "duplicate" }); + const events = await db.execute(sql`SELECT type, inbox_id, data FROM task_events`); + expect([...events]).toEqual([ + { + type: "message_queued", + inbox_id: (first as { roundId: string }).roundId, + data: { body: "more" }, + }, + ]); + }); + + // Moved from inbox.service.test ("enqueues when under the loop-breaker + // threshold" / "drops the CI failure once the loop-breaker threshold is + // reached"): the count now runs in the follow-up transaction. + it("enqueues when under the loop-breaker threshold", async () => { + const task = await insertTask({ key: "DBX-API-1" }); + for (let run = 1; run <= 19; run++) { + await intake.followUp({ task, message: ciResult(run), targets: [ghTarget(String(run))] }); + } + const res = await intake.followUp({ task, message: ciResult(20), targets: [ghTarget("20")] }); + expect(res).toEqual({ roundId: expect.any(String) }); + }); + + it("drops the CI failure once the loop-breaker threshold is reached", async () => { + const task = await insertTask({ key: "DBX-API-1" }); + for (let run = 1; run <= 20; run++) { + await intake.followUp({ task, message: ciResult(run), targets: [ghTarget(String(run))] }); + } + const res = await intake.followUp({ task, message: ciResult(21), targets: [ghTarget("21")] }); + expect(res).toEqual({ dropped: "ci_limit" }); + const [n] = await db.execute(sql`SELECT count(*)::int AS n FROM inbox`); + expect(n).toEqual({ n: 20 }); + }); + + it("checks payload.target_id against targets[0] (reviews exempt) and target kinds", async () => { + const task = await insertTask({ key: "DBX-API-1" }); + await expect( + intake.followUp({ task, message: ghComment("555"), targets: [ghTarget("556")] }), + ).rejects.toThrow("payload.target_id must equal targets[0].id"); + await expect( + intake.followUp({ task, message: ghComment("555"), targets: [session("555")] }), + ).rejects.toThrow("a github round can't carry a linear_session reply target"); + // A review's targets[0] is its first inline comment; it has no target_id. + const review = await intake.followUp({ + task, + message: { + source: "github", + kind: "review", + external_id: "88", + payload: { + review_body: { target_id: "88", body: "@bot" }, + inline_comments: [{ target_id: "11", path: "a.ts", line: 3, body: "nit" }], + pr: { repo: "r", pr_url: PR }, + created_at: "", + }, + }, + targets: [ghTarget("11")], + }); + expect(review).toEqual({ roundId: expect.any(String) }); + }); + }); + + it("restart is API-only", async () => { + const task = await insertTask({ source: "linear", externalId: "issue-1", key: "ISS-1" }); + const def = { title: "T2", description: null, flags: [] }; + await expect(intake.restart(task.id, def)).rejects.toThrow("is not an API task"); + const [row] = await db.execute(sql`SELECT title FROM tasks`); + expect(row).toEqual({ title: "t" }); + }); + + it("findTask matches on source and external id", async () => { + const t = await insertTask({ source: "linear", externalId: "issue-9", key: "ISS-9" }); + expect((await intake.findTask("linear", "issue-9"))?.id).toBe(t.id); + expect(await intake.findTask("api", "issue-9")).toBeNull(); + }); +}); diff --git a/control-plane/src/integrations/api/index.ts b/control-plane/src/integrations/api/index.ts new file mode 100644 index 0000000..bfeef3a --- /dev/null +++ b/control-plane/src/integrations/api/index.ts @@ -0,0 +1,21 @@ +import type { Integration } from "../types"; + +export type ApiStatus = { enabled: boolean }; + +// API tasks have no outbound channel: the task_events log carries replies and +// failures (failTaskInbox / markFailed write round_failed there). No webhook: +// the /api controller and ApiTaskService call TaskIntake. +export class ApiIntegration implements Integration { + readonly name = "api"; + + // API_ENABLED. + constructor(readonly enabled: boolean) {} + + async deliver(): Promise { + // Nothing to post. + } + + async status(): Promise { + return { enabled: this.enabled }; + } +} diff --git a/control-plane/src/integrations/github/index.ts b/control-plane/src/integrations/github/index.ts new file mode 100644 index 0000000..059185c --- /dev/null +++ b/control-plane/src/integrations/github/index.ts @@ -0,0 +1,254 @@ +import type { Context } from "hono"; +import { + addIssueCommentReaction, + addReviewCommentReaction, + parsePrUrl, + postIssueComment, + postReviewCommentReply, + requestReviewers, +} from "../../clients/github"; +import { config } from "../../config"; +import type { GitHubWebhookController } from "../../controllers/githubWebhook.controller"; +import type { Task } from "../../db/schema"; +import type { InboxRow, ReplyTarget, ReplyTargetSpec } from "../../repositories/inbox.repository"; +import type { GithubStatus, GithubStatusService } from "../../services/githubStatus.service"; +import { WORKSPACE_MISSING_MESSAGE } from "../../services/inbox.service"; +import { retryable } from "../http"; +import type { Outbound, SendCtx } from "../outbound"; +import type { HookCtx, Integration } from "../types"; + +type GithubSpec = Extract< + ReplyTargetSpec, + { kind: "issue_comment" | "review_comment" | "review_body" } +>; + +// GitHub rounds (comments, reviews, CI results) post on the PR. A PR shows no +// live progress, and it is public: a failure's raw detail never goes there. +// GitHub creates no tasks: its rounds come only from its webhooks, so it is +// enabled exactly when they are, and disabling them skips open rounds' replies. +export class GithubIntegration implements Integration { + readonly name = "github"; + // Today's webhook controller. It enqueues through InboxService, which needs + // this registry, so app.ts attaches it afterwards (only when enabled). + controller?: GitHubWebhookController; + + constructor( + // GITHUB_WEBHOOKS_ENABLED. + readonly enabled: boolean, + private statusService: Pick, + ) {} + + // Today's responses: 403 for untrusted actors, 503 when the trusted set + // can't be fetched. The controller already holds the intake. + async webhook(c: Context): Promise { + if (!this.controller) return c.notFound(); + return this.controller.handleWebhook(c); + } + + async status(): Promise { + return this.statusService.status(); + } + + async deliver(ctx: SendCtx, target: ReplyTarget, message: Outbound): Promise { + // Its own three kinds only (post() switches over them exhaustively). + const spec = target.spec; + if ( + spec.kind !== "issue_comment" && + spec.kind !== "review_comment" && + spec.kind !== "review_body" + ) { + throw new Error(`github cannot deliver to a ${spec.kind} target`); + } + switch (message.type) { + case "reply": + await post(ctx, spec, message.body); + return; + case "progress": + return; + case "failure": + await deliverFailure(ctx, spec, message); + return; + default: { + const _exhaustive: never = message; + void _exhaustive; + } + } + } + + // A finished review re-requests review from its reviewer, once (a replayed + // `completed` doesn't). Best-effort: a failure is logged, nothing waits on it. + async onRoundFinished( + ctx: HookCtx & { firstTransition: boolean }, + task: Task, + round: InboxRow, + ): Promise { + if (ctx.firstTransition && round.kind === "review") { + await requestReviewFromOriginalReviewer(ctx, round, task); + } + } +} + +async function requestReviewFromOriginalReviewer( + ctx: HookCtx, + inboxRow: Extract, + task: Task, +): Promise { + const payload = inboxRow.payload as Record | null | undefined; + const reviewerLoginRaw = payload?.reviewer_login; + const reviewerLogin = typeof reviewerLoginRaw === "string" ? reviewerLoginRaw.trim() : ""; + if (!reviewerLogin) { + ctx.log.info("review re-request skipped", { + event: "review_rerequest.result", + status: "skipped", + reason: "missing_reviewer_login", + taskId: task.id, + inboxId: inboxRow.id, + }); + return; + } + + const pr = reviewPrCoordinates(inboxRow); + if (!pr) { + ctx.log.warn("review re-request skipped", { + event: "review_rerequest.result", + status: "skipped", + reason: "missing_pr_coordinates", + taskId: task.id, + inboxId: inboxRow.id, + reviewerLogin, + }); + return; + } + + try { + await requestReviewers(pr.owner, pr.repo, pr.prNumber, [reviewerLogin], ctx.signal); + } catch (err) { + const e = err instanceof Error ? err : new Error(String(err)); + ctx.log.error("request reviewers failed", { + taskId: task.id, + inboxId: inboxRow.id, + owner: pr.owner, + repo: pr.repo, + prNumber: pr.prNumber, + reviewerLogin, + message: e.message, + stack: e.stack, + }); + } +} + +function reviewPrCoordinates( + inboxRow: Extract, +): { owner: string; repo: string; prNumber: number } | null { + const pr = inboxRow.payload.pr as Record | undefined; + const owner = typeof pr?.owner === "string" ? pr.owner : ""; + const repo = typeof pr?.repo === "string" ? pr.repo : ""; + const prNumber = typeof pr?.prNumber === "number" ? pr.prNumber : Number.NaN; + if (owner && repo && Number.isFinite(prNumber)) { + return { owner, repo, prNumber }; + } + + const target = inboxRow.reply_targets.find( + (t) => t.spec.kind === "review_body" || t.spec.kind === "review_comment", + ); + if (target && "owner" in target.spec) { + return { owner: target.spec.owner, repo: target.spec.repo, prNumber: target.spec.prNumber }; + } + + const prUrl = typeof pr?.pr_url === "string" ? pr.pr_url : ""; + return prUrl ? parsePrUrl(prUrl) : null; +} + +async function deliverFailure( + ctx: SendCtx, + spec: GithubSpec, + message: Extract, +): Promise { + const { round } = message; + switch (message.cause) { + case "task_failed": + // failTask's message is control-plane-generated and scrubbed: public-safe. + if (message.summary) await post(ctx, spec, message.summary); + return; + case "round_failed": { + // A failed CI-fix round stays silent. + if (round.kind === "ci_result") return; + // The retry hint is an @-mention, which only works with webhooks on. + const botLogin = config.github.webhooks?.botLogin; + if (!botLogin) return; + await post( + ctx, + spec, + `⚠️ Something went wrong while working on this. Mention @${botLogin} in a new comment to retry.`, + ); + return; + } + case "workspace_missing": + await workspaceMissing(ctx, spec, round); + return; + } +} + +// One notice per round. A comment gets a reply plus 😕. A review gets only a +// top-level PR comment for its summary, whatever target core passed (the +// review body has no reply / reactions API). A CI result is informational and +// dead-lettered silently. +async function workspaceMissing(ctx: SendCtx, spec: GithubSpec, round: InboxRow): Promise { + if (round.kind === "review") { + const body = round.reply_targets.find((t) => t.spec.kind === "review_body"); + if (body?.spec.kind === "review_body") await post(ctx, body.spec, WORKSPACE_MISSING_MESSAGE); + return; + } + if (round.kind !== "comment") return; + try { + await post(ctx, spec, WORKSPACE_MISSING_MESSAGE); + } finally { + // The reaction is independent of the reply: try it even if the reply failed. + await confusedReaction(ctx, spec).catch((err: unknown) => { + const e = err instanceof Error ? err : new Error(String(err)); + ctx.log.error("workspace_missing reaction failed", { message: e.message, stack: e.stack }); + }); + } +} + +async function confusedReaction(ctx: SendCtx, spec: GithubSpec): Promise { + if (spec.kind === "issue_comment") { + await addIssueCommentReaction(spec.owner, spec.repo, spec.commentId, "confused", ctx.signal); + } else if (spec.kind === "review_comment") { + await addReviewCommentReaction(spec.owner, spec.repo, spec.commentId, "confused", ctx.signal); + } +} + +// Posts `body` verbatim to one PR thread, by target kind. CP never formats it. +// It is the first request of every GitHub send (the 😕 reaction comes after and +// never fails the send), so a retryable failure here means nothing was posted. +async function post(ctx: SendCtx, spec: GithubSpec, body: string): Promise { + const { signal } = ctx; + try { + switch (spec.kind) { + case "issue_comment": + await postIssueComment(spec.owner, spec.repo, spec.prNumber, body, signal); + return; + case "review_comment": + await postReviewCommentReply( + spec.owner, + spec.repo, + spec.prNumber, + spec.inReplyTo, + body, + signal, + ); + return; + case "review_body": + // Top-level PR comment — GH has no reply-to-review-body API. + await postIssueComment(spec.owner, spec.repo, spec.prNumber, body, signal); + return; + default: { + const _exhaustive: never = spec; + void _exhaustive; + } + } + } catch (err) { + throw retryable(err) ?? err; + } +} diff --git a/control-plane/src/integrations/http.ts b/control-plane/src/integrations/http.ts new file mode 100644 index 0000000..dc54079 --- /dev/null +++ b/control-plane/src/integrations/http.ts @@ -0,0 +1,136 @@ +import { logger } from "../logger"; +import type { InboxRow, ReplyTarget } from "../repositories/inbox.repository"; +import { type Outbound, RetryableError } from "./outbound"; +import type { Integration, IntegrationName } from "./types"; + +// Shared rules for integrations' outbound HTTP: the typed errors clients +// throw, which failures may be retried, and how long one send may take. + +// Upper bound on one deliver() call. +export const SEND_TIMEOUT_MS = 20_000; + +// A non-2xx answer. The message keeps the client's own wording (tests and logs +// match on it); status, the wait before a retry and whether the platform +// rate-limited the call ride along for classification. +export class HttpError extends Error { + constructor( + message: string, + readonly status: number, + readonly retryAfterMs?: number, + // GitHub also answers a rate limit with a 403 (see githubFetch). + readonly rateLimited = status === 429, + ) { + super(message); + } +} + +// How long a rate limit without Retry-After waits: until the platform's reset +// time (epoch ms), and at least a minute. GitHub may ban integrations that +// retry sooner. +export function rateLimitWaitMs(resetAtMs?: number): number { + const untilReset = + resetAtMs !== undefined && Number.isFinite(resetAtMs) ? resetAtMs - Date.now() : 0; + return Math.max(untilReset, 60_000); +} + +// The client had no credential to send with, so nothing was sent. +export class TokenFetchError extends Error {} + +// What bun 1.4.4's fetch throws when the request never reached the server: a +// TypeError whose `code` names the cause (shapes pinned in http.test.ts). A +// reset (ECONNRESET) can come after the request went out, so it is not here. +const NOTHING_SENT = new Set([ + "ECONNREFUSED", // connection refused + "ENOTFOUND", // DNS: no such host + "EAI_AGAIN", // DNS: resolver unavailable (bun ships Node's code; not reproducible offline) + "EPROTO", // TLS handshake failed + // TLS: the server's certificate failed verification during the handshake + "DEPTH_ZERO_SELF_SIGNED_CERT", + "SELF_SIGNED_CERT_IN_CHAIN", + "UNABLE_TO_VERIFY_LEAF_SIGNATURE", + "UNABLE_TO_GET_ISSUER_CERT_LOCALLY", + "CERT_HAS_EXPIRED", + "ERR_TLS_CERT_ALTNAME_INVALID", +]); + +// The failure as a RetryableError when it is one of the exact retryable +// cases, where nothing was sent yet: a rate limit (429), connection refused, +// DNS, TLS handshake, token fetch. Anything else (timeouts, 5xx, other 4xx) +// is undefined: the caller logs and drops it. +export function retryable(err: unknown): RetryableError | undefined { + if (err instanceof RetryableError) return err; + if (err instanceof TokenFetchError) return new RetryableError(err.message); + if (err instanceof HttpError) { + return err.rateLimited ? new RetryableError(err.message, err.retryAfterMs) : undefined; + } + const code = (err as { code?: unknown } | null)?.code; + if (err instanceof TypeError && typeof code === "string" && NOTHING_SENT.has(code)) { + return new RetryableError(err.message); + } + return undefined; +} + +// Retry-After as delay-seconds or an HTTP date; undefined when absent or unreadable. +export function parseRetryAfter(value: string | null): number | undefined { + if (!value) return undefined; + const seconds = Number(value); + if (Number.isFinite(seconds) && seconds >= 0) return seconds * 1000; + const at = Date.parse(value); + return Number.isNaN(at) ? undefined : Math.max(0, at - Date.now()); +} + +// Runs `send` with an abort signal and stops waiting after `ms`, whether or not +// the client honours the signal. The send may still land after the timeout, so +// a timeout is a plain failure (dropped), never a retry. +export async function withTimeout( + ms: number, + send: (signal: AbortSignal) => Promise, +): Promise { + const ac = new AbortController(); + let timer: ReturnType | undefined; + const expired = new Promise((_, reject) => { + timer = setTimeout(() => { + const err = new Error(`timed out after ${ms} ms`); + ac.abort(err); + reject(err); + }, ms); + }); + try { + return await Promise.race([send(ac.signal), expired]); + } finally { + clearTimeout(timer); + } +} + +// Hands `message` for one of `round`'s targets to the round's integration, +// under the send timeout. An integration this build doesn't know (rows +// written by a newer build before a rollback) or a disabled one is skipped and +// logged: that counts as sent. Any other failure is logged and the message +// dropped, except a RetryableError (nothing was sent), which is rethrown: each +// caller decides whether it holds or drops. +export async function deliverToRound( + integrations: Record, + round: InboxRow, + target: ReplyTarget, + message: Outbound, + meta: Record, + timeoutMs = SEND_TIMEOUT_MS, +): Promise<"ok" | "dropped"> { + const log = { ...meta, inboxId: round.id, source: round.source, targetId: target.id }; + const integration: Integration | undefined = integrations[round.source]; + if (!integration?.enabled) { + logger.info(`${message.type} skipped: integration unknown or disabled`, log); + return "ok"; + } + try { + await withTimeout(timeoutMs, (signal) => + integration.deliver({ signal, log: logger }, target, message), + ); + return "ok"; + } catch (err) { + if (err instanceof RetryableError) throw err; + const e = err instanceof Error ? err : new Error(String(err)); + logger.error(`deliver ${message.type} failed`, { ...log, message: e.message, stack: e.stack }); + return "dropped"; + } +} diff --git a/control-plane/src/integrations/intake.ts b/control-plane/src/integrations/intake.ts new file mode 100644 index 0000000..f869bc7 --- /dev/null +++ b/control-plane/src/integrations/intake.ts @@ -0,0 +1,179 @@ +import type { Task } from "../db/schema"; +import { logger } from "../logger"; +import { + apiSessionTarget, + type InboxRepository, + type NewInboxMessage, + type ReplyTarget, + type ReplyTargetSpec, + taskIdentity, +} from "../repositories/inbox.repository"; +import type { TaskRepository } from "../repositories/task.repository"; +import type { IntegrationHooks } from "../services/integrationHooks"; +import type { SandboxManager } from "../services/sandboxManager.service"; +import type { InboxMessage, TaskIdentity } from "../types/inboxMessage"; +import type { IntegrationName, Person, TaskSource } from "./types"; + +// The only way modules create work. + +export type TaskRef = { id: string; key: string }; +type Draft = M extends InboxMessage + ? Pick & { payload: Omit } + : never; +export type InboxMessageDraft = Draft>; + +export interface TaskIntake { + assign(p: { + source: TaskSource; + externalId?: string; // required with a hint key; with a sequence key the new key is used + key: { hint: string } | { sequence: "api" }; + title: string; + description: string | null; // Markdown/plain; flags already stripped; bot mention stripped + flags: string[]; // modules parse with parseExperimentalFlags; intake never parses + url: (t: TaskRef) => string; + target: (t: TaskRef) => ReplyTarget; // target.id = delegation id; optional target.order (monotonic number) + requester?: Person; // tasks.metadata.requester + createdBy?: string; // API token id + }): Promise<{ + taskId: string; + key: string; + result: "created" | "restarted" | "duplicate"; + previousTargetId?: string; // a restart's replaced delegation id + }>; + + restart( + taskId: string, + def: { title: string; description: string | null; flags: string[] }, + ): Promise; + + followUp(p: { + task: Pick; + message: InboxMessageDraft; + targets: ReplyTarget[]; + prUrl?: string; + }): Promise<{ roundId: string } | { dropped: "duplicate" | "stale" | "ci_limit" }>; + + findTask(source: TaskSource, externalId: string): Promise; + findTaskByPrUrl(prUrl: string): Promise; +} + +// Which integration each reply-target kind belongs to. Exists only for the +// check that a round's targets belong to the round's own integration. +const TARGET_OWNER: Record = { + linear_session: "linear", + api_session: "api", + issue_comment: "github", + review_comment: "github", + review_body: "github", +}; + +export class Intake implements TaskIntake { + constructor( + private taskRepo: TaskRepository, + private inboxRepo: InboxRepository, + private sandboxManager: Pick, + private hooks: Pick, + private opts: { apiKeyPrefix: string; ciResultMaxPerTask: number }, + ) {} + + async assign(p: Parameters[0]): ReturnType { + if ("hint" in p.key && !p.externalId) throw new Error("assign: a hint key needs externalId"); + const { prevSandboxId, prevTargetId, ...t } = await this.taskRepo.assign({ + ...p, + key: + "hint" in p.key + ? { hint: p.key.hint.replace(/[^A-Za-z0-9-]/g, "-") } // branch-safe + : { sequencePrefix: this.opts.apiKeyPrefix }, + target: (ref) => { + const target = p.target(ref); + checkTargets(p.source, [target]); + // A NaN order is never "not above" the stored one, so every + // redelivery would restart the task. + const { order } = target; + if (order !== undefined && !(Number.isSafeInteger(order) && order >= 0)) { + throw new Error(`assign: target.order must be a non-negative integer, not ${order}`); + } + return target; + }, + }); + this.killPreempted(t.id, prevSandboxId, "preempted re-delegation"); + return { taskId: t.id, key: t.key, result: t.result, previousTargetId: prevTargetId }; + } + + // API only: today's API restart, with the definition replacing the current + // title, description and flags. + async restart( + taskId: string, + def: { title: string; description: string | null; flags: string[] }, + ): Promise { + const task = await this.taskRepo.getById(taskId); + if (!task) throw new Error(`restart: task ${taskId} not found`); + if (task.source !== "api") throw new Error(`restart: task ${taskId} is not an API task`); + const { prevSandboxId } = await this.taskRepo.restartTaskWithInbox({ + taskId, + inboxPayload: { + title: def.title, + description: def.description, + external_url: task.externalUrl, + target_id: taskId, + }, + inboxReplyTargets: [apiSessionTarget(taskId)], + api: { title: def.title, description: def.description, experimentalFlags: def.flags }, + }); + this.killPreempted(taskId, prevSandboxId, "API restart"); + } + + async followUp(p: Parameters[0]): ReturnType { + const { task, message, targets } = p; + // A review's payload carries one target id per thread instead. + if ("target_id" in message.payload && message.payload.target_id !== targets[0]?.id) { + throw new Error("followUp: payload.target_id must equal targets[0].id"); + } + checkTargets(message.source, targets); + return this.inboxRepo.appendFollowUp({ + taskId: task.id, + message: { + ...message, + payload: { ...message.payload, ...taskIdentity(task) }, + } as NewInboxMessage, + replyTargets: targets, + prUrl: p.prUrl, + ciResultMax: this.opts.ciResultMaxPerTask, + }); + } + + findTask(source: TaskSource, externalId: string): Promise { + return this.taskRepo.findByExternalId(source, externalId); + } + + findTaskByPrUrl(prUrl: string): Promise { + return this.taskRepo.findByPrUrl(prUrl); + } + + // A restart's preempted sandbox, killed after commit in the tracked detached + // set so shutdown drains it. The old workspace volume is kept (a known leak). + private killPreempted(taskId: string, sandboxId: string | null, why: string): void { + if (!sandboxId) return; + this.hooks.detach( + taskId, + "killSandbox", + this.sandboxManager.killSandbox(sandboxId).catch((err) => { + const e = err instanceof Error ? err : new Error(String(err)); + logger.error(`killSandbox(${why}) failed`, { + taskId, + sandboxId, + message: e.message, + stack: e.stack, + }); + }), + ); + } +} + +function checkTargets(source: IntegrationName, targets: ReplyTarget[]): void { + for (const t of targets) { + if (TARGET_OWNER[t.spec.kind] !== source) { + throw new Error(`a ${source} round can't carry a ${t.spec.kind} reply target`); + } + } +} diff --git a/control-plane/src/integrations/linear/index.ts b/control-plane/src/integrations/linear/index.ts new file mode 100644 index 0000000..f70b3d4 --- /dev/null +++ b/control-plane/src/integrations/linear/index.ts @@ -0,0 +1,188 @@ +import { type LinearClient, LinearError, RatelimitedLinearError } from "@linear/sdk"; +import type { Context } from "hono"; +import { getUserById } from "../../clients/github"; +import { + attachUrlToIssue, + emitAction, + emitError, + emitResponse, + emitThought, + getIssueHumanAssignee, + setIssueStarted, + uploadIssueAttachment, +} from "../../clients/linear"; +import type { LinearWebhookController } from "../../controllers/linearWebhook.controller"; +import type { Task } from "../../db/schema"; +import type { InboxRow, ReplyTarget } from "../../repositories/inbox.repository"; +import { WORKSPACE_MISSING_MESSAGE } from "../../services/inbox.service"; +import type { LinearStatus, LinearStatusService } from "../../services/linearStatus.service"; +import type { LinearTokenProvider } from "../../services/linearTokenProvider"; +import { rateLimitWaitMs, retryable } from "../http"; +import { type Outbound, RetryableError, type SendCtx } from "../outbound"; +import type { HookCtx, Integration, Person, PrRef } from "../types"; + +// Linear rounds post to the agent session. target.id IS the agent session id +// by construction (the enqueue paths stamp the live session id). The session +// is private to the workspace, so a failure shows its raw detail. +export class LinearIntegration implements Integration { + readonly name = "linear"; + // Today's webhook controller. It enqueues through InboxService, which needs + // this registry, so app.ts attaches it afterwards (only when enabled). + controller?: Pick; + + constructor( + private linearTokenProvider: Pick, + // LINEAR_ENABLED. + readonly enabled: boolean, + private statusService: Pick, + ) {} + + // Today's responses, including 503 session_not_found when a prompt arrives + // before its session (Linear retries). The controller already holds the intake. + async webhook(c: Context): Promise { + if (!this.controller) return c.notFound(); + return this.controller.handleWebhook(c); + } + + status(): Promise { + return this.statusService.status(); + } + + async deliver(ctx: SendCtx, target: ReplyTarget, message: Outbound): Promise { + if (target.spec.kind !== "linear_session") { + throw new Error(`linear cannot deliver to a ${target.spec.kind} target`); + } + const client = await this.client(ctx.signal); + // Each message is one activity (one request), so a retryable failure means + // nothing was posted. + try { + switch (message.type) { + case "reply": + await emitResponse(client, target.id, message.body); + return; + case "progress": + if (message.kind === "thought") await emitThought(client, target.id, message.body); + // The daemon's body is the full action label (e.g. "Planning", + // "Addressing comment"). Linear renders ` ` + // concatenated, so a non-empty parameter appends a noisy suffix + // ("Addressing comment task"). Pass empty parameter — the body + // already carries everything the user should see. + else if (message.kind === "action") await emitAction(client, target.id, message.body, ""); + else await emitError(client, target.id, message.body); + return; + case "failure": { + // task_failed and a failed assignment carry the summary; a crashed + // comment round carries only the detail. + const text = + message.cause === "workspace_missing" + ? WORKSPACE_MISSING_MESSAGE + : (message.summary ?? message.detail); + if (text) await emitError(client, target.id, text); + return; + } + default: { + const _exhaustive: never = message; + void _exhaustive; + } + } + } catch (err) { + throw linearRetryable(err) ?? err; + } + } + + // A fresh provision: moves the issue to its team's started state, then shows + // "Provisioning" on the session. Each is best-effort; a team without a + // started state still gets the action. + async onTaskStarted(ctx: HookCtx, task: Task, target: ReplyTarget): Promise { + const client = await this.client(ctx.signal); + await setIssueStarted(client, task.externalId).catch((err: unknown) => { + const e = err instanceof Error ? err : new Error(String(err)); + ctx.log.error(`setIssueStarted failed for task ${task.id}`, { + message: e.message, + stack: e.stack, + }); + }); + await emitAction(client, target.id, "Provisioning", "infra"); + } + + // Links the PR on the issue. Idempotent (Linear updates an existing link), + // so it runs on every pr_opened dispatch; a RetryableError holds the event. + async onPrOpened(ctx: HookCtx, task: Task, pr: PrRef): Promise { + const client = await this.client(ctx.signal); + try { + await attachUrlToIssue(client, task.externalId, pr.url, pr.repo); + } catch (err) { + throw linearRetryable(err) ?? err; + } + } + + // Uploads the run archive to the issue once, when this call finished the + // assignment (a replayed `completed` doesn't upload again). The sandbox is + // still alive here (it lives until the idle reaper). The archive is a + // forensic nicety: core logs a failure and nothing else depends on it. + async onRoundFinished( + ctx: HookCtx & { firstTransition: boolean; archive(): Promise }, + task: Task, + round: InboxRow, + ): Promise { + if (!ctx.firstTransition || round.kind !== "assignment") return; + const bytes = await ctx.archive(); + if (!bytes || bytes.byteLength === 0) return; + const client = await this.client(ctx.signal); + await uploadIssueAttachment(client, { + issueId: task.externalId, + title: "Harness run archive", + subtitle: `Task ${task.id}`, + filename: `harness-run-${task.id}.tar.gz`, + contentType: "application/gzip", + bytes, + metadata: { taskId: task.id }, + signal: ctx.signal, + }); + } + + // The issue's human assignee. Identity comes from Linear's own user↔GitHub + // mapping (User.gitHubUserId, set when a member connects GitHub under Linear + // Settings → Connected accounts), never email matching. + async assignee(ctx: HookCtx, task: Task): Promise { + const client = await this.client(ctx.signal); + const human = await getIssueHumanAssignee(client, task.externalId); + if (!human) return null; + const person = { id: human.id, displayName: human.displayName, email: human.email }; + if (!human.gitHubUserId) return { ...person, codeHost: "unlinked" }; + // Defensive about the field's format: numeric → GitHub account id + // (resolve to the current login), anything else → already a login. + const githubLogin = /^\d+$/.test(human.gitHubUserId) + ? (await getUserById(human.gitHubUserId, ctx.signal)).login + : human.gitHubUserId; + return { ...person, codeHost: { githubLogin } }; + } + + // Nothing has been sent when the token fetch fails, so the caller may retry. + // A per-call client: the signal bounds this send's requests only. + private async client(signal: AbortSignal): Promise { + try { + return await this.linearTokenProvider.getAuthorizedLinearClient(signal); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + throw new RetryableError(`Linear token fetch failed: ${msg}`); + } + } +} + +// The SDK wraps every failure in a LinearError whose `raw` is what fetch (or +// the token renewal after a 401) threw. A rate limit is the SDK's +// RatelimitedLinearError: an HTTP 429 or a "ratelimited" GraphQL error, with +// Retry-After in seconds. Without it, the wait runs to the later of Linear's +// request and complexity reset times (epoch ms), and at least a minute. +function linearRetryable(err: unknown): RetryableError | undefined { + if (err instanceof RatelimitedLinearError) { + const s = err.retryAfter; + const wait = + s !== undefined && s >= 0 + ? s * 1000 + : rateLimitWaitMs(Math.max(err.requestsResetAt ?? 0, err.complexityResetAt ?? 0)); + return new RetryableError(err.message, wait); + } + return retryable(err instanceof LinearError ? err.raw : err); +} diff --git a/control-plane/src/integrations/outbound.ts b/control-plane/src/integrations/outbound.ts new file mode 100644 index 0000000..47e9d6a --- /dev/null +++ b/control-plane/src/integrations/outbound.ts @@ -0,0 +1,29 @@ +import type { logger } from "../logger"; +import type { InboxRow } from "../repositories/inbox.repository"; + +type Logger = typeof logger; + +// What core hands to deliver(). Core says WHAT a message is; the integration +// decides WHERE (or whether) it shows up. +export type Outbound = + | { type: "reply"; body: string } // the agent's answer + | { type: "progress"; kind: "thought" | "action" | "error"; body: string } + | { + type: "failure"; + cause: "round_failed" | "task_failed" | "workspace_missing"; + round: InboxRow; + summary?: string; // user-safe text (the harness's "what failed" summary) + detail?: string; // raw error text (exception message): never post it on a public channel + }; +export type SendCtx = { signal: AbortSignal; log: Logger }; + +// Throw only before the first request is sent (token fetch failed, 429, +// connection refused, DNS, TLS handshake). +export class RetryableError extends Error { + constructor( + msg: string, + readonly retryAfterMs?: number, + ) { + super(msg); + } +} diff --git a/control-plane/src/integrations/types.ts b/control-plane/src/integrations/types.ts new file mode 100644 index 0000000..d04ffb8 --- /dev/null +++ b/control-plane/src/integrations/types.ts @@ -0,0 +1,68 @@ +import type { Context } from "hono"; +import type { Task, taskSource } from "../db/schema"; +import type { logger } from "../logger"; +import type { InboxRow, ReplyTarget } from "../repositories/inbox.repository"; +import type { ApiTaskSource } from "../types/api"; +import type { InboxSource } from "../types/inboxMessage"; +import type { TaskIntake } from "./intake"; +import type { Outbound, SendCtx } from "./outbound"; + +type Logger = typeof logger; + +// The sources a task can have (the task_source enum). +export type TaskSource = (typeof taskSource.enumValues)[number]; +// One per task/round source (InboxSource). +export type IntegrationName = "linear" | "api" | "github"; + +// Compile-time checks: the literal unions kept where db/schema can't be +// imported (types/api.ts is bundled into the CLI) match what they mirror, so a +// new source fails the build until each is updated. +type Equal = + (() => T extends A ? 1 : 2) extends () => T extends B ? 1 : 2 ? true : false; +type Assert = T; +export type SourceChecks = [ + Assert>, + Assert>, +]; + +export interface Integration { + name: IntegrationName; + // Core never calls webhook, deliver, hooks or assignee of a disabled module: + // it logs and skips (a skipped send counts as sent, never held). + enabled: boolean; + + // Incoming. Core mounts POST /webhook/ only when enabled (404 + // otherwise), behind the shared body limit and raw-body read. + webhook?(c: Context, intake: TaskIntake): Promise; + + // Outgoing. Core labels each message; the integration decides where (or + // whether) it shows up. Throws RetryableError only before anything was sent. + deliver(ctx: SendCtx, target: ReplyTarget, message: Outbound): Promise; + + // Lifecycle hooks. Core runs each through IntegrationHooks (catch, log, + // timeout, abort signal); see services/integrationHooks.ts for where and how. + onTaskStarted?(ctx: HookCtx, task: Task, target: ReplyTarget): Promise; + onPrOpened?(ctx: HookCtx & { firstTime: boolean }, task: Task, pr: PrRef): Promise; + onRoundFinished?( + ctx: HookCtx & { firstTransition: boolean; archive(): Promise }, + task: Task, + round: InboxRow, + outcome: Outcome | null, + ): Promise; + assignee?(ctx: HookCtx, task: Task): Promise; // throws when the lookup itself fails + + // Health. Core serves GET /api/integrations/ (behind API auth, only + // when the API is enabled) with this body plus the integration's held sends. + status(): Promise; +} + +// isCurrent() is false once a restart rotated the task's assignment id. +export type HookCtx = { signal: AbortSignal; log: Logger; isCurrent(): Promise }; +export type Person = { + id: string; // the platform account id: logs carry it, never the name or email + displayName: string; + email?: string; + codeHost: { githubLogin: string } | "unlinked" | "unsupported"; +}; +export type PrRef = { url: string; repo: string; branch: string }; +export type Outcome = "completed" | "partial_failure" | "no_action" | "failed"; diff --git a/control-plane/src/integrationsConfig.ts b/control-plane/src/integrationsConfig.ts index 93d2048..9c2a3bd 100644 --- a/control-plane/src/integrationsConfig.ts +++ b/control-plane/src/integrationsConfig.ts @@ -2,6 +2,9 @@ // testable without mocking ../config. config.ts calls these at startup; a bad // combination fails boot with an error that names what is missing. +import { taskSource } from "./db/schema"; +import type { TaskSource } from "./integrations/types"; + export type LinearConfig = { clientId: string; clientSecret: string; @@ -13,6 +16,8 @@ export type SlackConfig = { botToken: string; // Channel id (C…) the updates are posted to. channel: string; + // The task sources whose tasks post (SLACK_NOTIFY_SOURCES; default linear). + notifySources: TaskSource[]; }; export type GithubWebhooksConfig = { @@ -80,7 +85,8 @@ export function parseLinearConfig(env: Env): LinearConfig | undefined { } // Slack is on only with SLACK_ENABLED=true; then the token and channel are -// required. +// required. SLACK_NOTIFY_SOURCES lists the task sources that post (comma- or +// whitespace-separated, default linear); an unknown source fails startup. export function parseSlackConfig(env: Env): SlackConfig | undefined { if (!parseSwitch(env, "SLACK_ENABLED")) return undefined; const botToken = present(env, "SLACK_BOT_TOKEN"); @@ -89,7 +95,16 @@ export function parseSlackConfig(env: Env): SlackConfig | undefined { if (!botToken) missing.push("SLACK_BOT_TOKEN"); if (!channel) missing.push("SLACK_CHANNEL_ID"); if (!botToken || !channel) throw missingError("SLACK_ENABLED", missing); - return { botToken, channel }; + const sources = parseTrustList(env.SLACK_NOTIFY_SOURCES); + const known: readonly string[] = taskSource.enumValues; + const unknown = sources.find((s) => !known.includes(s)); + if (unknown !== undefined) { + throw new Error( + `SLACK_NOTIFY_SOURCES: unknown task source "${unknown}" (allowed: ${known.join(", ")})`, + ); + } + const notifySources: TaskSource[] = sources.length > 0 ? (sources as TaskSource[]) : ["linear"]; + return { botToken, channel, notifySources }; } export function parseGithubConfig(env: Env): GithubIntegrationConfig { diff --git a/control-plane/src/repositories/__tests__/apiTask.repository.test.ts b/control-plane/src/repositories/__tests__/apiTask.repository.test.ts index 1ad712d..a3ae765 100644 --- a/control-plane/src/repositories/__tests__/apiTask.repository.test.ts +++ b/control-plane/src/repositories/__tests__/apiTask.repository.test.ts @@ -9,7 +9,7 @@ mock.module("../../config", () => ({ config: { databaseUrl: TEST_DATABASE_URL } const { db } = await import("../../db"); const { ApiTaskRepository } = await import("../apiTask.repository"); const { TaskRepository } = await import("../task.repository"); -const { InboxRepository } = await import("../inbox.repository"); +const { InboxRepository, apiSessionTarget } = await import("../inbox.repository"); const { migrate } = await import("drizzle-orm/bun-sql/migrator"); const repo = new ApiTaskRepository(); @@ -17,16 +17,44 @@ const taskRepo = new TaskRepository(); const inboxRepo = new InboxRepository(); const T0 = "2026-10-07T12:00:00.000Z"; -const create = (o: Partial[0]> = {}) => - repo.createApiTask({ - keyPrefix: "DBX-API-", - publicBaseUrl: "https://api.example", +// An API create: assign with a sequence key, as TaskIntake does it. +const create = async (o: { createdBy?: string } = {}) => { + const t = await taskRepo.assign({ + source: "api", + key: { sequencePrefix: "DBX-API-" }, title: "Add a thing", description: "Please", - experimentalFlags: ["FOO"], - createdBy: null, - ...o, + flags: ["FOO"], + url: (ref) => `https://api.example/api/tasks/${ref.key}`, + target: (ref) => apiSessionTarget(ref.id), + createdBy: o.createdBy, }); + return { taskId: t.id, key: t.key }; +}; + +// An API message: a comment round through the follow-up transaction. +const appendMessage = async (p: { task: { id: string; key: string }; body: string }) => { + const result = await inboxRepo.appendFollowUp({ + taskId: p.task.id, + message: { + source: "api", + kind: "comment", + external_id: crypto.randomUUID(), + payload: { + target_id: p.task.id, + body: p.body, + pr: null, + created_at: T0, + task_id: p.task.id, + task_key: p.task.key, + }, + }, + replyTargets: [apiSessionTarget(p.task.id)], + ciResultMax: 0, + }); + if (!("roundId" in result)) throw new Error(`dropped: ${result.dropped}`); + return result; +}; // Pins every DB timestamp of a task to T0 (+ offsets for inbox rows) so whole // records compare exactly. @@ -64,7 +92,7 @@ describe("ApiTaskRepository", () => { await db.execute(sql`ALTER SEQUENCE api_task_key_seq RESTART`); }); - describe("createApiTask", () => { + describe("assign (API create)", () => { it("inserts the task, the assignment row and task_created in one transaction", async () => { const { taskId, key } = await create(); expect(key).toBe("DBX-API-1"); @@ -157,14 +185,10 @@ describe("ApiTaskRepository", () => { }); }); - describe("appendApiMessage", () => { + describe("appendFollowUp (API message)", () => { it("inserts a comment row and message_queued atomically", async () => { const { taskId, key } = await create(); - const { roundId } = await repo.appendApiMessage({ - task: { id: taskId, key }, - body: "more", - createdAt: T0, - }); + const { roundId } = await appendMessage({ task: { id: taskId, key }, body: "more" }); const [row] = await db.execute( sql`SELECT source, kind, payload, reply_targets FROM inbox WHERE id = ${roundId}`, ); @@ -195,9 +219,7 @@ describe("ApiTaskRepository", () => { sql`ALTER TABLE task_events ADD CONSTRAINT test_block CHECK (type <> 'message_queued')`, ); try { - await expect( - repo.appendApiMessage({ task: { id: taskId, key }, body: "x", createdAt: T0 }), - ).rejects.toThrow(); + await expect(appendMessage({ task: { id: taskId, key }, body: "x" })).rejects.toThrow(); const [n] = await db.execute<{ n: number }>( sql`SELECT count(*)::int AS n FROM inbox WHERE kind = 'comment'`, ); @@ -428,11 +450,11 @@ describe("ApiTaskRepository", () => { expect(await logTypes(taskId)).toEqual(["task_created", "agent_action", cpType]); } - it("appendApiMessage waits for a concurrent per-task writer to commit", async () => { + it("an API message waits for a concurrent per-task writer to commit", async () => { const { taskId, key } = await create(); await expectSerialized( taskId, - () => repo.appendApiMessage({ task: { id: taskId, key }, body: "more", createdAt: T0 }), + () => appendMessage({ task: { id: taskId, key }, body: "more" }), "message_queued", ); }); diff --git a/control-plane/src/repositories/__tests__/inbox.repository.test.ts b/control-plane/src/repositories/__tests__/inbox.repository.test.ts index 941a99b..c659f8c 100644 --- a/control-plane/src/repositories/__tests__/inbox.repository.test.ts +++ b/control-plane/src/repositories/__tests__/inbox.repository.test.ts @@ -1,5 +1,5 @@ import { beforeAll, beforeEach, describe, expect, it, mock } from "bun:test"; -import { sql } from "drizzle-orm"; +import { sql, TransactionRollbackError } from "drizzle-orm"; const TEST_DATABASE_URL = process.env.TEST_DATABASE_URL ?? "postgres://devbox:devbox@localhost:5432/devbox"; @@ -116,6 +116,101 @@ describe("InboxRepository", () => { const rows = await db.execute<{ n: string }>(sql`SELECT count(*) AS n FROM inbox`); expect(Number(rows[0]?.n)).toBe(1); }); + + const ghComment = (externalId: string) => ({ + source: "github" as const, + kind: "comment" as const, + external_id: externalId, + payload: { + target_id: externalId, + body: "@bot fix", + task_id: "task-1", + task_key: "T-1", + pr: { repo: "r", pr_url: "https://github.com/o/r/pull/7" }, + created_at: "", + }, + }); + + it("a redelivered comment adds nothing, even after its round finished", async () => { + const task = await insertTask(); + const first = await repo.appendInbox({ + taskId: task.id, + message: ghComment("555"), + replyTargets: [], + }); + await db.execute(sql`UPDATE inbox SET status = 'responded'`); + + expect( + await repo.appendInbox({ taskId: task.id, message: ghComment("555"), replyTargets: [] }), + ).toBeNull(); + const rows = await db.execute<{ id: string; status: string }>( + sql`SELECT id, status FROM inbox`, + ); + expect([...rows]).toEqual([{ id: first?.id as string, status: "responded" }]); + }); + + it("dedupes across kinds: a review whose id equals a stored comment id is dropped", async () => { + // characterization: changes once the old index is dropped. The + // (source, kind, external_id) index allows this row, but the old + // (source, external_id) one still rejects it. + const task = await insertTask(); + await repo.appendInbox({ taskId: task.id, message: ghComment("555"), replyTargets: [] }); + + const review = await repo.appendInbox({ + taskId: task.id, + message: { + source: "github", + kind: "review", + external_id: "555", + payload: { + review_body: { target_id: "555", body: "@bot" }, + inline_comments: [], + task_id: "task-1", + task_key: "T-1", + pr: { repo: "r", pr_url: "https://github.com/o/r/pull/7" }, + created_at: "", + }, + }, + replyTargets: [], + }); + + expect(review).toBeNull(); + const kinds = await db.execute<{ kind: string }>(sql`SELECT kind FROM inbox`); + expect([...kinds]).toEqual([{ kind: "comment" }]); + }); + + it("dedupes on (source, kind, external_id) once the old index is gone", async () => { + // A later release drops inbox_source_external_uniq; here it goes in a + // transaction that is rolled back, so the insert's conflict target + // must be the new index. + const task = await insertTask(); + const review = { + source: "github" as const, + kind: "review" as const, + external_id: "555", + payload: { + review_body: { target_id: "555", body: "@bot" }, + inline_comments: [], + task_id: "task-1", + task_key: "T-1", + pr: { repo: "r", pr_url: "https://github.com/o/r/pull/7" }, + created_at: "", + }, + }; + const results: unknown[] = []; + await expect( + db.transaction(async (tx) => { + await tx.execute(sql`DROP INDEX inbox_source_external_uniq`); + for (const message of [ghComment("555"), ghComment("555"), review]) { + results.push( + await repo.appendInbox({ taskId: task.id, message, replyTargets: [] }, tx), + ); + } + tx.rollback(); + }), + ).rejects.toThrow(TransactionRollbackError); + expect(results).toEqual([{ id: expect.any(String) }, null, { id: expect.any(String) }]); + }); }); describe("listPendingByTask", () => { @@ -240,6 +335,30 @@ describe("InboxRepository", () => { expect(await repo.listPendingByTask(task.id)).toEqual([]); }); + it("returns the rows it failed, whole, in arrival order", async () => { + const task = await insertTask({ ownerId: "pod-a" }); + const ids: string[] = []; + for (const external_id of ["first", "second"]) { + const row = await repo.appendInbox({ + taskId: task.id, + message: { + source: "linear", + kind: "assignment", + external_id, + payload: assignmentPayload, + }, + replyTargets: [linearTarget(`s-${external_id}`)], + }); + if (!row) throw new Error("insert failed"); + ids.push(row.id); + } + + const failed = await repo.failTaskInbox({ taskId: task.id, ownerId: "pod-a", reason: "x" }); + + expect(failed as unknown[]).toEqual(await Promise.all(ids.map((id) => repo.getById(id)))); + expect(failed[0]?.reply_targets).toEqual([linearTarget("s-first")]); + }); + it("does not touch an already-responded row", async () => { const task = await insertTask({ ownerId: "pod-a" }); const row = await repo.appendInbox({ @@ -303,6 +422,22 @@ describe("InboxRepository", () => { }); }); + describe("getAssignment", () => { + it("returns the task's assignment row, never another round or another task's", async () => { + const task = await insertTask(); + const other = await insertTask(); + await insertRow(task.id, { source: "github", kind: "comment", status: "pending" }); + const assignment = await insertRow(task.id, { + source: "linear", + kind: "assignment", + status: "delivered", + }); + + expect((await repo.getAssignment(task.id))?.id).toBe(assignment.id); + expect(await repo.getAssignment(other.id)).toBeNull(); + }); + }); + describe("markResponded", () => { it("owner match → status 'responded' + 1 row; owner mismatch / re-fire → 0 rows", async () => { const task = await insertTask({ ownerId: "pod-a" }); diff --git a/control-plane/src/repositories/__tests__/task.repository.test.ts b/control-plane/src/repositories/__tests__/task.repository.test.ts index 2a1f8ed..c79e44e 100644 --- a/control-plane/src/repositories/__tests__/task.repository.test.ts +++ b/control-plane/src/repositories/__tests__/task.repository.test.ts @@ -16,6 +16,9 @@ const { migrate } = await import("drizzle-orm/bun-sql/migrator"); const repo = new TaskRepository(); +// Every task source this build knows; claimNext and listInFlightOwnedBy filter on it. +const ALL_SOURCES: ("linear" | "api")[] = ["linear", "api"]; + const linearTarget = (targetId: string) => ({ id: targetId, spec: { kind: "linear_session" as const }, @@ -36,6 +39,22 @@ const assignmentContent = { target_id: "agent-session-1", }; +// A Linear `created` through assign: issue-1 / ISS-1. The delegation (agent +// session) defaults to assignmentContent's target_id. +const assignLinear = ( + o: { session?: string; title?: string; url?: string; key?: string; flags?: string[] } = {}, +) => + repo.assign({ + source: "linear", + externalId: "issue-1", + key: { hint: o.key ?? "ISS-1" }, + title: o.title ?? "Fix auth", + description: "desc", + flags: o.flags ?? [], + url: () => o.url ?? "https://linear.app/x/issue/ISS-1", + target: () => linearTarget(o.session ?? "agent-session-1"), + }); + // Inserts a bare tasks row directly. Returns the synthetic id. async function insertTask(overrides: Record = {}): Promise<{ id: string }> { const [row] = await db @@ -99,19 +118,11 @@ describe("TaskRepository", () => { await db.execute(sql`ALTER SEQUENCE api_task_key_seq RESTART`); }); - describe("upsertFromAgentSessionWithInbox", () => { - it("inserts task + inbox row in one tx, created=true on fresh insert", async () => { - const result = await repo.upsertFromAgentSessionWithInbox({ - taskKey: "ISS-1", - taskExternalId: "issue-1", - taskTitle: "Fix auth", - taskExternalUrl: "https://linear.app/x/issue/ISS-1", - inboxPayload: assignmentContent, - inboxReplyTargets: [linearTarget("sess-1")], - metadata: { experimentalFlags: ["FAST_PATH"] }, - }); + describe("assign (Linear created)", () => { + it("inserts task + inbox row in one tx; a fresh insert is created", async () => { + const result = await assignLinear({ session: "sess-1", flags: ["FAST_PATH"] }); - expect(result.created).toBe(true); + expect(result.result).toBe("created"); const tasks = await db.select().from(schema.tasks); expect(tasks).toHaveLength(1); @@ -127,72 +138,43 @@ describe("TaskRepository", () => { }); it("sets tasks.key on insert and never changes it on conflict", async () => { - const first = await repo.upsertFromAgentSessionWithInbox({ - taskExternalId: "issue-1", - taskKey: "ISS-1", - taskTitle: "Fix auth", - taskExternalUrl: "https://linear.app/x/issue/ISS-1", - inboxPayload: assignmentContent, - inboxReplyTargets: [linearTarget("sess-1")], - metadata: {}, - }); + const first = await assignLinear({ session: "sess-1" }); // The issue moved team: new URL, new identifier. The stored key stays. - await repo.upsertFromAgentSessionWithInbox({ - taskExternalId: "issue-1", - taskKey: "XYZ-9", - taskTitle: "Fix auth", - taskExternalUrl: "https://linear.app/x/issue/XYZ-9", - inboxPayload: assignmentContent, - inboxReplyTargets: [linearTarget("sess-2")], - metadata: {}, + await assignLinear({ + session: "sess-2", + key: "XYZ-9", + url: "https://linear.app/x/issue/XYZ-9", }); - const row = await repo.getById(first.taskId); + const row = await repo.getById(first.id); expect({ key: row?.key, createdBy: row?.createdBy }).toEqual({ key: "ISS-1", createdBy: null, }); }); - it("created=false on conflict; inbox INSERT is a no-op; existing reply_targets persist", async () => { - await repo.upsertFromAgentSessionWithInbox({ - taskKey: "ISS-1", - taskExternalId: "issue-1", - taskTitle: "Fix auth", - taskExternalUrl: "https://linear.app/x/issue/ISS-1", - inboxPayload: assignmentContent, - inboxReplyTargets: [linearTarget("sess-1")], - metadata: {}, - }); + it("a redelivery is a duplicate; inbox INSERT is a no-op; existing reply_targets persist", async () => { + await assignLinear({ session: "sess-1" }); // Seed a durable retry budget on the existing row. A duplicate `created` - // redelivery passes metadata={}; the upsert MERGES (metadata || EXCLUDED), - // so the budget must be PRESERVED — a blind overwrite would silently reset + // redelivery must PRESERVE it — a blind overwrite would silently reset // the sandbox-retry cap. await db.execute(sql` UPDATE tasks SET metadata = '{"retryCount":2}'::jsonb WHERE external_id = 'issue-1'; `); - const result = await repo.upsertFromAgentSessionWithInbox({ - taskKey: "ISS-1", - taskExternalId: "issue-1", - taskTitle: "Fix auth v2", - taskExternalUrl: "https://linear.app/x/issue/ISS-1", - inboxPayload: assignmentContent, - inboxReplyTargets: [linearTarget("sess-2")], - metadata: {}, - }); + // The same delegation (session) redelivered. + const result = await assignLinear({ session: "sess-1", title: "Fix auth v2" }); - expect(result.created).toBe(false); + expect(result.result).toBe("duplicate"); const [task] = await db.select().from(schema.tasks); - expect(task?.title).toBe("Fix auth v2"); - // retryCount survives the duplicate-redelivery upsert (merge, not replace). + // A duplicate changes nothing, not even the title. + expect(task?.title).toBe("Fix auth"); + // retryCount survives the duplicate redelivery. expect((task?.metadata as { retryCount?: number })?.retryCount).toBe(2); - // Still exactly one inbox row — INSERT was DO NOTHING on conflict, so - // the existing reply_targets carrying "sess-1" are preserved (the - // caller reads this via inboxRepo.getAssignmentReplyTargetId for - // duplicate-webhook detection). + // Still exactly one inbox row, and the existing reply_targets carrying + // "sess-1" are preserved (assign compares them for duplicate detection). const rows = await db.execute<{ id: string; reply_targets: Array<{ id: string }>; @@ -200,19 +182,88 @@ describe("TaskRepository", () => { expect(rows).toHaveLength(1); expect(rows[0]?.reply_targets[0]?.id).toBe("sess-1"); }); + + it("a duplicate created (same session) changes nothing: not the title, URL or flags, nor the run", async () => { + // It used to refresh title, URL and flags. + const { id: taskId } = await assignLinear({ session: "sess-1", flags: ["OLD"] }); + await db.execute(sql` + UPDATE tasks + SET owner_id = 'pod-a', heartbeat_at = now(), sandbox_id = 'sb-1', pvc_name = 'pvc-1', + last_observed_event_seq = 7, + pr_urls = '[{"repo":"r","url":"https://github.com/o/r/pull/1","branch":"b","commit_sha":"c"}]'::jsonb + WHERE id = ${taskId}; + `); + await db.execute(sql`UPDATE inbox SET status = 'delivered'`); + const [before] = await db.execute<{ id: string }>(sql`SELECT id FROM inbox`); + + const result = await assignLinear({ + session: "sess-1", + title: "Fix auth (edited)", + url: "https://linear.app/x/issue/ISS-1/fix-auth-edited", + flags: ["NEW"], + }); + + expect(result).toEqual({ + id: taskId, + key: "ISS-1", + result: "duplicate", + prevSandboxId: null, + }); + const task = await repo.getById(taskId); + expect({ title: task?.title, url: task?.externalUrl, metadata: task?.metadata }).toEqual({ + title: "Fix auth", + url: "https://linear.app/x/issue/ISS-1", + metadata: { experimentalFlags: ["OLD"] }, + }); + expect({ + ownerId: task?.ownerId, + sandboxId: task?.sandboxId, + pvcName: task?.pvcName, + seq: Number(task?.lastObservedEventSeq), + prs: task?.prUrls, + }).toEqual({ + ownerId: "pod-a", + sandboxId: "sb-1", + pvcName: "pvc-1", + seq: 7, + prs: [{ repo: "r", url: "https://github.com/o/r/pull/1", branch: "b", commit_sha: "c" }], + }); + const rows = await db.execute<{ id: string; status: string; payload: { title: string } }>( + sql`SELECT id, status, payload FROM inbox`, + ); + expect(rows).toHaveLength(1); + expect(rows[0]).toMatchObject({ + id: (before as { id: string }).id, + status: "delivered", + payload: { title: "Fix auth" }, + }); + }); }); describe("restartTaskWithInbox", () => { - it("stamps task_id/task_key on the assignment payload from the task row", async () => { - const { taskId } = await repo.upsertFromAgentSessionWithInbox({ - taskExternalId: "issue-1", - taskKey: "ISS-1", - taskTitle: "Fix auth", - taskExternalUrl: "https://linear.app/x/issue/ISS-1", - inboxPayload: assignmentContent, - inboxReplyTargets: [linearTarget("sess-1")], - metadata: {}, + it("re-delegation: assign restarts, refreshing title, URL and flags and dropping retryCount", async () => { + const { id: taskId } = await assignLinear({ session: "sess-1", flags: ["OLD"] }); + await db.execute(sql` + UPDATE tasks SET metadata = metadata || '{"retryCount":2}'::jsonb WHERE id = ${taskId}; + `); + + await assignLinear({ + session: "sess-2", + title: "Fix auth v2", + url: "https://linear.app/x/issue/ISS-1/v2", + flags: ["NEW"], }); + + const task = await repo.getById(taskId); + expect({ title: task?.title, url: task?.externalUrl, metadata: task?.metadata }).toEqual({ + title: "Fix auth v2", + url: "https://linear.app/x/issue/ISS-1/v2", + metadata: { experimentalFlags: ["NEW"] }, + }); + }); + + it("stamps task_id/task_key on the assignment payload from the task row", async () => { + const { id: taskId } = await assignLinear(); const created = await db.execute<{ payload: unknown }>( sql`SELECT payload FROM inbox WHERE task_id = ${taskId}`, ); @@ -240,15 +291,7 @@ describe("TaskRepository", () => { it("rotates and re-pends the task's single assignment inbox row, dead-letters open siblings, and resets the task", async () => { // Seed a task + assignment inbox row, then stage an in-flight run. - const { taskId } = await repo.upsertFromAgentSessionWithInbox({ - taskKey: "ISS-1", - taskExternalId: "issue-1", - taskTitle: "Fix auth", - taskExternalUrl: "https://linear.app/x/issue/ISS-1", - inboxPayload: assignmentContent, - inboxReplyTargets: [linearTarget("sess-1")], - metadata: {}, - }); + const { id: taskId } = await assignLinear({ session: "sess-1" }); await db.execute(sql` UPDATE tasks SET owner_id = 'pod-a', heartbeat_at = now(), sandbox_id = 'sb-old', @@ -344,15 +387,7 @@ describe("TaskRepository", () => { describe("requeueForRetry", () => { it("owner-guarded RESUME: keeps pvc/cursor/pr_urls + inbox rows, only drops sandbox_id + releases the lease", async () => { - const { taskId } = await repo.upsertFromAgentSessionWithInbox({ - taskKey: "ISS-1", - taskExternalId: "issue-1", - taskTitle: "Fix auth", - taskExternalUrl: "https://linear.app/x/issue/ISS-1", - inboxPayload: assignmentContent, - inboxReplyTargets: [linearTarget("sess-1")], - metadata: {}, - }); + const { id: taskId } = await assignLinear(); const prUrls = [ { repo: "api", @@ -443,6 +478,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 100, + sources: ALL_SOURCES, }), ).toBeNull(); }); @@ -455,6 +491,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 100, + sources: ALL_SOURCES, }); expect(task?.id).toBe(t.id); @@ -471,6 +508,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 100, + sources: ALL_SOURCES, }), ).toBeNull(); }); @@ -487,6 +525,7 @@ describe("TaskRepository", () => { ownerId: "pod-live", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 100, + sources: ALL_SOURCES, }); expect(task?.id).toBe(t.id); expect(task?.ownerId).toBe("pod-live"); @@ -509,6 +548,7 @@ describe("TaskRepository", () => { ownerId: "pod-live", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 100, + sources: ALL_SOURCES, }); expect(task?.id).toBe(t.id); expect(task?.ownerId).toBe("pod-live"); @@ -528,6 +568,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 100, + sources: ALL_SOURCES, }), ).toBeNull(); }); @@ -546,6 +587,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 100, + sources: ALL_SOURCES, }); expect(task?.id).toBe(b.id); }); @@ -561,6 +603,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 1, + sources: ALL_SOURCES, }), ).toBeNull(); }); @@ -573,6 +616,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 1, + sources: ALL_SOURCES, }); expect(task?.id).toBe(t.id); }); @@ -598,6 +642,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 1, + sources: ALL_SOURCES, }), ).toBeNull(); }); @@ -624,6 +669,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 1, + sources: ALL_SOURCES, }), ).toBeNull(); }); @@ -650,6 +696,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 1, + sources: ALL_SOURCES, }); // A claim succeeds — the stale lease did not consume the only slot. expect(task?.id).toBe(stale.id); @@ -674,6 +721,7 @@ describe("TaskRepository", () => { ownerId: "pod-a", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 2, + sources: ALL_SOURCES, }); // running.n == 1 (not 2) → the fresh task is claimable. expect(task?.id).toBe(fresh.id); @@ -736,6 +784,14 @@ describe("TaskRepository", () => { expect(await events()).toEqual([]); }); + it("renews the lease with each advance", async () => { + const stale = new Date(Date.now() - 100_000); + const t = await insertTask({ ownerId: "pod-a", heartbeatAt: stale }); + await repo.recordEventCursor({ taskId: t.id, ownerId: "pod-a", event: ev(1) }); + const [row] = await db.select().from(schema.tasks); + expect(row?.heartbeatAt?.getTime()).toBeGreaterThan(stale.getTime() + 90_000); + }); + it("API task: one log row per event, holding the full event", async () => { const t = await insertTask({ source: "api", ownerId: "pod-a" }); await repo.recordEventCursor({ taskId: t.id, ownerId: "pod-a", event: ev(1) }); @@ -850,7 +906,7 @@ describe("TaskRepository", () => { branch: "b", commitSha: "abc", }); - expect(first).toEqual([{ id: t.id }]); + expect(first).toEqual([{ id: t.id, firstTime: true }]); // Re-append same url — should not duplicate. await repo.appendPrUrl({ taskId: t.id, @@ -874,6 +930,20 @@ describe("TaskRepository", () => { expect(urls).toEqual(["https://github.com/o/r/pull/1", "https://github.com/o/r/pull/2"]); }); + it("re-appending a known url still returns the row, with firstTime false", async () => { + const t = await insertTask({ ownerId: "pod-a" }); + const pr = { + taskId: t.id, + ownerId: "pod-a", + prUrl: "https://github.com/o/r/pull/1", + repo: "r", + branch: "b", + commitSha: "abc", + }; + expect(await repo.appendPrUrl(pr)).toEqual([{ id: t.id, firstTime: true }]); + expect(await repo.appendPrUrl(pr)).toEqual([{ id: t.id, firstTime: false }]); + }); + it("owner-guarded: returns 0 rows on stale-lease takeover", async () => { const t = await insertTask({ ownerId: "pod-a" }); const wrong = await repo.appendPrUrl({ @@ -891,7 +961,7 @@ describe("TaskRepository", () => { }); }); - describe("setSandboxId / getById / findByLinearIssueId / findByPrUrl", () => { + describe("setSandboxId / getById / findByExternalId / findByPrUrl", () => { it("setSandboxId owner-guarded", async () => { const t = await insertTask({ ownerId: "pod-a" }); expect(await repo.setSandboxId({ taskId: t.id, ownerId: "pod-x", sandboxId: "s" })).toEqual( @@ -915,10 +985,11 @@ describe("TaskRepository", () => { expect(await repo.getByIdOwnedBy(t.id, "pod-other")).toBeNull(); }); - it("findByLinearIssueId matches on source + external_id", async () => { + it("findByExternalId matches on source + external_id", async () => { const t = await insertTask({ externalId: "linear-uuid-1" }); - expect((await repo.findByLinearIssueId("linear-uuid-1"))?.id).toBe(t.id); - expect(await repo.findByLinearIssueId("nope")).toBeNull(); + expect((await repo.findByExternalId("linear", "linear-uuid-1"))?.id).toBe(t.id); + expect(await repo.findByExternalId("linear", "nope")).toBeNull(); + expect(await repo.findByExternalId("api", "linear-uuid-1")).toBeNull(); }); it("findByPrUrl matches via the GIN-indexed jsonb predicate", async () => { @@ -942,11 +1013,46 @@ describe("TaskRepository", () => { await insertTask({ externalId: "b", ownerId: "pod-a" }); // no sandbox await insertTask({ externalId: "c", ownerId: "pod-b", sandboxId: "sbx-c" }); // other pod - const ids = (await repo.listInFlightOwnedBy("pod-a")).map((r) => r.id); + const ids = (await repo.listInFlightOwnedBy("pod-a", ALL_SOURCES)).map((r) => r.id); expect(ids).toEqual([withSbx.id]); }); }); + describe("source filter (an image never works a source it doesn't know)", () => { + it("claimNext and listInFlightOwnedBy skip a task whose source is not in the list", async () => { + const api = await insertTask({ source: "api", externalId: "api-unclaimed" }); + await insertInboxRow(api.id); + const apiOwned = await insertTask({ + source: "api", + externalId: "api-owned", + ownerId: "pod-a", + sandboxId: "sbx-api", + }); + const linear = await insertTask({ + externalId: "lin-owned", + ownerId: "pod-a", + sandboxId: "sbx-l", + }); + const opts = { + leaseTimeoutMs: 10_000, + maxConcurrentSandboxes: 100, + sources: ["linear" as const], + }; + + expect(await repo.claimNext({ ownerId: "pod-b", ...opts })).toBeNull(); + expect((await repo.listInFlightOwnedBy("pod-a", ["linear"])).map((r) => r.id)).toEqual([ + linear.id, + ]); + // The same rows with the source listed: claimed and listed. + expect((await repo.claimNext({ ownerId: "pod-b", ...opts, sources: ALL_SOURCES }))?.id).toBe( + api.id, + ); + expect( + (await repo.listInFlightOwnedBy("pod-a", ALL_SOURCES)).map((r) => r.id).sort(), + ).toEqual([apiOwned.id, linear.id].sort()); + }); + }); + describe("heartbeatOwnedBy / backdateHeartbeatsOwnedBy", () => { it("heartbeat updates only rows owned by us", async () => { const mine = await insertTask({ @@ -960,7 +1066,8 @@ describe("TaskRepository", () => { heartbeatAt: new Date("2020-01-01"), }); - await repo.heartbeatOwnedBy("pod-a"); + // The renewed tasks: a task taken over is missing. + expect(await repo.heartbeatOwnedBy("pod-a")).toEqual([{ id: mine.id }]); const rows = await db.select().from(schema.tasks); expect(rows.find((r) => r.id === mine.id)?.heartbeatAt?.getFullYear()).toBeGreaterThan(2020); @@ -976,6 +1083,7 @@ describe("TaskRepository", () => { ownerId: "pod-b", leaseTimeoutMs: 10_000, maxConcurrentSandboxes: 100, + sources: ALL_SOURCES, }); expect(task?.id).toBe(t.id); expect(task?.ownerId).toBe("pod-b"); diff --git a/control-plane/src/repositories/apiTask.repository.ts b/control-plane/src/repositories/apiTask.repository.ts index 67fc707..f19a498 100644 --- a/control-plane/src/repositories/apiTask.repository.ts +++ b/control-plane/src/repositories/apiTask.repository.ts @@ -1,16 +1,14 @@ import { type SQL, sql } from "drizzle-orm"; import { db } from "../db"; +import type { TaskSource } from "../integrations/types"; import type { ApiRound, ApiTaskState } from "../types/api"; -import type { AssignmentPayload, CommentPayload } from "../types/inboxMessage"; -import { apiSessionTarget, taskIdentity } from "./inbox.repository"; -import { insertCpEventIfApi } from "./taskEvent.repository"; -// API read model and the API task write transactions. +// The API read model. API tasks are written through TaskIntake. export type ApiTaskRecord = { id: string; key: string; - source: "linear" | "api"; + source: TaskSource; title: string; description: string | null; external_url: string; @@ -26,7 +24,7 @@ export type ApiTaskRecord = { }; export type ApiTaskListFilter = { - source?: "linear" | "api"; + source?: TaskSource; // undefined = no filter; null = matches nothing (created_by=me with auth off). createdBy?: string | null; state?: ApiTaskState; @@ -98,8 +96,11 @@ export class ApiTaskRepository { return asRows(rows)[0] ?? null; } + // Case-insensitive, like the key's unique index (lower(key)). async findByKey(key: string): Promise { - const rows = await db.execute(sql`${RESOURCE_SELECT} WHERE t.key = ${key}`); + const rows = await db.execute( + sql`${RESOURCE_SELECT} WHERE lower(t.key) = lower(${key})`, + ); return asRows(rows)[0] ?? null; } @@ -126,103 +127,6 @@ export class ApiTaskRepository { WHERE t.id = ${taskId}`); return asRows<{ state: ApiTaskState; sandbox_live: boolean }>(rows)[0] ?? null; } - - // POST /api/tasks: the tasks row, the assignment inbox row and task_created - // in ONE transaction. The key comes from api_task_key_seq. - async createApiTask(p: { - keyPrefix: string; - publicBaseUrl: string; - title: string; - description: string | null; - experimentalFlags: string[]; - createdBy: string | null; - }): Promise<{ taskId: string; key: string }> { - return db.transaction(async (tx) => { - const [seq] = asRows<{ n: string }>( - await tx.execute(sql`SELECT nextval('api_task_key_seq')::text AS n`), - ); - if (!seq) throw new Error("createApiTask: nextval returned no row"); - const key = `${p.keyPrefix}${seq.n}`; - const externalUrl = `${p.publicBaseUrl}/api/tasks/${key}`; - const metadataJson = JSON.stringify({ experimentalFlags: p.experimentalFlags }); - const [task] = asRows<{ id: string }>( - await tx.execute(sql` - INSERT INTO tasks (source, external_id, key, title, external_url, metadata, created_by) - VALUES ('api', ${key}, ${key}, ${p.title}, ${externalUrl}, - ${metadataJson}::text::jsonb, ${p.createdBy}) - RETURNING id;`), - ); - if (!task) throw new Error("createApiTask: tasks insert returned no row"); - - const payload: AssignmentPayload = { - title: p.title, - description: p.description, - external_url: externalUrl, - target_id: task.id, - ...taskIdentity({ id: task.id, key }), - }; - const [round] = asRows<{ id: string }>( - await tx.execute(sql` - INSERT INTO inbox (task_id, source, kind, external_id, payload, reply_targets) - VALUES (${task.id}, 'api', 'assignment', ${key}, - ${JSON.stringify(payload)}::text::jsonb, - ${JSON.stringify([apiSessionTarget(task.id)])}::text::jsonb) - RETURNING id;`), - ); - if (!round) throw new Error("createApiTask: inbox insert returned no row"); - - await insertCpEventIfApi(tx, { - taskId: task.id, - type: "task_created", - inboxId: round.id, - data: { - title: p.title, - description: p.description, - experimental_flags: p.experimentalFlags, - }, - }); - return { taskId: task.id, key }; - }); - } - - // POST /api/tasks/:ref/messages: a comment row (external_id = a new UUID) - // and message_queued in ONE transaction. A round in progress doesn't block - // it: the new row queues behind it, as on Linear. - async appendApiMessage(p: { - task: { id: string; key: string }; - body: string; - createdAt: string; - }): Promise<{ roundId: string }> { - const payload: CommentPayload = { - target_id: p.task.id, - body: p.body, - pr: null, - created_at: p.createdAt, - ...taskIdentity(p.task), - }; - return db.transaction(async (tx) => { - // Lock the task row first: task_events ids must be handed out in commit - // order per task, or an event cursor can skip past a still-open insert. - // Every other per-task log writer starts with UPDATE tasks. - await tx.execute(sql`SELECT 1 FROM tasks WHERE id = ${p.task.id} FOR NO KEY UPDATE`); - const [round] = asRows<{ id: string }>( - await tx.execute(sql` - INSERT INTO inbox (task_id, source, kind, external_id, payload, reply_targets) - VALUES (${p.task.id}, 'api', 'comment', ${crypto.randomUUID()}, - ${JSON.stringify(payload)}::text::jsonb, - ${JSON.stringify([apiSessionTarget(p.task.id)])}::text::jsonb) - RETURNING id;`), - ); - if (!round) throw new Error("appendApiMessage: inbox insert returned no row"); - await insertCpEventIfApi(tx, { - taskId: p.task.id, - type: "message_queued", - inboxId: round.id, - data: { body: p.body }, - }); - return { roundId: round.id }; - }); - } } export const apiTaskRepository = new ApiTaskRepository(); diff --git a/control-plane/src/repositories/inbox.repository.ts b/control-plane/src/repositories/inbox.repository.ts index b35c88a..cc9c953 100644 --- a/control-plane/src/repositories/inbox.repository.ts +++ b/control-plane/src/repositories/inbox.repository.ts @@ -1,6 +1,7 @@ import { sql } from "drizzle-orm"; import { db } from "../db"; import type { Task } from "../db/schema"; +import { logger } from "../logger"; import type { AssignmentPayload, InboxKind, @@ -8,7 +9,7 @@ import type { InboxSource, TaskIdentity, } from "../types/inboxMessage"; -import { insertCpEventIfApi } from "./taskEvent.repository"; +import { insertCpEventIfApi, type SqlExecutor, type Tx } from "./taskEvent.repository"; // Re-export contract types so existing repo-import sites stay valid without // reaching into types/ directly. @@ -65,8 +66,24 @@ export type ReplyTarget = { // event from the daemon routes to the right thread. id: string; spec: ReplyTargetSpec; + // An assignment's delegation order, when the platform has one: a delegation + // whose order is not above the stored one is a duplicate (TaskIntake.assign). + order?: number; }; +// A row to insert: the wire message without its id, per (source, kind) arm so +// source, kind and payload stay paired. +type WithoutId = M extends InboxMessage ? Omit : never; +export type NewInboxMessage = WithoutId; + +// A Postgres unique violation (23505) on `constraint`. drizzle wraps the +// driver's error in `cause`. +export function isUniqueViolation(err: unknown, constraint: string): boolean { + const e = (err as { cause?: unknown })?.cause ?? err; + const pg = e as { errno?: unknown; constraint?: unknown }; + return pg.errno === "23505" && pg.constraint === constraint; +} + // Assignment content as callers build it. The identity fields are stamped by // the repository from the tasks row (see TaskRepository). export type AssignmentContent = Omit; @@ -137,44 +154,119 @@ function asRows(result: unknown): T[] { export class InboxRepository { // INSERT (synthetic uuid PK auto-generated). Idempotent on - // (source, external_id) via the unique index. Returns the synthetic id on - // insert, null on conflict. - // - // `message` is the wire contract sans `id` — the tagged union forbids - // invalid (source, kind) combos at the type level. - async appendInbox(params: { - taskId: string; - message: Omit; - replyTargets: ReplyTarget[]; - }): Promise<{ id: string } | null> { + // (source, kind, external_id) via the unique index. Returns the synthetic id + // on insert, null on conflict. Given a transaction, it inserts in a + // savepoint, so a unique violation leaves that transaction usable. + async appendInbox( + params: { + taskId: string; + message: NewInboxMessage; + replyTargets: ReplyTarget[]; + }, + tx?: Tx, + ): Promise<{ id: string } | null> { const { source, kind, external_id, payload } = params.message; // bun-sql JSON-encodes a JS string param, so `${jsonStr}::jsonb` would // store a JSON *string scalar*, not the array/object. Casting via // `::text::jsonb` forces PG to parse the raw text into proper jsonb. const payloadJson = JSON.stringify(payload); const replyTargetsJson = JSON.stringify(params.replyTargets ?? []); - const result = await db.execute<{ id: string }>(sql` - INSERT INTO inbox (task_id, source, kind, external_id, payload, reply_targets) - VALUES (${params.taskId}, ${source}, ${kind}, ${external_id}, - ${payloadJson}::text::jsonb, ${replyTargetsJson}::text::jsonb) - ON CONFLICT (source, external_id) DO NOTHING - RETURNING id; - `); - return asRows<{ id: string }>(result)[0] ?? null; + const insert = (ex: SqlExecutor) => + ex.execute<{ id: string }>(sql` + INSERT INTO inbox (task_id, source, kind, external_id, payload, reply_targets) + VALUES (${params.taskId}, ${source}, ${kind}, ${external_id}, + ${payloadJson}::text::jsonb, ${replyTargetsJson}::text::jsonb) + ON CONFLICT (source, kind, external_id) DO NOTHING + RETURNING id; + `); + try { + const result = await (tx ? tx.transaction(insert) : insert(db)); + return asRows<{ id: string }>(result)[0] ?? null; + } catch (err) { + // Until a later release drops it, the old (source, external_id) index + // also rejects a row whose id another kind of the same source already + // uses, with a 23505 instead of DO NOTHING. Dedupe it as before; this + // goes with the index. + if (isUniqueViolation(err, "inbox_source_external_uniq")) return null; + throw err; + } } - // Count of ci_result rows already forwarded for a task — the loop breaker's - // denominator. Counts every ci_result row regardless of status: a 'failed' - // or 'responded' row still represents a CI failure harness already saw, so - // it must count toward the runaway threshold. Enqueue is idempotent on - // (source, external_id) = workflow_run:run:attempt:pr, so a redelivered - // webhook does not inflate this count. - async countCiResultsByTask(taskId: string): Promise { - const result = await db.execute<{ n: number }>(sql` - SELECT COUNT(*)::int AS n FROM inbox - WHERE task_id = ${taskId} AND source = 'github' AND kind = 'ci_result'; - `); - return asRows<{ n: number }>(result)[0]?.n ?? 0; + // TaskIntake.followUp's transaction. Locks the task row first: tasks before + // inbox, and task_events ids handed out in commit order per task. Drops the + // round when `prUrl` is no longer one of the task's PRs (a restart cleared + // it) or when the CI loop breaker trips, and as a duplicate when the round is + // already stored. message_queued is written for an inserted API message. + async appendFollowUp(p: { + taskId: string; + message: NewInboxMessage; + replyTargets: ReplyTarget[]; + prUrl?: string; + ciResultMax: number; + }): Promise<{ roundId: string } | { dropped: "duplicate" | "stale" | "ci_limit" }> { + const { message } = p; + const hasPr = + p.prUrl === undefined + ? sql`true` + : sql`pr_urls @> jsonb_build_array(jsonb_build_object('url', ${p.prUrl}::text))`; + return db.transaction(async (tx) => { + const [task] = asRows<{ has_pr: boolean }>( + await tx.execute( + sql`SELECT ${hasPr} AS has_pr FROM tasks WHERE id = ${p.taskId} FOR NO KEY UPDATE`, + ), + ); + if (!task) throw new Error(`appendFollowUp: task ${p.taskId} not found`); + if (!task.has_pr) { + logger.info("follow-up dropped: the PR is no longer on the task", { + taskId: p.taskId, + prUrl: p.prUrl, + }); + return { dropped: "stale" }; + } + + // Loop breaker. harness fixes CI → pushes → CI re-runs → may fail again, + // ad infinitum. Once a task has accumulated `ciResultMax` ci_result rows, + // stop forwarding new failures. Every row counts regardless of status: a + // 'failed' or 'responded' row is still a CI failure the harness saw. The + // count runs under the task lock, so concurrent webhooks can't overshoot. + // 0 / negative disables the breaker. + if (message.kind === "ci_result" && p.ciResultMax > 0) { + const [count] = asRows<{ n: number }>( + await tx.execute(sql` + SELECT COUNT(*)::int AS n FROM inbox + WHERE task_id = ${p.taskId} AND source = 'github' AND kind = 'ci_result'; + `), + ); + const seen = count?.n ?? 0; + if (seen >= p.ciResultMax) { + // external_id is workflow_run:::. + const [, runId, , prNumber] = message.external_id.split(":").map(Number); + logger.warn("ci_result loop breaker tripped — dropping CI failure", { + taskId: p.taskId, + seen, + max: p.ciResultMax, + runId, + prNumber, + }); + return { dropped: "ci_limit" }; + } + } + + const row = await this.appendInbox( + { taskId: p.taskId, message, replyTargets: p.replyTargets }, + tx, + ); + if (!row) return { dropped: "duplicate" }; + if (message.source === "api" && message.kind === "comment") { + await insertCpEventIfApi(tx, { + taskId: p.taskId, + type: "message_queued", + inboxId: row.id, + data: { body: message.payload.body }, + }); + } + return { roundId: row.id }; + }); } // Rows still awaiting delivery to the sandbox (status = 'pending'). @@ -187,9 +279,7 @@ export class InboxRepository { return asRows(result).map(mapRow); } - // Every non-terminal row of a task (pending or delivered). failTask walks - // these to broadcast the failure message to each reply target before - // dead-lettering. + // Every non-terminal row of a task (pending or delivered). async listNonTerminalByTask(taskId: string): Promise { const result = await db.execute(sql` SELECT * FROM inbox @@ -219,23 +309,14 @@ export class InboxRepository { return row ? mapRow(row) : null; } - // Reply target id for task-scoped CP-originated emits (provisioning - // lifecycle). Reads the first entry of the assignment row's reply_targets — - // the canonical store, written on `created` and overwritten on - // re-delegation. An assignment row carries exactly one reply target by - // construction. Returns null when the assignment row is missing or its - // reply_targets is empty. - async getAssignmentReplyTargetId(taskId: string): Promise { - const result = await db.execute<{ target_id: string | null }>(sql` - SELECT reply_targets->0->>'id' AS target_id - FROM inbox - WHERE task_id = ${taskId} AND kind = 'assignment' - LIMIT 1; + // The task's assignment row. A restart re-pends it under a new id, which is + // how hooks tell whether their run is still the current one. + async getAssignment(taskId: string): Promise { + const result = await db.execute(sql` + SELECT * FROM inbox WHERE task_id = ${taskId} AND kind = 'assignment' LIMIT 1; `); - const row = Array.isArray(result) - ? (result[0] as { target_id: string | null } | undefined) - : undefined; - return row?.target_id ?? null; + const row = asRows(result)[0]; + return row ? mapRow(row) : null; } // pending → delivered. Owner-guarded (a stale-lease pod must not advance a @@ -258,29 +339,33 @@ export class InboxRepository { // task-failure paths (provision failure, sandbox crash/timeout) call this so // a failed task cannot leave a row stuck pre-terminal — a non-terminal row // pins the idle reaper, and on a no-sandbox provision failure a 'pending' - // row would otherwise stay claimable forever. Returns the affected row ids - // for the caller to log. + // row would otherwise stay claimable forever. Returns the rows it failed, in + // arrival order: the caller logs them and sends the failure message only to + // those rows. async failTaskInbox(params: { taskId: string; ownerId: string; reason: string; - }): Promise<{ id: string }[]> { + }): Promise { return db.transaction(async (tx) => { // Lock the task row first so task_events ids are handed out in commit // order per task for stable event cursors. Tasks before inbox, // the same order as restartTaskWithInbox. await tx.execute(sql`SELECT 1 FROM tasks WHERE id = ${params.taskId} FOR NO KEY UPDATE`); - const result = await tx.execute<{ id: string }>(sql` - UPDATE inbox i - SET status = 'failed', failure_reason = ${params.reason}, updated_at = now() - FROM tasks t - WHERE i.task_id = ${params.taskId} - AND i.task_id = t.id - AND t.owner_id = ${params.ownerId} - AND i.status IN ('pending', 'delivered') - RETURNING i.id; + const result = await tx.execute(sql` + WITH failed AS ( + UPDATE inbox i + SET status = 'failed', failure_reason = ${params.reason}, updated_at = now() + FROM tasks t + WHERE i.task_id = ${params.taskId} + AND i.task_id = t.id + AND t.owner_id = ${params.ownerId} + AND i.status IN ('pending', 'delivered') + RETURNING i.* + ) + SELECT * FROM failed ORDER BY created_at ASC; `); - const failed = asRows<{ id: string }>(result); + const failed = asRows(result).map(mapRow); // Write one round_failed event per failed row in the same transaction. for (const row of failed) { await insertCpEventIfApi(tx, { diff --git a/control-plane/src/repositories/task.repository.ts b/control-plane/src/repositories/task.repository.ts index f2d12e1..f7cbc5a 100644 --- a/control-plane/src/repositories/task.repository.ts +++ b/control-plane/src/repositories/task.repository.ts @@ -1,91 +1,178 @@ -import { and, eq, sql } from "drizzle-orm"; +import { and, eq, inArray, sql } from "drizzle-orm"; import { db, schema } from "../db"; import type { Task } from "../db/schema"; +import type { TaskRef } from "../integrations/intake"; +import type { Person, TaskSource } from "../integrations/types"; import { logger } from "../logger"; import type { HarnessEvent } from "../types/harnessEvent"; -import type { AssignmentContent, ReplyTarget } from "./inbox.repository"; -import { insertCpEventIfApi, insertDaemonEvent } from "./taskEvent.repository"; +import type { AssignmentPayload } from "../types/inboxMessage"; +import { + type AssignmentContent, + isUniqueViolation, + type ReplyTarget, + taskIdentity, +} from "./inbox.repository"; +import { + insertCpEventIfApi, + insertDaemonEvent, + type SqlExecutor, + type Tx, +} from "./taskEvent.repository"; -export class TaskRepository { - // Single tx for a Linear `created` event. UPSERT the tasks row (infra) + - // INSERT the assignment inbox row (work content + reply destinations). - // Returns the task id and an insert-vs-update bit (`created`). - // - // A re-delegation conflict refreshes the display fields + metadata only — the - // inbox INSERT keys off the Linear issue UUID, so it DO-NOTHINGs and the - // assignment row is left untouched. The caller branches on `created` and - // routes a re-delegation to `restartTaskWithInbox` — duplicate-webhook - // detection (same session id) is the caller's job via - // `inboxRepo.getAssignmentReplyTargetId(taskId)`. - async upsertFromAgentSessionWithInbox(params: { - taskExternalId: string; - taskKey: string; - taskTitle: string; - taskExternalUrl: string; - inboxPayload: AssignmentContent; - inboxReplyTargets: ReplyTarget[]; - metadata: unknown; - }): Promise<{ taskId: string; created: boolean }> { - // bun-sql JSON-encodes a JS string param, so `${jsonStr}::jsonb` would - // store a JSON *string scalar* rather than the array/object. Casting via - // `::text::jsonb` forces PG to parse the raw text into proper jsonb. - // `?? {}` / `?? []` guard against `JSON.stringify(undefined)` returning - // `undefined`, which would bind as SQL NULL and violate NOT NULL. - const metadataJson = JSON.stringify(params.metadata ?? {}); - const payloadJson = JSON.stringify(params.inboxPayload); - const replyTargetsJson = JSON.stringify(params.inboxReplyTargets ?? []); +function asRows(result: unknown): T[] { + return Array.isArray(result) ? (result as T[]) : []; +} +export class TaskRepository { + // TaskIntake.assign's transaction. Inserts the task, or finds it by + // (source, external_id) and locks it, then compares the stored delegation + // (the assignment row's reply target) with the incoming one: when both carry + // an order, one not above the stored order is a duplicate; otherwise the + // same id is. A duplicate changes nothing; anything else restarts the task. + // A new task gets its assignment row and, for API tasks, task_created. + // Returns the preempted sandbox for the caller to kill after commit, and on + // a restart the replaced delegation's target id. + async assign(p: { + source: TaskSource; + externalId?: string; // required with a hint key; a sequence key is its own + key: { hint: string } | { sequencePrefix: string }; + title: string; + description: string | null; + flags: string[]; + url: (t: TaskRef) => string; + target: (t: TaskRef) => ReplyTarget; + requester?: Person; + createdBy?: string; + }): Promise< + TaskRef & { + result: "created" | "restarted" | "duplicate"; + prevSandboxId: string | null; + prevTargetId?: string; + } + > { return db.transaction(async (tx) => { - // 1. UPSERT tasks. RETURNING (xmax = 0) AS created: xmax=0 means a fresh - // INSERT, non-zero means an ON CONFLICT UPDATE path was taken. - const taskRows = await tx.execute<{ id: string; key: string; created: boolean }>(sql` - INSERT INTO tasks (source, external_id, key, title, external_url, metadata) - VALUES ('linear', ${params.taskExternalId}, ${params.taskKey}, ${params.taskTitle}, - ${params.taskExternalUrl}, ${metadataJson}::text::jsonb) - ON CONFLICT (source, external_id) DO UPDATE - SET title = EXCLUDED.title, - external_url = EXCLUDED.external_url, - -- MERGE, don't replace. A duplicate created-webhook redelivery - -- passes metadata={}; a blind = EXCLUDED.metadata would wipe the - -- durable retry budget (metadata.retryCount), silently resetting - -- the sandbox-retry cap. The jsonb || keeps existing keys while - -- still letting a real EXCLUDED value overlay. A genuine - -- re-delegation resets the budget via restartTaskWithInbox. - metadata = tasks.metadata || EXCLUDED.metadata, - updated_at = now() - RETURNING id, key, (xmax = 0) AS created; - `); - const taskRow = Array.isArray(taskRows) ? taskRows[0] : undefined; - if (!taskRow) - throw new Error("upsertFromAgentSessionWithInbox: tasks upsert returned no row"); + // In a savepoint so a key collision leaves the transaction usable: a + // concurrent insert can take the key between the check and the insert, + // and the retry then sees it. + const insert = () => tx.transaction((sp) => this.insertTask(sp, p)); + const inserted = await insert().catch((err) => { + if (!isUniqueViolation(err, "tasks_key_lower_uniq")) throw err; + return insert(); + }); - // 2. INSERT inbox. external_id is the Linear issue UUID, so a re-delegation - // conflicts here and DO NOTHING leaves the assignment row untouched — the - // caller routes the re-delegation to restartTaskWithInbox instead. - await tx.execute(sql` - INSERT INTO inbox (task_id, source, kind, external_id, payload, reply_targets) - VALUES (${taskRow.id}, 'linear', 'assignment', ${params.taskExternalId}, - ${payloadJson}::text::jsonb || jsonb_build_object( - 'task_id', ${taskRow.id}::text, - 'task_key', ${taskRow.key}::text), - ${replyTargetsJson}::text::jsonb) - ON CONFLICT (source, external_id) DO NOTHING; - `); + if (inserted) { + const t = { id: inserted.id, key: inserted.key }; + const target = p.target(t); + const payload: AssignmentPayload = { + title: p.title, + description: p.description, + external_url: inserted.url, + target_id: target.id, + ...taskIdentity(t), + }; + // external_id is the task's: the platform's id, or the API task key. + const [round] = asRows<{ id: string }>( + await tx.execute(sql` + INSERT INTO inbox (task_id, source, kind, external_id, payload, reply_targets) + VALUES (${t.id}, ${p.source}, 'assignment', ${inserted.externalId}, + ${JSON.stringify(payload)}::text::jsonb, + ${JSON.stringify([target])}::text::jsonb) + RETURNING id;`), + ); + if (!round) throw new Error("assign: inbox insert returned no row"); + await insertCpEventIfApi(tx, { + taskId: t.id, + type: "task_created", + inboxId: round.id, + data: { title: p.title, description: p.description, experimental_flags: p.flags }, + }); + return { ...t, result: "created", prevSandboxId: null }; + } - return { taskId: taskRow.id, created: taskRow.created }; + const [t] = asRows( + await tx.execute(sql` + SELECT id, key FROM tasks + WHERE source = ${p.source} AND external_id = ${p.externalId ?? null} + FOR NO KEY UPDATE`), + ); + if (!t) throw new Error("assign: task not found after a conflicting insert"); + // Read after the lock, so a restart that committed meanwhile is seen. + const [stored] = asRows<{ target: ReplyTarget | null }>( + await tx.execute(sql` + SELECT reply_targets->0 AS target FROM inbox + WHERE task_id = ${t.id} AND kind = 'assignment' LIMIT 1`), + ); + const prev = stored?.target ?? null; + const next = p.target(t); + const duplicate = + prev !== null && + (typeof prev.order === "number" && typeof next.order === "number" + ? next.order <= prev.order + : next.id === prev.id); + if (duplicate) return { ...t, result: "duplicate", prevSandboxId: null }; + + const url = p.url(t); + const { prevSandboxId } = await this.restartTaskWithInbox( + { + taskId: t.id, + inboxPayload: { + title: p.title, + description: p.description, + external_url: url, + target_id: next.id, + }, + inboxReplyTargets: [next], + delegation: { flags: p.flags, requester: p.requester }, + }, + tx, + ); + return { ...t, result: "restarted", prevSandboxId, prevTargetId: prev?.id }; }); } - // Re-delegation restart. The caller routes here when - // upsertFromAgentSessionWithInbox returned `created: false` AND the prior - // session id differs from the event's. PREEMPTS any in-flight run and - // restarts from scratch, in one tx: + // assign's insert. Picks the key: a hint is used as is unless another task + // holds it (case-insensitively), then `${SOURCE}-${hint}`; a sequence key is + // the prefix plus api_task_key_seq. Returns null when (source, external_id) + // already exists. + private async insertTask( + ex: SqlExecutor, + p: Parameters[0], + ): Promise<(TaskRef & { externalId: string; url: string }) | null> { + const picked = + "hint" in p.key + ? sql`CASE WHEN EXISTS ( + SELECT 1 FROM tasks + WHERE lower(key) = lower(${p.key.hint}) + AND NOT (source = ${p.source} AND external_id = ${p.externalId ?? null}) + ) THEN ${`${p.source.toUpperCase()}-${p.key.hint}`}::text ELSE ${p.key.hint}::text END` + : sql`${p.key.sequencePrefix}::text || nextval('api_task_key_seq')::text`; + const [pick] = asRows( + await ex.execute(sql`SELECT uuidv7()::text AS id, ${picked} AS key`), + ); + if (!pick) throw new Error("assign: key selection returned no row"); + const externalId = p.externalId ?? pick.key; + const url = p.url(pick); + const metadataJson = JSON.stringify({ experimentalFlags: p.flags, requester: p.requester }); + const [row] = asRows<{ id: string }>( + await ex.execute(sql` + INSERT INTO tasks (id, source, external_id, key, title, external_url, metadata, created_by) + VALUES (${pick.id}, ${p.source}, ${externalId}, ${pick.key}, ${p.title}, ${url}, + ${metadataJson}::text::jsonb, ${p.createdBy ?? null}) + ON CONFLICT (source, external_id) DO NOTHING + RETURNING id;`), + ); + return row ? { ...pick, externalId, url } : null; + } + + // Restart: assign's newer delegation, or TaskIntake.restart (API). PREEMPTS + // any in-flight run and restarts from scratch, in one tx (the caller's, when + // given: assign already holds the task row lock): // 1. Reset the task to a clean slate — pvc_name / sandbox_id / owner_id / // heartbeat_at cleared, cursor reset. The old owner pod self-evicts on // its next owner-guarded write (owner_id gone). // 2. Re-pend the task's single `assignment` inbox row (one per task, created - // by upsertFromAgentSessionWithInbox) with a fresh id plus the new - // event's payload + reply_targets. The inbox id is the daemon-visible run + // by assign) with a fresh id plus the new event's payload + + // reply_targets. The inbox id is the daemon-visible run // identity (`inbox/.json`) and feeds the harness PR branch suffixes; // the logical task row id and assignment external_id remain stable. // 3. Dead-letter every still-open NON-assignment inbox row (pending @@ -93,108 +180,119 @@ export class TaskRepository { // re-delivered. // Not owner-guarded — it forcibly supersedes whatever pod ran the old round. // 4. With `api`: also set the new title and experimental flags in step 1, - // and write `task_restarted` to the log. + // and write `task_restarted` to the log. With `delegation`: also set the + // payload's title and URL and the new flags and requester in step 1. // // Returns `prevSandboxId` — the preempted sandbox the caller must kill; // `assignmentId` — the re-pended assignment row's new id (null if the task // has none); and `supersededRoundIds` — the rows dead-lettered in step 3. - // Duplicate webhook deliveries do not route here; the service checks the - // prior session first. - async restartTaskWithInbox(params: { - taskId: string; - inboxPayload: AssignmentContent; - inboxReplyTargets: ReplyTarget[]; - api?: { title: string; description: string | null; experimentalFlags: string[] }; - }): Promise<{ + // Duplicate deliveries do not route here; assign compares delegations first. + async restartTaskWithInbox( + params: { + taskId: string; + inboxPayload: AssignmentContent; + inboxReplyTargets: ReplyTarget[]; + api?: { title: string; description: string | null; experimentalFlags: string[] }; + delegation?: { flags: string[]; requester?: Person }; + }, + tx?: Tx, + ): Promise<{ prevSandboxId: string | null; assignmentId: string | null; supersededRoundIds: string[]; }> { + if (!tx) return db.transaction((t) => this.restartTaskWithInbox(params, t)); + const { api, delegation } = params; const payloadJson = JSON.stringify(params.inboxPayload); - // An API restart also applies the new title and flags. The + // An API restart or a re-delegation also applies the new title and flags + // (a re-delegation its URL and requester too, dropping a stale one). The // description lives on the assignment payload (step 2). - const apiSet = params.api - ? sql`, title = ${params.api.title}, - metadata = jsonb_set(metadata - 'retryCount', '{experimentalFlags}', - ${JSON.stringify(params.api.experimentalFlags)}::text::jsonb)` - : sql`, metadata = metadata - 'retryCount'`; + const flags = api?.experimentalFlags ?? delegation?.flags; + const definition = JSON.stringify({ + experimentalFlags: flags, + requester: delegation?.requester, + }); + const definitionSet = + flags === undefined + ? sql`, metadata = metadata - 'retryCount'` + : sql`, title = ${params.inboxPayload.title}, + metadata = (metadata - 'retryCount' - 'requester') || ${definition}::text::jsonb + ${delegation ? sql`, external_url = ${params.inboxPayload.external_url}` : sql``}`; const replyTargetsJson = JSON.stringify(params.inboxReplyTargets ?? []); - return db.transaction(async (tx) => { - // 1. Reset the task to a clean slate. PG18 `RETURNING old.sandbox_id` is - // the preempted sandbox the caller kills. - const taskRows = await tx.execute<{ prev_sandbox_id: string | null }>(sql` - UPDATE tasks - SET pvc_name = NULL, - sandbox_id = NULL, - owner_id = NULL, - heartbeat_at = NULL, - last_observed_event_seq = 0, - pr_urls = '[]'::jsonb - -- Clean slate also resets the sandbox-retry budget (retryCount - -- is dropped in the metadata fragment) — a genuine - -- re-delegation is a fresh attempt, not a continuation. - ${apiSet}, - updated_at = now() - WHERE id = ${params.taskId} - RETURNING old.sandbox_id AS prev_sandbox_id; - `); - const taskRow = Array.isArray(taskRows) ? taskRows[0] : undefined; - if (!taskRow) throw new Error("restartTaskWithInbox: task not found"); + // 1. Reset the task to a clean slate. PG18 `RETURNING old.sandbox_id` is + // the preempted sandbox the caller kills. + const taskRows = await tx.execute<{ prev_sandbox_id: string | null }>(sql` + UPDATE tasks + SET pvc_name = NULL, + sandbox_id = NULL, + owner_id = NULL, + heartbeat_at = NULL, + last_observed_event_seq = 0, + pr_urls = '[]'::jsonb + -- Clean slate also resets the sandbox-retry budget (retryCount + -- is dropped in the metadata fragment) — a genuine + -- re-delegation is a fresh attempt, not a continuation. + ${definitionSet}, + updated_at = now() + WHERE id = ${params.taskId} + RETURNING old.sandbox_id AS prev_sandbox_id; + `); + const taskRow = Array.isArray(taskRows) ? taskRows[0] : undefined; + if (!taskRow) throw new Error("restartTaskWithInbox: task not found"); - // 2. Re-pend the task's single `assignment` inbox row with a new id plus - // the new event's payload + reply targets — this drives the fresh run. - // The row remains the one assignment for this task / Linear issue, but - // the daemon sees a new run identity via `inbox/.json`. - const reRows = await tx.execute<{ id: string }>(sql` - UPDATE inbox i - SET id = uuidv7(), - status = 'pending', - failure_reason = NULL, - -- a re-run never inherits the previous run's outcome - outcome = NULL, - payload = ${payloadJson}::text::jsonb || jsonb_build_object( - 'task_id', t.id::text, 'task_key', t.key), - reply_targets = ${replyTargetsJson}::text::jsonb, - updated_at = now() - FROM tasks t - WHERE t.id = i.task_id - AND i.task_id = ${params.taskId} - AND i.kind = 'assignment' - RETURNING i.id; - `); - const assignmentId = (Array.isArray(reRows) ? reRows[0]?.id : undefined) ?? null; + // 2. Re-pend the task's single `assignment` inbox row with a new id plus + // the new event's payload + reply targets — this drives the fresh run. + // The row remains the one assignment for this task / Linear issue, but + // the daemon sees a new run identity via `inbox/.json`. + const reRows = await tx.execute<{ id: string }>(sql` + UPDATE inbox i + SET id = uuidv7(), + status = 'pending', + failure_reason = NULL, + -- a re-run never inherits the previous run's outcome + outcome = NULL, + payload = ${payloadJson}::text::jsonb || jsonb_build_object( + 'task_id', t.id::text, 'task_key', t.key), + reply_targets = ${replyTargetsJson}::text::jsonb, + updated_at = now() + FROM tasks t + WHERE t.id = i.task_id + AND i.task_id = ${params.taskId} + AND i.kind = 'assignment' + RETURNING i.id; + `); + const assignmentId = (Array.isArray(reRows) ? reRows[0]?.id : undefined) ?? null; - // 3. Dead-letter every still-open NON-assignment inbox row (pending - // comment / review rows). The re-pended `assignment` row (step 2) - // is excluded by `kind <> 'assignment'`. - const supRows = await tx.execute<{ id: string }>(sql` - UPDATE inbox - SET status = 'failed', - failure_reason = 'superseded by re-delegation', - updated_at = now() - WHERE task_id = ${params.taskId} - AND status IN ('pending', 'delivered') - AND kind <> 'assignment' - RETURNING id; - `); - const supersededRoundIds = (Array.isArray(supRows) ? supRows : []).map((r) => r.id); + // 3. Dead-letter every still-open NON-assignment inbox row (pending + // comment / review rows). The re-pended `assignment` row (step 2) + // is excluded by `kind <> 'assignment'`. + const supRows = await tx.execute<{ id: string }>(sql` + UPDATE inbox + SET status = 'failed', + failure_reason = 'superseded by re-delegation', + updated_at = now() + WHERE task_id = ${params.taskId} + AND status IN ('pending', 'delivered') + AND kind <> 'assignment' + RETURNING id; + `); + const supersededRoundIds = (Array.isArray(supRows) ? supRows : []).map((r) => r.id); - if (params.api) { - await insertCpEventIfApi(tx, { - taskId: params.taskId, - type: "task_restarted", - inboxId: assignmentId, - data: { - title: params.api.title, - description: params.api.description, - experimental_flags: params.api.experimentalFlags, - superseded_round_ids: supersededRoundIds, - }, - }); - } - return { prevSandboxId: taskRow.prev_sandbox_id, assignmentId, supersededRoundIds }; - }); + if (params.api) { + await insertCpEventIfApi(tx, { + taskId: params.taskId, + type: "task_restarted", + inboxId: assignmentId, + data: { + title: params.api.title, + description: params.api.description, + experimental_flags: params.api.experimentalFlags, + superseded_round_ids: supersededRoundIds, + }, + }); + } + return { prevSandboxId: taskRow.prev_sandbox_id, assignmentId, supersededRoundIds }; } // Retry reset — RESUME, not restart. The backing sandbox failed (terminal / @@ -290,10 +388,15 @@ export class TaskRepository { // on that delivered row; throughout its recreate-from-PVC window sandbox_id // is NULL, so excluding 'delivered' here would let a retry wave over-admit // past the cap. Matches the claimable predicate's IN ('pending','delivered'). + // + // Only tasks of `sources` (the sources this build has an integration for) are + // claimed, so an older image never takes a task of a source it doesn't know: + // rollbacks and rolling overlaps stay safe without draining. async claimNext(params: { ownerId: string; leaseTimeoutMs: number; maxConcurrentSandboxes: number; + sources: Task["source"][]; }): Promise { const leaseCutoff = `${params.leaseTimeoutMs} milliseconds`; const rows = await db.execute(sql` @@ -315,7 +418,8 @@ export class TaskRepository { claimable AS ( SELECT t.id FROM tasks t, running - WHERE EXISTS ( + WHERE t.source IN ${params.sources} + AND EXISTS ( SELECT 1 FROM inbox i WHERE i.task_id = t.id AND i.status IN ('pending', 'delivered') @@ -352,21 +456,28 @@ export class TaskRepository { // Tasks this pod owns with a sandbox attached — the reconcile loop iterates // these every tick. Comment-pickup tasks with sandbox_id NULL are reconciled - // via claimNext + sweep, not here. - async listInFlightOwnedBy(ownerId: string): Promise { + // via claimNext + sweep, not here. Filtered to `sources` like claimNext. + async listInFlightOwnedBy(ownerId: string, sources: Task["source"][]): Promise { return db .select() .from(schema.tasks) - .where(and(eq(schema.tasks.ownerId, ownerId), sql`${schema.tasks.sandboxId} IS NOT NULL`)); + .where( + and( + eq(schema.tasks.ownerId, ownerId), + sql`${schema.tasks.sandboxId} IS NOT NULL`, + inArray(schema.tasks.source, sources), + ), + ); } // As long as a pod owns a task it heartbeats; if it loses interest it calls - // releaseLease. - async heartbeatOwnedBy(ownerId: string): Promise { - await db + // releaseLease. Returns the tasks renewed: one missing was taken over. + async heartbeatOwnedBy(ownerId: string): Promise<{ id: string }[]> { + return db .update(schema.tasks) .set({ heartbeatAt: sql`now()` }) - .where(eq(schema.tasks.ownerId, ownerId)); + .where(eq(schema.tasks.ownerId, ownerId)) + .returning({ id: schema.tasks.id }); } // Backdate heartbeats on own tasks so survivors become immediately @@ -433,6 +544,8 @@ export class TaskRepository { // a replay of a passed event, a pod without the lease, and the old owner // after a restart (owner_id cleared) all change 0 rows and log nothing. // Returns 0 rows when the lease is lost OR the cursor is already past seq. + // Each advance also renews the lease, so a long run of slow but successful + // sends can't outlive it mid-reconcile. async recordEventCursor(params: { taskId: string; ownerId: string; @@ -442,7 +555,7 @@ export class TaskRepository { return db.transaction(async (tx) => { const result = await tx.execute<{ id: string; source: string }>(sql` UPDATE tasks - SET last_observed_event_seq = ${event.seq}, updated_at = now() + SET last_observed_event_seq = ${event.seq}, heartbeat_at = now(), updated_at = now() WHERE id = ${params.taskId} AND owner_id = ${params.ownerId} AND last_observed_event_seq < ${event.seq} @@ -459,7 +572,8 @@ export class TaskRepository { // Owner-guarded append to pr_urls; dedupes on (url) within the array. // Returns 0 rows on stale-lease takeover — the caller (EventDispatcher's // pr_opened case) treats that as a self-eviction signal, matching every - // other state-advancing repo method. + // other state-advancing repo method. A URL already there still returns the + // row; `firstTime` (PG18 `old.`) says whether this call added it. async appendPrUrl(params: { taskId: string; ownerId: string; @@ -467,7 +581,7 @@ export class TaskRepository { repo: string; branch: string; commitSha: string; - }): Promise<{ id: string }[]> { + }): Promise<{ id: string; firstTime: boolean }[]> { const entry = sql`jsonb_build_object( 'repo', ${params.repo}::text, 'url', ${params.prUrl}::text, @@ -490,7 +604,10 @@ export class TaskRepository { updatedAt: new Date(), }) .where(this.ownedByUs(params.taskId, params.ownerId)) - .returning({ id: schema.tasks.id }); + .returning({ + id: schema.tasks.id, + firstTime: sql`old.pr_urls IS DISTINCT FROM new.pr_urls`, + }); } // GitHub webhook PR → task lookup. GIN-indexed. LIMIT 2 (not 1) so that a PR @@ -517,12 +634,13 @@ export class TaskRepository { return row ? mapTaskRow(row) : null; } - // tasks.external_id IS the Linear issue UUID. Replaces findByAgentSessionId. - async findByLinearIssueId(issueId: string): Promise { + // tasks.external_id is the platform's id (the Linear issue UUID, the API + // task key). + async findByExternalId(source: TaskSource, externalId: string): Promise { const rows = await db .select() .from(schema.tasks) - .where(and(eq(schema.tasks.source, "linear"), eq(schema.tasks.externalId, issueId))) + .where(and(eq(schema.tasks.source, source), eq(schema.tasks.externalId, externalId))) .limit(1); return rows[0] ?? null; } @@ -638,7 +756,7 @@ export class TaskRepository { // Raw row shape from db.execute (snake_case). type RawTaskRow = { id: string; - source: "linear" | "api"; + source: TaskSource; external_id: string; key: string; created_by: string | null; diff --git a/control-plane/src/repositories/taskEvent.repository.ts b/control-plane/src/repositories/taskEvent.repository.ts index e4b98c1..10c56ee 100644 --- a/control-plane/src/repositories/taskEvent.repository.ts +++ b/control-plane/src/repositories/taskEvent.repository.ts @@ -7,6 +7,8 @@ import type { HarnessEvent } from "../types/harnessEvent"; // Both are no-ops for non-API tasks; Linear tasks do not use this event log. export type SqlExecutor = Pick; +// A transaction handle, for writes that also run inside a caller's transaction. +export type Tx = Parameters[0]>[0]; export type CpEventType = | "task_created" diff --git a/control-plane/src/routes/api.ts b/control-plane/src/routes/api.ts index 6abfefa..cdd0202 100644 --- a/control-plane/src/routes/api.ts +++ b/control-plane/src/routes/api.ts @@ -1,11 +1,10 @@ import { Hono } from "hono"; import { bodyLimit } from "hono/body-limit"; import { type ApiController, type ApiEnv, errorBody } from "../controllers/api.controller"; +import type { Integration, IntegrationName } from "../integrations/types"; import { logger } from "../logger"; import { API_BODY_LIMIT_BYTES } from "../requestLimits"; import type { ApiAuthService } from "../services/apiAuth.service"; -import type { GithubStatus } from "../services/githubStatus.service"; -import type { LinearStatus } from "../services/linearStatus.service"; // /api. Mounted by app.ts only when API_ENABLED=true. // Order matters: the host check runs first (a disallowed host learns nothing, @@ -16,8 +15,8 @@ export function createApiRoutes( opts: { authEnabled: boolean; allowedHosts: string[]; - linearStatus: () => Promise; - githubStatus: () => GithubStatus; + // GET /api/integrations/, for every integration, enabled or not. + integrations: Record>; }, ): Hono { const app = new Hono(); @@ -66,7 +65,11 @@ export function createApiRoutes( app.get("/api/tasks/:ref/events/stream", (c) => controller.streamEvents(c)); app.get("/api/tasks/:ref/events", (c) => controller.listEvents(c)); app.get("/api/me", (c) => controller.me(c)); - app.get("/api/integrations/linear", async (c) => c.json(await opts.linearStatus())); - app.get("/api/integrations/github", (c) => c.json(opts.githubStatus())); + app.get("/api/integrations/:name", async (c) => { + const name = c.req.param("name"); + // An unknown name is a 404, as an unregistered route. + if (!Object.hasOwn(opts.integrations, name)) return c.notFound(); + return c.json(await opts.integrations[name as IntegrationName].status()); + }); return app; } diff --git a/control-plane/src/routes/webhook.ts b/control-plane/src/routes/webhook.ts index 2a9e652..6f014b7 100644 --- a/control-plane/src/routes/webhook.ts +++ b/control-plane/src/routes/webhook.ts @@ -1,17 +1,16 @@ import { type Context, Hono, type MiddlewareHandler } from "hono"; import { bodyLimit } from "hono/body-limit"; -import type { GitHubWebhookController } from "../controllers/githubWebhook.controller"; -import type { LinearWebhookController } from "../controllers/linearWebhook.controller"; +import type { TaskIntake } from "../integrations/intake"; +import type { Integration } from "../integrations/types"; import { WEBHOOK_BODY_LIMIT_BYTES } from "../requestLimits"; -// Each webhook is registered only when its integration is configured; an -// unconfigured one answers 404. -export function createWebhookRoutes(controllers: { - linear?: LinearWebhookController; - github?: GitHubWebhookController; -}): Hono { +// POST /webhook/ is registered only for an enabled integration that has +// a webhook; any other name answers 404. +export function createWebhookRoutes( + integrations: Pick[], + intake: TaskIntake, +): Hono { const app = new Hono(); - const { linear, github } = controllers; const payloadTooLarge = (c: Context) => c.json({ error: "payload too large" }, 413); const webhookBodyLimit = bodyLimit({ @@ -32,13 +31,12 @@ export function createWebhookRoutes(controllers: { await next(); }; - if (linear) - app.post("/webhook/linear", webhookBodyLimit, readLimitedWebhookBody, (c) => - linear.handleWebhook(c), - ); - if (github) - app.post("/webhook/github", webhookBodyLimit, readLimitedWebhookBody, (c) => - github.handleWebhook(c), + for (const integration of integrations) { + const webhook = integration.webhook?.bind(integration); + if (!integration.enabled || !webhook) continue; + app.post(`/webhook/${integration.name}`, webhookBodyLimit, readLimitedWebhookBody, (c) => + webhook(c, intake), ); + } return app; } diff --git a/control-plane/src/services/__tests__/agentSession.service.test.ts b/control-plane/src/services/__tests__/agentSession.service.test.ts index bdd7ff2..2566ee0 100644 --- a/control-plane/src/services/__tests__/agentSession.service.test.ts +++ b/control-plane/src/services/__tests__/agentSession.service.test.ts @@ -1,12 +1,11 @@ import { beforeEach, describe, expect, it, mock } from "bun:test"; import type { LinearClient } from "@linear/sdk"; import type { Task } from "../../db/schema"; -import type { InboxRepository } from "../../repositories/inbox.repository"; -import type { TaskRepository } from "../../repositories/task.repository"; +import type { TaskIntake, TaskRef } from "../../integrations/intake"; +import type { ReplyTarget } from "../../repositories/inbox.repository"; import type { AgentSessionCreated, AgentSessionPrompted } from "../../types/agentSession"; import type { InboxService } from "../inbox.service"; import type { LinearTokenProvider } from "../linearTokenProvider"; -import type { SandboxManager } from "../sandboxManager.service"; const emitResponse = mock(() => Promise.resolve()); const emitAction = mock(() => Promise.resolve()); @@ -63,45 +62,36 @@ const foundTask = { } as Task; describe("AgentSessionService", () => { - let taskRepo: TaskRepository; - let inboxRepo: InboxRepository; + let intake: TaskIntake; let linearTokenProvider: Pick; - let sandboxManager: SandboxManager; let inboxService: InboxService; - let upsertFromAgentSessionWithInbox: ReturnType; - let restartTaskWithInbox: ReturnType; - let findByLinearIssueId: ReturnType; - let getAssignmentReplyTargetId: ReturnType; + let assign: ReturnType; + let findTask: ReturnType; let enqueueLinearComment: ReturnType; let getAuthorizedLinearClient: ReturnType; - let killSandbox: ReturnType; let service: InstanceType; + type AssignArg = { + source: string; + externalId: string; + key: { hint: string }; + title: string; + description: string | null; + flags: string[]; + url: (t: TaskRef) => string; + target: (t: TaskRef) => ReplyTarget; + }; + const ref = { id: "task-1", key: "ISS-1" }; + beforeEach(() => { emitResponse.mockReset(); emitResponse.mockResolvedValue(undefined); emitAction.mockReset(); emitAction.mockResolvedValue(undefined); - upsertFromAgentSessionWithInbox = mock(() => - Promise.resolve({ taskId: "task-1", created: true }), - ); - restartTaskWithInbox = mock(() => - Promise.resolve({ - prevSandboxId: null, - assignmentId: null, - supersededRoundIds: [] as string[], - }), - ); - findByLinearIssueId = mock(() => Promise.resolve(foundTask)); - taskRepo = { - upsertFromAgentSessionWithInbox, - restartTaskWithInbox, - findByLinearIssueId, - } as unknown as TaskRepository; - - getAssignmentReplyTargetId = mock(() => Promise.resolve(null as string | null)); - inboxRepo = { getAssignmentReplyTargetId } as unknown as InboxRepository; + assign = mock(() => Promise.resolve({ taskId: "task-1", key: "ISS-1", result: "created" })); + findTask = mock(() => Promise.resolve(foundTask)); + intake = { assign, findTask } as unknown as TaskIntake; getAuthorizedLinearClient = mock(() => Promise.resolve({} as unknown as LinearClient)); linearTokenProvider = { getAuthorizedLinearClient } as unknown as Pick< @@ -109,55 +99,28 @@ describe("AgentSessionService", () => { "getAuthorizedLinearClient" >; - killSandbox = mock(() => Promise.resolve()); - sandboxManager = { killSandbox } as unknown as SandboxManager; - enqueueLinearComment = mock(() => Promise.resolve()); inboxService = { enqueueLinearComment } as unknown as InboxService; - service = new AgentSessionService( - taskRepo, - inboxRepo, - linearTokenProvider, - sandboxManager, - inboxService, - ); + service = new AgentSessionService(intake, linearTokenProvider, inboxService); }); describe("handleEvent created", () => { it("writes task + inbox row and emits Evaluating on fresh insert", async () => { - upsertFromAgentSessionWithInbox.mockResolvedValue({ - taskId: "task-1", - created: true, - }); - const result = await service.handleEvent(createdEvent()); expect(result).toEqual({ status: "ok" }); - expect(upsertFromAgentSessionWithInbox).toHaveBeenCalledTimes(1); - const arg = upsertFromAgentSessionWithInbox.mock.calls[0]?.[0] as { - taskExternalId: string; - taskKey: string; - taskTitle: string; - taskExternalUrl: string; - inboxPayload: { - title: string; - description: string | null; - target_id: string; - }; - inboxReplyTargets: Array<{ id: string; spec: { kind: string } }>; - metadata: { experimentalFlags: string[] }; - }; - expect(arg.taskExternalId).toBe("issue-1"); - expect(arg.taskKey).toBe("ISS-1"); - expect(arg.taskExternalUrl).toBe("https://linear.app/x/issue/ISS-1"); - expect(arg.inboxPayload.description).toBe("desc"); - expect(arg.metadata.experimentalFlags).toEqual([]); - // Agent session id is carried by the assignment row's reply_targets + - // payload.target_id — the canonical store after dropping the column. - expect(arg.inboxPayload.target_id).toBe("sess-1"); - expect(arg.inboxReplyTargets[0]?.id).toBe("sess-1"); - expect(arg.inboxReplyTargets[0]?.spec.kind).toBe("linear_session"); + expect(assign).toHaveBeenCalledTimes(1); + const arg = assign.mock.calls[0]?.[0] as AssignArg; + expect(arg.source).toBe("linear"); + expect(arg.externalId).toBe("issue-1"); + expect(arg.key).toEqual({ hint: "ISS-1" }); + expect(arg.url(ref)).toBe("https://linear.app/x/issue/ISS-1"); + expect(arg.description).toBe("desc"); + expect(arg.flags).toEqual([]); + // The agent session id is the delegation id: the assignment row's reply + // target (intake stamps it as payload.target_id too). + expect(arg.target(ref)).toEqual({ id: "sess-1", spec: { kind: "linear_session" } }); await flushMicrotasks(); expect(emitResponse).not.toHaveBeenCalled(); @@ -179,30 +142,18 @@ describe("AgentSessionService", () => { const result = await service.handleEvent(event); expect(result).toEqual({ status: "ok" }); - const arg = upsertFromAgentSessionWithInbox.mock.calls[0]?.[0] as { - inboxPayload: { description: string | null }; - metadata: { experimentalFlags: string[] }; - }; - expect(arg.metadata.experimentalFlags).toEqual(["FAST_PATH", "TRACE2"]); - expect(arg.inboxPayload.description).toBe( - "Please validate the new path.\n\n\n#devbox:exp:lowercase\n", - ); + const arg = assign.mock.calls[0]?.[0] as AssignArg; + expect(arg.flags).toEqual(["FAST_PATH", "TRACE2"]); + expect(arg.description).toBe("Please validate the new path.\n\n\n#devbox:exp:lowercase\n"); }); - it("restarts on conflict (new agent session id): calls restartTaskWithInbox, acks restart", async () => { - upsertFromAgentSessionWithInbox.mockResolvedValue({ - taskId: "task-1", - created: false, - }); - getAssignmentReplyTargetId.mockResolvedValue("sess-old"); - restartTaskWithInbox.mockResolvedValue({ prevSandboxId: null }); + it("restarts on a new agent session id: acks the restart", async () => { + assign.mockResolvedValue({ taskId: "task-1", key: "ISS-1", result: "restarted" }); const result = await service.handleEvent(createdEvent()); expect(result).toEqual({ status: "ok" }); - expect(restartTaskWithInbox).toHaveBeenCalledTimes(1); - const restartArg = restartTaskWithInbox.mock.calls[0]?.[0] as { taskId: string }; - expect(restartArg.taskId).toBe("task-1"); + expect(assign).toHaveBeenCalledTimes(1); await flushMicrotasks(); const respArgs = emitResponse.mock.calls[0] as unknown as [unknown, string, string]; @@ -213,48 +164,22 @@ describe("AgentSessionService", () => { expect(actArgs[3]).toBe("task"); }); - it("kills the preempted sandbox when a re-delegation restarts a live run", async () => { - upsertFromAgentSessionWithInbox.mockResolvedValue({ - taskId: "task-1", - created: false, - }); - getAssignmentReplyTargetId.mockResolvedValue("sess-old"); - restartTaskWithInbox.mockResolvedValue({ prevSandboxId: "sb-old" }); - - const result = await service.handleEvent(createdEvent()); - - expect(result).toEqual({ status: "ok" }); - await flushMicrotasks(); - expect(killSandbox).toHaveBeenCalledTimes(1); - expect(killSandbox.mock.calls[0]?.[0]).toBe("sb-old"); - }); - it("ignores a duplicate webhook redelivery (same agent session id): no restart", async () => { // Linear redelivers the same `created` event — the existing assignment // row's reply_targets already carries the event's session id, so it is - // not a new delegation. No restart, no sandbox kill, no re-emit. - upsertFromAgentSessionWithInbox.mockResolvedValue({ - taskId: "task-1", - created: false, - }); - getAssignmentReplyTargetId.mockResolvedValue("sess-1"); + // not a new delegation. No restart, no re-emit. + assign.mockResolvedValue({ taskId: "task-1", key: "ISS-1", result: "duplicate" }); const result = await service.handleEvent(createdEvent()); expect(result).toEqual({ status: "ok" }); - expect(restartTaskWithInbox).not.toHaveBeenCalled(); await flushMicrotasks(); expect(emitResponse).not.toHaveBeenCalled(); expect(emitAction).not.toHaveBeenCalled(); - expect(killSandbox).not.toHaveBeenCalled(); }); it("does not throw when linear emit fails (fire-and-forget)", async () => { - upsertFromAgentSessionWithInbox.mockResolvedValue({ - taskId: "task-1", - created: true, - }); getAuthorizedLinearClient.mockImplementation(() => Promise.reject(new Error("boom"))); const result = await service.handleEvent(createdEvent()); @@ -271,7 +196,7 @@ describe("AgentSessionService", () => { const result = await service.handleEvent(promptedEvent()); expect(result).toEqual({ status: "ok" }); - expect(findByLinearIssueId).toHaveBeenCalledWith("issue-1"); + expect(findTask).toHaveBeenCalledWith("linear", "issue-1"); expect(enqueueLinearComment).toHaveBeenCalledWith({ task: foundTask, agentActivityId: "activity-1", @@ -285,8 +210,17 @@ describe("AgentSessionService", () => { expect(emitAction).toHaveBeenCalledWith(expect.anything(), "sess-1", "Evaluating", ""); }); + it("a redelivered prompted re-acks and enqueues again (the inbox drops the duplicate row)", async () => { + await service.handleEvent(promptedEvent()); + await service.handleEvent(promptedEvent()); + await flushMicrotasks(); + expect(enqueueLinearComment).toHaveBeenCalledTimes(2); + expect(enqueueLinearComment.mock.calls[1]).toEqual(enqueueLinearComment.mock.calls[0]); + expect(emitAction).toHaveBeenCalledTimes(2); + }); + it("fires the Evaluating ack even when the task is not yet known (session_not_found)", async () => { - findByLinearIssueId.mockResolvedValue(null); + findTask.mockResolvedValue(null); const result = await service.handleEvent(promptedEvent()); diff --git a/control-plane/src/services/__tests__/apiTask.service.test.ts b/control-plane/src/services/__tests__/apiTask.service.test.ts index bb27bef..c9d8c13 100644 --- a/control-plane/src/services/__tests__/apiTask.service.test.ts +++ b/control-plane/src/services/__tests__/apiTask.service.test.ts @@ -7,7 +7,9 @@ mock.module("../../config", () => ({ config: { databaseUrl: TEST_DATABASE_URL } const { db } = await import("../../db"); const { ApiTaskRepository } = await import("../../repositories/apiTask.repository"); +const { InboxRepository } = await import("../../repositories/inbox.repository"); const { TaskRepository } = await import("../../repositories/task.repository"); +const { Intake } = await import("../../integrations/intake"); const { TaskEventRepository } = await import("../../repositories/taskEvent.repository"); const { ApiTaskService } = await import("../apiTask.service"); const { ApiError } = await import("../../errors"); @@ -41,17 +43,17 @@ describe("ApiTaskService", () => { ); principal.tokenId = (tok as { id: string }).id; killSandbox = mock(() => Promise.resolve()); - service = new ApiTaskService( - new ApiTaskRepository(), + const intake = new Intake( new TaskRepository(), - new TaskEventRepository(), + new InboxRepository(), { killSandbox }, - { - keyPrefix: "DBX-API-", - publicBaseUrl: "https://api.example", - now: () => new Date(T0), - }, + { detach: () => {} }, + { apiKeyPrefix: "DBX-API-", ciResultMaxPerTask: 20 }, ); + service = new ApiTaskService(new ApiTaskRepository(), intake, new TaskEventRepository(), { + publicBaseUrl: "https://api.example", + now: () => new Date(T0), + }); }); it("create returns the whole task resource", async () => { @@ -208,6 +210,64 @@ describe("ApiTaskService", () => { expect(killSandbox).toHaveBeenCalledWith("sbx-old"); }); + it("restart resets the run, re-pends a failed assignment and only kills the old sandbox", async () => { + const { id } = await service.create(principal, { + title: "T", + description: null, + experimentalFlags: ["FOO"], + }); + await db.execute(sql` + UPDATE tasks + SET owner_id = 'pod-a', heartbeat_at = now(), sandbox_id = 'sbx-old', pvc_name = 'pvc-old', + last_observed_event_seq = 9, metadata = metadata || '{"retryCount":2}'::jsonb, + pr_urls = '[{"repo":"r","url":"https://github.com/o/r/pull/1","branch":"b","commit_sha":"c"}]'::jsonb + WHERE id = ${id}`); + await db.execute(sql` + UPDATE inbox SET status = 'failed', failure_reason = 'boom', outcome = 'failed' + WHERE task_id = ${id}`); + const [before] = await db.execute<{ id: string }>(sql`SELECT id FROM inbox`); + + await service.restart("DBX-API-1", { title: "T2", description: null, experimentalFlags: [] }); + + const [task] = await db.execute(sql` + SELECT key, external_url, created_by, owner_id, heartbeat_at, sandbox_id, pvc_name, + last_observed_event_seq::int AS seq, pr_urls, metadata + FROM tasks WHERE id = ${id}`); + expect(task).toEqual({ + key: "DBX-API-1", + external_url: "https://api.example/api/tasks/DBX-API-1", + created_by: principal.tokenId, + owner_id: null, + heartbeat_at: null, + sandbox_id: null, + pvc_name: null, + seq: 0, + pr_urls: [], + metadata: { experimentalFlags: [] }, + }); + const rows = await db.execute<{ id: string }>( + sql`SELECT id, status, failure_reason, outcome FROM inbox`, + ); + expect([...rows]).toEqual([ + { id: expect.any(String), status: "pending", failure_reason: null, outcome: null }, + ]); + expect(rows[0]?.id).not.toBe((before as { id: string }).id); + // The old workspace volume is not deleted, only unlinked (a known leak). + expect(killSandbox).toHaveBeenCalledTimes(1); + expect(killSandbox).toHaveBeenCalledWith("sbx-old"); + }); + + it("sendMessage is not idempotent: the same body twice queues two rounds", async () => { + await service.create(null, { title: "T", description: null, experimentalFlags: [] }); + const a = await service.sendMessage("DBX-API-1", "again"); + const b = await service.sendMessage("DBX-API-1", "again"); + expect(a.round_id).not.toBe(b.round_id); + const queued = await db.execute( + sql`SELECT type FROM task_events WHERE type = 'message_queued'`, + ); + expect(queued).toHaveLength(2); + }); + it("restart with no previous sandbox kills nothing", async () => { await service.create(null, { title: "T", description: null, experimentalFlags: [] }); await service.restart("DBX-API-1", { title: "T2", description: null, experimentalFlags: [] }); diff --git a/control-plane/src/services/__tests__/eventDispatcher.test.ts b/control-plane/src/services/__tests__/eventDispatcher.test.ts index 38ad1eb..47a5049 100644 --- a/control-plane/src/services/__tests__/eventDispatcher.test.ts +++ b/control-plane/src/services/__tests__/eventDispatcher.test.ts @@ -1,9 +1,10 @@ import { beforeEach, describe, expect, it, mock } from "bun:test"; -import type { LinearClient } from "@linear/sdk"; +import { type LinearClient, LinearError, RatelimitedLinearError } from "@linear/sdk"; // Type-only import — erased at link time, so it can't collide with the // mock.module barrel factories below, and it pins the real contract. import type { LinearHumanUser } from "../../clients/linearApi"; import type { Task } from "../../db/schema"; +import { fakeIntegrations } from "../../integrations/__tests__/fakeIntegrations"; import type { InboxRepository, InboxRow, InboxStatus } from "../../repositories/inbox.repository"; import type { TaskRepository } from "../../repositories/task.repository"; import type { HarnessEvent } from "../../types/harnessEvent"; @@ -78,6 +79,12 @@ mock.module("../../clients/github", () => ({ })); const { EventDispatcher } = await import("../eventDispatcher"); +const { LinearIntegration } = await import("../../integrations/linear"); +const { GithubIntegration } = await import("../../integrations/github"); +const { ApiIntegration } = await import("../../integrations/api"); +const { HttpError } = await import("../../integrations/http"); +const { ReplyHolds } = await import("../replyHolds"); +const { IntegrationHooks } = await import("../integrationHooks"); const { config: mockedConfig } = (await import("../../config")) as unknown as { config: { github: { webhooks: { botLogin: string } | undefined }; @@ -150,9 +157,21 @@ describe("EventDispatcher", () => { let getByIdForTask: ReturnType; let markResponded: ReturnType; let notifyTerminal: ReturnType; + let extractArchive: ReturnType; let slackNotifier: SlackNotifierService; + let holds: InstanceType; let dispatcher: InstanceType; + // The registry, with GitHub's webhooks (and so its integration) on or off. + const makeIntegrations = (githubEnabled: boolean) => + fakeIntegrations({ + linear: new LinearIntegration(linearTokenProvider, true, { + status: async () => ({ configured: false }), + }), + api: new ApiIntegration(true), + github: new GithubIntegration(githubEnabled, { status: () => ({}) as never }), + }); + beforeEach(() => { for (const m of [ emitThought, @@ -161,6 +180,7 @@ describe("EventDispatcher", () => { emitError, setIssueStarted, attachUrlToIssue, + uploadIssueAttachment, ]) { m.mockReset(); m.mockResolvedValue(undefined); @@ -178,7 +198,7 @@ describe("EventDispatcher", () => { requestReviewers.mockReset(); requestReviewers.mockResolvedValue(undefined); - appendPrUrl = mock(() => Promise.resolve([{ id: "task-1" }])); + appendPrUrl = mock(() => Promise.resolve([{ id: "task-1", firstTime: true }])); taskRepo = { appendPrUrl } as unknown as TaskRepository; getByIdForTask = mock(() => Promise.resolve(linearRow())); @@ -186,22 +206,32 @@ describe("EventDispatcher", () => { inboxRepo = { getByIdForTask, markResponded, + getAssignment: mock(() => Promise.resolve(linearRow())), } as unknown as InboxRepository; - sandboxManager = {} as unknown as SandboxManager; + extractArchive = mock(() => Promise.resolve(null as Uint8Array | null)); + sandboxManager = { extractArchive } as unknown as SandboxManager; linearTokenProvider = { getAuthorizedLinearClient: mock(() => Promise.resolve({} as unknown as LinearClient)), } as unknown as Pick; notifyTerminal = mock(() => Promise.resolve()); - slackNotifier = { notifyTerminal } as unknown as SlackNotifierService; - + // Slack on with the default SLACK_NOTIFY_SOURCES (linear). + slackNotifier = { + notifyTerminal, + notifies: (source: string) => source === "linear", + } as unknown as SlackNotifierService; + + holds = new ReplyHolds(); + const integrations = makeIntegrations(true); dispatcher = new EventDispatcher( taskRepo, inboxRepo, sandboxManager, - linearTokenProvider, slackNotifier, + integrations, + holds, + new IntegrationHooks(integrations, inboxRepo), ); }); @@ -267,6 +297,7 @@ describe("EventDispatcher", () => { "Mention `@harness` and I'll act on it — on a top-level PR comment,", "in a review summary, or on an inline comment on the diff. Without a mention I won't see it.", ].join("\n"), + expect.any(AbortSignal), ); }); @@ -435,6 +466,28 @@ describe("EventDispatcher", () => { expect(emitThought).not.toHaveBeenCalled(); }); + it("Linear task: a token fetch failure posts nothing and holds the event", async () => { + (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( + new Error("token refresh failed"), + ); + expect( + await dispatcher.handle( + { + seq: 1, + ts: "t", + type: "agent_thought", + inbox_id: "inbox-1", + target_id: "sess-1", + body: "x", + }, + makeTask(), + ), + ).toBe("hold"); + for (const m of [emitThought, emitAction, emitResponse, emitError, postIssueComment]) { + expect(m).not.toHaveBeenCalled(); + } + }); + it("foreign / unknown inbox_id → cross-task leak: aborts, no emit", async () => { // A foreign inbox_id resolving to null means this task's stream is // carrying an event it doesn't own (cross-task leak). The dispatcher @@ -459,6 +512,174 @@ describe("EventDispatcher", () => { }); }); + describe("agent_error routing", () => { + // The harness's round-crash format (_emit_round_failure), aimed at the + // round's primary target. + const errorEvent = (targetId: string): Extract => ({ + seq: 1, + ts: "t", + type: "agent_error", + inbox_id: "inbox-1", + target_id: targetId, + body: "⚠ Something went wrong — RuntimeError: boom", + }); + const ghRow = (kind: string, target: InboxRow["reply_targets"][number]): InboxRow => + ({ ...linearRow(), source: "github", kind, reply_targets: [target] }) as InboxRow; + + it("a Linear comment round failure → emitError on the row's session", async () => { + getByIdForTask.mockResolvedValue({ ...linearRow(), kind: "comment" } as InboxRow); + expect(await dispatcher.handle(errorEvent("sess-1"), makeTask())).toBe("ok"); + expect(emitError).toHaveBeenCalledWith( + expect.anything(), + "sess-1", + "⚠ Something went wrong — RuntimeError: boom", + ); + }); + + it("a GitHub comment or review round failure posts the generic retry message, not the error", async () => { + getByIdForTask.mockResolvedValueOnce( + ghRow("comment", { + id: "555", + spec: { kind: "issue_comment", owner: "o", repo: "r", prNumber: 1, commentId: 555 }, + }), + ); + expect(await dispatcher.handle(errorEvent("555"), makeTask())).toBe("ok"); + getByIdForTask.mockResolvedValueOnce( + ghRow("review", { + id: "88", + spec: { kind: "review_body", owner: "o", repo: "r", prNumber: 1, reviewId: 88 }, + }), + ); + expect(await dispatcher.handle(errorEvent("88"), makeTask())).toBe("ok"); + + const retry = + "⚠️ Something went wrong while working on this. Mention @harness in a new comment to retry."; + // The comment's thread and the review summary (a top-level PR comment). + expect(postIssueComment.mock.calls as unknown[]).toEqual([ + ["o", "r", 1, retry, expect.any(AbortSignal)], + ["o", "r", 1, retry, expect.any(AbortSignal)], + ]); + expect(postReviewCommentReply).not.toHaveBeenCalled(); + expect(emitError).not.toHaveBeenCalled(); + }); + + it("a CI round failure is posted nowhere", async () => { + getByIdForTask.mockResolvedValue( + ghRow("ci_result", { + id: "9001", + spec: { kind: "issue_comment", owner: "o", repo: "r", prNumber: 1, commentId: 0 }, + }), + ); + expect(await dispatcher.handle(errorEvent("9001"), makeTask())).toBe("ok"); + expect(postIssueComment).not.toHaveBeenCalled(); + expect(emitError).not.toHaveBeenCalled(); + }); + + it("a GitHub comment round failure posts nothing when GitHub webhooks are off (no bot login)", async () => { + const webhooks = mockedConfig.github.webhooks; + mockedConfig.github.webhooks = undefined; + try { + getByIdForTask.mockResolvedValue( + ghRow("comment", { + id: "555", + spec: { kind: "issue_comment", owner: "o", repo: "r", prNumber: 1, commentId: 555 }, + }), + ); + expect(await dispatcher.handle(errorEvent("555"), makeTask())).toBe("ok"); + expect(postIssueComment).not.toHaveBeenCalled(); + expect(postReviewCommentReply).not.toHaveBeenCalled(); + } finally { + mockedConfig.github.webhooks = webhooks; + } + }); + }); + + describe("labelling", () => { + it("labels each agent emit and hands it to the round's integration with the event's target", async () => { + const delivered: unknown[] = []; + const record = (name: "linear" | "api" | "github") => ({ + name, + enabled: true, + status: async () => ({}), + deliver: async (_ctx: unknown, target: unknown, message: unknown) => { + delivered.push([name, target, message]); + }, + }); + const integrations = fakeIntegrations({ + linear: record("linear"), + api: record("api"), + github: record("github"), + }); + const d = new EventDispatcher( + taskRepo, + inboxRepo, + sandboxManager, + slackNotifier, + integrations, + new ReplyHolds(), + new IntegrationHooks(integrations, inboxRepo), + ); + const assignment = linearRow(); + const ghComment = { + ...linearRow(), + source: "github", + kind: "comment", + reply_targets: [ + { + id: "555", + spec: { kind: "issue_comment", owner: "o", repo: "r", prNumber: 1, commentId: 555 }, + }, + ], + } as InboxRow; + const emit = (type: string, targetId: string, extra = {}) => + ({ + seq: 1, + ts: "t", + type, + inbox_id: "inbox-1", + target_id: targetId, + body: "b", + ...extra, + }) as HarnessEvent; + const cases: Array<[InboxRow, HarnessEvent, unknown]> = [ + [assignment, emit("agent_response", "sess-1"), { type: "reply", body: "b" }], + [ + assignment, + emit("agent_thought", "sess-1"), + { type: "progress", kind: "thought", body: "b" }, + ], + [ + assignment, + emit("agent_action", "sess-1"), + { type: "progress", kind: "action", body: "b" }, + ], + [ + assignment, + emit("agent_error", "sess-1", { retryable_assignment_failure: true }), + { type: "progress", kind: "error", body: "b" }, + ], + [ + assignment, + emit("agent_error", "sess-1"), + { type: "failure", cause: "round_failed", round: assignment, summary: "b" }, + ], + [ + ghComment, + emit("agent_error", "555"), + { type: "failure", cause: "round_failed", round: ghComment, detail: "b" }, + ], + ]; + for (const [row, event] of cases) { + getByIdForTask.mockResolvedValueOnce(row); + expect(await d.handle(event, makeTask())).toBe("ok"); + } + // Routed by round.source (the task is a Linear task throughout). + expect(delivered).toEqual( + cases.map(([row, , message]) => [row.source, row.reply_targets[0], message]), + ); + }); + }); + describe("pr_opened", () => { it("appends the PR url and attaches it to the issue", async () => { const result = await dispatcher.handle(prEvent(), makeTask()); @@ -479,6 +700,29 @@ describe("EventDispatcher", () => { ); }); + it("Linear task: a token fetch failure holds the event at onPrOpened, after appendPrUrl", async () => { + (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( + new Error("token refresh failed"), + ); + expect(await dispatcher.handle(prEvent(), makeTask())).toBe("hold"); + expect(appendPrUrl).toHaveBeenCalledTimes(1); + expect(attachUrlToIssue).not.toHaveBeenCalled(); + }); + + it("Linear task: a 429 or a refused connection on the attach holds the event", async () => { + const rateLimited = new RatelimitedLinearError({ + response: { headers: new Headers({ "retry-after": "5" }) }, + } as never); + const refused = Object.assign(new TypeError("Unable to connect"), { code: "ECONNREFUSED" }); + attachUrlToIssue + .mockRejectedValueOnce(rateLimited) + .mockRejectedValueOnce(new LinearError(refused as never)); + expect(await dispatcher.handle(prEvent(), makeTask())).toBe("hold"); + expect(await dispatcher.handle(prEvent(), makeTask())).toBe("hold"); + expect(attachUrlToIssue).toHaveBeenCalledTimes(2); + expect(holds.of("linear").heldTasks).toBe(1); + }); + it("self-evicts when appendPrUrl returns 0 rows (lease lost); skips attachUrlToIssue", async () => { appendPrUrl.mockResolvedValue([]); const result = await dispatcher.handle(prEvent(), makeTask()); @@ -504,6 +748,7 @@ describe("EventDispatcher", () => { "Mention `@harness` and I'll act on it — on a top-level PR comment,", "in a review summary, or on an inline comment on the diff. Without a mention I won't see it.", ].join("\n"), + expect.any(AbortSignal), ); }); @@ -538,7 +783,7 @@ describe("EventDispatcher", () => { expect(result).toBe("ok"); expect(getIssueHumanAssignee).toHaveBeenCalledWith(expect.anything(), "issue-1"); - expect(getUserById).toHaveBeenCalledWith("583231"); + expect(getUserById).toHaveBeenCalledWith("583231", expect.any(AbortSignal)); expect(addPrAssignees).toHaveBeenCalledWith("o", "r", 1, ["octocat"]); }); @@ -684,7 +929,13 @@ describe("EventDispatcher", () => { it("routes an issue_comment target to postIssueComment", async () => { getByIdForTask.mockResolvedValue(ghCommentRow()); await dispatcher.handle(responseEvent("555"), makeTask()); - expect(postIssueComment).toHaveBeenCalledWith("o", "r", 1, "good catch, fixing"); + expect(postIssueComment).toHaveBeenCalledWith( + "o", + "r", + 1, + "good catch, fixing", + expect.any(AbortSignal), + ); expect(emitResponse).not.toHaveBeenCalled(); }); @@ -707,7 +958,14 @@ describe("EventDispatcher", () => { ]; getByIdForTask.mockResolvedValue(row); await dispatcher.handle(responseEvent("555"), makeTask()); - expect(postReviewCommentReply).toHaveBeenCalledWith("o", "r", 1, 555, "good catch, fixing"); + expect(postReviewCommentReply).toHaveBeenCalledWith( + "o", + "r", + 1, + 555, + "good catch, fixing", + expect.any(AbortSignal), + ); }); it("routes a linear_session target to emitResponse using target.id (the session id)", async () => { @@ -720,12 +978,101 @@ describe("EventDispatcher", () => { expect(emitResponse).toHaveBeenCalledWith(expect.anything(), "sess-1", "good catch, fixing"); }); + it("a GitHub reply that fails before sending (429) holds; any other failure is dropped", async () => { + getByIdForTask.mockResolvedValue(ghCommentRow()); + // githubFetch's error for a 429 carries the status and Retry-After. + postIssueComment.mockRejectedValueOnce( + new HttpError( + 'GitHub POST /repos/o/r/issues/1/comments failed: HTTP 429 {"message":"API rate limit exceeded"}', + 429, + 30_000, + ), + ); + expect(await dispatcher.handle(responseEvent("555"), makeTask())).toBe("hold"); + // A 5xx may have landed: dropped, the cursor advances past it. + postIssueComment.mockRejectedValueOnce( + new HttpError("GitHub POST /repos/o/r/issues/1/comments failed: HTTP 502 ", 502), + ); + expect(await dispatcher.handle(responseEvent("555"), makeTask())).toBe("dropped"); + expect(postIssueComment).toHaveBeenCalledTimes(2); + }); + + it("skips GitHub replies when GitHub webhooks are off: not posted, not held", async () => { + const integrations = makeIntegrations(false); + const d = new EventDispatcher( + taskRepo, + inboxRepo, + sandboxManager, + slackNotifier, + integrations, + holds, + new IntegrationHooks(integrations, inboxRepo), + ); + getByIdForTask.mockResolvedValue(ghCommentRow()); + expect(await d.handle(responseEvent("555"), makeTask())).toBe("ok"); + expect(postIssueComment).not.toHaveBeenCalled(); + expect(holds.of("github").heldTasks).toBe(0); + }); + + it("a Linear token failure does not hold a GitHub round's reply or completion on a Linear task", async () => { + (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( + new Error("token refresh failed"), + ); + getByIdForTask.mockResolvedValue(ghCommentRow()); + expect(await dispatcher.handle(responseEvent("555"), makeTask())).toBe("ok"); + expect( + await dispatcher.handle( + { seq: 2, ts: "t", type: "completed", inbox_id: "inbox-2" }, + makeTask(), + ), + ).toBe("ok"); + expect(postIssueComment).toHaveBeenCalledWith( + "o", + "r", + 1, + "good catch, fixing", + expect.any(AbortSignal), + ); + expect(markResponded).toHaveBeenCalledWith({ + inboxId: "inbox-2", + ownerId: "pod-test", + outcome: null, + }); + expect(linearTokenProvider.getAuthorizedLinearClient).not.toHaveBeenCalled(); + }); + it("drops a target_id absent from reply_targets (no throw, no post)", async () => { getByIdForTask.mockResolvedValue(ghCommentRow()); const result = await dispatcher.handle(responseEvent("not-a-target"), makeTask()); expect(result).toBe("ok"); expect(postIssueComment).not.toHaveBeenCalled(); }); + + it("a send holds on the round's integration; the Linear client fetch on the task's", async () => { + // A GitHub round on a Linear task: the 429 is GitHub's hold. + getByIdForTask.mockResolvedValue(ghCommentRow()); + postIssueComment.mockRejectedValueOnce(new HttpError("HTTP 429", 429)); + expect(await dispatcher.handle(responseEvent("555"), makeTask())).toBe("hold"); + expect(holds.of("github").heldTasks).toBe(1); + expect(holds.of("linear").heldTasks).toBe(0); + // pr_opened's Linear client fetch is the task's (Linear's) hold. + (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( + new Error("token refresh failed"), + ); + expect(await dispatcher.handle(prEvent(), makeTask({ id: "task-2" }))).toBe("hold"); + expect(holds.of("linear").heldTasks).toBe(1); + }); + + it("a Linear send gets a per-call client bound to the send's signal", async () => { + const row = ghCommentRow(); + row.source = "linear"; + row.reply_targets = [{ id: "sess-1", spec: { kind: "linear_session" } }]; + getByIdForTask.mockResolvedValue(row); + expect(await dispatcher.handle(responseEvent("sess-1"), makeTask())).toBe("ok"); + expect(linearTokenProvider.getAuthorizedLinearClient).toHaveBeenCalledWith( + expect.any(AbortSignal), + ); + }); }); describe("completed", () => { @@ -831,11 +1178,27 @@ describe("EventDispatcher", () => { expect(markResponded).not.toHaveBeenCalled(); }); + it("uploads the run archive to the Linear issue for an assignment round", async () => { + extractArchive.mockResolvedValue(new Uint8Array([1, 2])); + await dispatcher.handle(completedEvent(), makeTask()); + await flushMicrotasks(); + expect(extractArchive).toHaveBeenCalledWith("sbx-1"); + expect(uploadIssueAttachment).toHaveBeenCalledWith(expect.anything(), { + issueId: "issue-1", + title: "Harness run archive", + subtitle: "Task task-1", + filename: "harness-run-task-1.tar.gz", + contentType: "application/gzip", + bytes: new Uint8Array([1, 2]), + metadata: { taskId: "task-1" }, + signal: expect.any(AbortSignal), + }); + }); + it("posts a Slack completion update on the first transition (assignment round)", async () => { await dispatcher.handle(completedEvent(), makeTask()); expect(notifyTerminal).toHaveBeenCalledTimes(1); expect(notifyTerminal).toHaveBeenCalledWith( - expect.anything(), expect.objectContaining({ id: "task-1" }), "completed", ); @@ -843,11 +1206,7 @@ describe("EventDispatcher", () => { it("forwards the event status to the Slack notify (partial_failure)", async () => { await dispatcher.handle({ ...completedEvent(), status: "partial_failure" }, makeTask()); - expect(notifyTerminal).toHaveBeenCalledWith( - expect.anything(), - expect.anything(), - "partial_failure", - ); + expect(notifyTerminal).toHaveBeenCalledWith(expect.anything(), "partial_failure"); }); it("does NOT post to Slack on an idempotent re-fire (already responded)", async () => { @@ -879,7 +1238,13 @@ describe("EventDispatcher", () => { await flushMicrotasks(); expect(requestReviewers).toHaveBeenCalledTimes(1); - expect(requestReviewers).toHaveBeenCalledWith("o", "r", 7, ["reviewer"]); + expect(requestReviewers).toHaveBeenCalledWith( + "o", + "r", + 7, + ["reviewer"], + expect.any(AbortSignal), + ); }); it("falls back to review reply_targets for historical review rows without payload PR coordinates", async () => { @@ -890,7 +1255,13 @@ describe("EventDispatcher", () => { await dispatcher.handle(completedEvent(), makeTask()); await flushMicrotasks(); - expect(requestReviewers).toHaveBeenCalledWith("o", "r", 7, ["reviewer"]); + expect(requestReviewers).toHaveBeenCalledWith( + "o", + "r", + 7, + ["reviewer"], + expect.any(AbortSignal), + ); }); it("falls back to a review_comment reply_target when review_body is unavailable", async () => { @@ -916,7 +1287,13 @@ describe("EventDispatcher", () => { await dispatcher.handle(completedEvent(), makeTask()); await flushMicrotasks(); - expect(requestReviewers).toHaveBeenCalledWith("oc", "rc", 8, ["reviewer"]); + expect(requestReviewers).toHaveBeenCalledWith( + "oc", + "rc", + 8, + ["reviewer"], + expect.any(AbortSignal), + ); }); it("falls back to parsing review payload pr_url when reply_targets are unavailable", async () => { @@ -928,7 +1305,13 @@ describe("EventDispatcher", () => { await dispatcher.handle(completedEvent(), makeTask()); await flushMicrotasks(); - expect(requestReviewers).toHaveBeenCalledWith("fallback", "repo", 9, ["reviewer"]); + expect(requestReviewers).toHaveBeenCalledWith( + "fallback", + "repo", + 9, + ["reviewer"], + expect.any(AbortSignal), + ); }); it("does not re-request review on an idempotent review completion re-fire", async () => { @@ -992,7 +1375,13 @@ describe("EventDispatcher", () => { await flushMicrotasks(); expect(result).toBe("ok"); - expect(requestReviewers).toHaveBeenCalledWith("o", "r", 7, ["reviewer"]); + expect(requestReviewers).toHaveBeenCalledWith( + "o", + "r", + 7, + ["reviewer"], + expect.any(AbortSignal), + ); }); it("does not crash when markResponded returns 0 rows (lease lost; warned)", async () => { @@ -1001,6 +1390,20 @@ describe("EventDispatcher", () => { expect(markResponded).toHaveBeenCalled(); }); + it("Linear assignment: a token fetch failure no longer holds; it marks and notifies Slack", async () => { + // Slack's mention comes from assignee(), which fails (logged) and the + // post goes out without it; completed needs no Linear client. + (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( + new Error("token refresh failed"), + ); + expect(await dispatcher.handle(completedEvent(), makeTask())).toBe("ok"); + expect(markResponded).toHaveBeenCalled(); + expect(notifyTerminal).toHaveBeenCalledWith( + expect.objectContaining({ id: "task-1" }), + "completed", + ); + }); + it("still marks responded when linearClient is null", async () => { // The marker frees the sandbox regardless of Linear availability — it // never touches linearClient. @@ -1011,4 +1414,72 @@ describe("EventDispatcher", () => { expect(markResponded).toHaveBeenCalled(); }); }); + + // reconcileOne advances the cursor only after handle() returns, so a lost + // lease or a crash in between dispatches the same events again. + describe("replay (same event dispatched again)", () => { + it("completed: the run archive uploads once; the already-responded replay skips it", async () => { + extractArchive.mockResolvedValue(new Uint8Array([1])); + const done: HarnessEvent = { + seq: 5, + ts: "t", + type: "completed", + inbox_id: "inbox-1", + status: "completed", + }; + await dispatcher.handle(done, makeTask()); + getByIdForTask.mockResolvedValue(linearRow({ status: "responded" })); + await dispatcher.handle(done, makeTask()); + await flushMicrotasks(); + + expect(markResponded).toHaveBeenCalledTimes(1); + expect(extractArchive).toHaveBeenCalledTimes(1); + expect(uploadIssueAttachment).toHaveBeenCalledTimes(1); + }); + + it("pr_opened: the Linear attach re-runs (appendPrUrl returns the row for a known URL)", async () => { + await dispatcher.handle(prEvent(), makeTask()); + await dispatcher.handle(prEvent(), makeTask()); + expect(appendPrUrl).toHaveBeenCalledTimes(2); + expect(attachUrlToIssue).toHaveBeenCalledTimes(2); + }); + + it("pr_opened: the onboarding comment and the PR assignee mirror run only the first time", async () => { + // appendPrUrl finds the URL already there on the replay. + appendPrUrl + .mockResolvedValueOnce([{ id: "task-1", firstTime: true }]) + .mockResolvedValueOnce([{ id: "task-1", firstTime: false }]); + getIssueHumanAssignee.mockResolvedValue({ + id: "usr-1", + name: "Ada Lovelace", + displayName: "ada", + email: "ada@acme.com", + gitHubUserId: "octocat", + }); + addPrAssignees.mockResolvedValue({ assigned: ["octocat"] }); + await dispatcher.handle(prEvent(), makeTask()); + await dispatcher.handle(prEvent(), makeTask()); + await flushMicrotasks(); + + expect(postIssueComment).toHaveBeenCalledTimes(1); + expect(addPrAssignees).toHaveBeenCalledTimes(1); + // The owner's onPrOpened (the idempotent Linear attach) runs every time. + expect(attachUrlToIssue).toHaveBeenCalledTimes(2); + }); + + it("agent_response and agent_error post again on every dispatch", async () => { + const reply: HarnessEvent = { + seq: 2, + ts: "t", + type: "agent_response", + inbox_id: "inbox-1", + target_id: "sess-1", + body: "done", + }; + const error: HarnessEvent = { ...reply, seq: 3, type: "agent_error", body: "bad" }; + for (const e of [reply, reply, error, error]) await dispatcher.handle(e, makeTask()); + expect(emitResponse).toHaveBeenCalledTimes(2); + expect(emitError).toHaveBeenCalledTimes(2); + }); + }); }); diff --git a/control-plane/src/services/__tests__/inbox.service.test.ts b/control-plane/src/services/__tests__/inbox.service.test.ts index 0cee8a7..9526ae4 100644 --- a/control-plane/src/services/__tests__/inbox.service.test.ts +++ b/control-plane/src/services/__tests__/inbox.service.test.ts @@ -1,15 +1,15 @@ -import { beforeEach, describe, expect, it, mock } from "bun:test"; +import { beforeEach, describe, expect, it, mock, spyOn } from "bun:test"; import type { Task } from "../../db/schema"; +import { fakeIntegrations } from "../../integrations/__tests__/fakeIntegrations"; import type { InboxRepository, InboxRow } from "../../repositories/inbox.repository"; import type { LinearTokenProvider } from "../linearTokenProvider"; import type { SandboxManager } from "../sandboxManager.service"; -// Pin the two config values InboxService reads (podId, the ci_result cap) so -// results do not depend on the environment the suite runs in. +// Pin the config value InboxService reads (podId) so results do not depend on +// the environment the suite runs in. mock.module("../../config", () => ({ config: { podId: "pod-test", - github: { ciResultMaxPerTask: 20 }, }, })); @@ -68,6 +68,10 @@ mock.module("../../clients/github", () => ({ })); const { InboxService, WORKSPACE_MISSING_MESSAGE } = await import("../inbox.service"); +const { LinearIntegration } = await import("../../integrations/linear"); +const { GithubIntegration } = await import("../../integrations/github"); +const { ApiIntegration } = await import("../../integrations/api"); +const { IntegrationHooks } = await import("../integrationHooks"); const makeTask = (overrides: Partial = {}): Task => ({ id: "task-1", @@ -124,11 +128,11 @@ describe("InboxService", () => { let listNonTerminalByTask: ReturnType; let markDelivered: ReturnType; let markFailed: ReturnType; - let appendInbox: ReturnType; - let countCiResultsByTask: ReturnType; + let followUp: ReturnType; let writeHarnessFile: ReturnType; let killSandbox: ReturnType; let getById: ReturnType; + let linear: InstanceType; let service: InstanceType; beforeEach(() => { @@ -152,8 +156,7 @@ describe("InboxService", () => { listNonTerminalByTask = mock(() => Promise.resolve([] as InboxRow[])); markDelivered = mock(() => Promise.resolve()); markFailed = mock(() => Promise.resolve([{ id: "x" }])); - appendInbox = mock(() => Promise.resolve({ id: "inbox-new" })); - countCiResultsByTask = mock(() => Promise.resolve(0)); + followUp = mock(() => Promise.resolve({ roundId: "inbox-new" })); getById = mock((id: string) => Promise.resolve(ghCommentRow(id))); inboxRepo = { listPendingByTask, @@ -161,8 +164,6 @@ describe("InboxService", () => { getById, markDelivered, markFailed, - appendInbox, - countCiResultsByTask, } as unknown as InboxRepository; writeHarnessFile = mock(() => Promise.resolve()); @@ -176,7 +177,21 @@ describe("InboxService", () => { getAuthorizedLinearClient: mock(() => Promise.resolve({})), } as unknown as Pick; - service = new InboxService(inboxRepo, sandboxManager, linearTokenProvider); + linear = new LinearIntegration(linearTokenProvider, true, { + status: async () => ({ configured: false }), + }); + const integrations = fakeIntegrations({ + linear, + api: new ApiIntegration(true), + github: new GithubIntegration(true, { status: () => ({}) as never }), + }); + service = new InboxService( + inboxRepo, + sandboxManager, + integrations, + new IntegrationHooks(integrations, inboxRepo), + { followUp }, + ); }); describe("sweepUndelivered", () => { @@ -226,7 +241,13 @@ describe("InboxService", () => { await service.deadLetterWorkspaceMissing(makeTask({ sandboxId: null })); expect(postIssueComment).toHaveBeenCalledTimes(1); - expect(addIssueCommentReaction).toHaveBeenCalledWith("o", "r", 555, "confused"); + expect(addIssueCommentReaction).toHaveBeenCalledWith( + "o", + "r", + 555, + "confused", + expect.any(AbortSignal), + ); expect(markFailed).toHaveBeenCalledWith({ inboxId: "a", ownerId: "pod-test", @@ -235,6 +256,35 @@ describe("InboxService", () => { }); }); + it("finishes a dead-lettered assignment's round: outcome failed, first transition, no archive", async () => { + const onRoundFinished = spyOn(linear, "onRoundFinished"); + const assignment = { + ...ghCommentRow("as"), + source: "linear", + kind: "assignment", + reply_targets: [{ id: "sess-1", spec: { kind: "linear_session" } }], + } as InboxRow; + listNonTerminalByTask.mockResolvedValue([assignment, ghCommentRow("c")]); + getById.mockImplementation((id: string) => + Promise.resolve(id === "as" ? assignment : ghCommentRow(id)), + ); + + await service.deadLetterWorkspaceMissing(makeTask({ sandboxId: null })); + await new Promise((r) => setTimeout(r, 0)); + + expect(onRoundFinished).toHaveBeenCalledTimes(1); + const [ctx, , round, outcome] = onRoundFinished.mock.calls[0] as unknown as [ + { firstTransition: boolean; archive(): Promise }, + Task, + InboxRow, + string, + ]; + expect(round.id).toBe("as"); + expect(outcome).toBe("failed"); + expect(ctx.firstTransition).toBe(true); + expect(await ctx.archive()).toBeNull(); + }); + it("API rows: no Linear or GitHub post, row failed with the user message", async () => { const row = { ...ghCommentRow("r1"), @@ -298,6 +348,175 @@ describe("InboxService", () => { expect(emitError).not.toHaveBeenCalled(); expect(markFailed).not.toHaveBeenCalled(); }); + + // One open row per kind on a Linear task. getById echoes each row so the + // per-row status re-check passes. + const stageRows = (rows: InboxRow[]) => { + listNonTerminalByTask.mockResolvedValue(rows); + getById.mockImplementation((id: string) => + Promise.resolve(rows.find((r) => r.id === id) ?? null), + ); + }; + const linearRow = (id: string, sessionId: string, kind = "comment"): InboxRow => + ({ + ...ghCommentRow(id), + source: "linear", + kind, + reply_targets: [{ id: sessionId, spec: { kind: "linear_session" } }], + }) as InboxRow; + + it("notifies each row by kind: Linear session error, review-comment reply + 😕, review body as a PR comment, CI silent", async () => { + stageRows([ + linearRow("lin", "sess-2"), + { + ...ghCommentRow("rc"), + reply_targets: [ + { + id: "556", + spec: { + kind: "review_comment", + owner: "o", + repo: "r", + prNumber: 7, + commentId: 556, + inReplyTo: 556, + path: "a.ts", + line: 3, + }, + }, + ], + }, + { + ...ghCommentRow("rev"), + kind: "review", + reply_targets: [ + { + id: "11", + spec: { + kind: "review_comment", + owner: "o", + repo: "r", + prNumber: 8, + commentId: 11, + inReplyTo: 11, + path: "b.ts", + line: 9, + }, + }, + { + id: "88", + spec: { kind: "review_body", owner: "o", repo: "r", prNumber: 8, reviewId: 88 }, + }, + ], + } as InboxRow, + { + ...ghCommentRow("ci"), + kind: "ci_result", + reply_targets: [ + { + id: "9001", + spec: { kind: "issue_comment", owner: "o", repo: "r", prNumber: 9, commentId: 0 }, + }, + ], + } as InboxRow, + ]); + + await service.deadLetterWorkspaceMissing(makeTask({ sandboxId: null })); + + expect(emitError).toHaveBeenCalledTimes(1); + expect(emitError).toHaveBeenCalledWith( + expect.anything(), + "sess-2", + WORKSPACE_MISSING_MESSAGE, + ); + expect(postReviewCommentReply).toHaveBeenCalledTimes(1); + expect(postReviewCommentReply).toHaveBeenCalledWith( + "o", + "r", + 7, + 556, + WORKSPACE_MISSING_MESSAGE, + expect.any(AbortSignal), + ); + expect(addReviewCommentReaction).toHaveBeenCalledTimes(1); + expect(addReviewCommentReaction).toHaveBeenCalledWith( + "o", + "r", + 556, + "confused", + expect.any(AbortSignal), + ); + // Only the review body posts; its inline comments and the CI row get nothing. + expect(postIssueComment).toHaveBeenCalledTimes(1); + expect(postIssueComment).toHaveBeenCalledWith( + "o", + "r", + 8, + WORKSPACE_MISSING_MESSAGE, + expect.any(AbortSignal), + ); + expect(addIssueCommentReaction).not.toHaveBeenCalled(); + expect(markFailed.mock.calls.map((c) => (c[0] as { inboxId: string }).inboxId)).toEqual([ + "lin", + "rc", + "rev", + "ci", + ]); + }); + + it("posts per row, not per target: two Linear rows on one session get two errors", async () => { + // failTask dedupes by target id; this path does not. + stageRows([linearRow("a", "sess-1", "assignment"), linearRow("b", "sess-1")]); + + await service.deadLetterWorkspaceMissing(makeTask({ sandboxId: null })); + + expect(emitError).toHaveBeenCalledTimes(2); + expect(markFailed).toHaveBeenCalledTimes(2); + }); + + it("marks the row before posting, and posts nothing when markFailed transitions nothing", async () => { + const order: string[] = []; + postIssueComment.mockImplementation(() => { + order.push("post"); + return Promise.resolve({ id: 1, url: "u" }); + }); + addIssueCommentReaction.mockImplementation(() => { + order.push("react"); + return Promise.resolve(); + }); + // Row b went terminal under us (a racing restart or a late `completed`). + markFailed.mockImplementation(({ inboxId }: { inboxId: string }) => { + order.push(`mark ${inboxId}`); + return Promise.resolve(inboxId === "a" ? [{ id: "a" }] : []); + }); + stageRows([ghCommentRow("a"), ghCommentRow("b")]); + + await service.deadLetterWorkspaceMissing(makeTask({ sandboxId: null })); + + expect(order).toEqual(["mark a", "post", "react", "mark b"]); + }); + + it("a Linear token failure drops only that row's notice: every row is marked failed, later rows still post", async () => { + stageRows([linearRow("a", "sess-1"), ghCommentRow("b")]); + (linearTokenProvider.getAuthorizedLinearClient as ReturnType).mockRejectedValue( + new Error("token refresh failed"), + ); + + await service.deadLetterWorkspaceMissing(makeTask({ sandboxId: null })); + + expect(emitError).not.toHaveBeenCalled(); + expect(markFailed.mock.calls.map((c) => (c[0] as { inboxId: string }).inboxId)).toEqual([ + "a", + "b", + ]); + expect(postIssueComment).toHaveBeenCalledWith( + "o", + "r", + 7, + WORKSPACE_MISSING_MESSAGE, + expect.any(AbortSignal), + ); + }); }); describe("enqueueComment", () => { @@ -310,26 +529,28 @@ describe("InboxService", () => { createdAt: "2026-04-23T10:00:00Z", }); - const arg = appendInbox.mock.calls[0]?.[0] as { + const arg = followUp.mock.calls[0]?.[0] as { + task: { id: string; key: string }; message: { source: string; kind: string; external_id: string; - payload: { target_id: string; task_key: string; created_at: string }; + payload: { target_id: string; created_at: string }; }; - replyTargets: Array<{ id: string }>; + targets: Array<{ id: string }>; + prUrl: string; }; expect(arg.message.source).toBe("github"); expect(arg.message.kind).toBe("comment"); expect(arg.message.external_id).toBe("555"); expect(arg.message.payload.target_id).toBe("555"); - expect(arg.message.payload).toMatchObject({ - task_id: "task-1", - task_key: "XYZ-1", - }); + // followUp stamps task_id / task_key from the task. + expect(arg.task).toMatchObject({ id: "task-1", key: "XYZ-1" }); // The comment's created_at is carried into the payload for temporal scoping. expect(arg.message.payload.created_at).toBe("2026-04-23T10:00:00Z"); - expect(arg.replyTargets[0]?.id).toBe("555"); + expect(arg.targets[0]?.id).toBe("555"); + // Dropped as stale if a restart has since cleared the PR from the task. + expect(arg.prUrl).toBe("https://github.com/o/r/pull/7"); }); }); @@ -350,7 +571,7 @@ describe("InboxService", () => { createdAt: "2026-04-23T10:00:00Z", }); - const arg = appendInbox.mock.calls[0]?.[0] as { + const arg = followUp.mock.calls[0]?.[0] as { message: { kind: string; payload: { @@ -366,7 +587,7 @@ describe("InboxService", () => { }>; }; }; - replyTargets: Array<{ + targets: Array<{ id: string; spec: { kind: string; @@ -385,7 +606,7 @@ describe("InboxService", () => { pr_url: "https://github.com/o/r/pull/7", }); // 2 inline + 1 review_body - expect(arg.replyTargets).toHaveLength(3); + expect(arg.targets).toHaveLength(3); expect(arg.message.payload.inline_comments[0]).toMatchObject({ target_id: "11", path: "a.ts", @@ -393,11 +614,11 @@ describe("InboxService", () => { start_line: 1, }); expect(arg.message.payload.inline_comments[1]?.start_line).toBeUndefined(); - expect(arg.replyTargets[0]).toMatchObject({ + expect(arg.targets[0]).toMatchObject({ id: "11", spec: { kind: "review_comment", startLine: 1 }, }); - expect(arg.replyTargets[1]?.spec.startLine).toBeUndefined(); + expect(arg.targets[1]?.spec.startLine).toBeUndefined(); }); }); @@ -410,24 +631,22 @@ describe("InboxService", () => { body: "any update?", }); - const arg = appendInbox.mock.calls[0]?.[0] as { + const arg = followUp.mock.calls[0]?.[0] as { + task: { id: string; key: string }; message: { source: string; external_id: string; - payload: { task_key: string; target_id: string }; + payload: { target_id: string }; }; - replyTargets: Array<{ id: string; spec: { kind: string } }>; + targets: Array<{ id: string; spec: { kind: string } }>; }; expect(arg.message.source).toBe("linear"); expect(arg.message.external_id).toBe("act-1"); - expect(arg.message.payload).toMatchObject({ - task_id: "task-1", - task_key: "XYZ-1", - }); + expect(arg.task).toMatchObject({ id: "task-1", key: "XYZ-1" }); // Replies route to the session, not the activity. expect(arg.message.payload.target_id).toBe("sess-1"); - expect(arg.replyTargets[0]?.id).toBe("sess-1"); - expect(arg.replyTargets[0]?.spec.kind).toBe("linear_session"); + expect(arg.targets[0]?.id).toBe("sess-1"); + expect(arg.targets[0]?.spec.kind).toBe("linear_session"); }); }); @@ -448,7 +667,8 @@ describe("InboxService", () => { prNumber: 42, }); - const arg = appendInbox.mock.calls[0]?.[0] as { + const arg = followUp.mock.calls[0]?.[0] as { + task: { id: string; key: string }; message: { source: string; kind: string; @@ -461,11 +681,10 @@ describe("InboxService", () => { run_url: string; head_sha: string; head_branch: string; - task_key: string; pr: { repo: string; pr_url: string }; }; }; - replyTargets: Array<{ + targets: Array<{ id: string; spec: { kind: string; owner: string; repo: string; prNumber: number; commentId: number }; }>; @@ -480,57 +699,27 @@ describe("InboxService", () => { expect(arg.message.payload.run_url).toBe("https://github.com/o/r/actions/runs/123456789"); expect(arg.message.payload.head_sha).toBe("abc123"); expect(arg.message.payload.head_branch).toBe("devbox/XYZ-1234-fix"); - expect(arg.message.payload).toMatchObject({ - task_id: "task-1", - task_key: "XYZ-1", - }); + expect(arg.task).toMatchObject({ id: "task-1", key: "XYZ-1" }); expect(arg.message.payload.pr).toEqual({ repo: "r", pr_url: "https://github.com/o/r/pull/42", }); - expect(arg.replyTargets).toHaveLength(1); - expect(arg.replyTargets[0]).toMatchObject({ + expect(arg.targets).toHaveLength(1); + expect(arg.targets[0]).toMatchObject({ id: "123456789", spec: { kind: "issue_comment", owner: "o", repo: "r", prNumber: 42, commentId: 0 }, }); }); - - const ciParams = { - task: makeTask(), - runId: 123456789, - runAttempt: 2, - workflowName: "CI", - workflowPath: ".github/workflows/ci.yml", - conclusion: "failure", - runUrl: "https://github.com/o/r/actions/runs/123456789", - headSha: "abc123", - headBranch: "devbox/XYZ-1234-fix", - owner: "o", - repo: "r", - prNumber: 42, - }; - - it("enqueues when under the loop-breaker threshold", async () => { - countCiResultsByTask.mockResolvedValue(19); // < 20 - await service.enqueueCiResult(ciParams); - expect(appendInbox).toHaveBeenCalledTimes(1); - }); - - it("drops the CI failure once the loop-breaker threshold is reached", async () => { - countCiResultsByTask.mockResolvedValue(20); // >= 20 - await service.enqueueCiResult(ciParams); - expect(appendInbox).not.toHaveBeenCalled(); - }); }); - describe("enqueue paths carry task_id and task_key from the task row", () => { + // followUp stamps task_id and task_key from the task it is handed. + describe("enqueue paths hand followUp the task row", () => { const apiTask = makeTask({ id: "task-api", source: "api", key: "DBX-API-7", externalUrl: "https://cp.example/api/tasks/DBX-API-7", }); - const identity = { task_id: "task-api", task_key: "DBX-API-7" }; const spec = { kind: "issue_comment" as const, owner: "o", @@ -547,8 +736,8 @@ describe("InboxService", () => { spec, createdAt: "2026-10-07T12:00:00Z", }); - expect(appendInbox).toHaveBeenCalledWith({ - taskId: "task-api", + expect(followUp).toHaveBeenCalledWith({ + task: apiTask, message: { source: "github", kind: "comment", @@ -558,10 +747,10 @@ describe("InboxService", () => { body: "@harness fix", pr: { repo: "r", pr_url: "https://github.com/o/r/pull/7" }, created_at: "2026-10-07T12:00:00Z", - ...identity, }, }, - replyTargets: [{ id: "555", spec }], + targets: [{ id: "555", spec }], + prUrl: "https://github.com/o/r/pull/7", }); }); @@ -577,8 +766,8 @@ describe("InboxService", () => { reviewerLogin: "rev", createdAt: "2026-10-07T12:00:00Z", }); - expect(appendInbox).toHaveBeenCalledWith({ - taskId: "task-api", + expect(followUp).toHaveBeenCalledWith({ + task: apiTask, message: { source: "github", kind: "review", @@ -591,10 +780,10 @@ describe("InboxService", () => { pr: { owner: "o", repo: "r", prNumber: 7, pr_url: "https://github.com/o/r/pull/7" }, reviewer_login: "rev", created_at: "2026-10-07T12:00:00Z", - ...identity, }, }, - replyTargets: [ + prUrl: "https://github.com/o/r/pull/7", + targets: [ { id: "11", spec: { @@ -632,8 +821,8 @@ describe("InboxService", () => { repo: "r", prNumber: 7, }); - expect(appendInbox).toHaveBeenCalledWith({ - taskId: "task-api", + expect(followUp).toHaveBeenCalledWith({ + task: apiTask, message: { source: "github", kind: "ci_result", @@ -647,10 +836,10 @@ describe("InboxService", () => { head_sha: "abc", head_branch: "devbox/DBX-API-7-x", pr: { repo: "r", pr_url: "https://github.com/o/r/pull/7" }, - ...identity, }, }, - replyTargets: [ + prUrl: "https://github.com/o/r/pull/7", + targets: [ { id: "9", spec: { kind: "issue_comment", owner: "o", repo: "r", prNumber: 7, commentId: 0 }, @@ -660,14 +849,15 @@ describe("InboxService", () => { }); it("enqueueLinearComment", async () => { + const task = makeTask({ id: "task-1", key: "XYZ-1" }); await service.enqueueLinearComment({ - task: makeTask({ id: "task-1", key: "XYZ-1" }), + task, agentActivityId: "act-1", agentSessionId: "sess-2", body: "follow-up", }); - expect(appendInbox).toHaveBeenCalledWith({ - taskId: "task-1", + expect(followUp).toHaveBeenCalledWith({ + task, message: { source: "linear", kind: "comment", @@ -677,11 +867,9 @@ describe("InboxService", () => { body: "follow-up", pr: null, created_at: "", - task_id: "task-1", - task_key: "XYZ-1", }, }, - replyTargets: [{ id: "sess-2", spec: { kind: "linear_session" } }], + targets: [{ id: "sess-2", spec: { kind: "linear_session" } }], }); }); }); diff --git a/control-plane/src/services/__tests__/integrationHooks.test.ts b/control-plane/src/services/__tests__/integrationHooks.test.ts new file mode 100644 index 0000000..e91d622 --- /dev/null +++ b/control-plane/src/services/__tests__/integrationHooks.test.ts @@ -0,0 +1,214 @@ +import { describe, expect, it, mock, spyOn } from "bun:test"; +import type { Task } from "../../db/schema"; +import { fakeIntegrations } from "../../integrations/__tests__/fakeIntegrations"; +import type { HookCtx, Integration } from "../../integrations/types"; +import type { InboxRepository, InboxRow } from "../../repositories/inbox.repository"; + +const { IntegrationHooks } = await import("../integrationHooks"); +const { RetryableError } = await import("../../integrations/outbound"); +const { logger } = await import("../../logger"); + +const task = { id: "task-1", source: "linear" } as Task; +const row = (id: string, kind = "assignment", source = "linear") => + ({ + id, + kind, + source, + reply_targets: [{ id: "sess-1", spec: { kind: "linear_session" } }], + }) as unknown as InboxRow; +const noop: Integration = { name: "api", enabled: true, async deliver() {}, async status() {} }; +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); + +function hooksWith( + linear: Partial, + github: Partial = {}, + timeoutMs?: number, +) { + const inboxRepo = { getAssignment: mock(() => Promise.resolve(row("a-1"))) }; + const integrations = fakeIntegrations({ + linear: { ...noop, name: "linear" as const, ...linear }, + github: { ...noop, name: "github" as const, ...github }, + }); + const t = timeoutMs; + const timeouts = + t === undefined + ? undefined + : { onTaskStarted: t, onPrOpened: t, onRoundFinished: t, assignee: t }; + return { + hooks: new IntegrationHooks(integrations, inboxRepo as unknown as InboxRepository, timeouts), + inboxRepo, + }; +} + +describe("IntegrationHooks", () => { + it("never throws: a failing hook is logged and yields undefined", async () => { + const error = spyOn(logger, "error").mockImplementation(() => logger); + try { + const { hooks } = hooksWith({ + assignee: () => Promise.reject(new RetryableError("token fetch failed")), + }); + expect(await hooks.assignee(task, row("a-1"), { prUrl: "u" })).toBeUndefined(); + expect(error).toHaveBeenCalledWith( + "assignee hook failed", + expect.objectContaining({ taskId: "task-1", integration: "linear" }), + ); + } finally { + error.mockRestore(); + } + }); + + it("skips a disabled integration's hooks: logged, never called, never held", async () => { + const info = spyOn(logger, "info").mockImplementation(() => logger); + try { + const assignee = mock(() => Promise.resolve(null)); + const onPrOpened = mock(() => Promise.reject(new RetryableError("429"))); + const { hooks } = hooksWith({ enabled: false, assignee, onPrOpened }); + const pr = { url: "u", repo: "r", branch: "b" }; + expect(await hooks.assignee(task, row("a-1"), {})).toBeUndefined(); + expect(await hooks.prOpened(task, row("a-1"), pr, true)).toBeUndefined(); + expect(assignee).not.toHaveBeenCalled(); + expect(onPrOpened).not.toHaveBeenCalled(); + expect(info).toHaveBeenCalledWith( + "assignee skipped: integration disabled", + expect.objectContaining({ taskId: "task-1", integration: "linear" }), + ); + } finally { + info.mockRestore(); + } + }); + + it("onPrOpened alone lets a RetryableError escape, so pr_opened holds", async () => { + const error = spyOn(logger, "error").mockImplementation(() => logger); + try { + const pr = { url: "u", repo: "r", branch: "b" }; + const retry = hooksWith({ onPrOpened: () => Promise.reject(new RetryableError("429")) }); + await expect(retry.hooks.prOpened(task, row("a-1"), pr, true)).rejects.toThrow("429"); + const other = hooksWith({ onPrOpened: () => Promise.reject(new Error("HTTP 500")) }); + expect(await other.hooks.prOpened(task, row("a-1"), pr, true)).toBeUndefined(); + } finally { + error.mockRestore(); + } + }); + + it("cuts a hung hook at its timeout: the call resolves and the hook's signal aborts", async () => { + const error = spyOn(logger, "error").mockImplementation(() => logger); + try { + let signal: AbortSignal | undefined; + const { hooks } = hooksWith( + { + assignee: (ctx) => { + signal = ctx.signal; + return new Promise(() => {}); + }, + }, + {}, + 20, + ); + expect(await hooks.assignee(task, row("a-1"), {})).toBeUndefined(); + expect(signal?.aborted).toBe(true); + } finally { + error.mockRestore(); + } + }); + + it("a failed assignment lookup still runs the hook, with isCurrent() false", async () => { + const error = spyOn(logger, "error").mockImplementation(() => logger); + try { + let ctx: HookCtx | undefined; + const { hooks, inboxRepo } = hooksWith({ + async assignee(c) { + ctx = c; + return null; + }, + }); + inboxRepo.getAssignment.mockRejectedValue(new Error("connection reset")); + expect(await hooks.assignee(task, row("c-1", "comment"), {})).toBeNull(); + expect(await ctx?.isCurrent()).toBe(false); + } finally { + error.mockRestore(); + } + }); + + it("isCurrent() turns false once a restart rotated the assignment id", async () => { + let ctx: HookCtx | undefined; + const { hooks, inboxRepo } = hooksWith({ + async assignee(c) { + ctx = c; + return null; + }, + }); + // A comment round: the baseline is the task's assignment id at hook start. + await hooks.assignee(task, row("c-1", "comment"), {}); + expect(await ctx?.isCurrent()).toBe(true); + inboxRepo.getAssignment.mockResolvedValue(row("a-2")); + expect(await ctx?.isCurrent()).toBe(false); + }); + + it("onTaskStarted runs at most once per assignment id", async () => { + const onTaskStarted = mock(() => Promise.resolve()); + const { hooks, inboxRepo } = hooksWith({ onTaskStarted }); + await hooks.taskStarted(task); + await hooks.taskStarted(task); + expect(onTaskStarted).toHaveBeenCalledTimes(1); + // A restart re-pends the assignment under a new id: a new run starts. + inboxRepo.getAssignment.mockResolvedValue(row("a-2")); + await hooks.taskStarted(task); + expect(onTaskStarted).toHaveBeenCalledTimes(2); + }); + + it("drain also waits for work detached while it drains", async () => { + const warn = spyOn(logger, "warn").mockImplementation(() => logger); + try { + const { hooks } = hooksWith({}); + let finished = false; + hooks.detach( + "task-1", + "onRoundFinished", + sleep(10).then(() => + // A webhook still being served restarts a task meanwhile. + hooks.detach( + "task-2", + "killSandbox", + sleep(20).then(() => { + finished = true; + }), + ), + ), + ); + await hooks.drain(sleep(500)); + + expect(finished).toBe(true); + expect(warn).not.toHaveBeenCalled(); + } finally { + warn.mockRestore(); + } + }); + + it("onRoundFinished routes by round.source; drain logs hook_abandoned for what is still running", async () => { + const warn = spyOn(logger, "warn").mockImplementation(() => logger); + try { + const finished: string[] = []; + const record = async (_ctx: unknown, _task: Task, round: InboxRow) => { + finished.push(round.id); + }; + const { hooks } = hooksWith( + // Outlives the drain deadline below. + { onRoundFinished: (ctx, t, round) => sleep(200).then(() => record(ctx, t, round)) }, + { onRoundFinished: record }, + ); + hooks.roundFinished(task, row("gh-1", "review", "github"), null, true); + hooks.roundFinished(task, row("a-1"), "failed", true); + await hooks.drain(sleep(50)); + + expect(finished).toEqual(["gh-1"]); + expect(warn.mock.calls).toEqual([ + [ + "hook abandoned at shutdown", + { event: "hook_abandoned", taskId: "task-1", hook: "onRoundFinished" }, + ], + ]); + } finally { + warn.mockRestore(); + } + }); +}); diff --git a/control-plane/src/services/__tests__/replyHolds.test.ts b/control-plane/src/services/__tests__/replyHolds.test.ts new file mode 100644 index 0000000..1f227bc --- /dev/null +++ b/control-plane/src/services/__tests__/replyHolds.test.ts @@ -0,0 +1,84 @@ +import { afterEach, describe, expect, it, setSystemTime, spyOn } from "bun:test"; + +const { ReplyHolds } = await import("../replyHolds"); +const { RetryableError } = await import("../../integrations/outbound"); +const { logger } = await import("../../logger"); + +const T0 = new Date("2026-10-30T12:00:00Z").getTime(); +const at = (ms: number) => setSystemTime(new Date(T0 + ms)); +const event = (seq: number, inbox_id = "inbox-1") => ({ seq, inbox_id }); + +describe("ReplyHolds", () => { + afterEach(() => { + setSystemTime(); + }); + + it("without Retry-After the next attempt is the next tick; with it, after the delay", () => { + const holds = new ReplyHolds(); + at(0); + holds.hold("t1", event(1), "linear", new RetryableError("token fetch failed")); + expect(holds.waiting("t1")).toBe(false); + + holds.hold("t2", event(1), "github", new RetryableError("HTTP 429", 30_000)); + expect(holds.waiting("t2")).toBe(true); + at(29_999); + expect(holds.waiting("t2")).toBe(true); + at(30_000); + expect(holds.waiting("t2")).toBe(false); + }); + + it("warns reply_held at most every 10 minutes per task; a new event starts a new hold", () => { + const warn = spyOn(logger, "warn").mockImplementation(() => logger); + try { + const holds = new ReplyHolds(); + const err = new RetryableError("HTTP 429"); + at(0); + holds.hold("t1", event(5), "github", err); + at(9 * 60_000); + holds.hold("t1", event(5), "github", err); + at(10 * 60_000); + holds.hold("t1", event(5), "github", err); + expect(warn.mock.calls.map((c) => c[1])).toEqual([ + { + event: "reply_held", + taskId: "t1", + integration: "github", + heldForMin: 0, + reason: "HTTP 429", + }, + { + event: "reply_held", + taskId: "t1", + integration: "github", + heldForMin: 10, + reason: "HTTP 429", + }, + ]); + // The same task held on another event is a new hold: its age restarts. + at(15 * 60_000); + holds.hold("t1", event(6), "github", err); + expect(warn).toHaveBeenCalledTimes(3); + expect(holds.of("github")).toEqual({ heldTasks: 1, oldestHoldAgeMs: 0 }); + } finally { + warn.mockRestore(); + } + }); + + it("reports held tasks and the oldest hold age per integration; release and retain drop holds", () => { + const holds = new ReplyHolds(); + const err = new RetryableError("x"); + at(0); + holds.hold("a", event(1), "linear", err); + at(60_000); + holds.hold("b", event(1), "linear", err); + holds.hold("c", event(1), "github", err); + at(120_000); + expect(holds.of("linear")).toEqual({ heldTasks: 2, oldestHoldAgeMs: 120_000 }); + expect(holds.of("github")).toEqual({ heldTasks: 1, oldestHoldAgeMs: 60_000 }); + + holds.release("a"); + holds.retain(new Set(["a", "b"])); + expect(holds.of("linear")).toEqual({ heldTasks: 1, oldestHoldAgeMs: 60_000 }); + expect(holds.of("github")).toEqual({ heldTasks: 0, oldestHoldAgeMs: 0 }); + }); +}); diff --git a/control-plane/src/services/__tests__/sandboxManager.service.test.ts b/control-plane/src/services/__tests__/sandboxManager.service.test.ts index caa1e81..ca68ebe 100644 --- a/control-plane/src/services/__tests__/sandboxManager.service.test.ts +++ b/control-plane/src/services/__tests__/sandboxManager.service.test.ts @@ -833,6 +833,34 @@ describe("SandboxManager", () => { expect(err.body.length).toBeLessThanOrEqual(2001); }); + it("keeps the harness's retryable_assignment_failure flag on an agent_error only when it is true", async () => { + const line = (seq: number, flag: unknown) => + JSON.stringify({ + seq, + ts: "t", + type: "agent_error", + inbox_id: "i", + target_id: "tgt", + body: "RuntimeError: boom", + retryable_assignment_failure: flag, + }); + const events = await mgrWith([line(1, true), line(2, "true")].join("\n")).readEvents( + "sbx", + 0, + ); + expect(events).toHaveLength(2); + expect(events[0]).toEqual({ + seq: 1, + ts: "t", + type: "agent_error", + inbox_id: "i", + target_id: "tgt", + body: "RuntimeError: boom", + retryable_assignment_failure: true, + }); + expect("retryable_assignment_failure" in (events[1] ?? {})).toBe(false); + }); + it("agent_thought/action/response use the wider 60k cap (plan markdown + completion summaries can run long)", async () => { const huge = "y".repeat(5000); const raw = [ diff --git a/control-plane/src/services/__tests__/slackNotifier.service.test.ts b/control-plane/src/services/__tests__/slackNotifier.service.test.ts index 99975df..2a228da 100644 --- a/control-plane/src/services/__tests__/slackNotifier.service.test.ts +++ b/control-plane/src/services/__tests__/slackNotifier.service.test.ts @@ -1,5 +1,4 @@ import { beforeEach, describe, expect, it, mock } from "bun:test"; -import type { LinearClient } from "@linear/sdk"; import type { Task } from "../../db/schema"; import type { TaskRepository } from "../../repositories/task.repository"; @@ -39,23 +38,22 @@ const makeTask = (overrides: Partial = {}): Task => ({ ...overrides, }); -const client = {} as unknown as LinearClient; - describe("SlackNotifierService", () => { let post: ReturnType; - let resolveAssignee: ReturnType; + let assignee: ReturnType; let lookupSlackUser: ReturnType; let getById: ReturnType; let taskRepo: TaskRepository; - const make = (botToken = "xoxb-test", channel = "C1") => + const make = (botToken = "xoxb-test", channel = "C1", sources: Ctor[3] = ["linear"]) => new SlackNotifierService( taskRepo, botToken, channel, - post as unknown as Ctor[3], - resolveAssignee as unknown as Ctor[4], - lookupSlackUser as unknown as Ctor[5], + sources, + { assignee } as unknown as Ctor[4], + post as unknown as Ctor[5], + lookupSlackUser as unknown as Ctor[6], ); // The text posted by the single post() call. @@ -63,14 +61,20 @@ describe("SlackNotifierService", () => { beforeEach(() => { post = mock(() => Promise.resolve({ ts: "1" })); - resolveAssignee = mock(() => Promise.resolve(null)); + assignee = mock(() => Promise.resolve(null)); lookupSlackUser = mock(() => Promise.resolve(null)); getById = mock((id: string) => Promise.resolve(makeTask({ id, prUrls: [PR] }))); taskRepo = { getById } as unknown as TaskRepository; }); + it("bounds the post with an abort signal", async () => { + await make().notifyTerminal(makeTask(), "completed"); + const arg = (post.mock.calls[0] as unknown as [{ signal?: unknown }])[0]; + expect(arg.signal).toBeInstanceOf(AbortSignal); + }); + it("posts a ✅ completion into the configured channel with the ticket + PR links", async () => { - await make().notifyTerminal(client, makeTask(), "completed"); + await make().notifyTerminal(makeTask(), "completed"); expect(post).toHaveBeenCalledTimes(1); const arg = (post.mock.calls[0] as unknown as [{ channel: string; threadTs?: string }])[0]; @@ -84,7 +88,7 @@ describe("SlackNotifierService", () => { it("uses singular 'PR' with no bullet for a single PR", async () => { // Default getById returns exactly one PR. - await make().notifyTerminal(client, makeTask(), "completed"); + await make().notifyTerminal(makeTask(), "completed"); const text = postedText(); expect(text).toContain( "Here is the PR for review: ", @@ -99,7 +103,7 @@ describe("SlackNotifierService", () => { prUrls: [PR, { repo: "acme/web", url: "https://github.com/acme/web/pull/2" }], }), ); - await make().notifyTerminal(client, makeTask(), "completed"); + await make().notifyTerminal(makeTask(), "completed"); const text = postedText(); expect(text).toContain("Here are the PRs for review:"); expect(text).toContain("• "); @@ -107,29 +111,35 @@ describe("SlackNotifierService", () => { }); it("uses ⚠️ for partial_failure and still lists PRs", async () => { - await make().notifyTerminal(client, makeTask(), "partial_failure"); + await make().notifyTerminal(makeTask(), "partial_failure"); const text = postedText(); expect(text).toContain("⚠️"); expect(text).toContain("acme/api"); }); it("uses 💤 for no_action and omits PR lines", async () => { - await make().notifyTerminal(client, makeTask(), "no_action"); + await make().notifyTerminal(makeTask(), "no_action"); const text = postedText(); expect(text).toContain("💤"); expect(text).not.toContain("github.com"); }); it("uses ❌ for failed and includes the failure message", async () => { - await make().notifyTerminal(client, makeTask(), "failed", "sandbox ran out of memory"); + await make().notifyTerminal(makeTask(), "failed", "sandbox ran out of memory"); const text = postedText(); expect(text).toContain("❌"); expect(text).toContain("sandbox ran out of memory"); }); + it("labels the ticket link 'Linear ticket' when the task has no title", async () => { + getById.mockResolvedValue(makeTask({ title: "", prUrls: [PR] })); + await make().notifyTerminal(makeTask({ title: "" }), "completed"); + expect(postedText()).toContain(""); + }); + it("re-reads the task for the freshest pr_urls (stale in-memory task had none)", async () => { // The task handed in predates the pr_opened write; getById returns the PR. - await make().notifyTerminal(client, makeTask({ prUrls: [] }), "completed"); + await make().notifyTerminal(makeTask({ prUrls: [] }), "completed"); expect(getById).toHaveBeenCalledWith("task-1"); expect(postedText()).toContain("github.com/acme/api/pull/1"); }); @@ -138,44 +148,65 @@ describe("SlackNotifierService", () => { // A DB blip on the optional freshness re-read must NOT drop the whole post — // the thread is resolved and the in-memory task already carries the PR. getById.mockRejectedValue(new Error("db connection reset")); - await make().notifyTerminal(client, makeTask({ prUrls: [PR] }), "completed"); + await make().notifyTerminal(makeTask({ prUrls: [PR] }), "completed"); expect(post).toHaveBeenCalledTimes(1); expect(postedText()).toContain(""); }); it("no-ops when Slack is disabled (empty token) — never posts", async () => { - await make("").notifyTerminal(client, makeTask(), "completed"); + await make("").notifyTerminal(makeTask(), "completed"); expect(post).not.toHaveBeenCalled(); }); it("no-ops when no channel is configured", async () => { - await make("xoxb-test", "").notifyTerminal(client, makeTask(), "completed"); + await make("xoxb-test", "").notifyTerminal(makeTask(), "completed"); expect(post).not.toHaveBeenCalled(); }); it("skips a non-linear task", async () => { - await make().notifyTerminal( - client, - makeTask({ source: "github" as Task["source"] }), - "completed", - ); + await make().notifyTerminal(makeTask({ source: "github" as Task["source"] }), "completed"); expect(post).not.toHaveBeenCalled(); }); + it("skips an API task unless SLACK_NOTIFY_SOURCES has api", async () => { + const notifier = make(); + expect(notifier.notifies("api")).toBe(false); + await notifier.notifyTerminal(makeTask({ source: "api" }), "completed"); + expect(post).not.toHaveBeenCalled(); + }); + + it("posts for an opted-in API task: labelled by its key when untitled, never nudged", async () => { + const task = makeTask({ + source: "api", + key: "DBX-API-7", + title: "", + externalUrl: "https://devbox.example.com/api/tasks/DBX-API-7", + }); + getById.mockResolvedValue({ ...task, prUrls: [PR] }); + await make("xoxb-test", "C1", ["linear", "api"]).notifyTerminal(task, "completed"); + expect(postedText()).toContain(""); + expect(assignee).not.toHaveBeenCalled(); + }); + describe("connect-GitHub nudge", () => { + // Linear's assignee() for a member who hasn't connected GitHub in Linear. const unmapped = { id: "usr-1", - name: "Ada Lovelace", displayName: "ada", email: "ada@acme.com", - gitHubUserId: null, + codeHost: "unlinked", }; + const mapped = { ...unmapped, codeHost: { githubLogin: "ada" } }; it("appends the nudge when the human assignee has no GitHub connected and PRs exist", async () => { - resolveAssignee.mockResolvedValue(unmapped); - await make().notifyTerminal(client, makeTask(), "completed"); + assignee.mockResolvedValue(unmapped); + await make().notifyTerminal(makeTask(), "completed"); const text = postedText(); - expect(resolveAssignee).toHaveBeenCalledWith(client, "issue-1"); + expect(assignee).toHaveBeenCalledWith( + expect.objectContaining({ id: "task-1" }), + null, + expect.anything(), + ); expect(text).toContain("⚠️ ada: I couldn't assign the PR to you"); expect(text).toContain("*Linear → Settings → Connected accounts*"); // The nudge rides directly under the PR list. @@ -185,78 +216,79 @@ describe("SlackNotifierService", () => { }); it("omits the nudge when the assignee is already mapped", async () => { - resolveAssignee.mockResolvedValue({ ...unmapped, gitHubUserId: "583231" }); - await make().notifyTerminal(client, makeTask(), "completed"); + assignee.mockResolvedValue(mapped); + await make().notifyTerminal(makeTask(), "completed"); expect(postedText()).not.toContain("Connected accounts"); }); it("omits the nudge when there is no human assignee", async () => { - resolveAssignee.mockResolvedValue(null); - await make().notifyTerminal(client, makeTask(), "completed"); + assignee.mockResolvedValue(null); + await make().notifyTerminal(makeTask(), "completed"); expect(postedText()).not.toContain("Connected accounts"); }); it("never checks the assignee when there are no PRs (nothing to assign)", async () => { getById.mockResolvedValue(makeTask({ prUrls: [] })); - await make().notifyTerminal(client, makeTask(), "completed"); - expect(resolveAssignee).not.toHaveBeenCalled(); + await make().notifyTerminal(makeTask(), "completed"); + expect(assignee).not.toHaveBeenCalled(); expect(postedText()).not.toContain("Connected accounts"); }); it("still posts (without the nudge) when the assignee lookup fails", async () => { - resolveAssignee.mockRejectedValue(new Error("linear down")); - await make().notifyTerminal(client, makeTask(), "completed"); + // IntegrationHooks.assignee logs the failure and yields undefined. + assignee.mockResolvedValue(undefined); + await make().notifyTerminal(makeTask(), "completed"); expect(post).toHaveBeenCalledTimes(1); expect(postedText()).not.toContain("Connected accounts"); }); it("appends the nudge on partial_failure and failed too (PRs are listed there)", async () => { - resolveAssignee.mockResolvedValue(unmapped); - await make().notifyTerminal(client, makeTask(), "failed", "boom"); + assignee.mockResolvedValue(unmapped); + await make().notifyTerminal(makeTask(), "failed", "boom"); expect(postedText()).toContain("I couldn't assign the PR"); }); it("@-mentions the assignee when their Slack id resolves from the Linear email", async () => { - resolveAssignee.mockResolvedValue(unmapped); + assignee.mockResolvedValue(unmapped); lookupSlackUser.mockResolvedValue("U0ADA"); - await make().notifyTerminal(client, makeTask(), "completed"); - expect(lookupSlackUser).toHaveBeenCalledWith("ada@acme.com"); + await make().notifyTerminal(makeTask(), "completed"); + expect(lookupSlackUser).toHaveBeenCalledWith("ada@acme.com", expect.any(AbortSignal)); const text = postedText(); expect(text).toContain("⚠️ <@U0ADA>: I couldn't assign the PR"); expect(text).not.toContain("⚠️ ada:"); }); it("falls back to the display name when the Slack lookup finds nobody", async () => { - resolveAssignee.mockResolvedValue(unmapped); + assignee.mockResolvedValue(unmapped); lookupSlackUser.mockResolvedValue(null); - await make().notifyTerminal(client, makeTask(), "completed"); + await make().notifyTerminal(makeTask(), "completed"); expect(postedText()).toContain("⚠️ ada: I couldn't assign the PR"); }); it("falls back to the display name when the Slack lookup throws", async () => { - resolveAssignee.mockResolvedValue(unmapped); + assignee.mockResolvedValue(unmapped); lookupSlackUser.mockRejectedValue(new Error("missing_scope")); - await make().notifyTerminal(client, makeTask(), "completed"); + await make().notifyTerminal(makeTask(), "completed"); expect(postedText()).toContain("⚠️ ada: I couldn't assign the PR"); }); it("never looks up Slack when the assignee is already mapped", async () => { - resolveAssignee.mockResolvedValue({ ...unmapped, gitHubUserId: "583231" }); - await make().notifyTerminal(client, makeTask(), "completed"); + assignee.mockResolvedValue(mapped); + await make().notifyTerminal(makeTask(), "completed"); expect(lookupSlackUser).not.toHaveBeenCalled(); }); it("skips the Slack lookup when the assignee has no email (falls back to name)", async () => { - resolveAssignee.mockResolvedValue({ ...unmapped, email: "" }); - await make().notifyTerminal(client, makeTask(), "completed"); + assignee.mockResolvedValue({ ...unmapped, email: "" }); + await make().notifyTerminal(makeTask(), "completed"); expect(lookupSlackUser).not.toHaveBeenCalled(); expect(postedText()).toContain("⚠️ ada:"); }); it("escapes mrkdwn in the display-name fallback (no injection)", async () => { - resolveAssignee.mockResolvedValue({ ...unmapped, displayName: " <@U999>" }); + assignee.mockResolvedValue({ ...unmapped, displayName: " <@U999>" }); lookupSlackUser.mockResolvedValue(null); - await make().notifyTerminal(client, makeTask(), "completed"); + await make().notifyTerminal(makeTask(), "completed"); const text = postedText(); expect(text).toContain("<!channel> <@U999>"); expect(text).not.toContain(""); @@ -266,10 +298,10 @@ describe("SlackNotifierService", () => { it("never checks the assignee on no_action, even when the task carries PRs", async () => { // no_action renders no PR section, so there is nothing for the nudge to // ride on — the Linear/Slack lookups must be skipped entirely. - resolveAssignee.mockResolvedValue(unmapped); + assignee.mockResolvedValue(unmapped); getById.mockResolvedValue(makeTask({ prUrls: [PR] })); - await make().notifyTerminal(client, makeTask(), "no_action"); - expect(resolveAssignee).not.toHaveBeenCalled(); + await make().notifyTerminal(makeTask(), "no_action"); + expect(assignee).not.toHaveBeenCalled(); expect(postedText()).not.toContain("Connected accounts"); }); }); diff --git a/control-plane/src/services/__tests__/taskProcessor.service.test.ts b/control-plane/src/services/__tests__/taskProcessor.service.test.ts index 95e632a..2e22275 100644 --- a/control-plane/src/services/__tests__/taskProcessor.service.test.ts +++ b/control-plane/src/services/__tests__/taskProcessor.service.test.ts @@ -1,9 +1,11 @@ -import { beforeEach, describe, expect, it, mock, spyOn } from "bun:test"; +import { afterEach, beforeEach, describe, expect, it, mock, spyOn } from "bun:test"; import type { SandboxInfo } from "@alibaba-group/opensandbox"; import { SandboxApiException } from "@alibaba-group/opensandbox"; import type { LinearClient } from "@linear/sdk"; import type { Task } from "../../db/schema"; import { PvcNotFoundError, SandboxCreateError } from "../../errors"; +import { fakeIntegrations } from "../../integrations/__tests__/fakeIntegrations"; +import type { Integration, IntegrationName } from "../../integrations/types"; import type { InboxRepository } from "../../repositories/inbox.repository"; import type { TaskRepository } from "../../repositories/task.repository"; import type { HarnessEvent } from "../../types/harnessEvent"; @@ -73,11 +75,13 @@ mock.module("../../clients/linear", () => ({ // only). Mock the barrel before importing, like the other consumers' tests. // The mock leaks to later files too, so it stubs every export they import. // See control-plane/README.md (Conventions). +const postIssueComment = mock(() => Promise.resolve({ id: 1, url: "u" })); +const postReviewCommentReply = mock(() => Promise.resolve({ id: 1, url: "u" })); mock.module("../../clients/github", () => ({ addIssueCommentReaction: mock(() => Promise.resolve()), addReviewCommentReaction: mock(() => Promise.resolve()), - postIssueComment: mock(() => Promise.resolve({ id: 1, url: "u" })), - postReviewCommentReply: mock(() => Promise.resolve({ id: 1, url: "u" })), + postIssueComment, + postReviewCommentReply, getReviewComments: mock(() => Promise.resolve([])), fetchOrgMembers: mock(() => Promise.resolve(new Set())), addPrAssignees: mock(() => Promise.resolve({ assigned: [] })), @@ -88,7 +92,21 @@ mock.module("../../clients/github", () => ({ })); const { TaskProcessorService } = await import("../taskProcessor.service"); +const { LinearIntegration } = await import("../../integrations/linear"); +const { GithubIntegration } = await import("../../integrations/github"); +const { ApiIntegration } = await import("../../integrations/api"); const { logger } = await import("../../logger"); +const { ReplyHolds } = await import("../replyHolds"); +const { IntegrationHooks } = await import("../integrationHooks"); +const { EventDispatcher: RealEventDispatcher } = await import("../eventDispatcher"); +const { InboxService: RealInboxService, WORKSPACE_MISSING_MESSAGE } = await import( + "../inbox.service" +); +const { config: mockedConfig } = (await import("../../config")) as unknown as { + config: { shutdownDrainMs: number; leaseTimeoutMs: number }; +}; +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); +const { RetryableError } = await import("../../integrations/outbound"); const makeTask = (overrides: Partial = {}): Task => ({ id: "task-1", @@ -172,6 +190,11 @@ describe("TaskProcessorService", () => { let inboxService: InboxService; let notifyTerminal: ReturnType; let slackNotifier: SlackNotifierService; + let holds: InstanceType; + let integrations: Record & { + linear: InstanceType; + }; + let hooks: InstanceType; let service: InstanceType; beforeEach(() => { @@ -183,8 +206,13 @@ describe("TaskProcessorService", () => { emitThought.mockResolvedValue(undefined); emitError.mockReset(); emitError.mockResolvedValue(undefined); + postIssueComment.mockReset(); + postIssueComment.mockResolvedValue({ id: 1, url: "u" }); + postReviewCommentReply.mockReset(); + postReviewCommentReply.mockResolvedValue({ id: 1, url: "u" }); - heartbeatOwnedBy = mock(() => Promise.resolve()); + // Renews every task the tests reconcile. + heartbeatOwnedBy = mock(() => Promise.resolve([{ id: "task-1" }, { id: "task-2" }])); claimNext = mock(() => Promise.resolve(null)); attachSandboxResources = mock(() => Promise.resolve([{ id: "task-1" }])); backdateHeartbeatsOwnedBy = mock(() => Promise.resolve()); @@ -214,7 +242,7 @@ describe("TaskProcessorService", () => { failTaskInbox = mock(() => Promise.resolve([] as Array<{ id: string }>)); inboxRepo = { failTaskInbox, - getAssignmentReplyTargetId: mock(() => Promise.resolve("sess-1")), + getAssignment: mock(() => Promise.resolve(survivingNonTerminalRow())), // Default: one SURVIVING non-terminal row (with a Linear session reply // target). retryTask now treats "no surviving non-terminal row" as a clean // terminal (no requeue), so the retry-path tests need work left to resume; @@ -260,25 +288,40 @@ describe("TaskProcessorService", () => { inboxService = { sweepUndelivered, deadLetterWorkspaceMissing } as unknown as InboxService; notifyTerminal = mock(() => Promise.resolve()); - slackNotifier = { notifyTerminal } as unknown as SlackNotifierService; - + // Slack on with the default SLACK_NOTIFY_SOURCES (linear). + slackNotifier = { + notifyTerminal, + notifies: (source: string) => source === "linear", + } as unknown as SlackNotifierService; + + holds = new ReplyHolds(); + integrations = fakeIntegrations({ + linear: new LinearIntegration(linearTokenProvider, true, { + status: async () => ({ configured: false }), + }), + api: new ApiIntegration(true), + github: new GithubIntegration(true, { status: () => ({}) as never }), + }); + hooks = new IntegrationHooks(integrations, inboxRepo); service = new TaskProcessorService( taskRepo, inboxRepo, - linearTokenProvider, sandboxManager, eventDispatcher, inboxService, slackNotifier, + integrations, + holds, + ["linear", "api"], + hooks, ); }); - // failTask broadcasts to every non-terminal row's reply targets — stub one - // Linear session target so the user-facing failure message reaches emitError. + // failTask broadcasts to the reply targets of the rows failTaskInbox + // dead-letters — stub one with a Linear session target so the user-facing + // failure message reaches emitError. const stubLinearTarget = (task: Task): void => { - (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValueOnce([ - survivingNonTerminalRow(task.id, task.externalId), - ]); + failTaskInbox.mockResolvedValueOnce([survivingNonTerminalRow(task.id, task.externalId)]); }; describe("API tasks", () => { @@ -305,7 +348,7 @@ describe("TaskProcessorService", () => { it("failTask posts nothing to an api_session target and still dead-letters", async () => { const t = apiTask(); - (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValue([ + failTaskInbox.mockResolvedValue([ { ...survivingNonTerminalRow(t.id, t.externalId), source: "api", @@ -328,12 +371,12 @@ describe("TaskProcessorService", () => { }); describe("runTask — fresh provisioning", () => { - it("creates a sandbox, stamps sandbox_id, asserts InProgress + emits IDs thought", async () => { + it("creates a sandbox, stamps sandbox_id, asserts InProgress, emits no IDs thought", async () => { const task = makeTask({ sandboxId: null }); claimNext.mockResolvedValueOnce(task).mockResolvedValue(null); service.start(); - await waitFor(() => emitThought.mock.calls.length > 0); + await waitFor(() => attachSandboxResources.mock.calls.length > 0); await service.stop(); expect(createSandboxWithPvc).toHaveBeenCalledWith( @@ -347,6 +390,8 @@ describe("TaskProcessorService", () => { }); expect(setIssueStarted).toHaveBeenCalledWith(expect.anything(), "issue-1"); expect(emitAction).toHaveBeenCalledWith(expect.anything(), "sess-1", "Provisioning", "infra"); + // The post-create "Sandbox provisioned" thought is gone. + expect(emitThought).not.toHaveBeenCalled(); }); it("logs and still provisions when the team has no started state", async () => { @@ -359,7 +404,7 @@ describe("TaskProcessorService", () => { try { service.start(); - await waitFor(() => emitThought.mock.calls.length > 0); + await waitFor(() => attachSandboxResources.mock.calls.length > 0); await service.stop(); expect(error).toHaveBeenCalledTimes(1); @@ -381,6 +426,42 @@ describe("TaskProcessorService", () => { expect(failTaskInbox).not.toHaveBeenCalled(); }); + it("runs the owner's onTaskStarted once per assignment id, before the sandbox is created", async () => { + const order: string[] = []; + const onTaskStarted = mock(async () => { + order.push("started"); + }); + (integrations.linear as Integration).onTaskStarted = onTaskStarted; + const assignment = survivingNonTerminalRow(); + (inboxRepo as { getAssignment?: unknown }).getAssignment = mock(() => + Promise.resolve(assignment), + ); + // The first provision defers on a transient 503; the re-claim provisions + // the same assignment again. + createSandboxWithPvc + .mockImplementationOnce(() => { + order.push("create"); + return Promise.reject(new SandboxApiException({ statusCode: 503 })); + }) + .mockImplementation(() => { + order.push("create"); + return Promise.resolve({ sandboxId: "sbx-1", pvcName: "devbox-pvc-task-1-aaaaaa" }); + }); + const task = makeTask({ sandboxId: null }); + claimNext.mockResolvedValueOnce(task).mockResolvedValueOnce(task).mockResolvedValue(null); + + service.start(); + await waitFor(() => attachSandboxResources.mock.calls.length > 0); + await service.stop(); + + expect(order).toEqual(["started", "create", "create"]); + expect(onTaskStarted).toHaveBeenCalledWith( + expect.anything(), + task, + assignment.reply_targets[0], + ); + }); + it("threads experimental flags from task metadata into sandbox creation", async () => { const task = makeTask({ sandboxId: null, @@ -455,9 +536,9 @@ describe("TaskProcessorService", () => { createSandboxWithPvc.mockImplementation(() => Promise.reject(new Error("docker daemon gone with token=abcd")), ); - // failTask broadcasts the message to every non-terminal row's reply - // targets — stub one row with the assignment row's Linear session target. - (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValueOnce([ + // failTask broadcasts the message to the reply targets of the rows it + // dead-letters — stub one row with the assignment row's Linear session target. + failTaskInbox.mockResolvedValue([ { id: "inbox-1", task_id: task.id, @@ -493,17 +574,15 @@ describe("TaskProcessorService", () => { const task = makeTask({ sandboxId: null }); claimNext.mockResolvedValueOnce(task).mockResolvedValue(null); createSandboxWithPvc.mockImplementation(() => Promise.reject(new Error("provision boom"))); - stubLinearTarget(task); // failTaskInbox returning a non-empty set = rows actually dead-lettered on // THIS call; that transition is the gate for the (at-most-once) Slack post. - failTaskInbox.mockResolvedValueOnce([{ id: "inbox-1" }]); + stubLinearTarget(task); service.start(); await waitFor(() => notifyTerminal.mock.calls.length > 0); await service.stop(); expect(notifyTerminal).toHaveBeenCalledWith( - expect.anything(), expect.objectContaining({ id: "task-1" }), "failed", expect.stringContaining("control-plane logs"), @@ -532,10 +611,7 @@ describe("TaskProcessorService", () => { created_at: new Date(), updated_at: new Date(), }; - (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValueOnce([ - commentRow, - ]); - failTaskInbox.mockResolvedValueOnce([{ id: "inbox-c" }]); + failTaskInbox.mockResolvedValueOnce([commentRow]); service.start(); await waitFor(() => releaseLease.mock.calls.length > 0); @@ -549,7 +625,6 @@ describe("TaskProcessorService", () => { const task = makeTask({ sandboxId: null }); claimNext.mockResolvedValueOnce(task).mockResolvedValue(null); createSandboxWithPvc.mockImplementation(() => Promise.reject(new Error("provision boom"))); - stubLinearTarget(task); // Default failTaskInbox returns [] → no transition → no Slack post. service.start(); await waitFor(() => releaseLease.mock.calls.length > 0); @@ -625,6 +700,311 @@ describe("TaskProcessorService", () => { }); }); + describe("handleTaskError — pruned workspace", () => { + it("a throwing workspace-missing dead-letter is logged and the lease is still released", async () => { + deadLetterWorkspaceMissing.mockRejectedValue(new Error("db connection reset")); + const task = makeTask({ sandboxId: null, pvcName: "pvc-1" }); + + await ( + service as unknown as { handleTaskError(t: Task, e: unknown): Promise } + ).handleTaskError(task, new PvcNotFoundError("pvc-1")); + + expect(releaseLease).toHaveBeenCalledWith({ + taskId: "task-1", + ownerId: "pod-test", + alsoClearSandbox: true, + }); + }); + + it("a row whose integration is disabled gets no notice; it is still failed and the lease released", async () => { + integrations.github = new GithubIntegration(false, { status: () => ({}) as never }); + const row = { + ...survivingNonTerminalRow(), + id: "gc", + source: "github", + kind: "comment", + reply_targets: [ + { + id: "555", + spec: { kind: "issue_comment", owner: "o", repo: "r", prNumber: 7, commentId: 555 }, + }, + ], + }; + const markFailed = mock(() => Promise.resolve([{ id: "gc" }])); + (inboxRepo.listNonTerminalByTask as ReturnType).mockResolvedValue([row]); + Object.assign(inboxRepo, { getById: mock(() => Promise.resolve(row)), markFailed }); + // The real dead-letter path in place of the stub. + Object.assign(service, { + inboxService: new RealInboxService(inboxRepo, sandboxManager, integrations, hooks, { + followUp: mock(), + }), + }); + + await ( + service as unknown as { handleTaskError(t: Task, e: unknown): Promise } + ).handleTaskError( + makeTask({ sandboxId: null, pvcName: "pvc-1" }), + new PvcNotFoundError("pvc-1"), + ); + + expect(postIssueComment).not.toHaveBeenCalled(); + expect(holds.of("github").heldTasks).toBe(0); + expect(markFailed).toHaveBeenCalledWith({ + inboxId: "gc", + ownerId: "pod-test", + reason: "workspace_missing", + userMessage: WORKSPACE_MISSING_MESSAGE, + }); + expect(releaseLease).toHaveBeenCalledWith({ + taskId: "task-1", + ownerId: "pod-test", + alsoClearSandbox: true, + }); + }); + }); + + describe("dispatch", () => { + it("a throw from handleTaskError itself is logged, not left unhandled", async () => { + // Transient provision failure → releaseLeaseIfNoSandbox, whose release throws. + createSandboxWithPvc.mockRejectedValue(new SandboxApiException({ statusCode: 503 })); + releaseLease.mockRejectedValue(new Error("db connection reset")); + const error = spyOn(logger, "error").mockImplementation(() => logger); + claimNext.mockResolvedValueOnce(makeTask()).mockResolvedValue(null); + try { + service.start(); + await waitFor(() => + error.mock.calls.some(([msg]) => msg === "handleTaskError failed for task task-1"), + ); + await service.stop(); + } finally { + error.mockRestore(); + } + }); + }); + + describe("failTask — failure broadcast", () => { + const MSG = "Task failed — check control-plane logs for details."; + const failTask = (task: Task): Promise => + (service as unknown as { failTask(t: Task, m: string): Promise }).failTask(task, MSG); + // biome-ignore lint/suspicious/noExplicitAny: test stub shape, cast at use site + const row = (id: string, kind: string, source: string, replyTargets: unknown[]): any => ({ + ...survivingNonTerminalRow(), + id, + kind, + source, + reply_targets: replyTargets, + }); + const reviewComment = (id: number, prNumber: number) => ({ + id: String(id), + spec: { + kind: "review_comment", + owner: "o", + repo: "r", + prNumber, + commentId: id, + inReplyTo: id, + path: "a.ts", + line: 1, + }, + }); + const issueComment = (id: string, prNumber: number, commentId: number) => ({ + id, + spec: { kind: "issue_comment", owner: "o", repo: "r", prNumber, commentId }, + }); + + it("posts once per distinct target id across every open row, by target kind", async () => { + failTaskInbox.mockResolvedValue([ + row("a", "assignment", "linear", [{ id: "sess-1", spec: { kind: "linear_session" } }]), + // Same session as the assignment → deduped. + row("lc", "comment", "linear", [{ id: "sess-1", spec: { kind: "linear_session" } }]), + row("gc", "comment", "github", [issueComment("555", 7, 555)]), + row("rv", "review", "github", [ + reviewComment(11, 8), + reviewComment(12, 8), + { + id: "88", + spec: { kind: "review_body", owner: "o", repo: "r", prNumber: 8, reviewId: 88 }, + }, + ]), + // CI target: commentId 0 = "post a new PR comment". + row("ci", "ci_result", "github", [issueComment("9001", 9, 0)]), + ]); + + await failTask(makeTask()); + + expect(emitError).toHaveBeenCalledTimes(1); + expect(emitError).toHaveBeenCalledWith(expect.anything(), "sess-1", MSG); + // PR comments for the comment, the review body and the CI row, in row order. + const signal = expect.any(AbortSignal); + expect(postIssueComment.mock.calls as unknown[]).toEqual([ + ["o", "r", 7, MSG, signal], + ["o", "r", 8, MSG, signal], + ["o", "r", 9, MSG, signal], + ]); + expect(postReviewCommentReply.mock.calls as unknown[]).toEqual([ + ["o", "r", 8, 11, MSG, signal], + ["o", "r", 8, 12, MSG, signal], + ]); + expect(failTaskInbox).toHaveBeenCalledWith({ + taskId: "task-1", + ownerId: "pod-test", + reason: MSG, + }); + }); + + it("dedupes per integration: equal target ids of two sources each get a notice", async () => { + failTaskInbox.mockResolvedValue([ + row("a", "assignment", "linear", [{ id: "555", spec: { kind: "linear_session" } }]), + row("gc", "comment", "github", [issueComment("555", 7, 555)]), + ]); + + await failTask(makeTask()); + + expect(emitError).toHaveBeenCalledWith(expect.anything(), "555", MSG); + expect(postIssueComment).toHaveBeenCalledWith("o", "r", 7, MSG, expect.any(AbortSignal)); + }); + + it("notices every row failTaskInbox dead-letters, whatever a separate read of the open rows saw", async () => { + // A follow-up committed after such a read, or the read failing, must not + // cost a dead-lettered row its notice, hook or Slack post. + (inboxRepo.listNonTerminalByTask as ReturnType).mockRejectedValue( + new Error("connection reset"), + ); + failTaskInbox.mockResolvedValue([ + row("a", "assignment", "linear", [{ id: "sess-1", spec: { kind: "linear_session" } }]), + row("gc", "comment", "github", [issueComment("555", 7, 555)]), + ]); + + await failTask(makeTask()); + + expect(emitError).toHaveBeenCalledWith(expect.anything(), "sess-1", MSG); + expect(postIssueComment).toHaveBeenCalledWith("o", "r", 7, MSG, expect.any(AbortSignal)); + expect(notifyTerminal).toHaveBeenCalledWith( + expect.objectContaining({ id: "task-1" }), + "failed", + MSG, + ); + }); + + it("dead-letters before posting, and posts only to the rows it dead-lettered", async () => { + const order: string[] = []; + emitError.mockImplementation(() => { + order.push("post"); + return Promise.resolve(); + }); + getByIdOwnedBy.mockResolvedValue(makeTask({ sandboxId: "sbx-live" })); + killSandbox.mockImplementation(() => { + order.push("kill"); + return Promise.resolve(); + }); + failTaskInbox.mockImplementation(() => { + order.push("mark"); + // The GitHub round was responded by a late `completed` before + // failTaskInbox ran: not transitioned, so not returned. + return Promise.resolve([ + row("a", "assignment", "linear", [{ id: "sess-1", spec: { kind: "linear_session" } }]), + ]); + }); + releaseLease.mockImplementation(() => { + order.push("release"); + return Promise.resolve([{ id: "task-1" }]); + }); + + await failTask(makeTask()); + + expect(order).toEqual(["mark", "post", "kill", "release"]); + expect(postIssueComment).not.toHaveBeenCalled(); + }); + + it("finishes a dead-lettered assignment's round: outcome failed, first transition, no archive", async () => { + const onRoundFinished = spyOn(integrations.linear, "onRoundFinished"); + failTaskInbox.mockResolvedValue([ + row("a", "assignment", "linear", [{ id: "sess-1", spec: { kind: "linear_session" } }]), + row("gc", "comment", "github", [issueComment("555", 7, 555)]), + ]); + + await failTask(makeTask()); + await waitFor(() => onRoundFinished.mock.calls.length > 0); + + expect(onRoundFinished).toHaveBeenCalledTimes(1); + const [ctx, , round, outcome] = onRoundFinished.mock.calls[0] as unknown as [ + { firstTransition: boolean; archive(): Promise }, + Task, + { id: string }, + string, + ]; + expect(round.id).toBe("a"); + expect(outcome).toBe("failed"); + expect(ctx.firstTransition).toBe(true); + expect(await ctx.archive()).toBeNull(); + }); + + it("a row whose source has no integration is skipped; other rows post and the lease is released", async () => { + failTaskInbox.mockResolvedValue([ + row("x", "comment", "unknown-source", [issueComment("777", 7, 777)]), + row("gc", "comment", "github", [issueComment("555", 7, 555)]), + ]); + + await failTask(makeTask()); + + expect(postIssueComment.mock.calls as unknown[]).toEqual([ + ["o", "r", 7, MSG, expect.any(AbortSignal)], + ]); + expect(releaseLease).toHaveBeenCalledWith({ + taskId: "task-1", + ownerId: "pod-test", + alsoClearSandbox: true, + }); + }); + + it("a row whose integration is disabled gets no notice; it is still dead-lettered and the lease released", async () => { + integrations.github = new GithubIntegration(false, { status: () => ({}) as never }); + failTaskInbox.mockResolvedValue([ + row("gc", "comment", "github", [issueComment("555", 7, 555)]), + ]); + + await failTask(makeTask()); + + expect(postIssueComment).not.toHaveBeenCalled(); + expect(holds.of("github").heldTasks).toBe(0); + expect(failTaskInbox).toHaveBeenCalledWith({ + taskId: "task-1", + ownerId: "pod-test", + reason: MSG, + }); + expect(releaseLease).toHaveBeenCalledWith({ + taskId: "task-1", + ownerId: "pod-test", + alsoClearSandbox: true, + }); + }); + + it("a Linear token failure skips only the Linear post; GitHub and Slack posts, dead-letter and release go on", async () => { + getAuthorizedLinearClient.mockRejectedValue(new Error("token refresh failed")); + failTaskInbox.mockResolvedValue([ + row("a", "assignment", "linear", [{ id: "sess-1", spec: { kind: "linear_session" } }]), + row("gc", "comment", "github", [issueComment("555", 7, 555)]), + ]); + + await failTask(makeTask()); + + expect(emitError).not.toHaveBeenCalled(); + expect(postIssueComment).toHaveBeenCalledWith("o", "r", 7, MSG, expect.any(AbortSignal)); + expect(failTaskInbox).toHaveBeenCalled(); + expect(releaseLease).toHaveBeenCalledWith({ + taskId: "task-1", + ownerId: "pod-test", + alsoClearSandbox: true, + }); + // Sent without the assignee mention (its lookup fails). + expect(notifyTerminal).toHaveBeenCalledWith( + expect.objectContaining({ id: "task-1" }), + "failed", + MSG, + ); + }); + }); + describe("runTask — adoption (sandbox_id already set)", () => { it("attaches to the existing sandbox without provisioning a new one", async () => { const task = makeTask({ sandboxId: "sbx-alive" }); @@ -923,6 +1303,359 @@ describe("TaskProcessorService", () => { expect(dispatcherHandle.mock.calls.length).toBe(1); }); + it("a dispatcher throw (e.g. a database error) holds the cursor and fails nothing", async () => { + const task = makeTask({ sandboxId: "sbx-1" }); + listInFlightOwnedBy.mockResolvedValueOnce([task]).mockResolvedValue([]); + readEvents + .mockResolvedValueOnce([ + { seq: 1, ts: "t", type: "agent_thought", inbox_id: "i", target_id: "s", body: "x" }, + { seq: 2, ts: "t", type: "agent_thought", inbox_id: "i", target_id: "s", body: "y" }, + ] as HarnessEvent[]) + .mockResolvedValue([]); + dispatcherHandle.mockRejectedValueOnce(new Error("token refresh failed")); + + service.start(); + // assessReconcileError reads the pod (unknown here → log only). + await waitFor(() => getSandboxPodStatus.mock.calls.length > 0); + await service.stop(); + + // The throw ends this task's tick: seq 2 is not dispatched and seq 1 is + // re-read next tick from the unchanged cursor. + expect(dispatcherHandle).toHaveBeenCalledTimes(1); + expect(recordEventCursor).not.toHaveBeenCalled(); + expect(failTaskInbox).not.toHaveBeenCalled(); + expect(requeueForRetry).not.toHaveBeenCalled(); + }); + + it("a hung send is cut at the send timeout: dropped, cursor advanced, the task stops, the tick goes on", async () => { + const error = spyOn(logger, "error").mockImplementation(() => logger); + try { + const deliver = mock(() => new Promise(() => {})); + const round = { ...survivingNonTerminalRow(), source: "github", status: "delivered" }; + const dispatcher = new RealEventDispatcher( + taskRepo, + { getByIdForTask: mock(() => Promise.resolve(round)) } as unknown as InboxRepository, + sandboxManager, + slackNotifier, + { + ...integrations, + github: { name: "github", enabled: true, status: async () => ({}), deliver }, + }, + holds, + hooks, + 20, // send timeout (ms) + ); + service = new TaskProcessorService( + taskRepo, + inboxRepo, + sandboxManager, + dispatcher, + inboxService, + slackNotifier, + integrations, + holds, + ["linear", "api"], + hooks, + ); + const thought = (seq: number) => ({ + seq, + ts: "t", + type: "agent_thought", + inbox_id: "inbox-1", + target_id: "sess-1", + body: "x", + }); + listInFlightOwnedBy + .mockResolvedValueOnce([ + makeTask({ id: "task-1", sandboxId: "sbx-1" }), + makeTask({ id: "task-2", sandboxId: "sbx-2" }), + ]) + .mockResolvedValue([]); + readEvents.mockImplementation((sandboxId: string) => + Promise.resolve(sandboxId === "sbx-1" ? [thought(1), thought(2)] : [thought(1)]), + ); + + service.start(); + await waitFor(() => recordEventCursor.mock.calls.length >= 2); + await service.stop(); + + // task-1's seq 2 is never sent: the drop ended task-1's events; task-2 ran. + const cursors = recordEventCursor.mock.calls.map((c: unknown[]) => { + const p = c[0] as { taskId: string; event: { seq: number } }; + return [p.taskId, p.event.seq]; + }); + expect(cursors).toEqual([ + ["task-1", 1], + ["task-2", 1], + ]); + expect(deliver).toHaveBeenCalledTimes(2); + } finally { + error.mockRestore(); + } + }); + + it("releases a stale hold when this pod claims the task", async () => { + const task = makeTask({ sandboxId: "sbx-1" }); + // Still owned, so only the claim can drop the hold. + listInFlightOwnedBy.mockResolvedValue([task]); + holds.hold("task-1", { seq: 1, inbox_id: "i" }, "github", new RetryableError("429", 60_000)); + claimNext.mockResolvedValueOnce(task).mockResolvedValue(null); + + service.start(); + await waitFor(() => attachSandbox.mock.calls.length > 0); + await service.stop(); + + expect(holds.of("github").heldTasks).toBe(0); + }); + + it("once shutdown starts, reconciles no further owned task and claims nothing", async () => { + listInFlightOwnedBy.mockResolvedValue([ + makeTask({ id: "task-1", sandboxId: "sbx-1" }), + makeTask({ id: "task-2", sandboxId: "sbx-2" }), + ]); + let stopping: Promise | undefined; + getSandboxInfo.mockImplementation(() => { + stopping ??= service.stop(); + return Promise.resolve(makeSandboxInfo()); + }); + + service.start(); + await waitFor(() => stopping !== undefined); + await stopping; + + expect(getSandboxInfo).toHaveBeenCalledTimes(1); + expect(claimNext).not.toHaveBeenCalled(); + }); + + it("a hold stops before the cursor advances and is not a reconcile error", async () => { + const task = makeTask({ sandboxId: "sbx-1" }); + listInFlightOwnedBy.mockResolvedValueOnce([task]).mockResolvedValue([]); + readEvents + .mockResolvedValueOnce([ + { seq: 1, ts: "t", type: "agent_thought", inbox_id: "i", target_id: "s", body: "x" }, + { seq: 2, ts: "t", type: "agent_thought", inbox_id: "i", target_id: "s", body: "y" }, + ] as HarnessEvent[]) + .mockResolvedValue([]); + dispatcherHandle.mockResolvedValueOnce("hold"); + + service.start(); + await waitFor(() => dispatcherHandle.mock.calls.length >= 1); + await new Promise((r) => setTimeout(r, 30)); + await service.stop(); + + expect(dispatcherHandle).toHaveBeenCalledTimes(1); + expect(recordEventCursor).not.toHaveBeenCalled(); + expect(getSandboxPodStatus).not.toHaveBeenCalled(); + }); + + it("skips a held task's events while it waits out Retry-After (the sweep still runs)", async () => { + const task = makeTask({ sandboxId: "sbx-1" }); + listInFlightOwnedBy.mockResolvedValue([task]); + holds.hold("task-1", { seq: 1, inbox_id: "i" }, "github", new RetryableError("429", 60_000)); + + service.start(); + await waitFor(() => sweepUndelivered.mock.calls.length >= 3); + await service.stop(); + + expect(readEvents).not.toHaveBeenCalled(); + expect(holds.of("github").heldTasks).toBe(1); + }); + + it("a dropped send advances the cursor, then ends this task's events for the tick", async () => { + const task = makeTask({ sandboxId: "sbx-1" }); + listInFlightOwnedBy.mockResolvedValueOnce([task]).mockResolvedValue([]); + readEvents + .mockResolvedValueOnce([ + { seq: 1, ts: "t", type: "agent_thought", inbox_id: "i", target_id: "s", body: "x" }, + { seq: 2, ts: "t", type: "agent_thought", inbox_id: "i", target_id: "s", body: "y" }, + ] as HarnessEvent[]) + .mockResolvedValue([]); + dispatcherHandle.mockResolvedValueOnce("dropped"); + + service.start(); + await waitFor(() => recordEventCursor.mock.calls.length >= 1); + await new Promise((r) => setTimeout(r, 30)); + await service.stop(); + + expect(dispatcherHandle).toHaveBeenCalledTimes(1); + const seqs = recordEventCursor.mock.calls.map( + (c: unknown[]) => (c[0] as { event: { seq: number } }).event.seq, + ); + expect(seqs).toEqual([1]); + }); + + it("drops a hold once the cursor moves past its event, or once the task leaves this pod", async () => { + const task = makeTask({ sandboxId: "sbx-1" }); + // task-1 stays owned every tick, so only the cursor advance can drop its hold. + listInFlightOwnedBy.mockResolvedValue([task]); + readEvents + .mockResolvedValueOnce([ + { seq: 1, ts: "t", type: "agent_thought", inbox_id: "i", target_id: "s", body: "x" }, + ] as HarnessEvent[]) + .mockResolvedValue([]); + const err = new RetryableError("token fetch failed"); + holds.hold("task-1", { seq: 1, inbox_id: "i" }, "linear", err); + holds.hold("task-gone", { seq: 3, inbox_id: "j" }, "linear", err); + // The retry of task-1's held event succeeds; task-gone is not owned. + service.start(); + await waitFor(() => recordEventCursor.mock.calls.length >= 1); + await new Promise((r) => setTimeout(r, 30)); + await service.stop(); + + expect(holds.of("linear").heldTasks).toBe(0); + }); + + it("renews every lease this pod holds at tick start; a task it didn't renew is skipped", async () => { + const t1 = makeTask({ id: "task-1", sandboxId: "sbx-1" }); + const t2 = makeTask({ id: "task-2", sandboxId: "sbx-2" }); + listInFlightOwnedBy.mockResolvedValueOnce([t1, t2]).mockResolvedValue([]); + const order: string[] = []; + // task-2's lease was taken over. + heartbeatOwnedBy.mockImplementation(() => { + order.push("heartbeat"); + return Promise.resolve([{ id: "task-1" }, { id: "in-flight-provision" }]); + }); + getSandboxInfo.mockImplementation((id: string) => { + order.push(`reconcile ${id}`); + return Promise.resolve(makeSandboxInfo()); + }); + + service.start(); + await waitFor(() => order.length >= 2); + await new Promise((r) => setTimeout(r, 30)); + await service.stop(); + + expect(order.slice(0, 2)).toEqual(["heartbeat", "reconcile sbx-1"]); + expect(order).not.toContain("reconcile sbx-2"); + expect(heartbeatOwnedBy).toHaveBeenCalledWith("pod-test"); + }); + + describe("a reconcile longer than a quarter of the lease", () => { + const thought = (seq: number) => + ({ + seq, + ts: "t", + type: "agent_thought", + inbox_id: "inbox-1", + target_id: "sess-1", + body: "x", + }) as HarnessEvent; + let leaseMs: number; + beforeEach(() => { + leaseMs = mockedConfig.leaseTimeoutMs; + mockedConfig.leaseTimeoutMs = 40; // renew after 10 ms + listInFlightOwnedBy + .mockResolvedValueOnce([makeTask({ sandboxId: "sbx-1" })]) + .mockResolvedValue([]); + readEvents.mockResolvedValueOnce([thought(1), thought(2), thought(3)]); + }); + afterEach(() => { + mockedConfig.leaseTimeoutMs = leaseMs; + }); + + it("renews every lease again before a later event", async () => { + const order: string[] = []; + heartbeatOwnedBy.mockImplementation(() => { + order.push("heartbeat"); + return Promise.resolve([{ id: "task-1" }]); + }); + // Each send outlasts a quarter of the lease. + dispatcherHandle.mockImplementation(async (event: HarnessEvent) => { + order.push(`handle ${event.seq}`); + await sleep(15); + return "ok" as const; + }); + + service.start(); + await waitFor(() => recordEventCursor.mock.calls.length >= 3); + await service.stop(); + + expect(order.slice(order.indexOf("handle 1"), order.indexOf("handle 2"))).toContain( + "heartbeat", + ); + }); + + it("stops the task's events once a renewal shows its lease taken over", async () => { + // Ours until the first send; taken over by the renewal after it. + heartbeatOwnedBy.mockImplementation(() => + Promise.resolve(dispatcherHandle.mock.calls.length > 0 ? [] : [{ id: "task-1" }]), + ); + dispatcherHandle.mockImplementation(async () => { + await sleep(15); + return "ok" as const; + }); + + service.start(); + await waitFor(() => recordEventCursor.mock.calls.length >= 1); + await sleep(50); + await service.stop(); + + expect(dispatcherHandle).toHaveBeenCalledTimes(1); + }); + }); + + it("never provisions a task twice on this pod: a re-claim while its provision runs is skipped", async () => { + let release!: () => void; + createSandboxWithPvc.mockImplementation( + () => + new Promise((resolve) => { + release = () => resolve({ sandboxId: "sbx-1", pvcName: "devbox-pvc-task-1-aaaaaa" }); + }), + ); + // The lease of the running provision expired, so claimNext hands it back. + claimNext + .mockResolvedValueOnce(makeTask()) + .mockResolvedValueOnce(makeTask()) + .mockResolvedValue(null); + + service.start(); + await waitFor(() => claimNext.mock.calls.length >= 3); + release(); + await waitFor(() => attachSandboxResources.mock.calls.length > 0); + await service.stop(); + + expect(createSandboxWithPvc).toHaveBeenCalledTimes(1); + }); + + it("once shutdown starts, dispatches no further event of the task in hand", async () => { + const task = makeTask({ sandboxId: "sbx-1" }); + listInFlightOwnedBy.mockResolvedValue([task]); + const thought = (seq: number) => + ({ + seq, + ts: "t", + type: "agent_thought", + inbox_id: "inbox-1", + target_id: "sess-1", + body: "x", + }) as HarnessEvent; + readEvents.mockResolvedValue([thought(1), thought(2), thought(3)]); + let stopping: Promise | undefined; + dispatcherHandle.mockImplementation(() => { + stopping ??= service.stop(); + return Promise.resolve("ok" as const); + }); + + service.start(); + await waitFor(() => stopping !== undefined); + await stopping; + + // The event in hand finishes and its cursor is kept; the rest wait. + expect(dispatcherHandle).toHaveBeenCalledTimes(1); + expect(recordEventCursor).toHaveBeenCalledTimes(1); + }); + + it("claims and lists only the configured task sources", async () => { + service.start(); + await waitFor(() => claimNext.mock.calls.length > 0); + await service.stop(); + + expect(listInFlightOwnedBy).toHaveBeenCalledWith("pod-test", ["linear", "api"]); + expect(claimNext).toHaveBeenCalledWith( + expect.objectContaining({ sources: ["linear", "api"] }), + ); + }); + it("calls sweepUndelivered before reading events", async () => { const task = makeTask({ sandboxId: "sbx-1" }); listInFlightOwnedBy.mockResolvedValueOnce([task]).mockResolvedValue([]); @@ -1230,6 +1963,37 @@ describe("TaskProcessorService", () => { expect(backdateHeartbeatsOwnedBy).not.toHaveBeenCalled(); }); + it("stop() waits for detached hooks and logs hook_abandoned for one that outlives the drain", async () => { + const warn = spyOn(logger, "warn").mockImplementation(() => logger); + const drainMs = mockedConfig.shutdownDrainMs; + mockedConfig.shutdownDrainMs = 100; + try { + let finished = false; + service.start(); + await waitFor(() => heartbeatOwnedBy.mock.calls.length > 0); + hooks.detach( + "task-1", + "onRoundFinished", + sleep(20).then(() => { + finished = true; + }), + ); + hooks.detach("task-2", "assignee", sleep(400)); + await service.stop(); + + expect(finished).toBe(true); + expect(warn.mock.calls.filter(([msg]) => msg === "hook abandoned at shutdown")).toEqual([ + [ + "hook abandoned at shutdown", + { event: "hook_abandoned", taskId: "task-2", hook: "assignee" }, + ], + ]); + } finally { + mockedConfig.shutdownDrainMs = drainMs; + warn.mockRestore(); + } + }); + it("stop() backdates heartbeats owned by this pod before resolving", async () => { service.start(); await waitFor(() => heartbeatOwnedBy.mock.calls.length > 0); diff --git a/control-plane/src/services/agentSession.service.ts b/control-plane/src/services/agentSession.service.ts index 2fc1c26..d4fa768 100644 --- a/control-plane/src/services/agentSession.service.ts +++ b/control-plane/src/services/agentSession.service.ts @@ -1,16 +1,11 @@ import { emitAction, emitResponse, requireLinearTicketId } from "../clients/linear"; +import type { TaskIntake } from "../integrations/intake"; import { logger } from "../logger"; -import type { - AssignmentContent, - InboxRepository, - ReplyTarget, -} from "../repositories/inbox.repository"; -import type { TaskRepository } from "../repositories/task.repository"; +import type { ReplyTarget } from "../repositories/inbox.repository"; import type { HandleAgentSessionResult, ParsedAgentSessionEvent } from "../types/agentSession"; import { parseExperimentalFlags } from "./experimentalFlags"; import type { InboxService } from "./inbox.service"; import type { LinearTokenProvider } from "./linearTokenProvider"; -import type { SandboxManager } from "./sandboxManager.service"; const RESTARTED_BODY = [ "🔄 **Restarting from scratch.**", @@ -21,101 +16,58 @@ const RESTARTED_BODY = [ const STOP_NOT_SUPPORTED_BODY = "Stop is not supported yet — work will continue."; export class AgentSessionService { - private readonly taskRepo: TaskRepository; - private readonly inboxRepo: InboxRepository; + private readonly intake: Pick; private readonly linearTokenProvider: Pick; - private readonly sandboxManager: SandboxManager; private readonly inboxService: InboxService; constructor( - taskRepo: TaskRepository, - inboxRepo: InboxRepository, + intake: Pick, linearTokenProvider: Pick, - sandboxManager: SandboxManager, inboxService: InboxService, ) { - this.taskRepo = taskRepo; - this.inboxRepo = inboxRepo; + this.intake = intake; this.linearTokenProvider = linearTokenProvider; - this.sandboxManager = sandboxManager; this.inboxService = inboxService; } async handleEvent(event: ParsedAgentSessionEvent): Promise { switch (event.action) { case "created": { - const { flags: experimentalFlags, strippedDescription } = parseExperimentalFlags( - event.issue.description, - ); - const taskKey = requireLinearTicketId(event.issue.url); - const inboxPayload: AssignmentContent = { + const { flags, strippedDescription } = parseExperimentalFlags(event.issue.description); + // The agent session is the delegation id: Linear mints a new one per + // delegation and redelivers a `created` with the same one. + const target: ReplyTarget = { id: event.agentSessionId, spec: { kind: "linear_session" } }; + + // One transaction: insert the task and its assignment row, or compare + // the stored session with this one. The same session is a redelivery + // of a processed delegation (no change, no ack: a restart would kill + // the freshly-provisioned sandbox); a new one is a re-delegation, which + // preempts any in-flight run and restarts from a clean slate (intake + // kills the preempted sandbox). + const { taskId, result, previousTargetId } = await this.intake.assign({ + source: "linear", + externalId: event.issue.id, + key: { hint: requireLinearTicketId(event.issue.url) }, title: event.issue.title, description: strippedDescription, - external_url: event.issue.url, - target_id: event.agentSessionId, - }; - const inboxReplyTargets: ReplyTarget[] = [ - { id: event.agentSessionId, spec: { kind: "linear_session" } }, - ]; - - // Single tx: upsert the tasks row (infra) + insert the assignment inbox - // row. The inbox row's reply_targets[0].id is the canonical Linear - // agent session id; on a re-delegation `restartTaskWithInbox` - // overwrites it with the freshly-minted session and rotates the inbox - // id. Inbox row uses the Linear issue UUID as external_id; its id is the - // daemon-visible run identity. - const { taskId, created } = await this.taskRepo.upsertFromAgentSessionWithInbox({ - taskExternalId: event.issue.id, - taskKey, - taskTitle: event.issue.title, - taskExternalUrl: event.issue.url, - inboxPayload, - inboxReplyTargets, - metadata: { experimentalFlags }, + flags, + url: () => event.issue.url, + target: () => target, }); - if (created) { + if (result === "created") { // Fresh task — Linear sees Evaluating to satisfy the 10s SLA. this.fireAndForgetEmitAction(event.agentSessionId, "Evaluating", "task", "created-ack"); - return { status: "ok" }; - } - - // Conflict path — distinguish a duplicate webhook redelivery (same - // session id as the row already carries) from a genuine re-delegation - // (new session id). The upsert above DO-NOTHING'd the inbox INSERT, so - // the assignment row's existing reply_targets still carries the prior - // session id. - const prevSessionId = await this.inboxRepo.getAssignmentReplyTargetId(taskId); - if (prevSessionId !== event.agentSessionId) { - // Genuine re-delegation — Linear minted a NEW agent session for a new - // `created` event. Preempt any in-flight run and restart from scratch: - // restartTaskWithInbox resets the task + re-pends the `assignment` inbox - // row (with the new session id in reply_targets). Kill the preempted - // sandbox (if the run was still live) so the slot frees right away; - // the old owner pod self-evicts on its next owner-guarded write. Then - // ack the restart. + } else if (result === "restarted") { logger.info("re-delegation detected; restarting task from clean slate", { taskId, - previousSessionId: prevSessionId, + previousSessionId: previousTargetId, currentSessionId: event.agentSessionId, }); - const { prevSandboxId } = await this.taskRepo.restartTaskWithInbox({ - taskId, - inboxPayload, - inboxReplyTargets, - }); - if (prevSandboxId) { - this.fireAndForgetKillSandbox(prevSandboxId); - } this.fireAndForgetEmitResponse(event.agentSessionId, RESTARTED_BODY, "restart-ack"); this.fireAndForgetEmitAction(event.agentSessionId, "Evaluating", "task", "restart-ack"); } - // else: prevSessionId === event.agentSessionId — a duplicate webhook - // delivery of an already-processed delegation (Linear redelivers the same - // `created` event, same agent session id). The run is already (re)started - // and acked by the first delivery; no-op rather than run the restart - // twice (which would kill the freshly-provisioned sandbox). Fall through - // to the 200 below so Linear stops retrying. + // A duplicate falls through to the 200 so Linear stops retrying. return { status: "ok" }; } @@ -133,10 +85,10 @@ export class AgentSessionService { // (Linear mints a fresh agent session per re-prompt). Unknown issue → // session_not_found so the webhook returns 503 and Linear retries after // the `created` event lands. - const task = await this.taskRepo.findByLinearIssueId(event.issue.id); + const task = await this.intake.findTask("linear", event.issue.id); if (!task) return { status: "session_not_found" }; - // enqueueLinearComment INSERTs the inbox row (idempotent on activityId). + // enqueueLinearComment queues the round (idempotent on activityId). // The new comment row's reply_targets carry the fresh session id, so // replies route correctly; the assignment row's reply_targets is the // canonical session id for task-scoped lifecycle / failure emits and @@ -205,16 +157,4 @@ export class AgentSessionService { }); }); } - - // Fire-and-forget — the webhook handler must not block on the sandbox API. - // Kills a preempted sandbox when a re-delegation restarts a task. - private fireAndForgetKillSandbox(sandboxId: string): void { - void this.sandboxManager.killSandbox(sandboxId).catch((err) => { - const msg = err instanceof Error ? err.message : String(err); - logger.error("killSandbox(preempted re-delegation) failed", { - sandboxId, - message: msg, - }); - }); - } } diff --git a/control-plane/src/services/apiTask.service.ts b/control-plane/src/services/apiTask.service.ts index 415d268..9c5185b 100644 --- a/control-plane/src/services/apiTask.service.ts +++ b/control-plane/src/services/apiTask.service.ts @@ -1,8 +1,8 @@ import { ApiError } from "../errors"; -import { logger } from "../logger"; +import type { TaskIntake } from "../integrations/intake"; +import type { TaskSource } from "../integrations/types"; import type { ApiTaskRecord, ApiTaskRepository } from "../repositories/apiTask.repository"; import { apiSessionTarget } from "../repositories/inbox.repository"; -import type { TaskRepository } from "../repositories/task.repository"; import type { TaskEventRepository, TaskEventRow } from "../repositories/taskEvent.repository"; import type { ApiEventEnvelope, @@ -15,7 +15,6 @@ import type { SseStateData, } from "../types/api"; import { extractExperimentalFlags } from "./experimentalFlags"; -import type { SandboxManager } from "./sandboxManager.service"; export type Principal = { tokenId: string; name: string }; export type CreateTaskInput = { @@ -24,14 +23,13 @@ export type CreateTaskInput = { experimentalFlags: string[]; }; export type ListTasksInput = { - source?: "linear" | "api"; + source?: TaskSource; createdBy?: "me" | string; state?: ApiTaskState; before?: string; limit: number; }; export type ApiTaskServiceOptions = { - keyPrefix: string; publicBaseUrl: string; now: () => Date; }; @@ -41,26 +39,30 @@ const NOT_FOUND = () => new ApiError(404, "task_not_found", "Task not found"); const NOT_API = () => new ApiError(409, "not_api_task", "Only API tasks can be changed through the API"); -// The /api use cases read every task and write only API tasks. All settings -// arrive through the constructor so this module never imports ../config; that -// keeps process-global mock.module replacements from leaking into these tests. +// The /api use cases read every task and write only API tasks, through +// TaskIntake. All settings arrive through the constructor so this module never +// imports ../config; that keeps process-global mock.module replacements from +// leaking into these tests. export class ApiTaskService { constructor( private apiTaskRepo: ApiTaskRepository, - private taskRepo: TaskRepository, + private intake: Pick, private taskEventRepo: TaskEventRepository, - private sandboxManager: Pick, private opts: ApiTaskServiceOptions, ) {} + // The task row, its assignment round and task_created, in one transaction. + // The key comes from api_task_key_seq and is also the task's external id. async create(principal: Principal | null, input: CreateTaskInput): Promise { - const { taskId } = await this.apiTaskRepo.createApiTask({ - keyPrefix: this.opts.keyPrefix, - publicBaseUrl: this.opts.publicBaseUrl, + const { taskId } = await this.intake.assign({ + source: "api", + key: { sequence: "api" }, title: input.title, description: input.description, - experimentalFlags: input.experimentalFlags, - createdBy: principal?.tokenId ?? null, + flags: input.experimentalFlags, + url: (t) => `${this.opts.publicBaseUrl}/api/tasks/${t.key}`, + target: (t) => apiSessionTarget(t.id), + createdBy: principal?.tokenId, }); return this.get(taskId); } @@ -85,39 +87,36 @@ export class ApiTaskService { return this.toApiTask(await this.resolve(ref)); } + // A comment round (external_id = a new UUID) and message_queued in one + // transaction. A round in progress doesn't block it: the new row queues + // behind it, as on Linear. async sendMessage(ref: string, body: string): Promise { const task = await this.resolveApiTask(ref); - const { roundId } = await this.apiTaskRepo.appendApiMessage({ + const result = await this.intake.followUp({ task: { id: task.id, key: task.key }, - body, - createdAt: this.opts.now().toISOString(), + message: { + source: "api", + kind: "comment", + external_id: crypto.randomUUID(), + payload: { target_id: task.id, body, pr: null, created_at: this.opts.now().toISOString() }, + }, + targets: [apiSessionTarget(task.id)], }); - return { round_id: roundId }; + // A fresh external id and no PR guard: nothing can drop it. + if (!("roundId" in result)) throw new Error(`sendMessage: round dropped (${result.dropped})`); + return { round_id: result.roundId }; } - // Same transaction as Linear re-delegation (restartTaskWithInbox), plus - // the full new definition and task_restarted. The input replaces the current - // title/description/flags; nothing is merged. The previous sandbox is killed - // fire-and-forget afterwards. + // Same restart as a re-delegation, plus the full new definition and + // task_restarted. The input replaces the current title/description/flags; + // nothing is merged. The previous sandbox is killed afterwards. async restart(ref: string, input: CreateTaskInput): Promise { const task = await this.resolveApiTask(ref); - const { title, description, experimentalFlags } = input; - const { prevSandboxId } = await this.taskRepo.restartTaskWithInbox({ - taskId: task.id, - inboxPayload: { title, description, external_url: task.external_url, target_id: task.id }, - inboxReplyTargets: [apiSessionTarget(task.id)], - api: { title, description, experimentalFlags }, + await this.intake.restart(task.id, { + title: input.title, + description: input.description, + flags: input.experimentalFlags, }); - if (prevSandboxId) { - void this.sandboxManager.killSandbox(prevSandboxId).catch((err) => { - const e = err instanceof Error ? err : new Error(String(err)); - logger.error("killSandbox(API restart) failed", { - sandboxId: prevSandboxId, - message: e.message, - stack: e.stack, - }); - }); - } return this.get(task.id); } @@ -128,7 +127,7 @@ export class ApiTaskService { return { events, next_after: events[events.length - 1]?.id ?? after, logging: "available" }; } - async streamTarget(ref: string): Promise<{ id: string; source: "linear" | "api" }> { + async streamTarget(ref: string): Promise<{ id: string; source: TaskSource }> { const task = await this.resolve(ref); return { id: task.id, source: task.source }; } @@ -147,7 +146,7 @@ export class ApiTaskService { private async resolve(ref: string): Promise { const task = UUID_RE.test(ref) ? await this.apiTaskRepo.findById(ref) - : await this.apiTaskRepo.findByKey(ref.toUpperCase()); + : await this.apiTaskRepo.findByKey(ref); if (!task) throw NOT_FOUND(); return task; } diff --git a/control-plane/src/services/eventDispatcher.ts b/control-plane/src/services/eventDispatcher.ts index e607349..bbc8391 100644 --- a/control-plane/src/services/eventDispatcher.ts +++ b/control-plane/src/services/eventDispatcher.ts @@ -1,32 +1,27 @@ -import type { LinearClient } from "@linear/sdk"; -import { - addPrAssignees, - getUserById, - parsePrUrl, - postIssueComment, - postReviewCommentReply, - requestReviewers, -} from "../clients/github"; -import { - attachUrlToIssue, - emitAction, - emitError, - emitResponse, - emitThought, - getIssueHumanAssignee, - uploadIssueAttachment, -} from "../clients/linear"; +import { addPrAssignees, parsePrUrl, postIssueComment } from "../clients/github"; import { config } from "../config"; import type { Task } from "../db/schema"; +import { deliverToRound, SEND_TIMEOUT_MS, withTimeout } from "../integrations/http"; +import { type Outbound, RetryableError } from "../integrations/outbound"; +import type { Integration, IntegrationName } from "../integrations/types"; import { logger } from "../logger"; import type { InboxRepository, InboxRow } from "../repositories/inbox.repository"; import type { TaskRepository } from "../repositories/task.repository"; -import type { HarnessEvent } from "../types/harnessEvent"; -import type { LinearTokenProvider } from "./linearTokenProvider"; +import type { AgentEmit, HarnessEvent } from "../types/harnessEvent"; +import type { IntegrationHooks } from "./integrationHooks"; +import type { ReplyHolds } from "./replyHolds"; import type { SandboxManager } from "./sandboxManager.service"; import type { SlackNotifierService } from "./slackNotifier.service"; // Outcome of dispatching a single event. +// - `ok`: handled; reconcileOne advances the cursor and goes on. +// - `dropped`: the event's send timed out or failed for good. reconcileOne +// advances past it, then stops this task's events for the tick, so a slow +// or failing platform can't eat the tick. +// - `hold`: a send (or the owner's onPrOpened) failed before anything went out +// (RetryableError); the hold is recorded in ReplyHolds. reconcileOne stops +// this task's events BEFORE advancing the cursor, so the event is retried +// until it is sent or fails for good. // - `self_evict`: an owner-guarded write returned 0 rows — the lease was taken // over mid-tick — so reconcileOne must stop processing this task BEFORE // advancing the cursor (matching recordEventCursor's 0-row return). @@ -36,7 +31,7 @@ import type { SlackNotifierService } from "./slackNotifier.service"; // advancing the cursor and WITHOUT applying the side effect. We deliberately // stop-and-alarm rather than skip-and-continue, so a foreign event is visible // and cannot trigger a side effect. -export type DispatchResult = "ok" | "self_evict" | "abort"; +export type DispatchResult = "ok" | "dropped" | "hold" | "self_evict" | "abort"; // One-time onboarding comment posted on every PR harness opens — comment // interactivity is invisible unless users know it exists. Built per-call from @@ -57,20 +52,34 @@ function onboardingCommentBody(botLogin: string, taskKey: string): string { // // At-least-once dispatch semantics: the cursor advances AFTER handle() // completes. A crash between a side-effect and the cursor commit re-fires the -// event on the next reconcile tick — all side-effects are idempotent. +// event on the next reconcile tick: pr_opened and completed side effects are +// idempotent or run only the first time (appendPrUrl's firstTime, +// markResponded's first transition). export class EventDispatcher { constructor( private taskRepo: TaskRepository, private inboxRepo: InboxRepository, private sandboxManager: SandboxManager, - private linearTokenProvider: Pick, - private slackNotifier: SlackNotifierService, + private slackNotifier: Pick, + private integrations: Record, + private holds: ReplyHolds, + private hooks: IntegrationHooks, + private sendTimeoutMs = SEND_TIMEOUT_MS, ) {} async handle(event: HarnessEvent, task: Task): Promise { - const linearClient = - task.source === "linear" ? await this.linearTokenProvider.getAuthorizedLinearClient() : null; + try { + return await this.apply(event, task); + } catch (err) { + // The owner's onPrOpened failed before anything was sent: hold on the + // task's own integration. deliver holds on the round's integration itself. + if (!(err instanceof RetryableError)) throw err; + this.holds.hold(task.id, event, task.source, err); + return "hold"; + } + } + private async apply(event: HarnessEvent, task: Task): Promise { // event.inbox_id is sandbox-written; resolve it task-scoped. A null row // means this task's event stream is carrying an event whose inbox row it // does NOT own — a cross-task LEAK (a sibling's event in our stream, our own @@ -103,27 +112,21 @@ export class EventDispatcher { } } switch (event.type) { - // Streaming progress / errors → forwarded to the round's Linear session. - // The session id is pulled from inboxRow.reply_targets (the linear_session - // target), so a github-sourced round (whose reply_targets carry GH specs - // only) implicitly skips the Linear forward — no progress noise on the - // Linear UI when the user is driving from GitHub. + // Agent emits → labelled, then delivered by the round's integration + // (routed by round.source, not task.source) to the target named by + // event.target_id. A Linear outage therefore holds only Linear rounds' + // messages, never a GitHub round's. case "agent_thought": case "agent_action": - case "agent_error": - await this.forwardAgentEmit(event, inboxRow, linearClient); - break; - - // Per-item reply → routed by target_id through inboxRow.reply_targets[]. case "agent_response": - await this.postTargetedReply(event, inboxRow, task, linearClient); - break; + case "agent_error": + return this.deliver(event, inboxRow, task); case "pr_opened": { // appendPrUrl is owner-guarded; a 0-row return means the lease was - // taken over (stale-lease takeover). Self-evict BEFORE attachUrlToIssue + // taken over (stale-lease takeover). Self-evict BEFORE any side effect // and before the cursor advance — a sibling pod now owns this task. - const appended = await this.taskRepo.appendPrUrl({ + const [appended] = await this.taskRepo.appendPrUrl({ taskId: task.id, ownerId: config.podId, prUrl: event.pr_url, @@ -131,31 +134,35 @@ export class EventDispatcher { branch: event.branch, commitSha: event.commit_sha, }); - if (appended.length === 0) { + if (!appended) { logger.warn("appendPrUrl returned 0 rows (lease lost); self-evicting", { taskId: task.id, prUrl: event.pr_url, }); return "self_evict"; } - if (linearClient) { - try { - await attachUrlToIssue(linearClient, task.externalId, event.pr_url, event.repo); - } catch (err) { - logErr(`attachUrlToIssue failed for ${event.repo}`, err); - } + // Only the dispatch that added the URL posts the onboarding comment + // and mirrors the assignee; a replay (the cursor advances only after + // handle() returns) finds the URL already there. + if (appended.firstTime) { + // Onboarding comment so users discover they can drive harness with + // @harness comments + reviews. Best-effort, not load-bearing. + await this.postOnboardingComment(event, task); + // Detached (not awaited): up to 4 upstream round-trips must not + // stall tick() toward the lease window — the same hazard + // reconcileFilesTimeoutMs exists for. Nothing downstream reads the + // result, and the method never rejects (see its contract). + this.hooks.detach(task.id, "assignee", this.mirrorPrAssignee(event, task, inboxRow)); } - // Onboarding comment so users discover they can drive harness with - // @harness comments + reviews. Best-effort, no dedupe: at-least-once - // dispatch can re-fire on cursor-replay → duplicate help comments are - // possible. Acceptable: it's a help message, not load-bearing. - await this.postOnboardingComment(event, task); - // Mirror the Linear ticket's human assignee onto the PR. Detached - // (not awaited): up to 4 unbounded upstream round-trips (2× Linear, - // 2× GitHub) must not stall tick() toward the lease window — the same - // hazard reconcileFilesTimeoutMs exists for. Nothing downstream reads - // the result, and the method never rejects (see its contract). - void this.assignPrToLinearAssignee(event, task, linearClient); + // The owner's onPrOpened (Linear: attach the PR to the issue) runs on + // every dispatch and is idempotent. Unlike other hooks, its + // RetryableError escapes to handle(), which holds pr_opened. + await this.hooks.prOpened( + task, + inboxRow, + { url: event.pr_url, repo: event.repo, branch: event.branch }, + appended.firstTime, + ); break; } @@ -166,28 +173,31 @@ export class EventDispatcher { // No inbox row → nothing to mark. `marked` is true only on the FIRST // transition (a re-fire on cursor-replay returns false), which gates // at-most-once completion side effects below. - const marked = inboxRow - ? await this.markInboxResponded(inboxRow, event.status ?? null) - : false; - // Kind-specific side effect: assignment rounds upload a run archive to - // the Linear issue while the sandbox is still alive (it lives until - // the idle reaper). Fire-and-forget — the dispatch loop must not - // block on an in-sandbox archive command (~tens of seconds). - if (inboxRow?.kind === "assignment") { - void this.attachRunArchiveToLinearIssue(task); - // Slack completion update to the configured channel. - // Assignment-only (the top-level task, not comment/review rounds) and - // gated on the first inbox transition so a cursor-replay can't double- - // post. Fire-and-forget: a Slack outage must not stall the dispatch - // loop. The notifier no-ops when Slack is disabled. - if (marked && linearClient) { - void this.slackNotifier - .notifyTerminal(linearClient, task, event.status ?? "completed") - .catch((err) => logErr("slack completion notify failed", err)); - } - } - if (marked && inboxRow?.kind === "review") { - this.requestReviewFromOriginalReviewer(inboxRow, task); + if (!inboxRow) break; + const outcome = event.status ?? null; + const marked = await this.markInboxResponded(inboxRow, outcome); + // The round's integration finishes it (Linear: the run archive; GitHub: + // the review re-request), detached: the archive is an in-sandbox + // command (~tens of seconds), run only if the hook asks for it while + // the sandbox is still alive (it lives until the idle reaper). + const sandboxId = task.sandboxId; + this.hooks.roundFinished(task, inboxRow, outcome, marked, async () => + sandboxId ? this.sandboxManager.extractArchive(sandboxId) : null, + ); + // Slack completion update to the configured channel, for a task + // whose source is opted in (SLACK_NOTIFY_SOURCES). Assignment-only + // (the top-level task, not comment/review rounds) and gated on the + // first inbox transition so a cursor-replay can't double-post. + // Detached: a Slack outage (or the assignee lookup for its mention) + // must not stall the dispatch loop. + if (marked && inboxRow.kind === "assignment" && this.slackNotifier.notifies(task.source)) { + this.hooks.detach( + task.id, + "slack", + this.slackNotifier + .notifyTerminal(task, event.status ?? "completed") + .catch((err) => logErr("slack completion notify failed", err, { taskId: task.id })), + ); } break; } @@ -204,113 +214,50 @@ export class EventDispatcher { return "ok"; } - // Streaming forwarder for thought / action / error → Linear agent session. - // target_id on the event is informational; routing is by the inbox row's - // linear_session reply target — target.id IS the agent session id by - // construction (set by enqueue paths). A row whose reply_targets carry no - // linear_session (i.e. a GitHub-sourced comment/review round) yields a null - // session id → drop. A foreign / unknown inbox_id resolves inboxRow to null - // and also drops: a confused emit is safer dropped than routed to "the - // task's session" somewhere unrelated. Final user-facing replies route - // through postTargetedReply, which honors reply_targets and lands on GitHub - // for github-sourced rows. - private async forwardAgentEmit( - event: Extract, - inboxRow: InboxRow | null, - linearClient: LinearClient | null, - ): Promise { - const agentSessionId = - inboxRow?.reply_targets.find((t) => t.spec.kind === "linear_session")?.id ?? null; - if (!linearClient || !agentSessionId) return; - try { - switch (event.type) { - case "agent_thought": - await emitThought(linearClient, agentSessionId, event.body); - break; - case "agent_action": - // The daemon's body is the full action label (e.g. "Planning", - // "Addressing comment"). Linear renders ` ` - // concatenated, so a non-empty parameter appends a noisy suffix - // ("Addressing comment task"). Pass empty parameter — the body - // already carries everything the user should see. - await emitAction(linearClient, agentSessionId, event.body, ""); - break; - case "agent_error": - await emitError(linearClient, agentSessionId, event.body); - break; - } - } catch (err) { - logErr(`forward ${event.type} failed`, err); - } - } - - // Posts one agent reply to one thread — looks up the target on the inbox - // row's reply_targets[] by target_id, then posts event.body verbatim, routing - // by spec.kind. CP never formats the text. - private async postTargetedReply( - event: Extract, + // Delivers one agent emit to one thread: looks up the target on the inbox + // row's reply_targets[] by target_id, labels the emit, and hands both to the + // round's integration, which decides where (or whether) it shows up. Core + // never checks what an integration can show. A disabled integration's send + // is skipped and counts as sent. The send runs under sendTimeoutMs + // (SEND_TIMEOUT_MS). A RetryableError (nothing was sent) holds the event on + // the round's integration; a timeout or any other failure is logged and the + // event dropped. + private async deliver( + event: AgentEmit, inboxRow: InboxRow | null, task: Task, - linearClient: LinearClient | null, - ): Promise { + ): Promise { if (!inboxRow) { - logger.warn("agent_response references missing inbox row", { + logger.warn(`${event.type} references missing inbox row`, { taskId: task.id, inboxId: event.inbox_id, }); - return; + return "ok"; } const target = inboxRow.reply_targets.find((t) => t.id === event.target_id); if (!target) { - logger.warn("agent_response target_id not in reply_targets", { + logger.warn(`${event.type} target_id not in reply_targets`, { taskId: task.id, inboxId: inboxRow.id, targetId: event.target_id, }); - return; // cursor still advances — drop + return "ok"; // cursor still advances — drop } try { - switch (target.spec.kind) { - case "issue_comment": - await postIssueComment( - target.spec.owner, - target.spec.repo, - target.spec.prNumber, - event.body, - ); - break; - case "review_comment": - await postReviewCommentReply( - target.spec.owner, - target.spec.repo, - target.spec.prNumber, - target.spec.inReplyTo, - event.body, - ); - break; - case "review_body": - // Top-level PR comment — GH has no reply-to-review-body API. - await postIssueComment( - target.spec.owner, - target.spec.repo, - target.spec.prNumber, - event.body, - ); - break; - case "linear_session": - // target.id IS the agent session id by construction (enqueue paths - // stamp the live session id, refreshed on every `prompted`). - if (linearClient) { - await emitResponse(linearClient, target.id, event.body); - } - break; - case "api_session": - break; // API tasks: the event log carries the reply - } + return await deliverToRound( + this.integrations, + inboxRow, + target, + labelAgentEmit(event, inboxRow), + { taskId: task.id, eventType: event.type }, + this.sendTimeoutMs, + ); } catch (err) { - logErr(`post targeted reply failed for target ${target.id}`, err); - // At-least-once: cursor-replay re-fires this single agent_response retries. + // Nothing was sent: the event holds on the round's integration. + if (!(err instanceof RetryableError)) throw err; + this.holds.hold(task.id, event, inboxRow.source, err); + return "hold"; } } @@ -326,7 +273,7 @@ export class EventDispatcher { // Returns true iff THIS call transitioned the row to 'responded' (0-row // no-ops and already-responded re-fires return false). Callers use the // return as a first-transition gate for at-most-once side effects (the - // Slack completion notify). + // Slack completion notify, and onRoundFinished's firstTransition). private async markInboxResponded(inboxRow: InboxRow, outcome: string | null): Promise { if (inboxRow.status === "responded") return false; // already done — idempotent re-fire try { @@ -353,49 +300,6 @@ export class EventDispatcher { } } - private requestReviewFromOriginalReviewer( - inboxRow: Extract, - task: Task, - ): void { - const payload = inboxRow.payload as Record | null | undefined; - const reviewerLoginRaw = payload?.reviewer_login; - const reviewerLogin = typeof reviewerLoginRaw === "string" ? reviewerLoginRaw.trim() : ""; - if (!reviewerLogin) { - logger.info("review re-request skipped", { - event: "review_rerequest.result", - status: "skipped", - reason: "missing_reviewer_login", - taskId: task.id, - inboxId: inboxRow.id, - }); - return; - } - - const pr = reviewPrCoordinates(inboxRow); - if (!pr) { - logger.warn("review re-request skipped", { - event: "review_rerequest.result", - status: "skipped", - reason: "missing_pr_coordinates", - taskId: task.id, - inboxId: inboxRow.id, - reviewerLogin, - }); - return; - } - - void requestReviewers(pr.owner, pr.repo, pr.prNumber, [reviewerLogin]).catch((err) => - logErr("request reviewers failed", err, { - taskId: task.id, - inboxId: inboxRow.id, - owner: pr.owner, - repo: pr.repo, - prNumber: pr.prNumber, - reviewerLogin, - }), - ); - } - private async postOnboardingComment( event: Extract, task: Task, @@ -405,76 +309,72 @@ export class EventDispatcher { if (!webhooks) return; const pr = parsePrUrl(event.pr_url); if (!pr) return; // not a github.com PR URL — skip - await postIssueComment( - pr.owner, - pr.repo, - pr.prNumber, - onboardingCommentBody(webhooks.botLogin, task.key), + // Awaited inside the tick, so bounded like a send. + await withTimeout(this.sendTimeoutMs, (signal) => + postIssueComment( + pr.owner, + pr.repo, + pr.prNumber, + onboardingCommentBody(webhooks.botLogin, task.key), + signal, + ), ).catch((err) => logErr("onboarding comment post failed", err)); } - // Mirrors the Linear ticket's human assignee onto the GitHub PR as assignee. - // Assignee is the right primitive here: harness PRs open as DRAFTS and GitHub - // rejects requested_reviewers on drafts. Identity comes from Linear's own - // user↔GitHub mapping (User.gitHubUserId, populated when a member connects - // GitHub under Linear Settings → Connected accounts) — never email matching. + // Mirrors the task's assignee (the owner integration's assignee()) onto the + // GitHub PR as assignee. Assignee is the right primitive here: harness PRs + // open as DRAFTS and GitHub rejects requested_reviewers on drafts. // // Best-effort by contract: never throws, a failure must not block dispatch or - // the cursor. Idempotent on cursor-replay (POST assignees re-add is a GitHub - // no-op) — with the accepted quirk that a human who manually unassigned - // themselves gets re-added on a rare replay. - private async assignPrToLinearAssignee( + // the cursor. Runs once per PR (firstTime). + private async mirrorPrAssignee( event: Extract, task: Task, - linearClient: LinearClient | null, + round: InboxRow | null, ): Promise { - if (!linearClient) return; // GH-sourced task — no Linear identity to mirror. const pr = parsePrUrl(event.pr_url); if (!pr) return; // not a github.com PR URL — skip + // undefined: no assignee() (API tasks), or the lookup failed (logged). + const person = await this.hooks.assignee(task, round, { prUrl: event.pr_url }); + if (person === undefined) return; + if (!person) { + logger.info("PR assignee mirror skipped", { + event: "pr_assignee.result", + status: "skipped", + reason: "no_human_assignee", + taskId: task.id, + prUrl: event.pr_url, + }); + return; + } + if (typeof person.codeHost !== "object") { + // Linear's fix path: the member connects GitHub in Linear Settings → + // Connected accounts. Log the account id, not the name or email. + logger.info("PR assignee mirror skipped", { + event: "pr_assignee.result", + status: "skipped", + reason: person.codeHost === "unlinked" ? "no_github_mapping" : "unsupported", + taskId: task.id, + prUrl: event.pr_url, + linearUserId: person.id, + }); + return; + } + const login = person.codeHost.githubLogin; + if (!login) { + // getUserById normalizes a malformed 2xx body to login: "" — don't POST + // an empty assignee and misreport it as no_push_access. + logger.warn("PR assignee mirror skipped", { + event: "pr_assignee.result", + status: "skipped", + reason: "unresolved_login", + taskId: task.id, + prUrl: event.pr_url, + linearUserId: person.id, + }); + return; + } try { - const human = await getIssueHumanAssignee(linearClient, task.externalId); - if (!human) { - logger.info("PR assignee mirror skipped", { - event: "pr_assignee.result", - status: "skipped", - reason: "no_human_assignee", - taskId: task.id, - prUrl: event.pr_url, - }); - return; - } - if (!human.gitHubUserId) { - // Fix path: the member connects GitHub in Linear Settings → Connected - // accounts. Log id + displayName, not email. - logger.info("PR assignee mirror skipped", { - event: "pr_assignee.result", - status: "skipped", - reason: "no_github_mapping", - taskId: task.id, - prUrl: event.pr_url, - linearUserId: human.id, - linearDisplayName: human.displayName, - }); - return; - } - // Defensive about the field's format: numeric → GitHub account id - // (resolve to the current login), anything else → already a login. - const login = /^\d+$/.test(human.gitHubUserId) - ? (await getUserById(human.gitHubUserId)).login - : human.gitHubUserId; - if (!login) { - // getUserById normalizes a malformed 2xx body to login: "" — don't POST - // an empty assignee and misreport it as no_push_access. - logger.warn("PR assignee mirror skipped", { - event: "pr_assignee.result", - status: "skipped", - reason: "unresolved_login", - taskId: task.id, - prUrl: event.pr_url, - linearUserId: human.id, - }); - return; - } const { assigned } = await addPrAssignees(pr.owner, pr.repo, pr.prNumber, [login]); // GitHub logins are case-insensitive but the response echoes canonical // casing — compare case-folded so a cased mapping isn't a false drop. @@ -500,56 +400,33 @@ export class EventDispatcher { assigned, }); } catch (err) { - logErr("assignPrToLinearAssignee failed", err, { taskId: task.id, prUrl: event.pr_url }); - } - } - - // Best-effort run-archive upload. Runs the harness's `archive` command in the - // still-live sandbox, then uploads the tar.gz to the Linear issue as an - // attachment. Fully swallowed — the archive is a forensic nicety, never - // load-bearing; a failure must not affect dispatch. GH-source tasks have no - // Linear issue to attach to, so they are skipped. - private async attachRunArchiveToLinearIssue(task: Task): Promise { - if (!task.sandboxId || task.source !== "linear") return; - try { - const bytes = await this.sandboxManager.extractArchive(task.sandboxId); - if (!bytes || bytes.byteLength === 0) return; - const client = await this.linearTokenProvider.getAuthorizedLinearClient(); - await uploadIssueAttachment(client, { - issueId: task.externalId, - title: "Harness run archive", - subtitle: `Task ${task.id}`, - filename: `harness-run-${task.id}.tar.gz`, - contentType: "application/gzip", - bytes, - metadata: { taskId: task.id }, - }); - } catch (err) { - logErr(`attachRunArchiveToLinearIssue failed for task ${task.id}`, err); + logErr("PR assignee mirror failed", err, { taskId: task.id, prUrl: event.pr_url }); } } } -function reviewPrCoordinates( - inboxRow: Extract, -): { owner: string; repo: string; prNumber: number } | null { - const pr = inboxRow.payload.pr as Record | undefined; - const owner = typeof pr?.owner === "string" ? pr.owner : ""; - const repo = typeof pr?.repo === "string" ? pr.repo : ""; - const prNumber = typeof pr?.prNumber === "number" ? pr.prNumber : Number.NaN; - if (owner && repo && Number.isFinite(prNumber)) { - return { owner, repo, prNumber }; - } - - const target = inboxRow.reply_targets.find( - (t) => t.spec.kind === "review_body" || t.spec.kind === "review_comment", - ); - if (target && "owner" in target.spec) { - return { owner: target.spec.owner, repo: target.spec.repo, prNumber: target.spec.prNumber }; +// What an agent emit is, as the integration sees it. A non-retryable +// agent_error relies on the harness contract: on an assignment its body is the +// user-safe summary (_format_completion_body); on a comment, review or CI round +// it carries exception text (_emit_round_failure). Neither marks the row failed +// nor writes a task_event: the following `completed` marks it responded. +function labelAgentEmit(event: AgentEmit, round: InboxRow): Outbound { + switch (event.type) { + case "agent_response": + return { type: "reply", body: event.body }; + case "agent_thought": + return { type: "progress", kind: "thought", body: event.body }; + case "agent_action": + return { type: "progress", kind: "action", body: event.body }; + case "agent_error": + // A retried assignment attempt; the final failure arrives through failTask. + if (event.retryable_assignment_failure) { + return { type: "progress", kind: "error", body: event.body }; + } + return round.kind === "assignment" + ? { type: "failure", cause: "round_failed", round, summary: event.body } + : { type: "failure", cause: "round_failed", round, detail: event.body }; } - - const prUrl = typeof pr?.pr_url === "string" ? pr.pr_url : ""; - return prUrl ? parsePrUrl(prUrl) : null; } // Untrusted values (e.g. sandbox-written pr_url) belong in `meta`, never in diff --git a/control-plane/src/services/inbox.service.ts b/control-plane/src/services/inbox.service.ts index e9787f7..8f7e9c9 100644 --- a/control-plane/src/services/inbox.service.ts +++ b/control-plane/src/services/inbox.service.ts @@ -1,28 +1,24 @@ -import { - addIssueCommentReaction, - addReviewCommentReaction, - postIssueComment, - postReviewCommentReply, -} from "../clients/github"; -import { emitError } from "../clients/linear"; import { config } from "../config"; import type { Task } from "../db/schema"; +import { deliverToRound } from "../integrations/http"; +import type { TaskIntake } from "../integrations/intake"; +import type { Integration, IntegrationName } from "../integrations/types"; import { logger } from "../logger"; import type { InboxRepository, InboxRow, ReplyTarget, ReplyTargetSpec, - ReviewPayload, } from "../repositories/inbox.repository"; -import { taskIdentity } from "../repositories/inbox.repository"; import type { InboxMessage } from "../types/inboxMessage"; -import type { LinearTokenProvider } from "./linearTokenProvider"; +import type { IntegrationHooks } from "./integrationHooks"; import type { SandboxManager } from "./sandboxManager.service"; -// InboxService owns inbox enqueue (comment/review/Linear-comment), the sweep -// that delivers pending rows to the sandbox (recreating it from the preserved -// PVC when it has been reaped), and the PVC-missing per-row dead-letter. +// InboxService builds the GitHub and Linear follow-up rounds (comment, review, +// CI result, Linear comment) and queues them through TaskIntake.followUp; it +// owns the sweep that delivers pending rows to the sandbox (recreating it from +// the preserved PVC when it has been reaped), and the PVC-missing per-row +// dead-letter. export const WORKSPACE_MISSING_MESSAGE = "The workspace for this task has been cleaned up and can no longer be resumed. " + "Please reach out to the harness team or open a new ticket to continue."; @@ -31,7 +27,9 @@ export class InboxService { constructor( private inboxRepo: InboxRepository, private sandboxManager: SandboxManager, - private linearTokenProvider: Pick, + private integrations: Record, + private hooks: IntegrationHooks, + private intake: Pick, ) {} // ──── Enqueue paths ──── @@ -46,8 +44,9 @@ export class InboxService { createdAt: string; }): Promise { const externalId = String(params.commentId); - await this.inboxRepo.appendInbox({ - taskId: params.task.id, + const prUrl = `https://github.com/${params.spec.owner}/${params.spec.repo}/pull/${params.spec.prNumber}`; + await this.intake.followUp({ + task: params.task, message: { source: "github", kind: "comment", @@ -55,15 +54,12 @@ export class InboxService { payload: { target_id: externalId, // matches reply_targets[0].id body: params.body, - ...taskIdentity(params.task), - pr: { - repo: params.spec.repo, - pr_url: `https://github.com/${params.spec.owner}/${params.spec.repo}/pull/${params.spec.prNumber}`, - }, + pr: { repo: params.spec.repo, pr_url: prUrl }, created_at: params.createdAt, }, }, - replyTargets: [{ id: externalId, spec: params.spec }], + targets: [{ id: externalId, spec: params.spec }], + prUrl, }); } @@ -88,8 +84,9 @@ export class InboxService { createdAt: string; }): Promise { const reviewIdStr = String(params.reviewId); + const prUrl = `https://github.com/${params.owner}/${params.repo}/pull/${params.prNumber}`; - const payload: ReviewPayload = { + const payload = { review_body: { target_id: reviewIdStr, body: params.reviewBody }, inline_comments: params.inlineComments.map((c) => ({ target_id: String(c.id), @@ -98,13 +95,7 @@ export class InboxService { start_line: c.startLine, body: c.body, })), - ...taskIdentity(params.task), - pr: { - owner: params.owner, - repo: params.repo, - prNumber: params.prNumber, - pr_url: `https://github.com/${params.owner}/${params.repo}/pull/${params.prNumber}`, - }, + pr: { owner: params.owner, repo: params.repo, prNumber: params.prNumber, pr_url: prUrl }, reviewer_login: params.reviewerLogin, created_at: params.createdAt, }; @@ -139,40 +130,35 @@ export class InboxService { }, ]; - await this.inboxRepo.appendInbox({ - taskId: params.task.id, - message: { - source: "github", - kind: "review", - external_id: reviewIdStr, - payload, - }, - replyTargets, + await this.intake.followUp({ + task: params.task, + message: { source: "github", kind: "review", external_id: reviewIdStr, payload }, + targets: replyTargets, + prUrl, }); } // Linear `prompted` — a follow-up comment on the agent session. Same inbox // path as a GitHub comment; the single reply target routes to the Linear // session. Idempotent on the Linear activity UUID — a duplicate redelivery - // is a no-op insert. + // is dropped. async enqueueLinearComment(params: { task: Pick; agentActivityId: string; agentSessionId: string; body: string; }): Promise { - await this.inboxRepo.appendInbox({ - taskId: params.task.id, + await this.intake.followUp({ + task: params.task, message: { source: "linear", kind: "comment", - // Linear activity UUID — drives idempotency at (source, external_id). + // Linear activity UUID — drives idempotency at (source, kind, external_id). external_id: params.agentActivityId, payload: { // Replies route to the Linear agent session — matches reply_targets[0].id. target_id: params.agentSessionId, body: params.body, - ...taskIdentity(params.task), // Linear comment isn't PR-scoped — lives on the agent session. pr: null, // Linear-sourced → the daemon never injects PR discussion (gated on @@ -180,14 +166,14 @@ export class InboxService { created_at: "", }, }, - replyTargets: [{ id: params.agentSessionId, spec: { kind: "linear_session" } }], + targets: [{ id: params.agentSessionId, spec: { kind: "linear_session" } }], }); } // GitHub `workflow_run` completed — one row per task whose PR URL matches. // external_id includes runAttempt (re-runs share runId but bump attempt) and - // prNumber (one run can touch multiple PRs) for the global (source, external_id) - // unique index. + // prNumber (one run can touch multiple PRs). followUp applies the per-task + // loop breaker. async enqueueCiResult(params: { task: Pick; runId: number; @@ -202,30 +188,11 @@ export class InboxService { repo: string; prNumber: number; }): Promise { - // Loop breaker. harness fixes CI → pushes → CI re-runs → may fail again, - // ad infinitum. Once a task has accumulated `ciResultMaxPerTask` ci_result - // rows, stop forwarding new failures to the harness. The count/insert is - // not transactional, so concurrent webhooks for one task may overshoot by a - // few — fine for a circuit breaker. 0 / negative disables the breaker. - const max = config.github.ciResultMaxPerTask; - if (max > 0) { - const seen = await this.inboxRepo.countCiResultsByTask(params.task.id); - if (seen >= max) { - logger.warn("ci_result loop breaker tripped — dropping CI failure", { - taskId: params.task.id, - seen, - max, - runId: params.runId, - prNumber: params.prNumber, - }); - return; - } - } - const runIdStr = String(params.runId); const externalId = `workflow_run:${params.runId}:${params.runAttempt}:${params.prNumber}`; - await this.inboxRepo.appendInbox({ - taskId: params.task.id, + const prUrl = `https://github.com/${params.owner}/${params.repo}/pull/${params.prNumber}`; + await this.intake.followUp({ + task: params.task, message: { source: "github", kind: "ci_result", @@ -238,14 +205,10 @@ export class InboxService { run_url: params.runUrl, head_sha: params.headSha, head_branch: params.headBranch, - ...taskIdentity(params.task), - pr: { - repo: params.repo, - pr_url: `https://github.com/${params.owner}/${params.repo}/pull/${params.prNumber}`, - }, + pr: { repo: params.repo, pr_url: prUrl }, }, }, - replyTargets: [ + targets: [ { id: runIdStr, spec: { @@ -259,6 +222,7 @@ export class InboxService { }, }, ], + prUrl, }); } @@ -303,8 +267,8 @@ export class InboxService { } // A task whose PVC no longer exists cannot be resumed. Dead-letters every - // still-open ('pending'/'delivered') row of the task (per-row user notice + mark - // failed) so it stops pinning the reaper. Called by runTask's recreate-from-PVC + // still-open ('pending'/'delivered') row of the task (mark failed + per-row + // user notice) so it stops pinning the reaper. Called by runTask's recreate-from-PVC // path when createSandboxFromExistingPvc reports the PVC gone. // Re-checks each row's status from DB so a row a reassignment superseded mid-tick // is left untouched. @@ -313,83 +277,44 @@ export class InboxService { for (const row of rows) { const latest = await this.inboxRepo.getById(row.id); if (!latest || (latest.status !== "pending" && latest.status !== "delivered")) continue; - await this.handlePvcMissing(latest); + await this.handlePvcMissing(task, latest); } } - // PVC-missing per-row dead-letter. Once-per-row: emit a Linear error / GH - // reply + confused reaction, then mark the row failed so the sweep skips it. - private async handlePvcMissing(entry: InboxRow): Promise { - const userMessage = WORKSPACE_MISSING_MESSAGE; - - try { - if (entry.source === "api") { - // No outbound channel: the round_failed event markFailed writes below - // is how the user sees it. - } else if (entry.source === "linear") { - // reply_targets[0].id IS the agent session id by construction - // (assignment and prompted enqueue paths both stamp it). - const sessionTarget = entry.reply_targets.find((t) => t.spec.kind === "linear_session"); - if (sessionTarget) { - const linearClient = await this.linearTokenProvider.getAuthorizedLinearClient(); - await emitError(linearClient, sessionTarget.id, userMessage).catch(logErrSwallow); - } - } else if (entry.source === "github" && entry.kind === "comment") { - const target = entry.reply_targets[0]; - if (target?.spec.kind === "issue_comment") { - await postIssueComment( - target.spec.owner, - target.spec.repo, - target.spec.prNumber, - userMessage, - ).catch(logErrSwallow); - await addIssueCommentReaction( - target.spec.owner, - target.spec.repo, - target.spec.commentId, - "confused", - ).catch(logErrSwallow); - } else if (target?.spec.kind === "review_comment") { - await postReviewCommentReply( - target.spec.owner, - target.spec.repo, - target.spec.prNumber, - target.spec.inReplyTo, - userMessage, - ).catch(logErrSwallow); - await addReviewCommentReaction( - target.spec.owner, - target.spec.repo, - target.spec.commentId, - "confused", - ).catch(logErrSwallow); - } - } else if (entry.source === "github" && entry.kind === "review") { - // Top-level PR comment — the review body has no reply / reactions API. - const reviewBodyTarget = entry.reply_targets.find((t) => t.spec.kind === "review_body"); - if (reviewBodyTarget?.spec.kind === "review_body") { - await postIssueComment( - reviewBodyTarget.spec.owner, - reviewBodyTarget.spec.repo, - reviewBodyTarget.spec.prNumber, - userMessage, - ).catch(logErrSwallow); - } - } else if (entry.source === "github" && entry.kind === "ci_result") { - // CI result rows are informational — silently dead-letter on pruned - // workspace. No user-facing error is posted; the finally block below - // calls markFailed to clear the row from the sweep. - } - } finally { - await this.inboxRepo - .markFailed({ - inboxId: entry.id, - ownerId: config.podId, - reason: "workspace_missing", - userMessage, - }) - .catch((err) => logErr(`markFailed failed for inbox ${entry.id}`, err)); - } + // PVC-missing per-row dead-letter. Marks the row failed first, so the sweep + // skips it, then hands a workspace_missing failure to the round's integration + // only if this call transitioned the row (a racing restart or a late + // `completed` gets no contradictory notice). One notice per row, no target-id + // dedupe (unlike failTask); reply_targets[0] is the round's thread, and the + // integration words the notice by round kind. At most once: there is no + // cursor to hold, so a delivery error (RetryableError included) is logged and + // dropped, and the task's remaining rows still dead-letter. A dead-lettered + // assignment also finishes its round as failed (onRoundFinished, detached). + private async handlePvcMissing(task: Task, entry: InboxRow): Promise { + const marked = await this.inboxRepo + .markFailed({ + inboxId: entry.id, + ownerId: config.podId, + reason: "workspace_missing", + userMessage: WORKSPACE_MISSING_MESSAGE, + }) + .catch((err) => { + logErr(`markFailed failed for inbox ${entry.id}`, err); + return []; + }); + if (marked.length === 0) return; + if (entry.kind === "assignment") this.hooks.roundFinished(task, entry, "failed", true); + const target = entry.reply_targets[0]; + if (!target) return; + // A row whose integration is unknown or disabled gets no notice; it is + // still failed. No cursor to hold, so a RetryableError is dropped too. + await deliverToRound( + this.integrations, + entry, + target, + { type: "failure", cause: "workspace_missing", round: entry }, + { taskId: task.id }, + ).catch((err) => logErr(`workspace_missing notice for inbox ${entry.id} dropped`, err)); } } @@ -403,11 +328,6 @@ function logDebugErr(msg: string, err: unknown): void { logger.debug(msg, { message: e.message, stack: e.stack }); } -// 1-arg variant for `.catch(logErrSwallow)` on fire-and-forget calls. -function logErrSwallow(err: unknown): void { - logErr("fire-and-forget call failed", err); -} - // Strip CP-internal persistence metadata from a row, keeping only the wire // contract fields the daemon expects to read from `inbox/.json`. function toInboxMessage(row: InboxRow): InboxMessage { diff --git a/control-plane/src/services/integrationHooks.ts b/control-plane/src/services/integrationHooks.ts new file mode 100644 index 0000000..298b676 --- /dev/null +++ b/control-plane/src/services/integrationHooks.ts @@ -0,0 +1,190 @@ +import type { Task } from "../db/schema"; +import { withTimeout } from "../integrations/http"; +import { RetryableError } from "../integrations/outbound"; +import type { + HookCtx, + Integration, + IntegrationName, + Outcome, + Person, + PrRef, +} from "../integrations/types"; +import { logger } from "../logger"; +import type { InboxRepository, InboxRow } from "../repositories/inbox.repository"; + +// Upper bound on one hook call. +const HOOK_TIMEOUT_MS = { + onTaskStarted: 20_000, + onPrOpened: 30_000, + onRoundFinished: 120_000, + assignee: 15_000, +} as const; +type HookName = keyof typeof HOOK_TIMEOUT_MS; +// What a detached promise is doing, for the shutdown log. +type DetachedWork = HookName | "killSandbox" | "slack"; + +// Runs integrations' lifecycle hooks. onTaskStarted, onPrOpened and +// assignee go to the task's own integration; onRoundFinished goes to the +// round's. A disabled integration's hooks are skipped (logged). Each call runs +// under its hook's timeout with an abort signal and never throws: a failure is +// logged and the caller goes on. The one exception +// is a RetryableError from onPrOpened, which escapes so the dispatcher holds +// pr_opened as it holds a send (the hook must be idempotent). Detached work is +// tracked so TaskProcessor.stop() can drain it. +export class IntegrationHooks { + private readonly detached = new Map, { taskId: string; hook: DetachedWork }>(); + // Assignment ids whose onTaskStarted already ran on this pod: a provision + // deferred by a transient error runs again for the same assignment. + // ponytail: one id per started task for the pod's lifetime; prune if that ever matters. + private readonly started = new Set(); + + constructor( + private integrations: Record, + private inboxRepo: Pick, + private timeoutsMs: Record = HOOK_TIMEOUT_MS, + ) {} + + // Fresh provision, before the sandbox is created. Awaited; no hold. + async taskStarted(task: Task): Promise { + const owner = this.integrations[task.source]; + if (!owner.onTaskStarted) return; + const round = await this.inboxRepo.getAssignment(task.id).catch((err) => { + logErr("onTaskStarted skipped: assignment lookup failed", err, { taskId: task.id }); + return null; + }); + const target = round?.reply_targets[0]; + if (!round || !target || this.started.has(round.id)) return; + this.started.add(round.id); + await this.callHook("onTaskStarted", owner, task, round, (ctx) => + owner.onTaskStarted?.(ctx, task, target), + ); + } + + // pr_opened, after appendPrUrl, awaited before the cursor advances. Throws + // RetryableError (nothing was sent), which the dispatcher turns into a hold. + async prOpened(task: Task, round: InboxRow | null, pr: PrRef, firstTime: boolean): Promise { + const owner = this.integrations[task.source]; + await this.callHook("onPrOpened", owner, task, round, (ctx) => + owner.onPrOpened?.({ ...ctx, firstTime }, task, pr), + ); + } + + // The task's assignee, for detached work. undefined when the integration + // has no assignee, is disabled, or the lookup failed (logged here, with + // `logMeta`). + assignee( + task: Task, + round: InboxRow | null, + logMeta: Record, + ): Promise { + const owner = this.integrations[task.source]; + return this.callHook( + "assignee", + owner, + task, + round, + (ctx) => owner.assignee?.(ctx, task), + logMeta, + ); + } + + // Detached. Called (a) on `completed`, with firstTransition = whether that + // call marked the row responded, and (b) when a task failure or a missing + // workspace dead-letters an assignment row: outcome failed, first + // transition, no archive. Integrations gate their posts on firstTransition. + roundFinished( + task: Task, + round: InboxRow, + outcome: Outcome | null, + firstTransition: boolean, + archive: () => Promise = async () => null, + ): void { + // A source this build has no integration for (rows written by a newer + // build before a rollback) has no hook. + const owner: Integration | undefined = this.integrations[round.source]; + if (!owner?.onRoundFinished) return; + this.detach( + task.id, + "onRoundFinished", + this.callHook("onRoundFinished", owner, task, round, (ctx) => + owner.onRoundFinished?.({ ...ctx, firstTransition, archive }, task, round, outcome), + ), + ); + } + + // Tracks detached work (a hook, core work that calls one, or a restart's + // sandbox kill) for drain(). + detach(taskId: string, hook: DetachedWork, work: Promise): void { + const p: Promise = work + .then( + () => {}, + (err) => logErr(`detached ${hook} work failed`, err, { taskId }), + ) + .finally(() => this.detached.delete(p)); + this.detached.set(p, { taskId, hook }); + } + + // Waits for detached work until `deadline` settles, then logs what is still + // running; the process exits right after. Work detached meanwhile (say, a + // restart's sandbox kill from a webhook still being served) is waited for too. + async drain(deadline: Promise): Promise { + let expired = false; + void deadline.then(() => { + expired = true; + }); + while (this.detached.size > 0 && !expired) { + await Promise.race([Promise.allSettled(this.detached.keys()), deadline]); + } + for (const { taskId, hook } of this.detached.values()) { + logger.warn("hook abandoned at shutdown", { event: "hook_abandoned", taskId, hook }); + } + } + + private async callHook( + hook: HookName, + owner: Integration, + task: Task, + round: InboxRow | null, + run: (ctx: HookCtx) => Promise | undefined, + logMeta: Record = {}, + ): Promise { + if (!owner[hook]) return undefined; + const meta = { ...logMeta, taskId: task.id, integration: owner.name }; + if (!owner.enabled) { + logger.info(`${hook} skipped: integration disabled`, meta); + return undefined; + } + try { + return await withTimeout(this.timeoutsMs[hook], async (signal) => { + // A failed lookup still runs the hook; isCurrent() then answers false. + const startedOn = await this.assignmentId(task.id, round).catch((err) => { + logErr(`${hook}: assignment lookup failed; isCurrent() is false`, err, meta); + return undefined; + }); + return run({ + signal, + log: logger, + isCurrent: async () => + startedOn !== undefined && + (await this.assignmentId(task.id).catch(() => undefined)) === startedOn, + }); + }); + } catch (err) { + if (hook === "onPrOpened" && err instanceof RetryableError) throw err; + logErr(`${hook} hook failed`, err, meta); + return undefined; + } + } + + // The task's assignment id: a restart re-pends the assignment row under a + // new id. When the round in hand is the assignment, its id is that id. + private async assignmentId(taskId: string, round?: InboxRow | null): Promise { + if (round?.kind === "assignment") return round.id; + return (await this.inboxRepo.getAssignment(taskId))?.id ?? null; + } +} + +function logErr(msg: string, err: unknown, meta: Record): void { + const e = err instanceof Error ? err : new Error(String(err)); + logger.error(msg, { ...meta, message: e.message, stack: e.stack }); +} diff --git a/control-plane/src/services/linearTokenProvider.ts b/control-plane/src/services/linearTokenProvider.ts index 2b898f9..af1afec 100644 --- a/control-plane/src/services/linearTokenProvider.ts +++ b/control-plane/src/services/linearTokenProvider.ts @@ -77,9 +77,11 @@ export class LinearTokenProvider { // A LinearClient that, when Linear answers 401, renews the token and retries // the call once. Covers SDK calls (`request`) and raw queries (`rawRequest`). - async getAuthorizedLinearClient(): Promise { + // `signal` aborts this client's requests only; the shared token request + // never gets it, so one caller's timeout can't fail another's refresh. + async getAuthorizedLinearClient(signal?: AbortSignal): Promise { let current = await this.getAccessToken(); - const client = new LinearClient({ accessToken: current, apiUrl: this.apiUrl }); + const client = new LinearClient({ accessToken: current, apiUrl: this.apiUrl, signal }); // LinearClient sends every query through this instance's `request`, read at // call time, so wrapping it covers all calls made through the client. const gql = client.client; diff --git a/control-plane/src/services/replyHolds.ts b/control-plane/src/services/replyHolds.ts new file mode 100644 index 0000000..d2453d5 --- /dev/null +++ b/control-plane/src/services/replyHolds.ts @@ -0,0 +1,80 @@ +import type { RetryableError } from "../integrations/outbound"; +import type { IntegrationName } from "../integrations/types"; +import { logger } from "../logger"; +import type { HarnessEvent } from "../types/harnessEvent"; + +const WARN_EVERY_MS = 10 * 60 * 1000; + +type Hold = { + inboxId: string; + seq: number; + integration: IntegrationName; + since: number; + nextAttemptAt: number; + warnedAt: number; +}; + +// Events whose send failed before anything went out. A held task +// stops at that event, so it holds at most one: keyed by task, the hold is +// (taskId, inboxId, seq) and a different event starts a new one. In memory and +// with no time limit: the reconcile loop drops a hold when the cursor moves +// past its event or the task leaves this pod. +export class ReplyHolds { + private readonly byTask = new Map(); + + // Records (or renews) a hold. The next attempt honours Retry-After, else it + // is the next tick. Warns `reply_held` at most every 10 min per task. + hold( + taskId: string, + event: Pick, + integration: IntegrationName, + err: RetryableError, + ): void { + const now = Date.now(); + const { inbox_id: inboxId, seq } = event; + const prev = this.byTask.get(taskId); + const h = + prev?.inboxId === inboxId && prev.seq === seq + ? prev + : { inboxId, seq, integration, since: now, nextAttemptAt: now, warnedAt: -1 }; + h.nextAttemptAt = now + (err.retryAfterMs ?? 0); + if (h.warnedAt < 0 || now - h.warnedAt >= WARN_EVERY_MS) { + h.warnedAt = now; + logger.warn("reply held: its send failed before anything was sent; retrying", { + event: "reply_held", + taskId, + integration, + heldForMin: Math.floor((now - h.since) / 60_000), + reason: err.message, + }); + } + this.byTask.set(taskId, h); + } + + // True while the task's held event waits out its Retry-After. + waiting(taskId: string): boolean { + const h = this.byTask.get(taskId); + return h !== undefined && Date.now() < h.nextAttemptAt; + } + + release(taskId: string): void { + this.byTask.delete(taskId); + } + + // Drops the holds of tasks this pod no longer owns. + retain(ownedIds: Set): void { + for (const id of this.byTask.keys()) if (!ownedIds.has(id)) this.byTask.delete(id); + } + + // One integration's held tasks and oldest hold age, for its status body. + of(integration: IntegrationName): { heldTasks: number; oldestHoldAgeMs: number } { + const now = Date.now(); + const out = { heldTasks: 0, oldestHoldAgeMs: 0 }; + for (const h of this.byTask.values()) { + if (h.integration !== integration) continue; + out.heldTasks += 1; + out.oldestHoldAgeMs = Math.max(out.oldestHoldAgeMs, now - h.since); + } + return out; + } +} diff --git a/control-plane/src/services/sandboxManager.service.ts b/control-plane/src/services/sandboxManager.service.ts index 380fb24..ef687a9 100644 --- a/control-plane/src/services/sandboxManager.service.ts +++ b/control-plane/src/services/sandboxManager.service.ts @@ -864,6 +864,10 @@ export function validateHarnessEvent(parsed: unknown): HarnessEvent | null { inbox_id: parsed.inbox_id, target_id: parsed.target_id, body: capString(parsed.body, maxLen) ?? "", + // Kept only when exactly `true`; the dispatcher labels on it. + ...(parsed.type === "agent_error" && parsed.retryable_assignment_failure === true + ? { retryable_assignment_failure: true } + : {}), }; } diff --git a/control-plane/src/services/slackNotifier.service.ts b/control-plane/src/services/slackNotifier.service.ts index cc10016..f7cfe46 100644 --- a/control-plane/src/services/slackNotifier.service.ts +++ b/control-plane/src/services/slackNotifier.service.ts @@ -1,8 +1,9 @@ -import type { LinearClient } from "@linear/sdk"; -import { getIssueHumanAssignee } from "../clients/linear"; import { lookupSlackUserIdByEmail, postSlackMessage } from "../clients/slack"; import type { Task } from "../db/schema"; +import { SEND_TIMEOUT_MS, withTimeout } from "../integrations/http"; +import type { TaskSource } from "../integrations/types"; import type { TaskRepository } from "../repositories/task.repository"; +import type { IntegrationHooks } from "./integrationHooks"; // Terminal-state granularity. `completed | partial_failure | no_action` come // from the sandbox `completed` event's status field; `failed` is used by both @@ -13,11 +14,12 @@ export type TerminalStatus = "completed" | "partial_failure" | "no_action" | "fa type PrUrlEntry = { repo: string; url: string }; // Posts a single task-completion update to the configured Slack channel (a -// message per Linear-sourced task). Disabled unless both a bot token and a -// channel are configured. +// message per task whose source is opted in). Disabled unless both a bot token +// and a channel are configured. // -// Best-effort by contract: callers fire-and-forget with a `.catch`. A Slack -// outage or a missing thread must never affect task dispatch or teardown. +// Best-effort by contract: callers detach it (IntegrationHooks.detach logs a +// failure). A Slack outage or a missing thread must never affect task dispatch +// or teardown. export class SlackNotifierService { constructor( private readonly taskRepo: TaskRepository, @@ -27,19 +29,24 @@ export class SlackNotifierService { private readonly botToken: string, // config.slack.channel in production. Empty ⇒ Slack disabled (no-op). private readonly channel: string, + // config.slack.notifySources (SLACK_NOTIFY_SOURCES): the task sources + // whose tasks post. + private readonly notifySources: readonly TaskSource[], + // The owner integration's assignee(), for the connect-GitHub nudge. + private readonly hooks: Pick, // Injected for testability; default to the real implementations. private readonly post: typeof postSlackMessage = postSlackMessage, - private readonly resolveAssignee: typeof getIssueHumanAssignee = getIssueHumanAssignee, private readonly lookupSlackUser: typeof lookupSlackUserIdByEmail = lookupSlackUserIdByEmail, ) {} - async notifyTerminal( - linearClient: LinearClient, - task: Task, - status: TerminalStatus, - message?: string, - ): Promise { - if (!this.enabledFor(task)) return; + // Common gate for every Slack post: a token and a channel are provisioned + // and the source is opted in. Callers check it before notifyTerminal. + notifies(source: TaskSource): boolean { + return Boolean(this.botToken && this.channel) && this.notifySources.includes(source); + } + + async notifyTerminal(task: Task, status: TerminalStatus, message?: string): Promise { + if (!this.notifies(task.source)) return; // Re-read for the freshest pr_urls — the in-memory task handed in from the // reconcile loop can predate the pr_opened write that appended the PR. This @@ -49,52 +56,46 @@ export class SlackNotifierService { const fresh = await this.taskRepo.getById(task.id).catch(() => null); const t = fresh ?? task; const prUrls = parsePrUrls(t.prUrls); - // Nudge shown alongside PR links when the ticket's human assignee hasn't - // connected GitHub in Linear — the PR couldn't be assigned to them + // Nudge shown alongside PR links when a Linear ticket's human assignee + // hasn't connected GitHub in Linear — the PR couldn't be assigned to them // (eventDispatcher's pr_opened mirror skipped with no_github_mapping) and // self-service is the only fix. Re-checked lazily here so a user who // connected mid-run isn't nagged. Skipped for no_action, whose message // renders no PR section for the nudge to ride on. - const wantsNudge = status !== "no_action" && prUrls.length > 0; - const connectNote = wantsNudge ? await this.connectGitHubNote(linearClient, t) : ""; + const wantsNudge = t.source === "linear" && status !== "no_action" && prUrls.length > 0; + const connectNote = wantsNudge ? await this.connectGitHubNote(t) : ""; const text = buildMessage(status, t, prUrls, connectNote, message); - await this.post({ channel: this.channel, text }); + await withTimeout(SEND_TIMEOUT_MS, (signal) => + this.post({ channel: this.channel, text, signal }), + ); } // "" when there is nothing to nag about: no human assignee, already mapped, - // or the Linear lookup failed (best-effort — never blocks the post). - private async connectGitHubNote(linearClient: LinearClient, task: Task): Promise { - try { - const human = await this.resolveAssignee(linearClient, task.externalId); - if (!human || human.gitHubUserId) return ""; - // @-mention the assignee so the nudge notifies them (Slack only pings - // members of the channel — a non-member renders un-notified). Linear and - // Slack share the org email, so lookupByEmail is reliable here (it's only - // GitHub where personal emails diverge — the whole reason for this nudge). - // Degrades to the plain display name on any lookup miss (users_not_found, - // bot missing the users:read.email scope, transport error). displayName - // is member-editable — escape it so it can't inject mrkdwn (, - // forged <@mentions>, disguised links) into a trusted bot message. - const slackId = human.email - ? await this.lookupSlackUser(human.email).catch(() => null) - : null; - const mention = slackId ? `<@${slackId}>` : escapeMrkdwn(human.displayName); - return ( - `\n⚠️ ${mention}: I couldn't assign the PR to you — your GitHub account ` + - "isn't connected in Linear. Connect it under *Linear → Settings → Connected accounts* " + - "and future PRs will be assigned to you automatically." - ); - } catch { - return ""; - } - } - - // Common gate for every Slack post: a token and a channel are provisioned - // and the task is Linear-sourced. - private enabledFor(task: Task): boolean { - if (!this.botToken || !this.channel) return false; - return task.source === "linear"; + // or the lookup failed (IntegrationHooks logs it; never blocks the post). + private async connectGitHubNote(task: Task): Promise { + const person = await this.hooks.assignee(task, null, { purpose: "slack_nudge" }); + if (person?.codeHost !== "unlinked") return ""; + // @-mention the assignee so the nudge notifies them (Slack only pings + // members of the channel — a non-member renders un-notified). Linear and + // Slack share the org email, so lookupByEmail is reliable here (it's only + // GitHub where personal emails diverge — the whole reason for this nudge). + // Degrades to the plain display name on any lookup miss (users_not_found, + // bot missing the users:read.email scope, transport error). displayName + // is member-editable — escape it so it can't inject mrkdwn (, + // forged <@mentions>, disguised links) into a trusted bot message. + const email = person.email; + const slackId = email + ? await withTimeout(SEND_TIMEOUT_MS, (signal) => this.lookupSlackUser(email, signal)).catch( + () => null, + ) + : null; + const mention = slackId ? `<@${slackId}>` : escapeMrkdwn(person.displayName); + return ( + `\n⚠️ ${mention}: I couldn't assign the PR to you — your GitHub account ` + + "isn't connected in Linear. Connect it under *Linear → Settings → Connected accounts* " + + "and future PRs will be assigned to you automatically." + ); } } @@ -149,10 +150,12 @@ function prSection(prUrls: PrUrlEntry[]): string { return `\nHere are the PRs for review:\n${list}`; } -// The Linear-ticket link fragment shared by every message ("", or -// the bare title when the task has no external URL). +// The task's link fragment shared by every message ("", or the +// bare title when the task has no external URL). An untitled task is labelled +// "Linear ticket" on Linear and by its key elsewhere. function ticketLink(task: Task): string { - return task.externalUrl ? link(task.externalUrl, task.title || "Linear ticket") : task.title; + const label = task.title || (task.source === "linear" ? "Linear ticket" : task.key); + return task.externalUrl ? link(task.externalUrl, label) : task.title; } function buildMessage( diff --git a/control-plane/src/services/taskProcessor.service.ts b/control-plane/src/services/taskProcessor.service.ts index 70a60dc..711ec46 100644 --- a/control-plane/src/services/taskProcessor.service.ts +++ b/control-plane/src/services/taskProcessor.service.ts @@ -1,17 +1,18 @@ import type { SandboxInfo } from "@alibaba-group/opensandbox"; -import type { LinearClient } from "@linear/sdk"; -import { postIssueComment, postReviewCommentReply } from "../clients/github"; -import { emitAction, emitError, emitThought, setIssueStarted } from "../clients/linear"; import { config } from "../config"; import type { Task } from "../db/schema"; import { PvcNotFoundError, SandboxCreateError } from "../errors"; +import { deliverToRound } from "../integrations/http"; +import type { Outbound } from "../integrations/outbound"; +import type { Integration, IntegrationName } from "../integrations/types"; import { logger } from "../logger"; -import type { InboxRepository, ReplyTarget } from "../repositories/inbox.repository"; +import type { InboxRepository, InboxRow } from "../repositories/inbox.repository"; import type { TaskRepository } from "../repositories/task.repository"; import type { EventDispatcher } from "./eventDispatcher"; import { extractExperimentalFlags } from "./experimentalFlags"; import type { InboxService } from "./inbox.service"; -import type { LinearTokenProvider } from "./linearTokenProvider"; +import type { IntegrationHooks } from "./integrationHooks"; +import type { ReplyHolds } from "./replyHolds"; import { type PodStatus, SandboxManager } from "./sandboxManager.service"; import type { SlackNotifierService } from "./slackNotifier.service"; @@ -23,7 +24,11 @@ export class TaskProcessorService { private running = false; private loopPromise: Promise | null = null; private sleepAbort: AbortController | null = null; - private readonly inflight = new Set>(); + // Provisions running on this pod, by task id. + private readonly inflight = new Map>(); + // The last renewal of this pod's leases: when it was sent and what it renewed. + private leasesRenewedAt = 0; + private renewedTaskIds = new Set(); // First tick (per task) we saw the backing pod ABSENT (kubectl NotFound). A // pod can be briefly gone between create and scheduling, so absence gets a // grace window (config.sandboxAbsentGraceMs) before we fail. Cleared the @@ -37,28 +42,39 @@ export class TaskProcessorService { private readonly reconcileFilesUnavailableSince = new Map(); private readonly taskRepo: TaskRepository; private readonly inboxRepo: InboxRepository; - private readonly linearTokenProvider: Pick; private readonly sandboxManager: SandboxManager; private readonly eventDispatcher: EventDispatcher; private readonly inboxService: InboxService; - private readonly slackNotifier: SlackNotifierService; + private readonly slackNotifier: Pick; + private readonly integrations: Record; + private readonly holds: ReplyHolds; + // Sources this build has an integration for; claimNext and + // listInFlightOwnedBy see only these tasks. + private readonly taskSources: Task["source"][]; + private readonly hooks: IntegrationHooks; constructor( taskRepo: TaskRepository, inboxRepo: InboxRepository, - linearTokenProvider: Pick, sandboxManager: SandboxManager, eventDispatcher: EventDispatcher, inboxService: InboxService, - slackNotifier: SlackNotifierService, + slackNotifier: Pick, + integrations: Record, + holds: ReplyHolds, + taskSources: Task["source"][], + hooks: IntegrationHooks, ) { this.taskRepo = taskRepo; this.inboxRepo = inboxRepo; - this.linearTokenProvider = linearTokenProvider; this.sandboxManager = sandboxManager; this.eventDispatcher = eventDispatcher; this.inboxService = inboxService; this.slackNotifier = slackNotifier; + this.integrations = integrations; + this.holds = holds; + this.taskSources = taskSources; + this.hooks = hooks; } start(): void { @@ -76,9 +92,12 @@ export class TaskProcessorService { await this.loopPromise.catch(() => {}); } - const drain = Promise.allSettled([...this.inflight]); + // In-flight provisions, then detached hooks (some started by those + // provisions), share one shutdownDrainMs budget. + const drain = Promise.allSettled([...this.inflight.values()]); const timeout = new Promise((resolve) => setTimeout(resolve, config.shutdownDrainMs)); await Promise.race([drain, timeout]); + await this.hooks.drain(timeout); // Backdate own heartbeat so any survivors become immediately reclaimable // instead of waiting the full LEASE_TIMEOUT_MS. @@ -102,14 +121,25 @@ export class TaskProcessorService { private async tick(): Promise { // 1. Heartbeat refresh — keep owned-task leases warm. - await this.taskRepo.heartbeatOwnedBy(config.podId); + await this.renewLeases(); // 2. Idle reaper — kill sandboxes idle past the ceiling, release leases. await this.reapIdle().catch((err) => logErr("reapIdle failed", err)); - // 3. Reconcile owned tasks with live sandboxes. - const owned = await this.taskRepo.listInFlightOwnedBy(config.podId); + // 3. Reconcile owned tasks with live sandboxes. Before each reconcile and + // each event inside it, every lease this pod holds (in-flight provisions + // included) is renewed once a quarter of the lease has passed, so slow + // sends can't let a lease expire mid-tick; a task not renewed was taken + // over. + const owned = await this.taskRepo.listInFlightOwnedBy(config.podId, this.taskSources); for (const task of owned) { + // Shutting down: leave the rest to siblings (stop() backdates our + // heartbeats), so shutdown waits for one task's reconcile at most. + if (!this.running) break; + if (!(await this.leaseHeld(task.id))) { + logger.info("lease lost before reconcile; skipping", { taskId: task.id }); + continue; + } await this.reconcileOne(task) .then(() => { logger.debug("reconcileOne: succeeded", { taskId: task.id, sandboxId: task.sandboxId }); @@ -125,25 +155,49 @@ export class TaskProcessorService { }); } - // Drop grace/window timers for any task this pod no longer owns (lease handed - // off, retried away, or terminally failed). Keyed by task.id with no other GC, - // these Maps would otherwise leak for the process lifetime AND let a stale - // `since` from a prior ownership epoch skip the grace/window on a re-claimed - // task's first fault. `owned` is this tick's owned, sandbox-bound set. + // Drop grace/window timers and reply holds for any task this pod no longer + // owns (lease handed off, retried away, or terminally failed). Keyed by + // task.id with no other GC, these Maps would otherwise leak for the process + // lifetime AND let a stale `since` from a prior ownership epoch skip the + // grace/window on a re-claimed task's first fault. `owned` is this tick's + // owned, sandbox-bound set. const ownedIds = new Set(owned.map((t) => t.id)); for (const id of this.sandboxAbsentSince.keys()) if (!ownedIds.has(id)) this.sandboxAbsentSince.delete(id); for (const id of this.reconcileFilesUnavailableSince.keys()) if (!ownedIds.has(id)) this.reconcileFilesUnavailableSince.delete(id); + this.holds.retain(ownedIds); // 4. Claim new work (fresh-pending OR comment-pickup OR stale-lease - // takeover — single predicate). + // takeover — single predicate). None while shutting down. + if (!this.running) return; const claimed = await this.taskRepo.claimNext({ ownerId: config.podId, leaseTimeoutMs: config.leaseTimeoutMs, maxConcurrentSandboxes: config.maxConcurrentSandboxes, + sources: this.taskSources, }); - if (claimed) this.dispatch(claimed); + if (!claimed) return; + // A hold from before (say, a restart claimed in this same tick) belongs to + // the old run; it must not delay the new one. + this.holds.release(claimed.id); + this.dispatch(claimed); + } + + // Renews every lease this pod holds and notes which it renewed. + private async renewLeases(): Promise { + const at = Date.now(); + const rows = await this.taskRepo.heartbeatOwnedBy(config.podId); + this.leasesRenewedAt = at; + this.renewedTaskIds = new Set(rows.map((r) => r.id)); + } + + // Renews every lease once a quarter of the lease has passed since the last + // renewal (each step between checks is bounded well under a lease), then says + // whether `taskId` is still ours: one missing from the renewal was taken over. + private async leaseHeld(taskId: string): Promise { + if (Date.now() - this.leasesRenewedAt >= config.leaseTimeoutMs / 4) await this.renewLeases(); + return this.renewedTaskIds.has(taskId); } private async reapIdle(): Promise { @@ -164,16 +218,25 @@ export class TaskProcessorService { } private dispatch(task: Task): void { + // A provision still running here whose lease expired can be claimed back; + // a second one would orphan a sandbox and a PVC. + if (this.inflight.has(task.id)) { + logger.warn("claimed a task whose provision is still running here; skipping", { + taskId: task.id, + }); + return; + } const p = this.runTask(task) .catch((err) => this.handleTaskError(task, err)) - .finally(() => this.inflight.delete(p)); - this.inflight.add(p); + // Nothing awaits p, so a throw from handleTaskError itself (e.g. a DB + // error releasing the lease) would be an unhandled rejection. Log it. + .catch((err) => logErr(`handleTaskError failed for task ${task.id}`, err)) + .finally(() => this.inflight.delete(task.id)); + this.inflight.set(task.id, p); } // Creates or attaches a sandbox. Branches on `sandbox_id` nullability: // (sandbox_id=null) is a fresh provision; (sandbox_id set) is an adoption. - // - // Linear and API tasks share this path; Linear-only emits are gated below. private async runTask(task: Task): Promise { if (task.sandboxId) { // Adoption — resume the existing sandbox. Attach is best-effort: a failure @@ -232,36 +295,20 @@ export class TaskProcessorService { return; } - // Fresh provision. Linear tasks: assert issue state + lifecycle emits on - // the agent session. API tasks have no Linear surface; their log gets - // sandbox_started from attachSandboxResources instead. - const isLinear = task.source === "linear"; - if (isLinear) { - try { - const client = await this.linearTokenProvider.getAuthorizedLinearClient(); - await setIssueStarted(client, task.externalId); - } catch (err) { - logErr(`setIssueStarted failed for task ${task.id}`, err); - } - } - - // Reply target id on the assignment inbox row is the canonical destination - // for task-scoped lifecycle emits. A claimed task must have an assignment - // row (claimNext requires at least one non-terminal inbox row, and the - // `created` path always inserts one) — a missing target id means the row - // was deleted out from under us, which is a hard invariant break. - const replyTargetId = await this.inboxRepo.getAssignmentReplyTargetId(task.id); - if (!replyTargetId) { + // Fresh provision. The assignment row's reply target is the canonical + // destination for task-scoped lifecycle emits. A claimed task must have an + // assignment row (claimNext requires at least one non-terminal inbox row, + // and the `created` path always inserts one) — a missing target means the + // row was deleted out from under us, which is a hard invariant break. + const assignment = await this.inboxRepo.getAssignment(task.id); + if (!assignment?.reply_targets[0]) { throw new Error(`runTask: assignment inbox row missing reply target for task ${task.id}`); } - if (isLinear) { - try { - const client = await this.linearTokenProvider.getAuthorizedLinearClient(); - await emitAction(client, replyTargetId, "Provisioning", "infra"); - } catch (err) { - logErr(`emitAction(Provisioning) failed for task ${task.id}`, err); - } - } + + // The owner integration's onTaskStarted (Linear: issue state and the + // Provisioning action), at most once per assignment on this pod. Never + // throws. API tasks' logs get sandbox_started from attachSandboxResources. + await this.hooks.taskStarted(task); // WORKSPACE is the only task-specific input passed at creation; the daemon // derives its .harness dir from it and reads everything else from the inbox file @@ -299,20 +346,7 @@ export class TaskProcessorService { }); return; } - - if (isLinear) { - try { - const client = await this.linearTokenProvider.getAuthorizedLinearClient(); - await emitThought( - client, - replyTargetId, - `Sandbox provisioned.\n- sandboxId: \`${sandboxId}\`\n- pvcName: \`${pvcName}\``, - ); - } catch (err) { - logErr(`emitThought(provisioned-ids) failed for task ${task.id}`, err); - } - } - // No further Linear emit here — the daemon emits the Planning action via + // No Linear emit here — the daemon emits the Planning action via // events.jsonl once it starts processing the inbox file. } catch (err) { // A throw after a successful create (e.g. attachSandboxResources hit a @@ -359,6 +393,10 @@ export class TaskProcessorService { // Sweep: deliver any undelivered inbox rows to the sandbox. await this.inboxService.sweepUndelivered(task); + // A held event waiting out its Retry-After: skip this task's events this + // tick (the sweep above still hands new rounds to the sandbox). + if (this.holds.waiting(task.id)) return; + // Read events.jsonl since cursor; dispatch each new event. Re-fetch the // task between events because dispatcher cases mutate task state // (appendPrUrl, setSandboxId). @@ -366,34 +404,58 @@ export class TaskProcessorService { const events = await this.sandboxManager.readEvents(task.sandboxId, task.lastObservedEventSeq); for (const event of events) { if (currentTask === null) break; // self-evict: lease lost or task gone - - // handle() returns "self_evict" when an owner-guarded write inside the - // dispatcher (e.g. appendPrUrl) returned 0 rows — the lease was taken - // over mid-tick. Stop BEFORE advancing the cursor: a sibling pod owns - // this task and will resume from its own cursor. - const result = await this.eventDispatcher.handle(event, currentTask); - if (result === "self_evict") { - logger.info("dispatcher self-evicted (lease lost); stopping reconcile", { + // Shutting down: stop after the event in hand (the cursor is persisted + // per event), so shutdown waits for one event, not the backlog. + if (!this.running) break; + // A long run of slow sends must not let any lease expire; ours lost + // means a sibling took this task over. + if (!(await this.leaseHeld(currentTask.id))) { + logger.info("lease lost mid-reconcile; self-evicting", { taskId: currentTask.id, seq: event.seq, }); - break; + return; } - if (result === "abort") { - // The dispatcher found a foreign durable event in this task's stream (a - // cross-task leak). Stop here WITHOUT advancing the cursor — we neither - // apply nor skip past the leaked event. The task intentionally halts - // (and re-logs each tick) so the leak surfaces for an operator rather - // than being silently absorbed. Detail is in `cross_task_leak_detected`. - logger.error("reconcile halted: cross-task leak detected; not advancing cursor", { - event: "reconcile_halted_cross_task_leak", - taskId: currentTask.id, - taskExternalId: currentTask.externalId, - sandboxId: currentTask.sandboxId, - podId: config.podId, - seq: event.seq, - }); - break; + + const result = await this.eventDispatcher.handle(event, currentTask); + switch (result) { + case "ok": + case "dropped": + break; + case "hold": + // A send failed before anything went out. Stop BEFORE advancing the + // cursor and return normally (not a reconcile error): the event is + // retried on a later tick, and nothing after it runs ahead of it. + return; + case "self_evict": + // An owner-guarded write inside the dispatcher (e.g. appendPrUrl) + // returned 0 rows — the lease was taken over mid-tick. Stop BEFORE + // advancing the cursor: a sibling pod owns this task and will resume + // from its own cursor. + logger.info("dispatcher self-evicted (lease lost); stopping reconcile", { + taskId: currentTask.id, + seq: event.seq, + }); + return; + case "abort": + // The dispatcher found a foreign durable event in this task's stream (a + // cross-task leak). Stop here WITHOUT advancing the cursor — we neither + // apply nor skip past the leaked event. The task intentionally halts + // (and re-logs each tick) so the leak surfaces for an operator rather + // than being silently absorbed. Detail is in `cross_task_leak_detected`. + logger.error("reconcile halted: cross-task leak detected; not advancing cursor", { + event: "reconcile_halted_cross_task_leak", + taskId: currentTask.id, + taskExternalId: currentTask.externalId, + sandboxId: currentTask.sandboxId, + podId: config.podId, + seq: event.seq, + }); + return; + default: { + const _exhaustive: never = result; + void _exhaustive; + } } const advanced = await this.taskRepo.recordEventCursor({ @@ -410,6 +472,11 @@ export class TaskProcessorService { }); break; } + // The cursor is past any held event now. + this.holds.release(currentTask.id); + // A dropped send (timed out or failed for good) ends this task's events + // for the tick, so a slow or failing platform can't eat the tick. + if (result === "dropped") break; // Re-fetch (owner-scoped) to pick up dispatcher mutations (pr_urls // append, etc.). An owner-scoped fetch returns null on a takeover in the @@ -713,55 +780,71 @@ export class TaskProcessorService { } // Hard task failure (provision failure, sandbox timeout/crash, sandbox - // unreachable). Broadcast the failure message to every reply target on the - // task's non-terminal inbox rows (so each pending @harness comment + the - // Linear session sees why their round didn't complete), kill the sandbox, - // dead-letter the rows, release the lease. The dead-letter is what keeps - // the task from wedging: a 'pending'/'delivered' row left behind is - // un-claimable to no end (claimNext needs 'pending' but the task has no - // sandbox) and pins the idle reaper. + // unreachable). Dead-letter the task's open inbox rows, tell every reply + // target on the rows this call dead-lettered why their round didn't complete + // (each @harness comment + the Linear session), kill the sandbox, release the + // lease. The dead-letter is what keeps the task from wedging: a + // 'pending'/'delivered' row left behind is un-claimable to no end (claimNext + // needs 'pending' but the task has no sandbox) and pins the idle reaper. // killSandbox=false is for a crossed binding: the sandbox isn't ours to kill (a // sibling task owns it). We still dead-letter our rows + release the lease // (which clears our sandbox_id), so the task fails cleanly. The structured // failure log lives at each call site (with its pod/forensic detail); failTask - // only does the broadcast + teardown. + // only does the dead-letter, broadcast + teardown. private async failTask( task: Task, message: string, opts: { killSandbox?: boolean } = {}, ): Promise { const { killSandbox = true } = opts; - const nonTerminal = await this.inboxRepo - .listNonTerminalByTask(task.id) - .catch((e): Awaited> => { - logErr(`failTask: listNonTerminalByTask failed for ${task.id}`, e); + // Dead-letter the inbox rows FIRST, and BEFORE releaseLease — failTaskInbox + // is owner-guarded and needs owner_id still set. Only the rows it + // transitions (as it returns them) get a failure message, so a racing + // restart or a late `completed` can't draw a "Task failed" on a live or + // finished round, and every row it fails gets one. + const failed = await this.inboxRepo + .failTaskInbox({ taskId: task.id, ownerId: config.podId, reason: message }) + .catch((e): InboxRow[] => { + logErr(`failTaskInbox failed for task ${task.id}`, e); return []; }); - // Dedupe by target id — a Linear-source task has many rows sharing the - // same linear_session target id; posting once per id keeps the user from - // seeing the same error N times. + const assignment = failed.find((r) => r.kind === "assignment"); + if (failed.length > 0) { + logger.warn("inbox rows dead-lettered on task failure", { + taskId: task.id, + inboxIds: failed.map((r) => r.id), + reason: message, + }); + } + + // Dedupe by target — a Linear-source task has many rows sharing the + // same linear_session target id; posting once per target keeps the user + // from seeing the same error N times. Keyed by source too: two platforms' + // ids can collide. A row whose integration is unknown or disabled gets no + // notice; it is still failed. const seen = new Set(); - let cachedLinearClient: LinearClient | null = null; - const getLinearClient = async (): Promise => { - if (cachedLinearClient) return cachedLinearClient; - try { - cachedLinearClient = await this.linearTokenProvider.getAuthorizedLinearClient(); - return cachedLinearClient; - } catch (e) { - logErr("failTask: getAuthorizedLinearClient failed", e); - return null; - } - }; - for (const row of nonTerminal) { + for (const row of failed) { for (const t of row.reply_targets) { - if (seen.has(t.id)) continue; - seen.add(t.id); - await this.postFailureToTarget(t, message, getLinearClient).catch((e) => - logErr(`failTask: post to ${t.spec.kind} target ${t.id} failed`, e), + const key = `${row.source}:${t.id}`; + if (seen.has(key)) continue; + seen.add(key); + const notice: Outbound = { + type: "failure", + cause: "task_failed", + round: row, + summary: message, + }; + // At most once: there is no cursor to hold, so a RetryableError is + // dropped too. + await deliverToRound(this.integrations, row, t, notice, { taskId: task.id }).catch((e) => + logErr(`failTask: notice to target ${t.id} dropped`, e), ); } } + // The assignment's integration finishes its round as failed (detached). + if (assignment) this.hooks.roundFinished(task, assignment, "failed", true); + // Kill the sandbox the task ACTUALLY holds, read from the DB — NOT the // dispatch-time `task` closure, whose sandbox_id can be stale (a recreate // persists a fresh sandbox_id via setSandboxId after dispatch; the closure @@ -771,43 +854,24 @@ export class TaskProcessorService { const current = await this.taskRepo.getByIdOwnedBy(task.id, config.podId); if (current?.sandboxId) await this.killSandboxQuietly(current.sandboxId); } - // Dead-letter the inbox rows BEFORE releaseLease — failTaskInbox is - // owner-guarded and needs owner_id still set. - const failed = await this.inboxRepo - .failTaskInbox({ taskId: task.id, ownerId: config.podId, reason: message }) - .catch((e) => { - logErr(`failTaskInbox failed for task ${task.id}`, e); - return [] as { id: string }[]; - }); - if (failed.length > 0) { - logger.warn("inbox rows dead-lettered on task failure", { - taskId: task.id, - inboxIds: failed.map((r) => r.id), - reason: message, - }); - // Slack failure update to the configured channel. Gated on - // the ASSIGNMENT (top-level) round being among the rows just dead-lettered, - // mirroring the completion path's kind === "assignment" gate in - // eventDispatcher. Without this, a failing follow-up comment/review/ - // ci_result round (a new inbox row on the SAME Linear task, appended after - // the assignment already completed and posted "✅ devbox completed") would - // post a contradictory "❌ devbox failed" into that same thread — only the - // top-level delegated task should ever notify Slack. Tying it to this - // failTaskInbox transition also keeps it at-most-once. Fire-and-forget: a - // Slack outage must not affect teardown; the notifier no-ops when Slack is - // disabled. - const failedIds = new Set(failed.map((r) => r.id)); - const assignmentFailed = nonTerminal.some( - (r) => r.kind === "assignment" && failedIds.has(r.id), + // Slack failure update to the configured channel, for a task whose source + // is opted in (SLACK_NOTIFY_SOURCES). Gated on the ASSIGNMENT (top-level) + // round being among the rows just dead-lettered, mirroring the completion + // path's kind === "assignment" gate in eventDispatcher. Without this, a + // failing follow-up comment/review/ci_result round (a new inbox row on the + // SAME task, appended after the assignment already completed and posted + // "✅ devbox completed") would post a contradictory "❌ devbox failed" into + // that same thread — only the top-level delegated task should ever notify + // Slack. Tying it to this failTaskInbox transition also keeps it + // at-most-once. Detached: a Slack outage must not affect teardown. + if (assignment && this.slackNotifier.notifies(task.source)) { + this.hooks.detach( + task.id, + "slack", + this.slackNotifier + .notifyTerminal(task, "failed", message) + .catch((e) => logErr(`slack failure notify failed for task ${task.id}`, e)), ); - if (assignmentFailed && task.source === "linear") { - const client = await getLinearClient(); - if (client) { - void this.slackNotifier - .notifyTerminal(client, task, "failed", message) - .catch((e) => logErr(`slack failure notify failed for task ${task.id}`, e)); - } - } } await this.taskRepo.releaseLease({ taskId: task.id, @@ -816,42 +880,6 @@ export class TaskProcessorService { }); } - // Posts `message` to one reply target, switching on spec.kind. Used by - // failTask to broadcast a failure message to every pending thread before - // dead-lettering. Mirrors handlePvcMissing in inbox.service.ts — same per- - // spec branching; if a third caller appears, extract into a shared helper. - private async postFailureToTarget( - target: ReplyTarget, - message: string, - getLinearClient: () => Promise, - ): Promise { - switch (target.spec.kind) { - case "linear_session": { - const client = await getLinearClient(); - if (client) await emitError(client, target.id, message); - return; - } - case "issue_comment": - await postIssueComment(target.spec.owner, target.spec.repo, target.spec.prNumber, message); - return; - case "review_comment": - await postReviewCommentReply( - target.spec.owner, - target.spec.repo, - target.spec.prNumber, - target.spec.inReplyTo, - message, - ); - return; - case "review_body": - // No reply-to-review-body API — top-level PR comment instead. - await postIssueComment(target.spec.owner, target.spec.repo, target.spec.prNumber, message); - return; - case "api_session": - return; // API tasks: round_failed in the event log carries it - } - } - // THE single classifier for everything runTask raises (it's the dispatch // `.catch`). runTask is pure work; this is the only place that decides what a // provisioning failure means: @@ -863,7 +891,11 @@ export class TaskProcessorService { // stays in server logs only; the user sees a generic message. private async handleTaskError(task: Task, err: unknown): Promise { if (err instanceof PvcNotFoundError) { - await this.inboxService.deadLetterWorkspaceMissing(task); + // A throw here (a DB error) must not skip the release: an unreleased lease + // with no sandbox stays heartbeated by this pod and is never re-claimed. + await this.inboxService + .deadLetterWorkspaceMissing(task) + .catch((e) => logErr(`deadLetterWorkspaceMissing failed for task ${task.id}`, e)); await this.taskRepo .releaseLease({ taskId: task.id, ownerId: config.podId, alsoClearSandbox: true }) .catch((e) => logErr(`releaseLease after workspace-missing failed for task ${task.id}`, e)); diff --git a/control-plane/src/types/api.ts b/control-plane/src/types/api.ts index a303728..f4a6547 100644 --- a/control-plane/src/types/api.ts +++ b/control-plane/src/types/api.ts @@ -2,6 +2,8 @@ // values, so the devbox CLI can import it without pulling // in server modules. Guarded by src/types/__tests__/api.test.ts. +// The task_source enum, as a literal union: this file can't import db/schema. +// integrations/types.ts checks the two stay equal. export type ApiTaskSource = "linear" | "api"; export type ApiTaskState = "queued" | "running" | "idle" | "failed"; export type ApiRoundStatus = "pending" | "delivered" | "responded" | "failed"; diff --git a/control-plane/src/types/harnessEvent.ts b/control-plane/src/types/harnessEvent.ts index 5b006c2..20842cb 100644 --- a/control-plane/src/types/harnessEvent.ts +++ b/control-plane/src/types/harnessEvent.ts @@ -3,16 +3,18 @@ // // Two categories: // - Generic agent emits (agent_thought / agent_action / agent_response / -// agent_error): the dispatcher forwards each to the source-specific emit. +// agent_error): the dispatcher labels each and hands it to the round's +// integration. // - Structural events (pr_opened, completed): source-independent CP-side // state changes. -// Generic agent emits — dispatcher forwards to source-aware emit. All four -// share one shape, parametrized over `type` so `Extract` still narrows correctly at call sites. +// Generic agent emits. All four share one shape, parametrized over `type` so +// `Extract` still narrows correctly at call +// sites. // // `target_id` is always present and matches an inbox row's reply_targets[] // entry: -// - assignment: the Linear agent session id (AssignmentPayload.target_id) +// - assignment: the delegation id (AssignmentPayload.target_id; Linear's +// is the agent session id) // - comment: the GH comment id (CommentPayload.target_id) // - review: a review_body or inline-comment target_id (the agent picks one // per item for response; for thought/action it conventionally targets @@ -31,7 +33,10 @@ export type AgentEmit = | AgentEmitOf<"agent_thought"> | AgentEmitOf<"agent_action"> | AgentEmitOf<"agent_response"> - | AgentEmitOf<"agent_error">; + // retryable_assignment_failure: one failed assignment attempt. The daemon + // exits and the control plane retries the sandbox from the preserved inbox; + // the final failure, if any, arrives through failTask. + | (AgentEmitOf<"agent_error"> & { retryable_assignment_failure?: boolean }); export type HarnessEvent = | AgentEmit diff --git a/control-plane/src/types/inboxMessage.ts b/control-plane/src/types/inboxMessage.ts index f71896d..10f605f 100644 --- a/control-plane/src/types/inboxMessage.ts +++ b/control-plane/src/types/inboxMessage.ts @@ -33,8 +33,8 @@ export type AssignmentPayload = TaskIdentity & { title: string; description: string | null; external_url: string; - // Linear agent session id. Matches the row's reply_targets[0].id (a - // linear_session target); daemon stamps it as the per-round `--target` for + // The delegation id (Linear: the agent session id). Matches the row's + // reply_targets[0].id; daemon stamps it as the per-round `--target` for // emits. target_id: string; }; @@ -107,7 +107,7 @@ export type CiResultPayload = TaskIdentity & { // external_id — Upstream identifier (GH comment/review id stringified, // Linear activity UUID, Linear issue UUID for assignment, the task key for an API assignment; // a new UUID for an API comment). -// Drives idempotency at CP's `(source, external_id)` unique +// Drives idempotency at CP's `(source, kind, external_id)` unique // index AND flows through to event reply-target routing. // payload — Kind-specific body. See the payload types above. export type InboxMessage = diff --git a/deploy/helm/devbox/README.md b/deploy/helm/devbox/README.md index bdc9b84..eab9b4d 100644 --- a/deploy/helm/devbox/README.md +++ b/deploy/helm/devbox/README.md @@ -25,6 +25,7 @@ docker run --rm -v "$PWD/deploy/helm/devbox:/helm-docs" jnorwood/helm-docs:v1.14 | controlPlane.resources.limits.memory | string | `"1Gi"` | | | controlPlane.resources.requests.cpu | string | `"100m"` | | | controlPlane.resources.requests.memory | string | `"256Mi"` | | +| controlPlane.terminationGracePeriodSeconds | int | `150` | Seconds between SIGTERM and SIGKILL. At least the rest of one task's reconcile (up to 90s when platforms are slow: shutdown stops after the event in hand) plus SHUTDOWN_DRAIN_MS (10s) for in-flight provisions and integration hooks; raise it with SHUTDOWN_DRAIN_MS. | | database.existingSecret | string | `""` | With postgresql.enabled false: a Secret with key `DATABASE_URL`. | | extraEnv | list | `[]` | Extra control-plane env. Only explicitly allowlisted HARNESS_* settings reach sandboxes; never put a real secret in a forwarded name. | | extraManifests | list | `[]` | Extra manifests, rendered with tpl. | @@ -59,6 +60,7 @@ docker run --rm -v "$PWD/deploy/helm/devbox:/helm-docs" jnorwood/helm-docs:v1.14 | integrations.linear.enabled | bool | `false` | Turns Linear on (LINEAR_ENABLED). Its secrets go in the integrations Secret. | | integrations.slack.channelId | string | `""` | Required when enabled: the Slack channel ID. | | integrations.slack.enabled | bool | `false` | Turns Slack on (SLACK_ENABLED). SLACK_BOT_TOKEN goes in the integrations Secret. | +| integrations.slack.notifySources | list | `[]` | Task sources whose finished tasks post (SLACK_NOTIFY_SOURCES): `linear`, `api`. Empty: `linear`. | | ironProxy.clusterIP | string | `""` | Required. A free address in the cluster's Service range; sandboxes use it as their DNS server. `bun run k8s:install` picks one. | | ironProxy.extraAllowlist | list | `[]` | Extra allowlist rules ({host, methods, paths}), appended as-is. | | ironProxy.image.repository | string | `"ironsh/iron-proxy"` | | diff --git a/deploy/helm/devbox/templates/control-plane.yaml b/deploy/helm/devbox/templates/control-plane.yaml index 0cda6f4..528d2ea 100644 --- a/deploy/helm/devbox/templates/control-plane.yaml +++ b/deploy/helm/devbox/templates/control-plane.yaml @@ -62,6 +62,8 @@ spec: serviceAccountName: devbox-control-plane {{ include "devbox.controlPlaneImagePullSecrets" . | indent 6 }} securityContext: { runAsNonRoot: true, runAsUser: 1000, runAsGroup: 1000 } + # Shutdown finishes the task it is reconciling, then drains for SHUTDOWN_DRAIN_MS. + terminationGracePeriodSeconds: {{ .Values.controlPlane.terminationGracePeriodSeconds }} containers: - name: control-plane image: {{ include "devbox.controlPlaneImage" . }} @@ -142,6 +144,9 @@ spec: {{- with $slack.channelId }} - { name: SLACK_CHANNEL_ID, value: {{ . | quote }} } {{- end }} + {{- with $slack.notifySources }} + - { name: SLACK_NOTIFY_SOURCES, value: {{ join "," . | quote }} } + {{- end }} {{- /* A disabled App block may hold an App not yet installed: none of it renders. */}} {{- if $app.enabled }} - { name: GITHUB_WEBHOOKS_ENABLED, value: "true" } diff --git a/deploy/helm/devbox/tests/access.test.ts b/deploy/helm/devbox/tests/access.test.ts index e5167b0..4d7f747 100644 --- a/deploy/helm/devbox/tests/access.test.ts +++ b/deploy/helm/devbox/tests/access.test.ts @@ -252,6 +252,14 @@ describe("integration env", () => { expect(await envOf(["slack.yaml"])).toEqual({ SLACK_ENABLED: "true", SLACK_CHANNEL_ID: "C0123456789" }); }); + it("slack notifySources renders SLACK_NOTIFY_SOURCES", async () => { + expect(await envOf(["slack.yaml"], [], ['integrations.slack.notifySources=["linear","api"]'])).toEqual({ + SLACK_ENABLED: "true", + SLACK_CHANNEL_ID: "C0123456789", + SLACK_NOTIFY_SOURCES: "linear,api", + }); + }); + it("the GitHub App on renders the webhook switch, bot identity and CI results", async () => { expect(await envOf(["github-app.yaml"])).toEqual({ GITHUB_WEBHOOKS_ENABLED: "true", diff --git a/deploy/helm/devbox/tests/golden/byo-secrets.json b/deploy/helm/devbox/tests/golden/byo-secrets.json index 1ac8a3b..cf318a4 100644 --- a/deploy/helm/devbox/tests/golden/byo-secrets.json +++ b/deploy/helm/devbox/tests/golden/byo-secrets.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/defaults.json b/deploy/helm/devbox/tests/golden/defaults.json index 9b71ca6..70060f0 100644 --- a/deploy/helm/devbox/tests/golden/defaults.json +++ b/deploy/helm/devbox/tests/golden/defaults.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/external-db.json b/deploy/helm/devbox/tests/golden/external-db.json index a3c5f7c..c307850 100644 --- a/deploy/helm/devbox/tests/golden/external-db.json +++ b/deploy/helm/devbox/tests/golden/external-db.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/github-app-external.json b/deploy/helm/devbox/tests/golden/github-app-external.json index 97d7894..a3245f9 100644 --- a/deploy/helm/devbox/tests/golden/github-app-external.json +++ b/deploy/helm/devbox/tests/golden/github-app-external.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/github-app.json b/deploy/helm/devbox/tests/golden/github-app.json index ceea984..3431690 100644 --- a/deploy/helm/devbox/tests/golden/github-app.json +++ b/deploy/helm/devbox/tests/golden/github-app.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/ingress-api.json b/deploy/helm/devbox/tests/golden/ingress-api.json index 44d601d..8312390 100644 --- a/deploy/helm/devbox/tests/golden/ingress-api.json +++ b/deploy/helm/devbox/tests/golden/ingress-api.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/ingress-both.json b/deploy/helm/devbox/tests/golden/ingress-both.json index dcea67e..a92a867 100644 --- a/deploy/helm/devbox/tests/golden/ingress-both.json +++ b/deploy/helm/devbox/tests/golden/ingress-both.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/ingress-webhooks.json b/deploy/helm/devbox/tests/golden/ingress-webhooks.json index e628228..4a567a0 100644 --- a/deploy/helm/devbox/tests/golden/ingress-webhooks.json +++ b/deploy/helm/devbox/tests/golden/ingress-webhooks.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/linear.json b/deploy/helm/devbox/tests/golden/linear.json index 1a51e61..8b00fbe 100644 --- a/deploy/helm/devbox/tests/golden/linear.json +++ b/deploy/helm/devbox/tests/golden/linear.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/llm-stages.json b/deploy/helm/devbox/tests/golden/llm-stages.json index 13d8c9c..4d585bf 100644 --- a/deploy/helm/devbox/tests/golden/llm-stages.json +++ b/deploy/helm/devbox/tests/golden/llm-stages.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/replicas.json b/deploy/helm/devbox/tests/golden/replicas.json index 9a34f4b..018083d 100644 --- a/deploy/helm/devbox/tests/golden/replicas.json +++ b/deploy/helm/devbox/tests/golden/replicas.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 180, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/sandbox-options.json b/deploy/helm/devbox/tests/golden/sandbox-options.json index 71465c5..67f2843 100644 --- a/deploy/helm/devbox/tests/golden/sandbox-options.json +++ b/deploy/helm/devbox/tests/golden/sandbox-options.json @@ -607,6 +607,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/golden/slack.json b/deploy/helm/devbox/tests/golden/slack.json index 389362a..d538f36 100644 --- a/deploy/helm/devbox/tests/golden/slack.json +++ b/deploy/helm/devbox/tests/golden/slack.json @@ -597,6 +597,7 @@ "runAsUser": 1000, "runAsGroup": 1000 }, + "terminationGracePeriodSeconds": 150, "containers": [ { "name": "control-plane", diff --git a/deploy/helm/devbox/tests/validation.test.ts b/deploy/helm/devbox/tests/validation.test.ts index 230bbcd..d40f721 100644 --- a/deploy/helm/devbox/tests/validation.test.ts +++ b/deploy/helm/devbox/tests/validation.test.ts @@ -73,6 +73,12 @@ describe("chart validation", () => { } }); + it("refuses an unknown task source in integrations.slack.notifySources", async () => { + const stderr = await failure(["integrations.slack.notifySources={linear,github}"]); + expect(stderr).toContain("values don't meet the specifications of the schema"); + expect(stderr).toContain("notifySources"); + }); + it("refuses the removed integrations.githubWebhooks", async () => { const stderr = await failure(["integrations.githubWebhooks.enabled=true"]); expect(stderr).toContain("values don't meet the specifications of the schema"); diff --git a/deploy/helm/devbox/tests/values/replicas.yaml b/deploy/helm/devbox/tests/values/replicas.yaml index d801db2..3499f0c 100644 --- a/deploy/helm/devbox/tests/values/replicas.yaml +++ b/deploy/helm/devbox/tests/values/replicas.yaml @@ -1,2 +1,2 @@ -controlPlane: { replicas: 2 } +controlPlane: { replicas: 2, terminationGracePeriodSeconds: 180 } ironProxy: { replicas: 2 } diff --git a/deploy/helm/devbox/values.schema.json b/deploy/helm/devbox/values.schema.json index d28cf9e..b7ce8c5 100644 --- a/deploy/helm/devbox/values.schema.json +++ b/deploy/helm/devbox/values.schema.json @@ -117,7 +117,11 @@ "slack": { "type": "object", "additionalProperties": false, - "properties": { "enabled": { "type": "boolean" }, "channelId": { "type": "string" } } + "properties": { + "enabled": { "type": "boolean" }, + "channelId": { "type": "string" }, + "notifySources": { "type": "array", "items": { "enum": ["linear", "api"] } } + } }, "githubApp": { "type": "object", @@ -149,7 +153,8 @@ "properties": { "replicas": { "type": "integer", "minimum": 1 }, "publicBaseUrl": { "type": "string" }, - "resources": { "type": "object" } + "resources": { "type": "object" }, + "terminationGracePeriodSeconds": { "type": "integer", "minimum": 0 } } }, "postgresql": { diff --git a/deploy/helm/devbox/values.yaml b/deploy/helm/devbox/values.yaml index 4001deb..745ce4f 100644 --- a/deploy/helm/devbox/values.yaml +++ b/deploy/helm/devbox/values.yaml @@ -112,6 +112,8 @@ integrations: enabled: false # -- Required when enabled: the Slack channel ID. channelId: "" + # -- Task sources whose finished tasks post (SLACK_NOTIFY_SOURCES): `linear`, `api`. Empty: `linear`. + notifySources: [] githubApp: # -- Turns the GitHub App on: webhooks at /webhook/github (GITHUB_WEBHOOKS_ENABLED) and, with # `tokens: devbox`, the token minter. GITHUB_WEBHOOK_SECRET goes in the integrations Secret. @@ -149,6 +151,11 @@ controlPlane: resources: requests: { cpu: 100m, memory: 256Mi } limits: { memory: 1Gi } + # -- Seconds between SIGTERM and SIGKILL. At least the rest of one task's + # reconcile (up to 90s when platforms are slow: shutdown stops after the event + # in hand) plus SHUTDOWN_DRAIN_MS (10s) for in-flight provisions and + # integration hooks; raise it with SHUTDOWN_DRAIN_MS. + terminationGracePeriodSeconds: 150 postgresql: # -- A minimal bundled Postgres for trying devbox: no backups, HA or automated upgrades. diff --git a/docs/configuration.md b/docs/configuration.md index 5d065ea..6e3e9af 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -307,7 +307,7 @@ kube context, so a command never acts on a cluster you did not name. | `context` | yes | The kube context of the cluster. | | `profile` | no, default `k8s-` | The CLI profile the installer writes. Lower-case letters, digits and `-`; the default is the context made lower-case with other characters replaced by `-`. | | `values` | no | Helm values for the chart, passed as they are. `storage.storageClass` and `storage.volumeSnapshotClass` are required by the chart. Sandbox sizing goes under `values.sandbox`; the Compose-only `devbox.config.json` block is ignored. `repos` and `llm` are refused: the installer copies the full `llm` block, including effort and stage overrides, from `devbox.config.json`. `integrations` is refused: it goes at the top level. | -| `integrations` | no | The deployment's integrations: `linear` (`enabled`, `clientId`), `slack` (`enabled`, `channelId`) and `githubApp` (the fields [above](#integrations) except `tokenFile`; `slug` is copied as `botLogin`), with the same `enabled` rule. The installer copies them into the chart values. The top-level `publicUrl` (the public HTTPS URL webhooks reach, shared by all integrations) is read only by the CLI, never copied ([customizing.md](kubernetes/customizing.md#integrations)). The trust lists go under `values.github`. | +| `integrations` | no | The deployment's integrations: `linear` (`enabled`, `clientId`), `slack` (`enabled`, `channelId`, `notifySources`) and `githubApp` (the fields [above](#integrations) except `tokenFile`; `slug` is copied as `botLogin`), with the same `enabled` rule. The installer copies them into the chart values. The top-level `publicUrl` (the public HTTPS URL webhooks reach, shared by all integrations) is read only by the CLI, never copied ([customizing.md](kubernetes/customizing.md#integrations)). The trust lists go under `values.github`. | Any other key is an error: chart values go under `values`. The installer adds `repos`, `llm`, `integrations`, `ironProxy.clusterIP`, diff --git a/docs/github.md b/docs/github.md index 9066c84..804c5d8 100644 --- a/docs/github.md +++ b/docs/github.md @@ -696,8 +696,9 @@ bun run github:disable [--target ] [--no-keychain] [--forget [--yes]] - **`github:disable`** sets `integrations.githubApp.enabled` to `false` and keeps the settings and the saved secrets. The App and its webhooks are off, and devbox falls back to a personal token, so the next apply needs - `GITHUB_TOKEN`. Turn the App back on with `bun run github:setup`, or by - setting `enabled` to `true`. It asks nothing. A new App waiting for its + `GITHUB_TOKEN`. Comments and reviews already queued still run, but their + replies are not posted. Turn the App back on with `bun run github:setup`, or + by setting `enabled` to `true`. It asks nothing. A new App waiting for its install stays, and it prints `The new App is still waiting for its install: bun run github:setup resumes it, bun run github:disable --forget removes it`. - **`github:disable --forget`** removes the `integrations.githubApp` block (a diff --git a/docs/kubernetes/README.md b/docs/kubernetes/README.md index 6a8feff..1196c04 100644 --- a/docs/kubernetes/README.md +++ b/docs/kubernetes/README.md @@ -428,6 +428,38 @@ kubectl -n devbox-sandboxes create job devbox-snapshot-builder-manual-$(date +%s The snapshot builder also runs on a schedule (`snapshotBuilder.schedule`, every 6 hours by default). +### Database migrations + +A new control plane pod migrates the database when it starts, while the old +pods keep serving. The migrations hold short locks on the `tasks` and `inbox` +tables and wait at most 5 seconds for each. If a lock times out, the new pod +fails, restarts and tries again. Postgres may instead break a deadlock between +the migration and a request: then either the pod restarts, or the request +fails with a 500 and, for a webhook, the platform retries it. + +Task keys are unique regardless of case. The migration that enforces this +fails with `could not create unique index "tasks_key_lower_uniq"` and +`Key (lower(key))=(…) is duplicated` only if tasks were inserted by hand with +keys that differ only in case: devbox upper-cases Linear keys, and +`API_TASK_KEY_PREFIX` must match `^[A-Z][A-Z0-9-]*-$`. To list them: + +```sql +SELECT lower(key) AS key, array_agg(id) AS task_ids + FROM tasks GROUP BY lower(key) HAVING count(*) > 1; +``` + +Rename all but one key in each group; the pod migrates on its next start. + +While old and new pods overlap, and after a rollback to a release without the +`inbox_source_kind_external_uniq` index, an old pod can answer a webhook that +races its own redelivery with a 500: `duplicate key value violates unique +constraint "inbox_source_kind_external_uniq"`. The other request stored the +round and the platform retries, so nothing is lost. Leave the index in place +after a rollback: the database has recorded its migration, so a dropped index +is not rebuilt on the next upgrade, and the new release then fails to store +any follow-up (a comment, review, CI result or API message) with `there is no +unique or exclusion constraint matching the ON CONFLICT specification`. + ## Uninstall `k8s:uninstall` acts on the cluster named in the `--target` file, and on diff --git a/docs/kubernetes/customizing.md b/docs/kubernetes/customizing.md index eb9b14e..8515245 100644 --- a/docs/kubernetes/customizing.md +++ b/docs/kubernetes/customizing.md @@ -251,6 +251,7 @@ enabled from that block ([Check the Secrets](README.md#check-the-secrets)). | `integrations.linear.clientId` | The Linear app's client ID. Required when enabled. | | `integrations.slack.enabled` | Turns on Slack notifications when a task finishes. | | `integrations.slack.channelId` | The Slack channel ID. Required when enabled. | +| `integrations.slack.notifySources` | The task sources whose finished tasks post, from `linear` and `api`. Default: `linear`. | | `integrations.githubApp.enabled` | Turns on the GitHub App: webhooks at `/webhook/github` (`@` mentions, reviews and failed CI runs reach the agent) and, with `tokens: devbox`, the token minter. See [docs/github.md](../github.md). | | `integrations.githubApp.appId`, `integrations.githubApp.installationId`, `integrations.githubApp.botUserId` | The App's ID, its installation's ID and the bot's numeric GitHub user ID. Required when enabled. | | `integrations.githubApp.botLogin` | The App's slug. The installer copies it from `slug` in the target file's block. Required when enabled. | diff --git a/docs/linear.md b/docs/linear.md index ebbdbe8..e3e3d11 100644 --- a/docs/linear.md +++ b/docs/linear.md @@ -297,9 +297,10 @@ bun run linear:disable [--target ] [--no-keychain] [--forget [--yes]] ``` - **`linear:disable`** sets `integrations.linear.enabled` to `false` and keeps - the client ID, the public URL and the saved secrets. Turn Linear back on with - the same app with `bun run linear:setup`, or by setting `enabled` to `true`. - It asks nothing. + the client ID, the public URL and the saved secrets. Open Linear tasks keep + running, but nothing more is posted to Linear. Turn Linear back on with the + same app with `bun run linear:setup`, or by setting `enabled` to `true`. It + asks nothing. - **`linear:disable --forget`** removes the `integrations.linear` block and the deployment's saved `LINEAR_CLIENT_SECRET` and `LINEAR_WEBHOOK_SECRET`. It lists what it deletes and asks `[y/N]`; Enter or Ctrl-D answers no, and diff --git a/harness/src/harness/daemon.py b/harness/src/harness/daemon.py index 3d549a5..64244fd 100644 --- a/harness/src/harness/daemon.py +++ b/harness/src/harness/daemon.py @@ -274,10 +274,12 @@ def _emit_round_failure( # comment / review — the round died before (or part-way through) the agent # posting its replies. Derive the primary target_id from `payload` (the # daemon's inbox file carries no reply_targets — that is a CP-only column). - # Failures ride on agent_error (not agent_response): CP routes them to the - # Linear session via emitError, the same lane the assignment-stage terminal - # failure uses — keeps "this round broke" as a structured error block - # rather than masquerading as a normal reply. + # Failures ride on agent_error (not agent_response), so "this round broke" + # never masquerades as a normal reply. CP labels it a round failure with + # this exception text as its raw detail, and the round's integration + # decides what to show: Linear shows it on the agent session; GitHub posts a + # generic retry message instead (a PR is public); a CI round stays silent. + # test_daemon pins this format (TestAgentErrorContract). if entry["kind"] == "review": primary_target_id = (payload.get("review_body") or {}).get("target_id") else: # comment / ci_result diff --git a/harness/src/harness/deepagents/agents/followup.py b/harness/src/harness/deepagents/agents/followup.py index d4b1e60..51a0595 100644 --- a/harness/src/harness/deepagents/agents/followup.py +++ b/harness/src/harness/deepagents/agents/followup.py @@ -61,7 +61,7 @@ Each round, the user message hands you one of: -- a **single comment**, from GitHub or Linear; or +- a **single comment**, from any channel; or - a **review**: a review summary plus zero or more inline comments, each pinned to a file and a line or line range. @@ -77,8 +77,10 @@ - **GitHub**: pull request comments and reviews. The reviewer sees your reply on the comment or review it answers. - **Linear**: comments on the agent session. The user may be watching live. +- **Any other channel**: comments or messages on the task, where it was filed. + The user sees your reply there. -Post progress updates as you work, on either channel. +Post progress updates as you work, on any channel. ## What "address it" means @@ -123,7 +125,7 @@ task did not already complete planning and implementation, even if it is checked out in the workspace. Never create or clone a repository. Instead, reply that you can't take on new work in another repository here, and ask the - user to re-delegate it as a new task from Linear with a proper description. + user to re-delegate it as a new task with a proper description. 7. **Reply once per item, when its work is done.** For a review summary, stay silent by default: skip it when its body is empty, whitespace-only or just an diff --git a/harness/src/harness/handlers/assignment.py b/harness/src/harness/handlers/assignment.py index 3992344..7a0c496 100644 --- a/harness/src/harness/handlers/assignment.py +++ b/harness/src/harness/handlers/assignment.py @@ -605,8 +605,9 @@ def _format_completion_body( Two sections: a per-repo "Planned / Done" recap — the plan.json repo `goal` and the implementation.json repo `summary`, matched by repo name; a repo with neither is dropped from the recap — then a "Pull requests" list of - every opened PR. CP relays this body verbatim — see CP eventDispatcher - handleAssignmentCompleted. + every opened PR. CP relays this body verbatim; for a failed run it is the + failure's user-safe summary, so it must never carry exception text (see the + control plane's labelAgentEmit; test_daemon's TestAgentErrorContract pins it). """ plan_by_name = { r["name"]: r diff --git a/harness/tests/test_daemon.py b/harness/tests/test_daemon.py index c59c21e..e518cf2 100644 --- a/harness/tests/test_daemon.py +++ b/harness/tests/test_daemon.py @@ -820,9 +820,8 @@ def test_stage_exception_emits_error_and_exits_for_assignment_retry(self) -> Non def test_comment_round_failure_emits_agent_error_not_response(self) -> None: # Round-failure messages ride on agent_error, not agent_response — CP - # routes agent_error through emitError on the Linear session - # (structured error block) rather than as a normal targeted reply. - # Lines up with the assignment-stage terminal-failure path. + # labels it a round failure (Linear shows it as a structured error + # block) rather than a normal targeted reply. entry = { "id": "inbox-cf", "kind": "comment", @@ -1107,6 +1106,98 @@ def test_assignment_reads_task_key(self) -> None: self.assertTrue((self.processed / f.name).exists()) +class TestAgentErrorContract(unittest.TestCase): + """The two non-retryable agent_error formats the control plane routes on. + + The control plane's labelAgentEmit relies on them: on an assignment row + the body is the user-safe summary (shown on public channels as is); on a + comment, review or CI row it is exception text (shown only on private + channels). Changing either format requires changing that routing. + """ + + setUp = TestProcessOneInboxFile.setUp + tearDown = TestProcessOneInboxFile.tearDown + _write_inbox = TestProcessOneInboxFile._write_inbox + _events = TestProcessOneInboxFile._events + + def test_failed_assignment_is_the_summary_without_exception_text(self) -> None: + entry = { + "id": "inbox-f", + "kind": "assignment", + "source": "linear", + "payload": {"title": "T", "task_key": "ABC-9", "target_id": "sess-9"}, + } + f = self._write_inbox(entry) + plan_data = {"repos": [{"name": "repo-a", "path": "/tmp/repo-a", "goal": "g"}]} + impl_data = {"repos": []} + # Exception text the commit stage might carry must not reach the body. + commit_repos = [ + { + "name": "repo-a", + "pr_url": None, + "pr_status": "push_failed", + "error": "RuntimeError: push rejected for token=abc", + } + ] + plan_md = self.dir / "plan.md" + plan_md.write_text("# Plan", encoding="utf-8") + + with ( + patch.object(assignment_module, "setup_workspace"), + patch.object( + assignment_module, "_run_planning_stage", return_value=plan_data + ), + patch.object( + assignment_module, "_run_implementation_stage", return_value=impl_data + ), + patch.object( + assignment_module, "run_self_review_loop", return_value=({}, impl_data) + ), + patch.object(assignment_module, "run_distillation_stage"), + patch.object( + assignment_module, + "run_commit_stage", + return_value=("failed", commit_repos), + ), + patch.object(assignment_module, "PLAN_MD", plan_md), + ): + process_one_inbox_file(f, self.state) + + errors = [e for e in self._events() if e["type"] == "agent_error"] + self.assertEqual(len(errors), 1) + self.assertNotIn(RETRYABLE_ASSIGNMENT_ERROR_EVENT_FIELD, errors[0]) + self.assertEqual( + errors[0]["body"], + assignment_module._format_completion_body( + commit_repos, plan_data, impl_data, "failed" + ), + ) + self.assertNotIn("RuntimeError", errors[0]["body"]) + + def test_round_crash_carries_the_exception_text(self) -> None: + entry = { + "id": "inbox-c", + "kind": "comment", + "source": "github", + "payload": {"target_id": "555", "body": "fix it", "task_key": "ABC-9"}, + } + f = self._write_inbox(entry) + + with patch( + "harness.daemon.process_comment", + side_effect=RuntimeError("tool call blew up"), + ): + process_one_inbox_file(f, self.state) + + errors = [e for e in self._events() if e["type"] == "agent_error"] + self.assertEqual(len(errors), 1) + self.assertNotIn(RETRYABLE_ASSIGNMENT_ERROR_EVENT_FIELD, errors[0]) + self.assertEqual( + errors[0]["body"], + "⚠ Something went wrong — RuntimeError: tool call blew up", + ) + + class TestArchiveMoveFailureDoesNotCrashLoop(unittest.TestCase): """An FS error from the post-dispatch archive move must not escape.""" diff --git a/skills/devbox/SKILL.md b/skills/devbox/SKILL.md index 7136683..e636de8 100644 --- a/skills/devbox/SKILL.md +++ b/skills/devbox/SKILL.md @@ -91,6 +91,7 @@ Say so instead of looking for a way around it: self-hosted model needs a server that speaks the Responses API with reasoning, on port 80 or 443 under a DNS name. - Work comes in from the CLI, Linear and GitHub. Slack only gets completion - updates for Linear tasks. No Jira, no web UI. + updates, for Linear tasks unless `notifySources` adds API tasks. No Jira, no + web UI. - A task can't be stopped from Linear yet. - No stable extension interface ([docs/extending.md](../../docs/extending.md)). diff --git a/skills/devbox/references/integrations.md b/skills/devbox/references/integrations.md index b7e4b27..6ca9fb8 100644 --- a/skills/devbox/references/integrations.md +++ b/skills/devbox/references/integrations.md @@ -67,6 +67,8 @@ works with one Linear workspace. Everything: Kubernetes only: `integrations.slack` in the target file and `SLACK_BOT_TOKEN` ([docs/kubernetes/customizing.md](../../../docs/kubernetes/customizing.md#integrations)). +Finished Linear tasks post by default; `notifySources: ["linear", "api"]` adds +API tasks. ## Turning one off