From 69dbb139ed670525e2dfa6c25c130e8ddcf4eb2e Mon Sep 17 00:00:00 2001 From: Duy Nguyen Date: Tue, 20 Oct 2026 01:23:24 +0700 Subject: [PATCH 1/3] feat(personal-kits): answer kit.api_disabled while off and add an enablement smoke While SKILLX_PERSONAL_KITS_API is off every kit path still answers 404, now with the code kit.api_disabled, before any authentication or lookup. A client can then say the deployment does not offer personal kits instead of reporting a missing kit; the answer names no kit, owner or state. scripts/smoke-personal-kits.mjs stores one throwaway private kit and checks anonymous 401, idempotent replay, 412 on a stale If-Match, activation, the lock read-back and 404 for another account, then prints a JSON receipt without keys. It refuses a non-loopback base unless --allow-remote is passed and exits 2 when the deployment answers kit.api_disabled. A vitest runs it against the real route modules over the in-memory D1/R2. docs/personal-kits-api.md gains an enablement and rollback runbook. The production flag value in wrangler.jsonc is unchanged. --- apps/web/app/lib/personal-kits/kit-http.ts | 20 +- .../kit-enablement-smoke.test.ts | 68 +++++++ apps/web/tests/personal-kits/kit-harness.ts | 26 ++- .../tests/personal-kits/kit-revisions.test.ts | 14 +- docs/personal-kits-api.md | 35 +++- scripts/smoke-personal-kits.mjs | 175 ++++++++++++++++++ 6 files changed, 328 insertions(+), 10 deletions(-) create mode 100644 apps/web/tests/personal-kits/kit-enablement-smoke.test.ts create mode 100644 scripts/smoke-personal-kits.mjs diff --git a/apps/web/app/lib/personal-kits/kit-http.ts b/apps/web/app/lib/personal-kits/kit-http.ts index 545b36c..9effd37 100644 --- a/apps/web/app/lib/personal-kits/kit-http.ts +++ b/apps/web/app/lib/personal-kits/kit-http.ts @@ -1,6 +1,6 @@ /** * HTTP boundary of the personal kit API (`/api/v1/kits/@handle/name/...`). - * 404 while `SKILLX_PERSONAL_KITS_API` is off. The caller is the release + * 404 `kit.api_disabled` while `SKILLX_PERSONAL_KITS_API` is off. The caller is the release * viewer (unscoped account API key or web session; scoped publish keys are * refused); a kit is visible only to its owner, everyone else gets the same * 404 as an unknown kit. Reads count against `RELEASE_READ_LIMITER`, writes @@ -9,7 +9,7 @@ */ import { PersonalKitError } from "@skillx/contracts"; import { IDENTITY_PATTERN, validateIdentity } from "@skillx/skill-tree"; -import { ReleaseHttpError, errorResponse, notFoundResponse, releaseError } from "~/lib/releases/release-http"; +import { ReleaseHttpError, errorResponse, releaseError } from "~/lib/releases/release-http"; import { checkReleaseRateLimit, rateLimitCaller } from "~/lib/releases/release-rate-limit"; import { resolveViewer } from "~/lib/releases/release-read"; import { NO_STORE, failureHeaders, readJsonObject, resolveLimitedActor } from "~/lib/releases/release-state-http"; @@ -27,6 +27,20 @@ export function personalKitsApiEnabled(env: { SKILLX_PERSONAL_KITS_API?: string } export const kitNotFound = () => releaseError(404, "not_found", "Not found"); + +/** Code of the flag-off answer. */ +export const KIT_API_DISABLED = "kit.api_disabled"; + +/** + * The flag-off answer. It stays a 404, so a client that only reads the + * status keeps treating the API as absent, and its code tells a client the + * deployment does not offer personal kits, so it can say so instead of + * "no such kit". It is answered before any authentication or lookup and + * names no kit, owner or state: only the deployment's flag, which the routes + * do not hide either (other methods still get the framework's 405). + */ +export const kitApiDisabledResponse = () => + errorResponse(releaseError(404, KIT_API_DISABLED, "Personal kits are not enabled on this SkillX deployment")); export const badRequest = (message: string) => releaseError(400, "invalid_request", message); export const conflict = (code: string, message: string) => releaseError(409, code, message); @@ -160,7 +174,7 @@ export async function serveKit( handler: (caller: KitCaller) => Promise, now: () => Date = () => new Date(), ): Promise { - if (!personalKitsApiEnabled(env)) return notFoundResponse(); + if (!personalKitsApiEnabled(env)) return kitApiDisabledResponse(); try { let userId: string; if (access === "read") { diff --git a/apps/web/tests/personal-kits/kit-enablement-smoke.test.ts b/apps/web/tests/personal-kits/kit-enablement-smoke.test.ts new file mode 100644 index 0000000..2bcb7db --- /dev/null +++ b/apps/web/tests/personal-kits/kit-enablement-smoke.test.ts @@ -0,0 +1,68 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { parseBaseSkill, runPersonalKitSmoke } from "../../../../scripts/smoke-personal-kits.mjs"; +import { BODY, createKitHarness, skillText } from "./kit-harness"; +import type { KitHarness } from "./kit-harness"; + +vi.mock("~/lib/auth/session-helpers", () => ({ getSession: async () => null })); + +/** + * The enablement smoke run against the real kit and release route modules + * over the in-memory D1/R2: the local receipt that the smoke itself works. + * It says nothing about a deployed SkillX. + */ +let k: KitHarness; + +beforeEach(async () => { + k = await createKitHarness(); + await k.publish("@alpha/demo", "1.0.0", { "SKILL.md": skillText(BODY) }); +}); + +const run = (extra: Record = {}) => + runPersonalKitSmoke({ + fetchImpl: k.serve, + base: "https://skillx.test", + ownerKey: k.ownerKey, + otherKey: k.otherKey, + handle: "alpha", + baseSkill: "@alpha/demo@1.0.0", + name: "smoke-local", + ...extra, + }); + +describe("personal kit enablement smoke", () => { + it("passes every check against an enabled API and leaves one private kit", async () => { + const receipt = await run(); + expect(receipt.enabled).toBe(true); + expect(receipt.checks.filter((c: { ok: boolean }) => !c.ok)).toEqual([]); + expect(receipt.checks.map((c: { label: string }) => c.label)).toEqual([ + "unknown kit is 404 not_found", + "anonymous is 401", + "base release readable", + "store revision is 201", + "idempotent replay is 200", + "stale If-Match is 412", + "activate is 200 with a lock", + "lock reads back", + "other account is 404", + ]); + expect(receipt.ok).toBe(true); + expect(JSON.stringify(receipt)).not.toContain(k.ownerKey); + expect(k.count("personal_kits")).toBe(1); + expect(k.count("personal_kit_revisions")).toBe(1); + }); + + it("reports a flag-off deployment as disabled and writes nothing", async () => { + const receipt = await run({ fetchImpl: k.serveWith({ SKILLX_PERSONAL_KITS_API: "0" }) }); + expect(receipt).toMatchObject({ enabled: false, ok: false, checks: [{ label: "api enabled", ok: false, status: 404, code: "kit.api_disabled" }] }); + expect(k.count("personal_kits")).toBe(0); + }); + + it("refuses bad input before any request", async () => { + const fetchImpl = vi.fn(); + await expect(run({ fetchImpl, ownerKey: "" })).rejects.toThrow(/owner API key/); + await expect(run({ fetchImpl, handle: "Alpha" })).rejects.toThrow(/--handle/); + await expect(run({ fetchImpl, baseSkill: "@alpha/demo" })).rejects.toThrow(/--base-skill/); + expect(fetchImpl).not.toHaveBeenCalled(); + expect(parseBaseSkill("@alpha/demo@1.0.0-beta.1")).toEqual({ pkg: "@alpha/demo", version: "1.0.0-beta.1" }); + }); +}); diff --git a/apps/web/tests/personal-kits/kit-harness.ts b/apps/web/tests/personal-kits/kit-harness.ts index 081efcf..5d469ae 100644 --- a/apps/web/tests/personal-kits/kit-harness.ts +++ b/apps/web/tests/personal-kits/kit-harness.ts @@ -8,6 +8,7 @@ import { compareUtf8, memoryTree, overlayBetween, PERSONAL_KIT_SCHEMA } from "@s import type { KitTree, SkillOverlay } from "@skillx/contracts"; import { releasePublishScope } from "~/lib/auth/api-key-scopes"; import { loader, action } from "~/routes/api.v1.kits"; +import { loader as manifestLoader } from "~/routes/api.v1.release-manifest"; import { FakeRateLimiter } from "../releases/fake-rate-limiter"; import { createHarness, file, tarOf } from "../releases/release-harness"; @@ -93,11 +94,34 @@ export async function createKitHarness() { return { status: res.status, headers: res.headers, body: (await res.json()) as Record }; } + /** + * A `fetch` over the real kit and release-manifest route modules, for + * clients (the enablement smoke) that speak HTTP, with `extra` over the + * harness env. Other paths are 599. + */ + const serveWith = (extra: Record = {}) => async (input: string | URL | Request, init?: RequestInit): Promise => { + const request = new Request(input, init); + const path = new URL(request.url).pathname; + const context = { cloudflare: { env: env(extra) } }; + const kitPath = /^\/api\/v1\/kits\/(@[^/]+)\/([^/]+)(?:\/(.+))?$/.exec(path); + if (kitPath !== null) { + const params = { scope: kitPath[1], name: kitPath[2], ...(kitPath[3] === undefined ? {} : { "*": kitPath[3] }) }; + const args = { request, params, context } as never; + return request.method === "GET" || request.method === "HEAD" ? loader(args) : action(args); + } + const manifest = /^\/api\/v1\/packages\/(@[^/]+)\/([^/]+)\/releases\/([^/]+)$/.exec(path); + if (manifest !== null) { + return manifestLoader({ request, params: { scope: manifest[1], slug: manifest[2], version: manifest[3] }, context } as never); + } + return new Response("no route in the kit harness", { status: 599 }); + }; + const serve = serveWith(); + const count = (table: string) => (h.d1.raw.prepare(`SELECT count(*) AS n FROM ${table}`).get() as { n: number }).n; const sql = (statement: string, ...params: Array) => h.d1.raw.prepare(statement).run(...params); let keyCounter = 0; const idem = () => `idem-key-${String(++keyCounter).padStart(4, "0")}`; - return { h, ownerKey, otherKey, publish, treeOf, overlayTo, entry, doc, call, count, sql, idem }; + return { h, ownerKey, otherKey, publish, treeOf, overlayTo, entry, doc, call, serve, serveWith, count, sql, idem }; } diff --git a/apps/web/tests/personal-kits/kit-revisions.test.ts b/apps/web/tests/personal-kits/kit-revisions.test.ts index 8ef0f7e..b57ed69 100644 --- a/apps/web/tests/personal-kits/kit-revisions.test.ts +++ b/apps/web/tests/personal-kits/kit-revisions.test.ts @@ -24,11 +24,19 @@ async function createKit(files = MINE) { } describe("personal kit revisions", () => { - it("is 404 everywhere while the flag is off", async () => { - for (const res of [await k.call("", undefined, { env: { SKILLX_PERSONAL_KITS_API: "0" } }), await k.call("revisions", {}, { env: { SKILLX_PERSONAL_KITS_API: "0" } })]) { + it("is 404 kit.api_disabled everywhere while the flag is off, before any auth or lookup", async () => { + const off = { env: { SKILLX_PERSONAL_KITS_API: "0" } }; + for (const res of [ + await k.call("", undefined, off), + await k.call("revisions", {}, off), + await k.call("lock", undefined, { ...off, key: null }), + await k.call("", undefined, { ...off, kit: "@nobody/missing" }), + ]) { expect(res.status).toBe(404); - expect(res.body.error.code).toBe("not_found"); + expect(res.body.error.code).toBe("kit.api_disabled"); + expect(res.headers.get("Cache-Control")).toBe("no-store"); } + expect(k.count("personal_kits") + k.count("personal_kit_audit")).toBe(0); }); it("stores a draft whose composition the server re-derives, then activates it only with approval", async () => { diff --git a/docs/personal-kits-api.md b/docs/personal-kits-api.md index 9f748ef..49d247a 100644 --- a/docs/personal-kits-api.md +++ b/docs/personal-kits-api.md @@ -11,8 +11,11 @@ without an explicit user approval. - Mount: `/api/v1/kits/@{handle}/{name}[/...]`. Source: `apps/web/app/routes/api.v1.kits.ts` and `apps/web/app/lib/personal-kits/`. - Feature flag: `SKILLX_PERSONAL_KITS_API` (`"1"` enables it). While it is off, - every path returns the JSON 404. Reading base releases also needs the release - stack (`SKILLX_RELEASES_API`, D1, R2). + every path returns the JSON 404 `kit.api_disabled`, before any + authentication or lookup, so a client can say "this deployment does not offer + personal kits" instead of "no such kit". Reading base releases also needs the + release stack (`SKILLX_RELEASES_API`, D1, R2). Enabling and rolling back are + in [Enablement and rollback](#enablement-and-rollback). - Wire formats: `packages/contracts/src/personal-kit/`. The golden vector `packages/contracts/src/fixtures/personal-kit-golden.json` must re-derive byte for byte on both the SkillX (TypeScript) and AgentKit (Go) sides. @@ -527,6 +530,31 @@ quarantined), `reviewed` moves only to `quarantined`, nothing returns to with the owner's consent proof in `approval_json`, which is never cleared. It never holds transcripts, prompts, environment or file content. Its rollback SQL is in its header. +## Enablement and rollback + +The flag is the only switch; enabling it runs no migration (`0015` and `0016` +are applied with the rest) and touches no existing data. The production value +in `apps/web/wrangler.jsonc` stays `"0"` until an operator turns it on. + +1. Turn it on for one environment (`SKILLX_PERSONAL_KITS_API` = `"1"`) and + deploy that environment. +2. Run the enablement smoke against it with a throwaway owner account that + holds a published base skill: + `SKILLX_SMOKE_OWNER_KEY=... SKILLX_SMOKE_OTHER_KEY=... node scripts/smoke-personal-kits.mjs --base --allow-remote --handle --base-skill @pub/slug@version`. + It stores one private kit `@handle/smoke-`, checks 401 for + anonymous, idempotent replay, 412 on a stale `If-Match`, activation, the lock + read-back and 404 for another account, and prints a JSON receipt (no keys). + Exit 2 means the deployment still answers `kit.api_disabled`. +3. Roll back by setting the flag to `"0"` and redeploying. Every kit path then + answers 404 `kit.api_disabled` again; stored revisions, locks and audit rows + stay in D1 untouched and come back unchanged when the flag returns. Clients + keep their local kits: AgentKit maps `kit.api_disabled` to "unsupported" + (exit 2) and never treats it as a deleted kit. + +`apps/web/tests/personal-kits/kit-enablement-smoke.test.ts` runs the same +smoke against the real route modules over the in-memory test D1/R2; it proves +the smoke and the flag-off answer, not a deployment. + ## Error codes | Code | Status | @@ -538,4 +566,5 @@ or file content. Its rollback SQL is in its header. | `kit.invalid`, `overlay.invalid`, `schema.unsupported_version`, `kit.duplicate_identity`, `kit.mode_violation`, `kit.alias_collision`, `overlay.shared_alias`, `identity.reserved_alias`, `kit.request_too_large`, `kit.cycle`, `kit.multi_parent`, `kit.public_derivative_forbidden`, `kit.base_unavailable` (409 when resolving a conflict) | 422 | | `kit.publisher_not_owned` | 403 | | `invalid_request` (malformed body, ids, updates or `If-Match`) | 400 | -| `not_found` (flag off, not the owner, unknown id) | 404 | +| `kit.api_disabled` (flag off, on every path) | 404 | +| `not_found` (not the owner, unknown id) | 404 | diff --git a/scripts/smoke-personal-kits.mjs b/scripts/smoke-personal-kits.mjs new file mode 100644 index 0000000..feb7729 --- /dev/null +++ b/scripts/smoke-personal-kits.mjs @@ -0,0 +1,175 @@ +#!/usr/bin/env node +/** + * Enablement smoke of the personal kit API (`/api/v1/kits/...`). + * + * It answers one question: on this deployment, with `SKILLX_PERSONAL_KITS_API` + * on, can an owner store a revision, replay it idempotently, activate it under + * compare-and-set and read its lock, while anonymous and foreign callers see + * nothing? It writes one throwaway private kit (`@handle/smoke-`) and + * prints a JSON receipt. It never prints a key. + * + * LOCAL BY DEFAULT: a non-loopback base URL is refused unless `--allow-remote` + * is passed, so a staging run is always a deliberate choice. It never deploys + * and never changes a flag. + * + * Usage: + * SKILLX_SMOKE_OWNER_KEY= \ + * SKILLX_SMOKE_OTHER_KEY= \ + * node scripts/smoke-personal-kits.mjs --handle alpha --base-skill @alpha/demo@1.0.0 \ + * [--base http://localhost:5173] [--allow-remote] + * + * The base skill must be a published release the owner can read. Exit code 0 + * when every check passes, 2 when the deployment answers `kit.api_disabled` + * (the flag is off), 1 on any other failure. + */ +import { randomBytes } from "node:crypto"; +import { pathToFileURL } from "node:url"; + +const LOOPBACK = new Set(["localhost", "127.0.0.1", "[::1]"]); +const OVERLAY_SCHEMA = "agentkit.skill-overlay/v1"; +const KIT_SCHEMA = "skillx.personal-kit/v1"; + +/** `@pub/slug@1.2.3` into its package and version. */ +export function parseBaseSkill(raw) { + const m = /^(@[a-z0-9][a-z0-9-]*\/[a-z0-9][a-z0-9-]*)@([0-9A-Za-z.+-]+)$/.exec(raw ?? ""); + if (m === null) throw new Error(`--base-skill must be @publisher/slug@version, got ${JSON.stringify(raw)}`); + return { pkg: m[1], version: m[2] }; +} + +/** + * Runs the smoke against `base` through `fetchImpl` and returns the receipt. + * Every check is recorded with its status and error code; the run stops at + * the first failure whose later checks depend on it. + */ +export async function runPersonalKitSmoke({ fetchImpl = fetch, base, ownerKey, otherKey, handle, baseSkill, name, idempotencyKey }) { + if (typeof ownerKey !== "string" || ownerKey === "") throw new Error("an owner API key is required"); + if (!/^[a-z0-9][a-z0-9-]*$/.test(handle ?? "")) throw new Error(`--handle must be a lowercase publisher handle, got ${JSON.stringify(handle)}`); + const { pkg, version } = parseBaseSkill(baseSkill); + const kitName = name ?? `smoke-${randomBytes(4).toString("hex")}`; + const kit = `@${handle}/${kitName}`; + const origin = new URL(base).origin; + const kitUrl = (sub = "") => `${origin}/api/v1/kits/${kit}${sub === "" ? "" : `/${sub}`}`; + const checks = []; + const receipt = { schema: "skillx.personal-kits-smoke/v1", target: origin, kit, baseSkill: `${pkg}@${version}`, enabled: null, checks, ok: false }; + + async function request(url, { key = ownerKey, method = "GET", body, headers = {} } = {}) { + const h = new Headers({ Accept: "application/json", ...headers }); + if (key !== null) h.set("Authorization", `Bearer ${key}`); + if (body !== undefined) h.set("Content-Type", "application/json"); + const res = await fetchImpl(url, { method, headers: h, body: body === undefined ? undefined : JSON.stringify(body) }); + let json = null; + try { + json = await res.json(); + } catch { + json = null; + } + return { status: res.status, body: json, code: json?.error?.code ?? null }; + } + + function check(label, res, ok) { + checks.push({ label, ok, status: res.status, code: res.code }); + return ok; + } + + // 1. Is the API on? Flag off answers 404 kit.api_disabled before any auth or lookup. + const probe = await request(kitUrl()); + if (probe.status === 404 && probe.code === "kit.api_disabled") { + receipt.enabled = false; + check("api enabled", probe, false); + return receipt; + } + receipt.enabled = true; + if (!check("unknown kit is 404 not_found", probe, probe.status === 404 && probe.code === "not_found")) return receipt; + + // 2. Anonymous callers are refused. + const anon = await request(kitUrl(), { key: null }); + check("anonymous is 401", anon, anon.status === 401); + + // 3. Resolve the pinned base release digest. + const manifest = await request(`${origin}/api/v1/packages/${pkg}/releases/${version}`); + const releaseDigest = manifest.body?.releaseDigest; + if (!check("base release readable", manifest, manifest.status === 200 && typeof releaseDigest === "string")) return receipt; + + // 4. Store a draft revision: one override entry with no overlay ops. + const document = { + schema: KIT_SCHEMA, + kit, + localId: `local/${kitName}`, + visibility: "private", + skills: [ + { + identity: pkg, + mode: "override", + base: { package: pkg, version, releaseDigest }, + overlay: { schema: OVERLAY_SCHEMA, baseTreeDigest: releaseDigest, ops: [] }, + compositionDigest: releaseDigest, + }, + ], + exclude: [], + runtimes: {}, + }; + const idem = idempotencyKey ?? `smoke-${randomBytes(8).toString("hex")}`; + const created = await request(kitUrl("revisions"), { method: "POST", body: { expectedGeneration: 0, idempotencyKey: idem, document } }); + const revision = created.body?.revision; + if (!check("store revision is 201", created, created.status === 201 && created.body?.generation === 1 && typeof revision?.id === "string")) return receipt; + + // 5. Replaying the same request changes nothing. + const replay = await request(kitUrl("revisions"), { method: "POST", body: { expectedGeneration: 0, idempotencyKey: idem, document } }); + check("idempotent replay is 200", replay, replay.status === 200 && replay.body?.replayed === true && replay.body?.revision?.id === revision.id); + + // 6. Activation under compare-and-set: a stale If-Match is 412, the current one activates. + const approve = { revisionId: revision.id, approval: { revisionDigest: revision.digest } }; + const stale = await request(kitUrl("activate"), { method: "POST", body: approve, headers: { "If-Match": '"g0"' } }); + check("stale If-Match is 412", stale, stale.status === 412 && stale.code === "kit.generation_conflict"); + const activated = await request(kitUrl("activate"), { method: "POST", body: approve, headers: { "If-Match": '"g1"' } }); + if (!check("activate is 200 with a lock", activated, activated.status === 200 && activated.body?.lock?.kit === kit)) return receipt; + + // 7. The lock reads back identically. + const lock = await request(kitUrl("lock")); + check("lock reads back", lock, lock.status === 200 && JSON.stringify(lock.body?.lock) === JSON.stringify(activated.body.lock)); + + // 8. Another account sees the same 404 as an unknown kit. + if (typeof otherKey === "string" && otherKey !== "") { + const foreign = await request(kitUrl(), { key: otherKey }); + check("other account is 404", foreign, foreign.status === 404 && foreign.code === "not_found"); + } + + receipt.ok = checks.every((c) => c.ok); + return receipt; +} + +function argValue(args, flag) { + const i = args.indexOf(flag); + return i >= 0 ? args[i + 1] : undefined; +} + +async function main(args) { + const base = new URL(argValue(args, "--base") ?? "http://localhost:5173"); + if (!LOOPBACK.has(base.hostname) && !args.includes("--allow-remote")) { + console.error(`refusing non-loopback base URL ${base.origin}; pass --allow-remote to smoke a deployed SkillX on purpose`); + return 1; + } + const receipt = await runPersonalKitSmoke({ + base: base.href, + ownerKey: process.env.SKILLX_SMOKE_OWNER_KEY, + otherKey: process.env.SKILLX_SMOKE_OTHER_KEY, + handle: argValue(args, "--handle"), + baseSkill: argValue(args, "--base-skill"), + }); + console.log(JSON.stringify({ ...receipt, at: new Date().toISOString() }, null, 2)); + if (receipt.enabled === false) { + console.error("this deployment answers 404 kit.api_disabled: SKILLX_PERSONAL_KITS_API is off"); + return 2; + } + return receipt.ok ? 0 : 1; +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { + main(process.argv.slice(2)).then( + (code) => process.exit(code), + (error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); + }, + ); +} From d8480db0a7b0935d928e8286b4cb46f471574999 Mon Sep 17 00:00:00 2001 From: Duy Nguyen Date: Tue, 20 Oct 2026 02:38:36 +0700 Subject: [PATCH 2/3] fix(personal-kits): harden the enablement smoke and name kit.api_disabled in the CLI The smoke now checks cross-account writes as well as reads: a second account gets 404 storing or activating a revision, 403 kit.publisher_not_owned creating a kit under the owner's handle, and leaves the owner's kit unchanged. A remote run refuses to start without SKILLX_SMOKE_OTHER_KEY, and no request follows a redirect (a 3xx fails the check). The API cannot delete a kit (immutable revisions and audit, ON DELETE RESTRICT), so the smoke reuses one fixed private kit (--kit-name, default agentkit-smoke) and appends one revision per run instead of a new kit each time; the runbook gains the cleanup step. The docs state both accepted enable values ("1" or "true"). The CLI kit client maps 404 kit.api_disabled to KitApiDisabledError, a clear "this deployment does not offer personal kits yet" error, while a plain 404 stays an ordinary API error. --- .../kit-enablement-smoke.test.ts | 50 ++++++- docs/personal-kits-api.md | 28 +++- packages/cli/src/lib/api-client.ts | 14 +- packages/cli/src/lib/kit-api.test.ts | 22 ++- packages/cli/src/lib/kit-api.ts | 25 +++- scripts/smoke-personal-kits.mjs | 140 +++++++++++++----- 6 files changed, 221 insertions(+), 58 deletions(-) diff --git a/apps/web/tests/personal-kits/kit-enablement-smoke.test.ts b/apps/web/tests/personal-kits/kit-enablement-smoke.test.ts index 2bcb7db..162f710 100644 --- a/apps/web/tests/personal-kits/kit-enablement-smoke.test.ts +++ b/apps/web/tests/personal-kits/kit-enablement-smoke.test.ts @@ -1,5 +1,5 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { parseBaseSkill, runPersonalKitSmoke } from "../../../../scripts/smoke-personal-kits.mjs"; +import { main, parseBaseSkill, runPersonalKitSmoke } from "../../../../scripts/smoke-personal-kits.mjs"; import { BODY, createKitHarness, skillText } from "./kit-harness"; import type { KitHarness } from "./kit-harness"; @@ -25,32 +25,73 @@ const run = (extra: Record = {}) => otherKey: k.otherKey, handle: "alpha", baseSkill: "@alpha/demo@1.0.0", - name: "smoke-local", ...extra, }); +const failed = (receipt: { checks: Array<{ ok: boolean }> }) => receipt.checks.filter((c) => !c.ok); + describe("personal kit enablement smoke", () => { it("passes every check against an enabled API and leaves one private kit", async () => { const receipt = await run(); expect(receipt.enabled).toBe(true); - expect(receipt.checks.filter((c: { ok: boolean }) => !c.ok)).toEqual([]); + expect(receipt.kit).toBe("@alpha/agentkit-smoke"); + expect(failed(receipt)).toEqual([]); expect(receipt.checks.map((c: { label: string }) => c.label)).toEqual([ "unknown kit is 404 not_found", "anonymous is 401", "base release readable", + "smoke kit generation readable", "store revision is 201", "idempotent replay is 200", "stale If-Match is 412", "activate is 200 with a lock", "lock reads back", - "other account is 404", + "other account read is 404", + "other account write is 404", + "other account activate is 404", + "other account cannot create under the handle", + "owner kit unchanged by the other account", ]); expect(receipt.ok).toBe(true); + expect(receipt.crossAccount).toBe(true); expect(JSON.stringify(receipt)).not.toContain(k.ownerKey); + expect(JSON.stringify(receipt)).not.toContain(k.otherKey); + // The foreign writes stored nothing: one kit, one revision, all the owner's. expect(k.count("personal_kits")).toBe(1); expect(k.count("personal_kit_revisions")).toBe(1); }); + it("reuses the one smoke kit on a later run instead of creating another", async () => { + expect((await run()).ok).toBe(true); + const second = await run(); + expect(failed(second)).toEqual([]); + expect(k.count("personal_kits")).toBe(1); + expect(k.count("personal_kit_revisions")).toBe(2); + }); + + it("treats a redirect as a failed check and never follows it", async () => { + const seen: Array = []; + const fetchImpl = async (_input: string | URL | Request, init?: RequestInit) => { + seen.push(init); + return new Response(null, { status: 302, headers: { Location: "https://elsewhere.test/" } }); + }; + const receipt = await run({ fetchImpl }); + expect(receipt.ok).toBe(false); + expect(receipt.checks).toEqual([{ label: "unknown kit is 404 not_found", ok: false, status: 302, code: "redirect" }]); + expect(seen.every((init) => init?.redirect === "manual")).toBe(true); + }); + + it("refuses a remote run without a second account key, before any request", async () => { + const fetchImpl = vi.fn(); + const errors: string[] = []; + const io = { fetchImpl, log: () => {}, error: (m: string) => errors.push(m) }; + const args = ["--base", "https://staging.skillx.test", "--handle", "alpha", "--base-skill", "@alpha/demo@1.0.0"]; + expect(await main(args, { SKILLX_SMOKE_OWNER_KEY: "k" }, io)).toBe(1); + expect(await main([...args, "--allow-remote"], { SKILLX_SMOKE_OWNER_KEY: "k" }, io)).toBe(1); + expect(errors.join("\n")).toMatch(/SKILLX_SMOKE_OTHER_KEY/); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + it("reports a flag-off deployment as disabled and writes nothing", async () => { const receipt = await run({ fetchImpl: k.serveWith({ SKILLX_PERSONAL_KITS_API: "0" }) }); expect(receipt).toMatchObject({ enabled: false, ok: false, checks: [{ label: "api enabled", ok: false, status: 404, code: "kit.api_disabled" }] }); @@ -62,6 +103,7 @@ describe("personal kit enablement smoke", () => { await expect(run({ fetchImpl, ownerKey: "" })).rejects.toThrow(/owner API key/); await expect(run({ fetchImpl, handle: "Alpha" })).rejects.toThrow(/--handle/); await expect(run({ fetchImpl, baseSkill: "@alpha/demo" })).rejects.toThrow(/--base-skill/); + await expect(run({ fetchImpl, name: "Smoke_Kit" })).rejects.toThrow(/--kit-name/); expect(fetchImpl).not.toHaveBeenCalled(); expect(parseBaseSkill("@alpha/demo@1.0.0-beta.1")).toEqual({ pkg: "@alpha/demo", version: "1.0.0-beta.1" }); }); diff --git a/docs/personal-kits-api.md b/docs/personal-kits-api.md index 49d247a..f59b530 100644 --- a/docs/personal-kits-api.md +++ b/docs/personal-kits-api.md @@ -10,7 +10,8 @@ without an explicit user approval. - Mount: `/api/v1/kits/@{handle}/{name}[/...]`. Source: `apps/web/app/routes/api.v1.kits.ts` and `apps/web/app/lib/personal-kits/`. -- Feature flag: `SKILLX_PERSONAL_KITS_API` (`"1"` enables it). While it is off, +- Feature flag: `SKILLX_PERSONAL_KITS_API` (`"1"` or `"true"` enables it, as + for `SKILLX_RELEASES_API`; any other value or no value is off). While it is off, every path returns the JSON 404 `kit.api_disabled`, before any authentication or lookup, so a client can say "this deployment does not offer personal kits" instead of "no such kit". Reading base releases also needs the @@ -538,14 +539,25 @@ in `apps/web/wrangler.jsonc` stays `"0"` until an operator turns it on. 1. Turn it on for one environment (`SKILLX_PERSONAL_KITS_API` = `"1"`) and deploy that environment. -2. Run the enablement smoke against it with a throwaway owner account that - holds a published base skill: +2. Run the enablement smoke against it with a dedicated smoke owner account + whose publisher holds a published base skill, and a second account: `SKILLX_SMOKE_OWNER_KEY=... SKILLX_SMOKE_OTHER_KEY=... node scripts/smoke-personal-kits.mjs --base --allow-remote --handle --base-skill @pub/slug@version`. - It stores one private kit `@handle/smoke-`, checks 401 for - anonymous, idempotent replay, 412 on a stale `If-Match`, activation, the lock - read-back and 404 for another account, and prints a JSON receipt (no keys). - Exit 2 means the deployment still answers `kit.api_disabled`. -3. Roll back by setting the flag to `"0"` and redeploying. Every kit path then + A remote run refuses to start without `SKILLX_SMOKE_OTHER_KEY`. The smoke + checks 401 for anonymous, idempotent replay, 412 on a stale `If-Match`, + activation and the lock read-back, then that the second account gets 404 on + reading the kit, storing a revision and activating it, gets 403 + `kit.publisher_not_owned` creating a kit under the owner's handle, and leaves + the owner's kit unchanged. It never follows a redirect (a 3xx fails the + check) and prints a JSON receipt with no keys. Exit 2 means the deployment + still answers `kit.api_disabled`. +3. Clean up: the API cannot delete a kit (revisions and audit rows are + immutable, and kits are `ON DELETE RESTRICT`), so the smoke reuses one + private kit, `@handle/agentkit-smoke` (`--kit-name` to choose another), and + each run only appends and activates one revision there; the receipt names + the kit and the run. Keep that kit on the dedicated smoke account; do not + point the smoke at a real user's handle. The squat probe of step 2 stores + nothing (403 before any write). +4. Roll back by setting the flag to `"0"` and redeploying. Every kit path then answers 404 `kit.api_disabled` again; stored revisions, locks and audit rows stay in D1 untouched and come back unchanged when the flag returns. Clients keep their local kits: AgentKit maps `kit.api_disabled` to "unsupported" diff --git a/packages/cli/src/lib/api-client.ts b/packages/cli/src/lib/api-client.ts index 8b47416..6c90b46 100644 --- a/packages/cli/src/lib/api-client.ts +++ b/packages/cli/src/lib/api-client.ts @@ -1,10 +1,20 @@ import { getApiKey, getBaseUrl } from '../utils/config-store.js'; export class ApiError extends Error { - constructor(public status: number, message: string) { + constructor(public status: number, message: string, public body = '') { super(message); this.name = 'ApiError'; } + + /** The `error.code` of a JSON error body, or null. */ + get code(): string | null { + try { + const code = (JSON.parse(this.body) as { error?: { code?: unknown } })?.error?.code; + return typeof code === 'string' ? code : null; + } catch { + return null; + } + } } export async function apiRequest( @@ -28,7 +38,7 @@ export async function apiRequest( if (!res.ok) { const body = await res.text(); - throw new ApiError(res.status, `API error ${res.status}: ${body}`); + throw new ApiError(res.status, `API error ${res.status}: ${body}`, body); } return res.json() as Promise; diff --git a/packages/cli/src/lib/kit-api.test.ts b/packages/cli/src/lib/kit-api.test.ts index 5c0b438..6fca5db 100644 --- a/packages/cli/src/lib/kit-api.test.ts +++ b/packages/cli/src/lib/kit-api.test.ts @@ -5,7 +5,7 @@ vi.mock('../utils/config-store.js', () => ({ getBaseUrl: () => 'https://skillx.test', })); -import { kitApi } from './kit-api.js'; +import { KitApiDisabledError, kitApi } from './kit-api.js'; import { ApiError } from './api-client.js'; let calls: Array<{ url: string; method: string; body: unknown; auth: string | null }>; @@ -53,6 +53,24 @@ describe('kit API client', () => { expect(() => kitApi.get('../etc')).toThrow(/invalid kit name/); reply = () => new Response(JSON.stringify({ error: { code: 'kit.generation_conflict' } }), { status: 409 }); await expect(kitApi.get('@alpha/my-kit')).rejects.toBeInstanceOf(ApiError); - expect(calls).toHaveLength(1); + await expect(kitApi.get('@alpha/my-kit')).rejects.not.toBeInstanceOf(KitApiDisabledError); + expect(calls).toHaveLength(2); + }); + + it('reports a deployment with the kit API off as unsupported, not as a missing kit', async () => { + reply = () => new Response(JSON.stringify({ error: { code: 'kit.api_disabled', message: 'personal kits are not enabled' } }), { status: 404 }); + const err = await kitApi.storeRevision('@alpha/my-kit', {}, 0, 'idem-key-0005').catch((e: unknown) => e); + expect(err).toBeInstanceOf(KitApiDisabledError); + expect(err).toBeInstanceOf(ApiError); + expect(err).toMatchObject({ status: 404, code: 'kit.api_disabled' }); + expect((err as Error).message).toMatch(/does not offer personal kits yet/); + expect((err as Error).message).toMatch(/local kits are unaffected/); + + // A plain 404 (an unknown or foreign kit) stays an ordinary API error. + reply = () => new Response(JSON.stringify({ error: { code: 'not_found' } }), { status: 404 }); + const missing = await kitApi.get('@alpha/my-kit').catch((e: unknown) => e); + expect(missing).toBeInstanceOf(ApiError); + expect(missing).not.toBeInstanceOf(KitApiDisabledError); + expect(missing).toMatchObject({ status: 404, code: 'not_found' }); }); }); diff --git a/packages/cli/src/lib/kit-api.ts b/packages/cli/src/lib/kit-api.ts index a906cc0..38a3ac4 100644 --- a/packages/cli/src/lib/kit-api.ts +++ b/packages/cli/src/lib/kit-api.ts @@ -1,4 +1,4 @@ -import { apiRequest } from './api-client.js'; +import { ApiError, apiRequest as rawRequest } from './api-client.js'; /** * Typed client for the personal kit API (`/api/v1/kits/@handle/name`, see @@ -29,6 +29,29 @@ export interface Update { version: string; } +/** The code a deployment with the personal kit API turned off answers on every path. */ +export const KIT_API_DISABLED = 'kit.api_disabled'; + +/** + * The deployment does not offer personal kits (its flag is off). It is not a + * missing kit: nothing was deleted and local kits keep working. + */ +export class KitApiDisabledError extends ApiError { + constructor(body: string) { + super(404, 'this SkillX deployment does not offer personal kits yet (kit.api_disabled); nothing was changed and your local kits are unaffected', body); + this.name = 'KitApiDisabledError'; + } +} + +async function apiRequest(endpoint: string, options?: RequestInit): Promise { + try { + return await rawRequest(endpoint, options); + } catch (error) { + if (error instanceof ApiError && error.status === 404 && error.code === KIT_API_DISABLED) throw new KitApiDisabledError(error.body); + throw error; + } +} + const KIT = /^@[a-z0-9][a-z0-9-]{0,38}\/[a-z0-9][a-z0-9._-]{0,63}$/; function kitPath(kit: string, sub = ''): string { diff --git a/scripts/smoke-personal-kits.mjs b/scripts/smoke-personal-kits.mjs index feb7729..9a4a1fd 100644 --- a/scripts/smoke-personal-kits.mjs +++ b/scripts/smoke-personal-kits.mjs @@ -4,19 +4,26 @@ * * It answers one question: on this deployment, with `SKILLX_PERSONAL_KITS_API` * on, can an owner store a revision, replay it idempotently, activate it under - * compare-and-set and read its lock, while anonymous and foreign callers see - * nothing? It writes one throwaway private kit (`@handle/smoke-`) and - * prints a JSON receipt. It never prints a key. + * compare-and-set and read its lock, while anonymous and foreign callers can + * neither read nor write it? It prints a JSON receipt and never prints a key. * * LOCAL BY DEFAULT: a non-loopback base URL is refused unless `--allow-remote` - * is passed, so a staging run is always a deliberate choice. It never deploys - * and never changes a flag. + * is passed, so a staging run is always a deliberate choice, and a remote run + * also needs a second account key so the cross-account checks always run. It + * never deploys, never changes a flag and never follows a redirect. + * + * CLEANUP: the API cannot delete a kit (revisions and the audit are immutable + * and kits are ON DELETE RESTRICT), so the smoke reuses one fixed private kit, + * `@handle/<--kit-name>` (default `agentkit-smoke`), instead of creating a new + * one per run. Every run appends one revision to it and activates that + * revision; the receipt names the kit. Run it from a dedicated smoke account + * and publisher so the kit never sits next to a real user's kits. * * Usage: * SKILLX_SMOKE_OWNER_KEY= \ - * SKILLX_SMOKE_OTHER_KEY= \ + * SKILLX_SMOKE_OTHER_KEY= \ * node scripts/smoke-personal-kits.mjs --handle alpha --base-skill @alpha/demo@1.0.0 \ - * [--base http://localhost:5173] [--allow-remote] + * [--kit-name agentkit-smoke] [--base http://localhost:5173] [--allow-remote] * * The base skill must be a published release the owner can read. Exit code 0 * when every check passes, 2 when the deployment answers `kit.api_disabled` @@ -28,6 +35,8 @@ import { pathToFileURL } from "node:url"; const LOOPBACK = new Set(["localhost", "127.0.0.1", "[::1]"]); const OVERLAY_SCHEMA = "agentkit.skill-overlay/v1"; const KIT_SCHEMA = "skillx.personal-kit/v1"; +const NAME = /^[a-z0-9]+(-[a-z0-9]+)*$/; +export const DEFAULT_KIT_NAME = "agentkit-smoke"; /** `@pub/slug@1.2.3` into its package and version. */ export function parseBaseSkill(raw) { @@ -36,27 +45,34 @@ export function parseBaseSkill(raw) { return { pkg: m[1], version: m[2] }; } +const hex = (n) => randomBytes(n).toString("hex"); + /** * Runs the smoke against `base` through `fetchImpl` and returns the receipt. * Every check is recorded with its status and error code; the run stops at - * the first failure whose later checks depend on it. + * the first failure whose later checks depend on it. The cross-account + * checks run when `otherKey` is given. */ -export async function runPersonalKitSmoke({ fetchImpl = fetch, base, ownerKey, otherKey, handle, baseSkill, name, idempotencyKey }) { +export async function runPersonalKitSmoke({ fetchImpl = fetch, base, ownerKey, otherKey, handle, baseSkill, name = DEFAULT_KIT_NAME, idempotencyKey, runId }) { if (typeof ownerKey !== "string" || ownerKey === "") throw new Error("an owner API key is required"); if (!/^[a-z0-9][a-z0-9-]*$/.test(handle ?? "")) throw new Error(`--handle must be a lowercase publisher handle, got ${JSON.stringify(handle)}`); + if (!NAME.test(name ?? "") || name.length > 48) throw new Error(`--kit-name must be lowercase [a-z0-9] words joined by hyphens, got ${JSON.stringify(name)}`); const { pkg, version } = parseBaseSkill(baseSkill); - const kitName = name ?? `smoke-${randomBytes(4).toString("hex")}`; - const kit = `@${handle}/${kitName}`; + const run = runId ?? hex(6); + const kit = `@${handle}/${name}`; const origin = new URL(base).origin; - const kitUrl = (sub = "") => `${origin}/api/v1/kits/${kit}${sub === "" ? "" : `/${sub}`}`; + const kitUrl = (sub = "", of = kit) => `${origin}/api/v1/kits/${of}${sub === "" ? "" : `/${sub}`}`; const checks = []; - const receipt = { schema: "skillx.personal-kits-smoke/v1", target: origin, kit, baseSkill: `${pkg}@${version}`, enabled: null, checks, ok: false }; + const receipt = { schema: "skillx.personal-kits-smoke/v1", target: origin, kit, run, baseSkill: `${pkg}@${version}`, enabled: null, crossAccount: false, checks, ok: false }; async function request(url, { key = ownerKey, method = "GET", body, headers = {} } = {}) { const h = new Headers({ Accept: "application/json", ...headers }); if (key !== null) h.set("Authorization", `Bearer ${key}`); if (body !== undefined) h.set("Content-Type", "application/json"); - const res = await fetchImpl(url, { method, headers: h, body: body === undefined ? undefined : JSON.stringify(body) }); + // Never follow a redirect: it would carry the key to another location, + // and a 3xx is a failed check, not an answer. + const res = await fetchImpl(url, { method, headers: h, body: body === undefined ? undefined : JSON.stringify(body), redirect: "manual" }); + if (res.type === "opaqueredirect" || (res.status >= 300 && res.status < 400)) return { status: res.status, body: null, code: "redirect" }; let json = null; try { json = await res.json(); @@ -72,7 +88,8 @@ export async function runPersonalKitSmoke({ fetchImpl = fetch, base, ownerKey, o } // 1. Is the API on? Flag off answers 404 kit.api_disabled before any auth or lookup. - const probe = await request(kitUrl()); + const unknown = `@${handle}/smoke-probe-${hex(4)}`; + const probe = await request(kitUrl("", unknown)); if (probe.status === 404 && probe.code === "kit.api_disabled") { receipt.enabled = false; check("api enabled", probe, false); @@ -90,11 +107,17 @@ export async function runPersonalKitSmoke({ fetchImpl = fetch, base, ownerKey, o const releaseDigest = manifest.body?.releaseDigest; if (!check("base release readable", manifest, manifest.status === 200 && typeof releaseDigest === "string")) return receipt; - // 4. Store a draft revision: one override entry with no overlay ops. - const document = { + // 4. The reused smoke kit's current generation (0 when it does not exist yet). + const current = await request(kitUrl()); + const generation = current.status === 404 && current.code === "not_found" ? 0 : current.body?.generation; + if (!check("smoke kit generation readable", current, Number.isInteger(generation))) return receipt; + + // 5. Store a draft revision: one override entry with no overlay ops. The + // profile names the run, so a reused kit gets a new revision every run. + const documentFor = (of) => ({ schema: KIT_SCHEMA, - kit, - localId: `local/${kitName}`, + kit: of, + localId: `local/${of.slice(of.indexOf("/") + 1)}`, visibility: "private", skills: [ { @@ -106,32 +129,59 @@ export async function runPersonalKitSmoke({ fetchImpl = fetch, base, ownerKey, o }, ], exclude: [], + profile: `smoke-${run}`, runtimes: {}, - }; - const idem = idempotencyKey ?? `smoke-${randomBytes(8).toString("hex")}`; - const created = await request(kitUrl("revisions"), { method: "POST", body: { expectedGeneration: 0, idempotencyKey: idem, document } }); + }); + const document = documentFor(kit); + const idem = idempotencyKey ?? `smoke-${run}`; + const store = { method: "POST", body: { expectedGeneration: generation, idempotencyKey: idem, document } }; + const created = await request(kitUrl("revisions"), store); const revision = created.body?.revision; - if (!check("store revision is 201", created, created.status === 201 && created.body?.generation === 1 && typeof revision?.id === "string")) return receipt; + if (!check("store revision is 201", created, created.status === 201 && created.body?.generation === generation + 1 && typeof revision?.id === "string")) return receipt; - // 5. Replaying the same request changes nothing. - const replay = await request(kitUrl("revisions"), { method: "POST", body: { expectedGeneration: 0, idempotencyKey: idem, document } }); + // 6. Replaying the same request changes nothing. + const replay = await request(kitUrl("revisions"), store); check("idempotent replay is 200", replay, replay.status === 200 && replay.body?.replayed === true && replay.body?.revision?.id === revision.id); - // 6. Activation under compare-and-set: a stale If-Match is 412, the current one activates. + // 7. Activation under compare-and-set: a stale If-Match is 412, the current one activates. const approve = { revisionId: revision.id, approval: { revisionDigest: revision.digest } }; - const stale = await request(kitUrl("activate"), { method: "POST", body: approve, headers: { "If-Match": '"g0"' } }); + const stale = await request(kitUrl("activate"), { method: "POST", body: approve, headers: { "If-Match": `"g${generation}"` } }); check("stale If-Match is 412", stale, stale.status === 412 && stale.code === "kit.generation_conflict"); - const activated = await request(kitUrl("activate"), { method: "POST", body: approve, headers: { "If-Match": '"g1"' } }); + const activated = await request(kitUrl("activate"), { method: "POST", body: approve, headers: { "If-Match": `"g${generation + 1}"` } }); if (!check("activate is 200 with a lock", activated, activated.status === 200 && activated.body?.lock?.kit === kit)) return receipt; - // 7. The lock reads back identically. + // 8. The lock reads back identically. const lock = await request(kitUrl("lock")); check("lock reads back", lock, lock.status === 200 && JSON.stringify(lock.body?.lock) === JSON.stringify(activated.body.lock)); - // 8. Another account sees the same 404 as an unknown kit. + // 9. Another account can neither read nor write the kit, nor create one + // under the owner's handle, and the owner's kit is unchanged afterwards. if (typeof otherKey === "string" && otherKey !== "") { + receipt.crossAccount = true; + const before = await request(kitUrl()); const foreign = await request(kitUrl(), { key: otherKey }); - check("other account is 404", foreign, foreign.status === 404 && foreign.code === "not_found"); + check("other account read is 404", foreign, foreign.status === 404 && foreign.code === "not_found"); + const foreignStore = await request(kitUrl("revisions"), { + key: otherKey, + method: "POST", + body: { expectedGeneration: before.body?.generation, idempotencyKey: `smoke-${run}-other`, document: { ...document, profile: `smoke-${run}-other` } }, + }); + check("other account write is 404", foreignStore, foreignStore.status === 404 && foreignStore.code === "not_found"); + const foreignActivate = await request(kitUrl("activate"), { key: otherKey, method: "POST", body: approve, headers: { "If-Match": `"g${before.body?.generation}"` } }); + check("other account activate is 404", foreignActivate, foreignActivate.status === 404 && foreignActivate.code === "not_found"); + const squat = `@${handle}/smoke-squat-${hex(4)}`; + const foreignCreate = await request(kitUrl("revisions", squat), { + key: otherKey, + method: "POST", + body: { expectedGeneration: 0, idempotencyKey: `smoke-${run}-squat`, document: documentFor(squat) }, + }); + check("other account cannot create under the handle", foreignCreate, foreignCreate.status === 403 && foreignCreate.code === "kit.publisher_not_owned"); + const after = await request(kitUrl()); + check( + "owner kit unchanged by the other account", + after, + before.status === 200 && after.status === 200 && JSON.stringify(after.body) === JSON.stringify(before.body), + ); } receipt.ok = checks.every((c) => c.ok); @@ -143,22 +193,30 @@ function argValue(args, flag) { return i >= 0 ? args[i + 1] : undefined; } -async function main(args) { +export async function main(args, env = process.env, { fetchImpl = fetch, log = console.log, error = console.error } = {}) { const base = new URL(argValue(args, "--base") ?? "http://localhost:5173"); - if (!LOOPBACK.has(base.hostname) && !args.includes("--allow-remote")) { - console.error(`refusing non-loopback base URL ${base.origin}; pass --allow-remote to smoke a deployed SkillX on purpose`); - return 1; + if (!LOOPBACK.has(base.hostname)) { + if (!args.includes("--allow-remote")) { + error(`refusing non-loopback base URL ${base.origin}; pass --allow-remote to smoke a deployed SkillX on purpose`); + return 1; + } + if (!env.SKILLX_SMOKE_OTHER_KEY) { + error("--allow-remote needs SKILLX_SMOKE_OTHER_KEY (a second account's key) so the cross-account read and write checks run"); + return 1; + } } const receipt = await runPersonalKitSmoke({ + fetchImpl, base: base.href, - ownerKey: process.env.SKILLX_SMOKE_OWNER_KEY, - otherKey: process.env.SKILLX_SMOKE_OTHER_KEY, + ownerKey: env.SKILLX_SMOKE_OWNER_KEY, + otherKey: env.SKILLX_SMOKE_OTHER_KEY, handle: argValue(args, "--handle"), baseSkill: argValue(args, "--base-skill"), + name: argValue(args, "--kit-name") ?? DEFAULT_KIT_NAME, }); - console.log(JSON.stringify({ ...receipt, at: new Date().toISOString() }, null, 2)); + log(JSON.stringify({ ...receipt, at: new Date().toISOString() }, null, 2)); if (receipt.enabled === false) { - console.error("this deployment answers 404 kit.api_disabled: SKILLX_PERSONAL_KITS_API is off"); + error("this deployment answers 404 kit.api_disabled: SKILLX_PERSONAL_KITS_API is off"); return 2; } return receipt.ok ? 0 : 1; @@ -167,8 +225,8 @@ async function main(args) { if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) { main(process.argv.slice(2)).then( (code) => process.exit(code), - (error) => { - console.error(error instanceof Error ? error.message : String(error)); + (err) => { + console.error(err instanceof Error ? err.message : String(err)); process.exit(1); }, ); From cc7f69a45d9fb3d3b6e1e190bce08bcd3c4371b7 Mon Sep 17 00:00:00 2001 From: Duy Nguyen Date: Tue, 20 Oct 2026 02:53:40 +0700 Subject: [PATCH 3/3] fix(personal-kits): require https for a remote smoke and pin the flag values A remote enablement smoke now refuses a base URL that is not https, since it sends account keys. Tests pin the accepted flag values: "1" or "true", trimmed and in any case, and nothing else. The wrangler.jsonc comment and the runbook state those values (the production value stays "0"), and the runbook names the cross-account create probe precisely. --- .../personal-kits/kit-enablement-smoke.test.ts | 10 ++++++++++ apps/web/tests/personal-kits/kit-revisions.test.ts | 14 ++++++++++++++ apps/web/wrangler.jsonc | 3 ++- docs/personal-kits-api.md | 13 ++++++++----- scripts/smoke-personal-kits.mjs | 7 ++++++- 5 files changed, 40 insertions(+), 7 deletions(-) diff --git a/apps/web/tests/personal-kits/kit-enablement-smoke.test.ts b/apps/web/tests/personal-kits/kit-enablement-smoke.test.ts index 162f710..c10a461 100644 --- a/apps/web/tests/personal-kits/kit-enablement-smoke.test.ts +++ b/apps/web/tests/personal-kits/kit-enablement-smoke.test.ts @@ -92,6 +92,16 @@ describe("personal kit enablement smoke", () => { expect(fetchImpl).not.toHaveBeenCalled(); }); + it("refuses a remote run over plain http even with both keys, before any request", async () => { + const fetchImpl = vi.fn(); + const errors: string[] = []; + const io = { fetchImpl, log: () => {}, error: (m: string) => errors.push(m) }; + const args = ["--base", "http://staging.skillx.test", "--allow-remote", "--handle", "alpha", "--base-skill", "@alpha/demo@1.0.0"]; + expect(await main(args, { SKILLX_SMOKE_OWNER_KEY: "k", SKILLX_SMOKE_OTHER_KEY: "o" }, io)).toBe(1); + expect(errors.join("\n")).toMatch(/must be https/); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + it("reports a flag-off deployment as disabled and writes nothing", async () => { const receipt = await run({ fetchImpl: k.serveWith({ SKILLX_PERSONAL_KITS_API: "0" }) }); expect(receipt).toMatchObject({ enabled: false, ok: false, checks: [{ label: "api enabled", ok: false, status: 404, code: "kit.api_disabled" }] }); diff --git a/apps/web/tests/personal-kits/kit-revisions.test.ts b/apps/web/tests/personal-kits/kit-revisions.test.ts index b57ed69..2a7fa04 100644 --- a/apps/web/tests/personal-kits/kit-revisions.test.ts +++ b/apps/web/tests/personal-kits/kit-revisions.test.ts @@ -1,4 +1,5 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; +import { personalKitsApiEnabled } from "~/lib/personal-kits/kit-http"; import { BODY, createKitHarness, skillText } from "./kit-harness"; import type { KitHarness } from "./kit-harness"; @@ -39,6 +40,19 @@ describe("personal kit revisions", () => { expect(k.count("personal_kits") + k.count("personal_kit_audit")).toBe(0); }); + it('enables on "1" or "true", trimmed and in any case, and nothing else', async () => { + for (const on of ["1", "true", "TRUE", " 1 ", "\ttrue\n"]) { + expect(personalKitsApiEnabled({ SKILLX_PERSONAL_KITS_API: on }), JSON.stringify(on)).toBe(true); + const res = await k.call("", undefined, { env: { SKILLX_PERSONAL_KITS_API: on } }); + expect(res.body.error?.code, JSON.stringify(on)).toBe("not_found"); + } + for (const off of [undefined, "", " ", "0", "false", "yes", "on", "1 1", "truee"]) { + expect(personalKitsApiEnabled({ SKILLX_PERSONAL_KITS_API: off }), JSON.stringify(off)).toBe(false); + const res = await k.call("", undefined, { env: { SKILLX_PERSONAL_KITS_API: off } }); + expect(res.body.error?.code, JSON.stringify(off)).toBe("kit.api_disabled"); + } + }); + it("stores a draft whose composition the server re-derives, then activates it only with approval", async () => { const created = await createKit(); expect(created).toMatchObject({ generation: 1, replayed: false, revision: { number: 1, state: "draft", origin: "manual" } }); diff --git a/apps/web/wrangler.jsonc b/apps/web/wrangler.jsonc index 90e0fac..6bf2911 100644 --- a/apps/web/wrangler.jsonc +++ b/apps/web/wrangler.jsonc @@ -68,7 +68,8 @@ // Subrequests (D1 + R2 calls) one release publish may spend; above it the upload is refused with 413 // before any R2 write. Workers Paid allows 10,000 per request; on a free account (1,000) set ~600. "SKILLX_RELEASE_SUBREQUEST_BUDGET": "5000", - // Personal kit API (/api/v1/kits/@handle/name/...). "0" = off (404 everywhere). + // Personal kit API (/api/v1/kits/@handle/name/...). On only for "1" or "true" (trimmed, any + // case); any other value is off and every kit path answers 404 kit.api_disabled. "SKILLX_PERSONAL_KITS_API": "0" } } diff --git a/docs/personal-kits-api.md b/docs/personal-kits-api.md index f59b530..13f0fe3 100644 --- a/docs/personal-kits-api.md +++ b/docs/personal-kits-api.md @@ -10,8 +10,9 @@ without an explicit user approval. - Mount: `/api/v1/kits/@{handle}/{name}[/...]`. Source: `apps/web/app/routes/api.v1.kits.ts` and `apps/web/app/lib/personal-kits/`. -- Feature flag: `SKILLX_PERSONAL_KITS_API` (`"1"` or `"true"` enables it, as - for `SKILLX_RELEASES_API`; any other value or no value is off). While it is off, +- Feature flag: `SKILLX_PERSONAL_KITS_API` (`"1"` or `"true"`, trimmed and in + any case, enables it, as for `SKILLX_RELEASES_API`; any other value or no + value is off). While it is off, every path returns the JSON 404 `kit.api_disabled`, before any authentication or lookup, so a client can say "this deployment does not offer personal kits" instead of "no such kit". Reading base releases also needs the @@ -542,7 +543,8 @@ in `apps/web/wrangler.jsonc` stays `"0"` until an operator turns it on. 2. Run the enablement smoke against it with a dedicated smoke owner account whose publisher holds a published base skill, and a second account: `SKILLX_SMOKE_OWNER_KEY=... SKILLX_SMOKE_OTHER_KEY=... node scripts/smoke-personal-kits.mjs --base --allow-remote --handle --base-skill @pub/slug@version`. - A remote run refuses to start without `SKILLX_SMOKE_OTHER_KEY`. The smoke + A remote run refuses to start without `SKILLX_SMOKE_OTHER_KEY` or with a + base URL that is not `https`. The smoke checks 401 for anonymous, idempotent replay, 412 on a stale `If-Match`, activation and the lock read-back, then that the second account gets 404 on reading the kit, storing a revision and activating it, gets 403 @@ -555,8 +557,9 @@ in `apps/web/wrangler.jsonc` stays `"0"` until an operator turns it on. private kit, `@handle/agentkit-smoke` (`--kit-name` to choose another), and each run only appends and activates one revision there; the receipt names the kit and the run. Keep that kit on the dedicated smoke account; do not - point the smoke at a real user's handle. The squat probe of step 2 stores - nothing (403 before any write). + point the smoke at a real user's handle. The second account's attempt to + create `@handle/smoke-squat-` leaves no kit behind: it is refused + with 403 `kit.publisher_not_owned` before anything is written. 4. Roll back by setting the flag to `"0"` and redeploying. Every kit path then answers 404 `kit.api_disabled` again; stored revisions, locks and audit rows stay in D1 untouched and come back unchanged when the flag returns. Clients diff --git a/scripts/smoke-personal-kits.mjs b/scripts/smoke-personal-kits.mjs index 9a4a1fd..32eed69 100644 --- a/scripts/smoke-personal-kits.mjs +++ b/scripts/smoke-personal-kits.mjs @@ -9,7 +9,8 @@ * * LOCAL BY DEFAULT: a non-loopback base URL is refused unless `--allow-remote` * is passed, so a staging run is always a deliberate choice, and a remote run - * also needs a second account key so the cross-account checks always run. It + * also needs an https base URL and a second account key so the keys never + * travel in clear text and the cross-account checks always run. It * never deploys, never changes a flag and never follows a redirect. * * CLEANUP: the API cannot delete a kit (revisions and the audit are immutable @@ -200,6 +201,10 @@ export async function main(args, env = process.env, { fetchImpl = fetch, log = c error(`refusing non-loopback base URL ${base.origin}; pass --allow-remote to smoke a deployed SkillX on purpose`); return 1; } + if (base.protocol !== "https:") { + error(`refusing ${base.origin}: a remote run sends account keys, so its base URL must be https`); + return 1; + } if (!env.SKILLX_SMOKE_OTHER_KEY) { error("--allow-remote needs SKILLX_SMOKE_OTHER_KEY (a second account's key) so the cross-account read and write checks run"); return 1;