From 604183891908c2e451929d03729826adf052d37a Mon Sep 17 00:00:00 2001 From: Byte-Naut Date: Fri, 16 Oct 2026 19:03:25 +0800 Subject: [PATCH] feat(test): add export and import for backend definitions --- CHANGELOG.md | 4 + DOCUMENTATION.md | 20 + README.md | 2 + src/commands/test.definition.spec.ts | 914 ++++++++++++++++++ src/commands/test.test.ts | 2 + src/commands/test.ts | 272 +++++- src/lib/test-definition.ts | 266 +++++ test/__snapshots__/help.snapshot.test.ts.snap | 47 + test/help.snapshot.test.ts | 2 + 9 files changed, 1528 insertions(+), 1 deletion(-) create mode 100644 src/commands/test.definition.spec.ts create mode 100644 src/lib/test-definition.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 3d309346..bfdbb96b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to `@testsprite/testsprite-cli` are documented here. The for ## [Unreleased] +### Added + +- `test export [--out ]` and `test import ` put individual backend test definitions under version control. Exports combine the existing test metadata and code API shapes with deterministic JSON and saved code-version provenance. Imports create by default; `--test-id` updates code with optimistic concurrency before metadata. Frontend definitions are explicitly unsupported in this first version. + ## [0.14.0] - 2026-10-06 ### Added diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index aac4beeb..db6ff05a 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -389,6 +389,26 @@ testsprite test create --plan-from ./checkout.plan.json --dry-run --output json `--plan-from` owns the complete test definition and does not support this setting; a supplied `--step-timeout` is warned about and ignored. Set it later with `test update --step-timeout `. +#### `testsprite test export ` / `testsprite test import ` + +Export one **backend** test definition for version control, review, backup, or migration. The JSON object contains `test` (the supported definition fields projected from `GET /tests/{id}`) and `code` (the `GET /tests/{id}/code` object with inline Python source). Keys are sorted recursively, arrays keep their order, and source whitespace is preserved. A presigned code URL is downloaded without API authentication headers before export, so the artifact contains durable source. There is no separate file-format version. + +```bash +testsprite test export test_backend --out ./login.testsprite.json +testsprite test export test_backend > ./login.testsprite.json +testsprite test import ./login.testsprite.json # create a copy +testsprite test import ./login.testsprite.json --project project_new # create in another project +testsprite test import ./login.testsprite.json --test-id test_backend --idempotency-key restore-login +``` + +`test` includes `id`, `projectId`, `type`, `name`, and any returned `description`, `priority`, `stepTimeoutMs`, `produces`, `consumes`, or `category`. Status, run results, timestamps, project display names, and debug details are outside the definition projection. `test.id`, `code.testId`, `code.codeVersion`, and `code.etag` record provenance; creating a copy assigns a new identity and version. Null optional metadata is unset on create. Updating patches the fields present in the file, including explicit null clears; omitted fields remain unchanged. Unknown import fields are rejected before any API request rather than discarded. Frontend export and import fail explicitly: this first version does not restore frontend plans or generated code. + +An import creates by default, even if the exported `test.id` is present. `--test-id` explicitly selects update, must identify a backend test in the original project, and uses the saved `code.codeVersion` as `If-Match`. A null, empty, or wildcard version cannot silently force an update. `--expected-version ` supplies a reviewed target version and is required when the target differs from the saved `test.id`. `--project` applies only to creation. Definition input must be a regular UTF-8 JSON file of at most 1 MB (an optional BOM is accepted); Python/pytest code must be non-empty and at most 350 KB. Unsupported legacy code languages must be converted before import. + +Updates write code first, so a stale version fails with `PRECONDITION_FAILED` (exit 6) before metadata changes. Code and metadata remain two separate API writes. If metadata fails after code succeeds, the CLI reports the partial update on stderr and preserves the underlying error. Retry with the same `--idempotency-key` to replay completed stages safely; changing the key or expected version starts a different operation. The base key is printed on stderr for JSON, verbose, or debug output and accepts at most 247 ASCII characters to leave space for stage suffixes. Create uses `:create`; update uses `:code` and `:metadata`. + +Export prints the JSON definition to stdout regardless of display mode unless `--out` is supplied. `--out` uses the code-download path's sibling temporary file and atomic rename: an existing regular file is replaced only after export completes, and failures preserve its bytes. Dry-run export prints a sample without reading the API or writing a file. Dry-run import validates the supplied file and previews mutation responses without credentials or network calls. Definition export is separate from JUnit **result** reports and does not define suites or Test Lists. + #### `testsprite test create-batch` Bulk-create frontend tests from a JSONL plan-steps file (or a directory of plan files with `--plan-from-dir`). Optional `--run --max-concurrency ` fans out triggers. Without `--wait`, each run is dispatched (`status: "queued"`) and the command exits 0 when every trigger is accepted — mirroring single `test run` without `--wait`; a trigger error still exits non-zero. With `--wait`, it polls every run to terminal and exits non-zero if any run does not pass. diff --git a/README.md b/README.md index 55b1b4e1..4a881214 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,8 @@ See [the full local-testing reference](./DOCUMENTATION.md#local-frontend-testing 📚 **Full reference — every command, flag, and example:** [DOCUMENTATION.md](./DOCUMENTATION.md), including [configuration & profiles](./DOCUMENTATION.md#configuration), [scripting](./DOCUMENTATION.md#output--scripting), and [exit codes](./DOCUMENTATION.md#exit-codes). +Backend definitions can be stored in version control with `testsprite test export --out test.testsprite.json` and restored with `testsprite test import test.testsprite.json`. Import creates a copy by default; add `--test-id ` to update using the saved code version. See the [definition export/import reference](DOCUMENTATION.md#testsprite-test-export-test-id--testsprite-test-import-file) for supported fields, concurrency, and retry behavior. + ## Why a CLI for coding agents? - 🧪 **Tests like a real user.** Runs against a live browser or API in the cloud — real clicks, real navigation, real assertions. Not a mock. diff --git a/src/commands/test.definition.spec.ts b/src/commands/test.definition.spec.ts new file mode 100644 index 00000000..d0afc31c --- /dev/null +++ b/src/commands/test.definition.spec.ts @@ -0,0 +1,914 @@ +import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { execFileSync } from 'node:child_process'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { ApiError } from '../lib/errors.js'; +import { runExport, runImport } from './test.js'; + +const TEST_ID = 'test_backend_1'; +const PROJECT_ID = 'project_alice'; +const CODE = 'def test_健康():\n assert "猫" == "猫"\n\n'; +const testRow = { + id: TEST_ID, + projectId: PROJECT_ID, + name: '健康检查', + type: 'backend', + createdFrom: 'portal', + status: 'passed', + createdAt: '2026-04-01T00:00:00Z', + updatedAt: '2026-05-01T00:00:00Z', + description: null, + priority: 'p1', + stepTimeoutMs: 30000, + produces: ['token'], + consumes: ['seed'], + category: 'setup', +}; +const codeRow = { + testId: TEST_ID, + language: 'python', + framework: 'pytest', + code: CODE, + codeVersion: 'v7', + etag: 'v7', +}; +const definition = { + test: { + id: TEST_ID, + projectId: PROJECT_ID, + type: 'backend', + name: '健康检查', + description: null, + priority: 'p1', + stepTimeoutMs: 30000, + produces: ['token'], + consumes: ['seed'], + category: 'setup', + }, + code: codeRow, +}; + +type ExportOptions = Parameters[0]; +type ImportOptions = Parameters[0]; + +interface Call { + url: string; + method: string; + headers: Headers; + body: unknown; +} + +const dirs: string[] = []; +afterEach(() => { + for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); +}); + +function tempDir(): string { + const dir = mkdtempSync(join(tmpdir(), 'definition-')); + dirs.push(dir); + return dir; +} + +function fixtureFile(value: unknown, raw?: string): string { + const path = join(tempDir(), 'definition.json'); + writeFileSync(path, raw ?? `${JSON.stringify(value)}\n`, 'utf8'); + return path; +} + +function credentials(): string { + const path = join(tempDir(), 'credentials'); + writeFileSync( + path, + '[default]\napi_url = http://localhost:13502\napi_key = sk-user-local-fixture\n', + { + mode: 0o600, + }, + ); + return path; +} + +function io() { + const stdout: string[] = []; + const stderr: string[] = []; + return { + stdout, + stderr, + stdoutWriter: (line: string) => stdout.push(line), + stderrWriter: (line: string) => stderr.push(line), + }; +} + +function json(status: number, body: unknown): Response { + return new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); +} + +function mockApi(responder: (call: Call) => Response | Promise): { + fetchImpl: typeof globalThis.fetch; + calls: Call[]; +} { + const calls: Call[] = []; + const fetchImpl = (async (input: unknown, init: RequestInit = {}) => { + const url = + typeof input === 'string' + ? input + : input instanceof URL + ? input.toString() + : (input as { url: string }).url; + const call: Call = { + url, + method: (init.method ?? 'GET').toUpperCase(), + headers: new Headers(init.headers), + body: typeof init.body === 'string' ? JSON.parse(init.body) : init.body, + }; + calls.push(call); + return responder(call); + }) as typeof globalThis.fetch; + return { fetchImpl, calls }; +} + +function exportOptions(overrides: Record = {}): ExportOptions { + return { + profile: 'default', + output: 'json', + debug: false, + testId: TEST_ID, + ...overrides, + } as ExportOptions; +} + +function importOptions(file: string, overrides: Record = {}): ImportOptions { + return { profile: 'default', output: 'json', debug: false, file, ...overrides } as ImportOptions; +} + +function exportApi() { + return mockApi(call => { + if (call.method === 'GET' && call.url.endsWith(`/tests/${TEST_ID}`)) return json(200, testRow); + if (call.method === 'GET' && call.url.endsWith(`/tests/${TEST_ID}/code`)) + return json(200, codeRow); + throw new Error(`unexpected request: ${call.method} ${call.url}`); + }); +} + +describe('individual backend definition export', () => { + it('projects writable metadata and exact code with deterministic sorted JSON and no format version', async () => { + const h = exportApi(); + const output = io(); + const got = await runExport(exportOptions(), { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + stdout: output.stdoutWriter, + stderr: output.stderrWriter, + }); + expect(got).toEqual(definition); + expect( + h.calls.map(call => `${call.method} ${new URL(call.url).pathname.split('/api/cli/v1')[1]}`), + ).toEqual([`GET /tests/${TEST_ID}`, `GET /tests/${TEST_ID}/code`]); + expect(output.stdout).toHaveLength(1); + const printed = `${output.stdout[0]}\n`; + expect(JSON.parse(printed)).toEqual(definition); + expect(printed).not.toContain('schemaVersion'); + expect(printed).not.toContain('formatVersion'); + expect(printed).not.toContain('createdAt'); + expect((JSON.parse(printed) as typeof definition).code.code).toBe(CODE); + expect(printed.endsWith('\n')).toBe(true); + expect(printed).toBe(stableJson(definition)); + }); + + it('keeps nullable optional values and safely commits an --out file only after both reads', async () => { + const h = exportApi(); + const output = io(); + const dir = tempDir(); + const path = join(dir, 'definition.json'); + writeFileSync(path, 'OUTDATED\n'); + await runExport(exportOptions({ out: path }), { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + stdout: output.stdoutWriter, + stderr: output.stderrWriter, + }); + expect(readFileSync(path, 'utf8')).toBe(stableJson(definition)); + expect(output.stdout).toEqual([]); + expect(JSON.parse(readFileSync(path, 'utf8')).test.description).toBeNull(); + expect(readdirSync(dir)).toEqual(['definition.json']); + }); + + it('rejects frontend metadata before reading code and leaves no artifact', async () => { + const h = mockApi(call => { + if (call.method === 'GET' && call.url.endsWith(`/tests/${TEST_ID}`)) { + return json(200, { + ...testRow, + type: 'frontend', + planSteps: [{ type: 'action', description: 'click' }], + }); + } + throw new Error('frontend export must stop after metadata'); + }); + const path = join(tempDir(), 'definition.json'); + await expect( + runExport(exportOptions({ out: path }), { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(h.calls).toHaveLength(1); + expect(existsSync(path)).toBe(false); + }); + + it('refuses to combine metadata or code returned for another test id', async () => { + for (const wrong of ['metadata', 'code'] as const) { + const h = mockApi(call => { + if (call.url.endsWith('/code')) { + return json(200, { ...codeRow, testId: wrong === 'code' ? 'test_other' : TEST_ID }); + } + return json(200, { ...testRow, id: wrong === 'metadata' ? 'test_other' : TEST_ID }); + }); + const output = io(); + await expect( + runExport(exportOptions(), { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + stdout: output.stdoutWriter, + }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + expect(h.calls).toHaveLength(wrong === 'metadata' ? 1 : 2); + expect(output.stdout).toEqual([]); + } + }); + + it('resolves a presigned body without forwarding facade credentials and refuses oversize source', async () => { + const url = 'https://signed.example.test/code.py?sig=fixture'; + const h = mockApi(call => { + if (call.url.endsWith(`/tests/${TEST_ID}`)) return json(200, testRow); + if (call.url.endsWith(`/tests/${TEST_ID}/code`)) return json(200, { ...codeRow, code: url }); + if (call.url === url) return new Response(CODE, { status: 200 }); + throw new Error(`unexpected request: ${call.url}`); + }); + const output = io(); + const got = await runExport(exportOptions(), { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + stdout: output.stdoutWriter, + stderr: output.stderrWriter, + }); + expect(got.code.code).toBe(CODE); + expect(h.calls[2]?.url).toBe(url); + expect(h.calls[2]?.headers.has('authorization')).toBe(false); + expect(h.calls[2]?.headers.has('idempotency-key')).toBe(false); + + const tooLarge = mockApi(call => { + if (call.url.endsWith(`/tests/${TEST_ID}`)) return json(200, testRow); + if (call.url.endsWith(`/tests/${TEST_ID}/code`)) return json(200, { ...codeRow, code: url }); + if (call.url === url) return new Response('x'.repeat(350 * 1024 + 1), { status: 200 }); + throw new Error('unexpected request'); + }); + const path = join(tempDir(), 'too-large.json'); + writeFileSync(path, 'PREVIOUS\n'); + await expect( + runExport(exportOptions({ out: path }), { + credentialsPath: credentials(), + fetchImpl: tooLarge.fetchImpl, + }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect(readFileSync(path, 'utf8')).toBe('PREVIOUS\n'); + expect(readdirSync(join(path, '..'))).toEqual(['too-large.json']); + }); + + it('preserves the validation exit code when streamed presigned source crosses 350 KB', async () => { + const url = 'https://signed.example.test/code.py?sig=fixture'; + const chunks = ['x'.repeat(350 * 1024), 'y']; + const h = mockApi(call => { + if (call.url.endsWith(`/tests/${TEST_ID}`)) return json(200, testRow); + if (call.url.endsWith(`/tests/${TEST_ID}/code`)) return json(200, { ...codeRow, code: url }); + if (call.url === url) { + return new Response( + new ReadableStream({ + pull(controller) { + const chunk = chunks.shift(); + if (chunk === undefined) controller.close(); + else controller.enqueue(new TextEncoder().encode(chunk)); + }, + }), + ); + } + throw new Error(`unexpected request: ${call.url}`); + }); + const path = join(tempDir(), 'definition.json'); + writeFileSync(path, 'PREVIOUS\n'); + const output = io(); + const error = await runExport(exportOptions({ out: path }), { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + stdout: output.stdoutWriter, + stderr: output.stderrWriter, + }).catch((err: unknown) => err); + + expect(error).toBeInstanceOf(ApiError); + expect(error).toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + expect( + (error as ApiError).getDetail('field', (value): value is string => typeof value === 'string'), + ).toBe('code.code'); + expect(output.stdout).toEqual([]); + expect(readFileSync(path, 'utf8')).toBe('PREVIOUS\n'); + expect(readdirSync(join(path, '..'))).toEqual(['definition.json']); + }); + + it('preserves an existing --out file when the second API read fails', async () => { + const path = join(tempDir(), 'existing.json'); + writeFileSync(path, 'PREVIOUS\n'); + const h = mockApi(call => { + if (call.url.endsWith(`/tests/${TEST_ID}`)) return json(200, testRow); + return json(400, { + error: { code: 'VALIDATION_ERROR', message: 'fixture failure', requestId: 'fixture' }, + }); + }); + await expect( + runExport(exportOptions({ out: path }), { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + }), + ).rejects.toBeDefined(); + expect(readFileSync(path, 'utf8')).toBe('PREVIOUS\n'); + expect(readdirSync(join(path, '..'))).toEqual(['existing.json']); + }); + + it('dry-run emits a backend sample without reading credentials, calling fetch, or writing --out', async () => { + const path = join(tempDir(), 'dry-run.json'); + const output = io(); + const fetchImpl = (() => { + throw new Error('dry-run reached fetch'); + }) as typeof globalThis.fetch; + const got = await runExport(exportOptions({ dryRun: true, out: path }), { + fetchImpl, + stdout: output.stdoutWriter, + stderr: output.stderrWriter, + }); + expect(got.test.type).toBe('backend'); + expect(output.stdout).toHaveLength(1); + expect(JSON.parse(output.stdout[0]!)).toEqual(got); + expect(existsSync(path)).toBe(false); + }); + + it('produces identical literal bytes from differently ordered API objects', async () => { + const firstMeta = { + id: TEST_ID, + projectId: PROJECT_ID, + type: 'backend', + name: 'A', + status: 'passed', + }; + const secondMeta = { + status: 'passed', + name: 'A', + type: 'backend', + projectId: PROJECT_ID, + id: TEST_ID, + }; + const firstCode = { + testId: TEST_ID, + language: 'python', + framework: 'pytest', + code: 'def test_ok():\n assert True\n', + codeVersion: 'v1', + }; + const secondCode = { + codeVersion: 'v1', + code: 'def test_ok():\n assert True\n', + framework: 'pytest', + language: 'python', + testId: TEST_ID, + }; + const bodies: string[] = []; + for (const [metadata, source] of [ + [firstMeta, firstCode], + [secondMeta, secondCode], + ] as const) { + const h = mockApi(call => json(200, call.url.endsWith('/code') ? source : metadata)); + const output = io(); + await runExport(exportOptions(), { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + stdout: output.stdoutWriter, + }); + bodies.push(`${output.stdout[0]}\n`); + } + const literal = + '{\n' + + ' "code": {\n' + + ' "code": "def test_ok():\\n assert True\\n",\n' + + ' "codeVersion": "v1",\n' + + ' "framework": "pytest",\n' + + ' "language": "python",\n' + + ' "testId": "test_backend_1"\n' + + ' },\n' + + ' "test": {\n' + + ' "id": "test_backend_1",\n' + + ' "name": "A",\n' + + ' "projectId": "project_alice",\n' + + ' "type": "backend"\n' + + ' }\n' + + '}\n'; + expect(bodies).toEqual([literal, literal]); + }); +}); + +function stableJson(value: unknown): string { + const sort = (item: unknown): unknown => { + if (Array.isArray(item)) return item.map(sort); + if (item && typeof item === 'object') { + return Object.fromEntries( + Object.entries(item) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([k, v]) => [k, sort(v)]), + ); + } + return item; + }; + return `${JSON.stringify(sort(value), null, 2)}\n`; +} + +describe('individual backend definition import', () => { + it('creates by default, even with saved provenance id, and omits nullable create fields', async () => { + const file = fixtureFile(definition); + const output = io(); + const h = mockApi(call => { + expect(call.method).toBe('POST'); + expect(call.url).toMatch(/\/tests$/); + return json(201, { + testId: 'test_new', + type: 'backend', + codeVersion: 'v1', + createdAt: 'now', + }); + }); + const result = await runImport(importOptions(file, { idempotencyKey: 'replay-1' }), { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + stdout: output.stdoutWriter, + stderr: output.stderrWriter, + }); + expect(result).toEqual({ testId: 'test_new', action: 'created', codeVersion: 'v1' }); + expect(h.calls).toHaveLength(1); + expect(h.calls[0]?.headers.get('idempotency-key')).toBe('replay-1:create'); + expect(h.calls[0]?.body).toEqual({ + projectId: PROJECT_ID, + type: 'backend', + name: '健康检查', + priority: 'p1', + stepTimeoutMs: 30000, + produces: ['token'], + consumes: ['seed'], + category: 'setup', + code: CODE, + }); + expect(output.stdout).toHaveLength(1); + expect(JSON.parse(output.stdout[0]!)).toEqual(result); + }); + + it('prints a replayable minted key to stderr and derives the create request key', async () => { + const output = io(); + const h = mockApi(() => + json(201, { testId: 'test_new', type: 'backend', codeVersion: 'v1', createdAt: 'now' }), + ); + await runImport(importOptions(fixtureFile(definition)), { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + stdout: output.stdoutWriter, + stderr: output.stderrWriter, + }); + const minted = output.stderr.find(line => line.startsWith('idempotency-key: '))?.slice(17); + expect(minted).toMatch(/^cli-import-[0-9a-f-]{36}$/); + expect(h.calls[0]?.headers.get('idempotency-key')).toBe(`${minted}:create`); + expect(output.stdout).toHaveLength(1); + }); + + it('updates code before metadata with saved version and derived replay keys', async () => { + const file = fixtureFile(definition); + const output = io(); + const h = mockApi(call => { + if (call.method === 'GET') return json(200, testRow); + if (call.url.endsWith('/code')) + return json(200, { testId: TEST_ID, codeVersion: 'v8', updatedAt: 'now' }); + return json(200, { testId: TEST_ID, updatedFields: ['name'], updatedAt: 'now' }); + }); + const result = await runImport( + importOptions(file, { testId: TEST_ID, idempotencyKey: 'same-key' }), + { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + stdout: output.stdoutWriter, + stderr: output.stderrWriter, + }, + ); + expect(result).toEqual({ testId: TEST_ID, action: 'updated', codeVersion: 'v8' }); + expect(h.calls.map(call => call.method)).toEqual(['GET', 'PUT', 'PUT']); + expect(h.calls[1]?.url).toMatch(/\/tests\/test_backend_1\/code$/); + expect(h.calls[1]?.headers.get('if-match')).toBe('v7'); + expect(h.calls[1]?.headers.get('idempotency-key')).toBe('same-key:code'); + expect(h.calls[1]?.body).toEqual({ code: CODE, language: 'python' }); + expect(h.calls[2]?.headers.get('idempotency-key')).toBe('same-key:metadata'); + expect(h.calls[2]?.body).toMatchObject({ + name: '健康检查', + description: null, + priority: 'p1', + stepTimeoutMs: 30000, + produces: ['token'], + consumes: ['seed'], + category: 'setup', + }); + expect(JSON.parse(output.stdout[0]!)).toEqual(result); + }); + + it('uses an explicit version for the same or a different target and applies a create project override', async () => { + const file = fixtureFile(definition); + const create = mockApi(() => + json(201, { testId: 'test_new', type: 'backend', codeVersion: 'v1', createdAt: 'now' }), + ); + await runImport(importOptions(file, { projectId: 'project_migrated' }), { + credentialsPath: credentials(), + fetchImpl: create.fetchImpl, + stdout: () => undefined, + }); + expect((create.calls[0]?.body as { projectId: string }).projectId).toBe('project_migrated'); + + for (const targetId of [TEST_ID, 'test_other']) { + const target = mockApi(call => { + if (call.method === 'GET') return json(200, { ...testRow, id: targetId }); + if (call.url.endsWith('/code')) + return json(200, { testId: targetId, codeVersion: 'v12', updatedAt: 'now' }); + return json(200, { testId: targetId, updatedFields: ['name'], updatedAt: 'now' }); + }); + await runImport(importOptions(file, { testId: targetId, expectedVersion: 'v11' }), { + credentialsPath: credentials(), + fetchImpl: target.fetchImpl, + stdout: () => undefined, + }); + expect(target.calls[1]?.headers.get('if-match')).toBe('v11'); + } + }); + + it('does not write metadata after a code-version conflict', async () => { + const file = fixtureFile(definition); + const h = mockApi(call => { + if (call.method === 'GET') return json(200, testRow); + return json(412, { + error: { + code: 'PRECONDITION_FAILED', + message: 'version changed', + requestId: 'fixture', + details: { currentCodeVersion: 'v9' }, + }, + }); + }); + await expect( + runImport(importOptions(file, { testId: TEST_ID }), { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + }), + ).rejects.toMatchObject({ code: 'PRECONDITION_FAILED', exitCode: 6 }); + expect(h.calls.map(call => call.method)).toEqual(['GET', 'PUT']); + expect(h.calls[1]?.headers.get('if-match')).toBe('v7'); + }); + + it('surfaces metadata failure after a successful code PUT with a partial-update warning', async () => { + const file = fixtureFile(definition); + const output = io(); + const h = mockApi(call => { + if (call.method === 'GET') return json(200, testRow); + if (call.url.endsWith('/code')) + return json(200, { testId: TEST_ID, codeVersion: 'v8', updatedAt: 'now' }); + return json(400, { + error: { code: 'VALIDATION_ERROR', message: 'metadata rejected', requestId: 'fixture' }, + }); + }); + await expect( + runImport(importOptions(file, { testId: TEST_ID }), { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + stdout: output.stdoutWriter, + stderr: output.stderrWriter, + }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + expect(h.calls.map(call => call.method)).toEqual(['GET', 'PUT', 'PUT']); + expect(output.stdout).toEqual([]); + expect(output.stderr.join('\n')).toMatch(/partial|code.*updated|metadata/i); + }); + + it('replays both mutation keys after an ambiguous metadata failure and rejects a new stale key', async () => { + const file = fixtureFile(definition); + let currentVersion = 'v7'; + let firstMetadataFailure = true; + const codeReceipts = new Map< + string, + { testId: string; codeVersion: string; updatedAt: string } + >(); + const h = mockApi(call => { + if (call.method === 'GET') return json(200, testRow); + if (call.url.endsWith('/code')) { + const key = call.headers.get('idempotency-key')!; + const cached = codeReceipts.get(key); + if (cached) return json(200, cached); + if (call.headers.get('if-match') !== currentVersion) { + return json(412, { + error: { code: 'PRECONDITION_FAILED', message: 'stale code', requestId: 'fixture' }, + }); + } + currentVersion = 'v8'; + const receipt = { testId: TEST_ID, codeVersion: currentVersion, updatedAt: 'now' }; + codeReceipts.set(key, receipt); + return json(200, receipt); + } + if (firstMetadataFailure) { + firstMetadataFailure = false; + return json(400, { + error: { + code: 'VALIDATION_ERROR', + message: 'ambiguous first metadata response', + requestId: 'fixture', + }, + }); + } + return json(200, { testId: TEST_ID, updatedFields: ['name'], updatedAt: 'now' }); + }); + const opts = importOptions(file, { testId: TEST_ID, idempotencyKey: 'replay-base' }); + await expect( + runImport(opts, { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + stdout: () => undefined, + stderr: () => undefined, + }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + const result = await runImport(opts, { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + stdout: () => undefined, + }); + expect(result).toEqual({ testId: TEST_ID, action: 'updated', codeVersion: 'v8' }); + expect( + h.calls + .filter(call => call.url.endsWith('/code')) + .map(call => call.headers.get('idempotency-key')), + ).toEqual(['replay-base:code', 'replay-base:code']); + expect( + h.calls + .filter(call => call.method === 'PUT' && !call.url.endsWith('/code')) + .map(call => call.headers.get('idempotency-key')), + ).toEqual(['replay-base:metadata', 'replay-base:metadata']); + + const countBeforeStaleRetry = h.calls.length; + await expect( + runImport(importOptions(file, { testId: TEST_ID, idempotencyKey: 'new-base' }), { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + stdout: () => undefined, + }), + ).rejects.toMatchObject({ code: 'PRECONDITION_FAILED' }); + expect(h.calls.slice(countBeforeStaleRetry).map(call => call.method)).toEqual(['GET', 'PUT']); + }); + + it('requires explicit version for a different update target and rejects target project/type mismatch', async () => { + const file = fixtureFile(definition); + const noCalls = mockApi(() => { + throw new Error('must fail locally'); + }); + await expect( + runImport(importOptions(file, { testId: 'test_other' }), { + credentialsPath: credentials(), + fetchImpl: noCalls.fetchImpl, + }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + expect(noCalls.calls).toEqual([]); + await expect( + runImport(importOptions(file, { testId: TEST_ID, projectId: 'another_project' }), { + credentialsPath: credentials(), + fetchImpl: noCalls.fetchImpl, + }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + expect(noCalls.calls).toEqual([]); + + const mismatch = mockApi(() => + json(200, { ...testRow, id: 'test_other', projectId: 'project_elsewhere' }), + ); + await expect( + runImport(importOptions(file, { testId: 'test_other', expectedVersion: 'v4' }), { + credentialsPath: credentials(), + fetchImpl: mismatch.fetchImpl, + }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + expect(mismatch.calls).toHaveLength(1); + }); + + it('rejects absent/null code versions and overlong derived idempotency keys before network', async () => { + const h = mockApi(() => { + throw new Error('must fail locally'); + }); + for (const version of [null, undefined]) { + const file = fixtureFile({ ...definition, code: { ...codeRow, codeVersion: version } }); + await expect( + runImport(importOptions(file, { testId: TEST_ID }), { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + } + const file = fixtureFile(definition); + await expect( + runImport(importOptions(file, { idempotencyKey: 'x'.repeat(254) }), { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + for (const expectedVersion of ['*', ' ']) { + await expect( + runImport(importOptions(file, { testId: TEST_ID, expectedVersion }), { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + } + expect(h.calls).toEqual([]); + }); + + it.skipIf(process.platform === 'win32')( + 'rejects directories, devices, and FIFOs as import files before network', + async () => { + const dir = tempDir(); + const fifo = join(dir, 'pipe'); + execFileSync('mkfifo', [fifo]); + const h = mockApi(() => { + throw new Error('non-regular file reached network'); + }); + for (const file of [dir, '/dev/null', fifo]) { + await expect( + runImport(importOptions(file), { fetchImpl: h.fetchImpl }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR', exitCode: 5 }); + } + expect(h.calls).toEqual([]); + }, + ); + + it('validates the file shape and syntax before any client request', async () => { + const h = mockApi(() => { + throw new Error('must fail locally'); + }); + const cases = [ + fixtureFile(definition, '{bad json'), + fixtureFile({ ...definition, schemaVersion: 1 }), + fixtureFile({ ...definition, test: { ...definition.test, unexpected: true } }), + fixtureFile({ ...definition, test: { ...definition.test, type: 'frontend' } }), + fixtureFile({ ...definition, code: { ...codeRow, code: 'x'.repeat(350 * 1024 + 1) } }), + fixtureFile(definition, 'x'.repeat(1024 * 1024 + 1)), + ]; + for (const file of cases) { + await expect( + runImport(importOptions(file), { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + } + await expect( + runImport(importOptions(join(tempDir(), 'missing.json')), { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + expect(h.calls).toEqual([]); + }); + + it('rejects wrong optional and required field types without dropping them', async () => { + const h = mockApi(() => { + throw new Error('malformed file reached network'); + }); + const variants: unknown[] = [ + { ...definition, test: { ...definition.test, description: {} } }, + { ...definition, test: { ...definition.test, priority: 'urgent' } }, + { ...definition, test: { ...definition.test, stepTimeoutMs: [] } }, + { ...definition, test: { ...definition.test, produces: [null] } }, + { ...definition, test: { ...definition.test, consumes: {} } }, + { ...definition, test: { ...definition.test, category: [] } }, + { ...definition, test: { ...definition.test, name: null } }, + { ...definition, code: { ...codeRow, code: '' } }, + { ...definition, code: { ...codeRow, code: null } }, + { ...definition, code: { ...codeRow, codeVersion: [] } }, + { ...definition, code: { ...codeRow, language: 'typescript' } }, + { ...definition, code: { ...codeRow, testId: 'another_test' } }, + { ...definition, code: { ...codeRow, framework: 'playwright' } }, + ]; + for (const value of variants) { + const file = fixtureFile(value); + await expect( + runImport(importOptions(file), { fetchImpl: h.fetchImpl }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + } + const withoutCode = { test: definition.test }; + const withoutProject = { ...definition, test: { type: 'backend', name: 'A' } }; + for (const value of [withoutCode, withoutProject]) { + await expect( + runImport(importOptions(fixtureFile(value)), { fetchImpl: h.fetchImpl }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + } + expect(h.calls).toEqual([]); + }); + + it('preserves nullable exported fields while omitting absent fields from a sparse metadata update', async () => { + const nullable = { + ...definition, + test: { + ...definition.test, + description: null, + priority: null, + stepTimeoutMs: null, + produces: null, + consumes: null, + category: null, + }, + }; + const create = mockApi(() => + json(201, { testId: 'test_new', type: 'backend', codeVersion: 'v1', createdAt: 'now' }), + ); + await runImport(importOptions(fixtureFile(nullable)), { + credentialsPath: credentials(), + fetchImpl: create.fetchImpl, + stdout: () => undefined, + }); + expect(create.calls[0]?.body).toEqual({ + projectId: PROJECT_ID, + type: 'backend', + name: '健康检查', + code: CODE, + }); + + const sparse = { + test: { id: TEST_ID, projectId: PROJECT_ID, type: 'backend', name: 'Renamed' }, + code: codeRow, + }; + const update = mockApi(call => { + if (call.method === 'GET') return json(200, testRow); + if (call.url.endsWith('/code')) + return json(200, { testId: TEST_ID, codeVersion: 'v8', updatedAt: 'now' }); + return json(200, { testId: TEST_ID, updatedFields: ['name'], updatedAt: 'now' }); + }); + await runImport(importOptions(fixtureFile(sparse), { testId: TEST_ID }), { + credentialsPath: credentials(), + fetchImpl: update.fetchImpl, + stdout: () => undefined, + }); + expect(update.calls[2]?.body).toEqual({ name: 'Renamed' }); + }); + + it('accepts a BOM-prefixed exported definition and preserves code whitespace', async () => { + const file = fixtureFile(definition, `\uFEFF${JSON.stringify(definition)}`); + const h = mockApi(call => { + expect((call.body as { code: string }).code).toBe(CODE); + return json(201, { + testId: 'test_new', + type: 'backend', + codeVersion: 'v1', + createdAt: 'now', + }); + }); + await runImport(importOptions(file), { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + stdout: () => undefined, + }); + expect(h.calls).toHaveLength(1); + }); + + it('allows null code-version provenance on create without sending concurrency headers', async () => { + const file = fixtureFile({ ...definition, code: { ...codeRow, codeVersion: null } }); + const h = mockApi(() => + json(201, { testId: 'test_new', type: 'backend', codeVersion: 'v1', createdAt: 'now' }), + ); + await runImport(importOptions(file), { + credentialsPath: credentials(), + fetchImpl: h.fetchImpl, + stdout: () => undefined, + }); + expect(h.calls).toHaveLength(1); + expect(h.calls[0]?.headers.has('if-match')).toBe(false); + }); + + it('dry-run reads and validates the file without credentials or network', async () => { + const file = fixtureFile(definition); + const output = io(); + const result = await runImport(importOptions(file, { dryRun: true }), { + stdout: output.stdoutWriter, + stderr: output.stderrWriter, + }); + expect(result.action).toBe('created'); + expect(output.stdout).toHaveLength(1); + expect(JSON.parse(output.stdout[0]!)).toEqual(result); + const malformed = fixtureFile(definition, '{bad json'); + await expect( + runImport(importOptions(malformed, { dryRun: true }), { + stdout: output.stdoutWriter, + }), + ).rejects.toMatchObject({ code: 'VALIDATION_ERROR' }); + }); +}); diff --git a/src/commands/test.test.ts b/src/commands/test.test.ts index 97ce59e1..3708b8a7 100644 --- a/src/commands/test.test.ts +++ b/src/commands/test.test.ts @@ -160,9 +160,11 @@ describe('createTestCommand — surface', () => { 'delete', 'delete-batch', 'diff', + 'export', 'failure', 'flaky', 'get', + 'import', 'lint', 'list', 'open', diff --git a/src/commands/test.ts b/src/commands/test.ts index 31f3b107..8f0b4223 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -184,6 +184,15 @@ import { } from '../lib/gh-output.js'; import { loadConfig, resolveProfileName } from '../lib/config.js'; import { historyRetentionNote } from '../lib/history-retention.js'; +import { + definitionMetadata, + MAX_DEFINITION_BYTES, + MAX_DEFINITION_CODE_BYTES, + projectTestDefinition, + readDefinitionFileGuarded, + stringifyTestDefinition, + type TestDefinition, +} from '../lib/test-definition.js'; import { flakyExitCode, renderFlakyText, @@ -228,6 +237,7 @@ export interface CliTest { */ projectName?: string | null; name: string; + description?: string | null; type: 'frontend' | 'backend'; createdFrom: 'portal' | 'mcp' | 'cli'; status: CliPublicStatus; @@ -5374,6 +5384,220 @@ export async function runCodeGet(opts: CodeGetOptions, deps: TestDeps = {}): Pro } } +export interface ExportOptions extends CommonOptions { + testId: string; + out?: string; +} + +/** Export a single backend definition, including durable source rather than an expiring URL. */ +export async function runExport(opts: ExportOptions, deps: TestDeps = {}): Promise { + requireNonEmpty('test-id', opts.testId); + let sink = !opts.dryRun && opts.out !== undefined ? openOutputFile(opts.out) : null; + try { + let definition: TestDefinition; + if (opts.dryRun) { + definition = { + test: { + id: opts.testId, + projectId: 'project_dry_run', + type: 'backend', + name: 'Backend definition preview', + }, + code: { + testId: opts.testId, + language: 'python', + framework: 'pytest', + code: DRY_RUN_PLACEHOLDER_CODE, + codeVersion: 'v3', + }, + }; + if (opts.out !== undefined) { + (deps.stderr ?? (line => process.stderr.write(`${line}\n`)))( + `[dry-run] would write definition to ${opts.out}`, + ); + } + } else { + const client = makeClient(opts, deps); + const path = `/tests/${encodeURIComponent(opts.testId)}`; + const test = await client.get(path); + if (test.type !== 'backend') { + throw localValidationError( + 'test-id', + 'definition export/import currently supports backend tests only; frontend plans cannot be restored by this command', + ); + } + if (test.id !== opts.testId) { + throw localValidationError('test-id', 'server returned metadata for a different test'); + } + const code = await client.get(`${path}/code`); + if (code.testId !== opts.testId) { + throw localValidationError( + 'code.testId', + 'server returned code for a different test', + undefined, + 'field', + ); + } + let body = code.code; + if (typeof body === 'string' && isPresignedCodeUrl(body)) { + const chunks: string[] = []; + let bytes = 0; + await streamPresignedBody( + body, + new Output('text', { + rawStdout: chunk => { + bytes += Buffer.byteLength(chunk, 'utf8'); + if (bytes > MAX_DEFINITION_CODE_BYTES) { + throw localValidationError( + 'code.code', + 'exceeds the 350 KB code input limit', + undefined, + 'field', + ); + } + chunks.push(chunk); + }, + }), + deps, + ); + body = chunks.join(''); + } + definition = projectTestDefinition(test, { ...code, code: body }); + } + const serialized = stringifyTestDefinition(definition); + if (Buffer.byteLength(serialized, 'utf8') > MAX_DEFINITION_BYTES) { + throw localValidationError( + 'definition', + 'exceeds the 1 MB definition input limit', + undefined, + 'field', + ); + } + if (sink) { + await makeFileOutput('json', sink).writeChunk(serialized); + await closeOutputFile(sink, true); + sink = null; + } else { + (deps.stdout ?? (line => process.stdout.write(`${line}\n`)))(serialized.slice(0, -1)); + } + return definition; + } catch (error) { + if (sink) await abortOutputFile(sink).catch(() => undefined); + throw error; + } +} + +export interface ImportOptions extends CommonOptions { + file: string; + projectId?: string; + testId?: string; + expectedVersion?: string; + idempotencyKey?: string; +} + +export interface ImportResult { + testId: string; + action: 'created' | 'updated'; + codeVersion: string; +} + +/** Validate everything locally, then create or explicitly update one backend test. */ +export async function runImport(opts: ImportOptions, deps: TestDeps = {}): Promise { + requireNonEmpty('file', opts.file); + const definition = readDefinitionFileGuarded(opts.file); + assertIdempotencyKey(opts.idempotencyKey); + const key = opts.idempotencyKey ?? `cli-import-${randomUUID()}`; + for (const suffix of [':create', ':code', ':metadata']) assertIdempotencyKey(`${key}${suffix}`); + if (opts.projectId !== undefined) requireNonEmpty('project', opts.projectId); + if (opts.testId !== undefined) requireNonEmpty('test-id', opts.testId); + if (opts.testId !== undefined && opts.projectId !== undefined) { + throw localValidationError( + 'project', + 'is only supported when creating a test; omit --test-id to create', + ); + } + if (opts.testId === undefined && opts.expectedVersion !== undefined) { + throw localValidationError('expected-version', 'requires --test-id'); + } + const expected = opts.expectedVersion ?? definition.code.codeVersion; + if (opts.testId !== undefined) { + if ( + expected === null || + expected === undefined || + !expected.trim() || + expected.trim() === '*' + ) { + throw localValidationError( + 'expected-version', + 'updates require a non-empty saved codeVersion or --expected-version; wildcard force is not supported', + ); + } + if (opts.testId !== definition.test.id && opts.expectedVersion === undefined) { + throw localValidationError( + 'expected-version', + 'is required when updating a different test from the exported test.id', + ); + } + } + const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); + // Expose a replayable base key. Each stage derives the same stable suffix on retries. + if (opts.idempotencyKey === undefined && (opts.output === 'json' || opts.verbose || opts.debug)) { + stderr(`idempotency-key: ${key}`); + } + const client = makeClient(opts, deps); + let result: ImportResult; + if (opts.testId === undefined) { + const created = await client.post('/tests', { + body: { + ...definitionMetadata(definition.test, true), + projectId: opts.projectId ?? definition.test.projectId, + type: 'backend', + code: definition.code.code, + }, + headers: { 'idempotency-key': `${key}:create` }, + }); + emitResponseWarnings(created.warnings, deps); + result = { testId: created.testId, action: 'created', codeVersion: created.codeVersion }; + } else { + const path = `/tests/${encodeURIComponent(opts.testId)}`; + if (!opts.dryRun) { + const target = await client.get(path); + if ( + target.id !== opts.testId || + target.type !== 'backend' || + target.projectId !== definition.test.projectId + ) { + throw localValidationError( + 'test-id', + 'target must be a backend test in the definition project; use create with --project to migrate', + ); + } + } + const code = await client.put(`${path}/code`, { + body: { code: definition.code.code, language: 'python' }, + headers: { 'idempotency-key': `${key}:code`, 'if-match': expected! }, + }); + emitResponseWarnings(code.warnings, deps); + try { + await client.put(path, { + body: definitionMetadata(definition.test, false), + headers: { 'idempotency-key': `${key}:metadata` }, + }); + } catch (error) { + stderr( + `Import partially applied: code for ${opts.testId} was updated to ${code.codeVersion}, but metadata did not complete. Retry with the same --idempotency-key ${key}; inspect the test before changing its expected version.`, + ); + throw error; + } + result = { testId: opts.testId, action: 'updated', codeVersion: code.codeVersion }; + } + makeOutput(opts.output, deps).print(result, value => { + const imported = value as ImportResult; + return `${imported.action} ${imported.testId}\ncodeVersion ${imported.codeVersion}`; + }); + return result; +} + /** * §6.X / M3.2 piece-4 `PutTestCodeResponse` shape. `codeVersion` is * the freshly bumped value the server stamped on the entity. Used as @@ -15161,6 +15385,52 @@ export function createTestCommand(deps: TestDeps = {}): Command { ); test.addCommand(createTestCodeCommand(deps)); + test + .command('export ') + .description( + 'Export one backend test definition as deterministic JSON (metadata and Python source)', + ) + .option('--out ', 'write a .testsprite.json file atomically; defaults to stdout') + .addHelpText( + 'after', + '\nBackend tests only. This exports definitions for version control; run results are excluded.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action(async (testId: string, flags: { out?: string }, command: Command) => { + await runExport({ ...resolveCommonOptions(command, deps.env), testId, out: flags.out }, deps); + }); + test + .command('import ') + .description( + 'Create a backend test from an exported definition, or update an explicit --test-id', + ) + .option('--project ', 'override the project when creating a test') + .option('--test-id ', 'update this existing backend test instead of creating') + .option( + '--expected-version ', + 'explicit codeVersion for an update; defaults to the saved version', + ) + .option( + '--idempotency-key ', + 'replayable base token; at most 247 ASCII characters including stage suffix budget', + ) + .addHelpText( + 'after', + '\nBackend tests only. Updates write guarded code before metadata; the two writes are not atomic.\nUse the same idempotency key to retry a partial import. A different target requires --expected-version.', + ) + .addHelpText('after', GLOBAL_OPTS_HINT) + .action( + async ( + file: string, + flags: Omit & { project?: string }, + command: Command, + ) => { + await runImport( + { ...resolveCommonOptions(command, deps.env), ...flags, projectId: flags.project, file }, + deps, + ); + }, + ); test.addCommand(createTestPlanCommand(deps)); test.addCommand(createTestFailureCommand(deps)); test.addCommand(createTestArtifactCommand(deps)); @@ -16064,7 +16334,7 @@ async function readPresignedBody( if (shutdownSignal.aborted) throw shutdownSignal.reason; } catch (err) { if (shutdownSignal.aborted) throw shutdownSignal.reason; - if (err instanceof InterruptError) throw err; + if (err instanceof InterruptError || err instanceof ApiError) throw err; const message = err instanceof Error ? err.message : String(err); throw new TransportError(`Failed mid-download of presigned code body: ${message}`); } diff --git a/src/lib/test-definition.ts b/src/lib/test-definition.ts new file mode 100644 index 00000000..8d601530 --- /dev/null +++ b/src/lib/test-definition.ts @@ -0,0 +1,266 @@ +/** Individual backend definitions projected from the test and code API responses. */ +import { closeSync, constants, fstatSync, openSync, readSync } from 'node:fs'; +import { resolve } from 'node:path'; +import type { CliTest, CliTestCode } from '../commands/test.js'; +import { localValidationError } from './errors.js'; + +export const MAX_DEFINITION_BYTES = 1024 * 1024; +export const MAX_DEFINITION_CODE_BYTES = 350 * 1024; + +const METADATA_FIELDS = [ + 'name', + 'description', + 'priority', + 'stepTimeoutMs', + 'produces', + 'consumes', + 'category', +] as const; + +export interface BackendDefinitionMetadata { + id?: string; + projectId: string; + type: 'backend'; + name: string; + description?: string | null; + priority?: 'p0' | 'p1' | 'p2' | 'p3' | null; + stepTimeoutMs?: number | null; + produces?: string[] | null; + consumes?: string[] | null; + category?: string | null; +} + +export interface TestDefinition { + test: BackendDefinitionMetadata; + code: CliTestCode & { language: 'python'; framework: 'pytest' }; +} + +function object(value: unknown, field: string): asserts value is Record { + if (value === null || typeof value !== 'object' || Array.isArray(value)) { + throw localValidationError(field, 'must be an object', undefined, 'field'); + } +} + +function keys(value: Record, allowed: readonly string[], field: string): void { + const unknown = Object.keys(value).filter(key => !allowed.includes(key)); + if (unknown.length) { + throw localValidationError( + field, + `contains unsupported fields: ${unknown.join(', ')}; refusing to discard them`, + undefined, + 'field', + ); + } +} + +function string(value: unknown, field: string, max?: number): asserts value is string { + if (typeof value !== 'string' || !value.trim() || (max !== undefined && value.length > max)) { + throw localValidationError( + field, + `must be a non-empty string${max === undefined ? '' : ` of at most ${max} characters`}`, + undefined, + 'field', + ); + } +} + +/** Reject unsupported fields before any request so imports cannot silently lose data. */ +export function parseTestDefinition(value: unknown): TestDefinition { + object(value, 'definition'); + keys(value, ['test', 'code'], 'definition'); + object(value.test, 'test'); + object(value.code, 'code'); + const test = value.test; + const code = value.code; + keys(test, ['id', 'projectId', 'type', ...METADATA_FIELDS], 'test'); + keys(code, ['testId', 'language', 'framework', 'code', 'codeVersion', 'etag'], 'code'); + if (test.type !== 'backend') { + throw localValidationError( + 'test.type', + 'definition export/import currently supports backend tests only; frontend plans cannot be restored by this command', + undefined, + 'field', + ); + } + string(test.projectId, 'test.projectId'); + string(test.name, 'test.name', 200); + if (test.id !== undefined) string(test.id, 'test.id'); + for (const field of ['description', 'category'] as const) { + const entry = test[field]; + if (entry !== undefined && entry !== null && typeof entry !== 'string') { + throw localValidationError(`test.${field}`, 'must be a string or null', undefined, 'field'); + } + } + if (typeof test.description === 'string' && test.description.length > 2000) { + throw localValidationError( + 'test.description', + 'must be at most 2000 characters', + undefined, + 'field', + ); + } + if ( + test.priority !== undefined && + test.priority !== null && + (typeof test.priority !== 'string' || !['p0', 'p1', 'p2', 'p3'].includes(test.priority)) + ) { + throw localValidationError( + 'test.priority', + 'must be p0, p1, p2, p3, or null', + undefined, + 'field', + ); + } + if ( + test.stepTimeoutMs !== undefined && + test.stepTimeoutMs !== null && + (!Number.isInteger(test.stepTimeoutMs) || + Number(test.stepTimeoutMs) < 1 || + Number(test.stepTimeoutMs) > 60000) + ) { + throw localValidationError( + 'test.stepTimeoutMs', + 'must be an integer from 1 to 60000 or null', + undefined, + 'field', + ); + } + for (const field of ['produces', 'consumes'] as const) { + const entry = test[field]; + if ( + entry !== undefined && + entry !== null && + (!Array.isArray(entry) || entry.some(item => typeof item !== 'string' || !item.trim())) + ) { + throw localValidationError( + `test.${field}`, + 'must be an array of non-empty strings or null', + undefined, + 'field', + ); + } + } + string(code.testId, 'code.testId'); + if (test.id !== undefined && test.id !== code.testId) { + throw localValidationError('code.testId', 'must match test.id', undefined, 'field'); + } + if (code.language !== 'python' || code.framework !== 'pytest') { + throw localValidationError( + 'code', + 'backend imports require Python/pytest source', + undefined, + 'field', + ); + } + assertDefinitionCode(code.code); + if (code.codeVersion !== null) string(code.codeVersion, 'code.codeVersion'); + if (code.etag !== undefined && code.etag !== null) string(code.etag, 'code.etag'); + return value as unknown as TestDefinition; +} + +export function assertDefinitionCode(value: unknown): asserts value is string { + string(value, 'code.code'); + if (value.startsWith('https://')) { + throw localValidationError( + 'code.code', + 'must contain source, not an expiring code URL', + undefined, + 'field', + ); + } + if (Buffer.byteLength(value, 'utf8') > MAX_DEFINITION_CODE_BYTES) { + throw localValidationError( + 'code.code', + 'exceeds the 350 KB code input limit', + undefined, + 'field', + ); + } +} + +/** Runtime state and result fields are outside the definition projection. */ +export function projectTestDefinition(test: CliTest, code: CliTestCode): TestDefinition { + const projected: Record = { + id: test.id, + projectId: test.projectId, + type: test.type, + }; + for (const field of METADATA_FIELDS) { + if (Object.hasOwn(test, field)) projected[field] = test[field]; + } + return parseTestDefinition({ test: projected, code }); +} + +/** Create treats null optional values as unset; update preserves explicit clears. */ +export function definitionMetadata( + test: BackendDefinitionMetadata, + create: boolean, +): Record { + return Object.fromEntries( + METADATA_FIELDS.filter( + field => test[field] !== undefined && (!create || test[field] !== null), + ).map(field => [field, test[field]]), + ); +} + +export function stringifyTestDefinition(definition: TestDefinition): string { + const sort = (value: unknown): unknown => { + if (Array.isArray(value)) return value.map(sort); + if (value !== null && typeof value === 'object') { + return Object.fromEntries( + Object.entries(value) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)) + .map(([key, entry]) => [key, sort(entry)]), + ); + } + return value; + }; + return `${JSON.stringify(sort(definition), null, 2)}\n`; +} + +/** Use one nonblocking descriptor, rejecting devices/FIFOs and bounding actual bytes. */ +export function readDefinitionFileGuarded(path: string): TestDefinition { + const absolute = resolve(path); + let fd: number | undefined; + let raw: string; + try { + // eslint-disable-next-line security/detect-non-literal-fs-filename -- caller-selected file; fstat and bounded reads use this single descriptor, without resolving the path again. + fd = openSync(absolute, constants.O_RDONLY | (constants.O_NONBLOCK ?? 0)); + const info = fstatSync(fd); + if (!info.isFile()) throw localValidationError('file', `not a regular file: ${path}`); + if (info.size > MAX_DEFINITION_BYTES) + throw localValidationError('file', 'exceeds the 1 MB definition input limit'); + const buffer = Buffer.alloc(MAX_DEFINITION_BYTES + 1); + let count = 0; + while (count < buffer.length) { + const size = readSync(fd, buffer, count, buffer.length - count, null); + if (size === 0) break; + count += size; + } + if (count > MAX_DEFINITION_BYTES) + throw localValidationError('file', 'exceeds the 1 MB definition input limit'); + raw = buffer + .subarray(0, count) + .toString('utf8') + .replace(/^\uFEFF/, ''); + } catch (error) { + if (error instanceof Error && 'exitCode' in error) throw error; + const code = (error as NodeJS.ErrnoException).code; + const reason = + code === 'ENOENT' + ? 'file does not exist' + : code === 'EACCES' || code === 'EPERM' + ? 'permission denied reading file' + : 'cannot read regular file'; + throw localValidationError('file', `${reason}: ${path}`); + } finally { + if (fd !== undefined) closeSync(fd); + } + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch { + throw localValidationError('file', 'not valid JSON'); + } + return parseTestDefinition(parsed); +} diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index 966acfaf..30034d5a 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -663,6 +663,12 @@ Commands: 4 test not found (no replayable run — trigger \`testsprite test run \` first) 5 validation error code Inspect and edit generated test code + export [options] Export one backend test definition as + deterministic JSON (metadata and Python + source) + import [options] Create a backend test from an exported + definition, or update an explicit + --test-id plan Generate + accept AI test plans (V3), and manage FE plan-steps failure Export the latest-failure agent bundle @@ -889,6 +895,23 @@ Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeou " `; +exports[`--help snapshots > test export 1`] = ` +"Usage: testsprite test export [options] + +Export one backend test definition as deterministic JSON (metadata and Python +source) + +Options: + --out write a .testsprite.json file atomically; defaults to stdout + -h, --help display help for command + +Backend tests only. This exports definitions for version control; run results are excluded. + +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): + testsprite --help +" +`; + exports[`--help snapshots > test failure get 1`] = ` "Usage: testsprite test failure get [options] @@ -955,6 +978,30 @@ Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeou " `; +exports[`--help snapshots > test import 1`] = ` +"Usage: testsprite test import [options] + +Create a backend test from an exported definition, or update an explicit +--test-id + +Options: + --project override the project when creating a test + --test-id update this existing backend test instead of + creating + --expected-version explicit codeVersion for an update; defaults to + the saved version + --idempotency-key replayable base token; at most 247 ASCII + characters including stage suffix budget + -h, --help display help for command + +Backend tests only. Updates write guarded code before metadata; the two writes are not atomic. +Use the same idempotency key to retry a partial import. A different target requires --expected-version. + +Global options (--dry-run, --output, --profile, --endpoint-url, --request-timeout, --verbose, --debug): + testsprite --help +" +`; + exports[`--help snapshots > test list 1`] = ` "Usage: testsprite test list [options] diff --git a/test/help.snapshot.test.ts b/test/help.snapshot.test.ts index 8a87524e..dcc47c57 100644 --- a/test/help.snapshot.test.ts +++ b/test/help.snapshot.test.ts @@ -45,6 +45,8 @@ const cases: Array<[string, string[]]> = [ ['test', ['test', '--help']], ['test list', ['test', 'list', '--help']], ['test get', ['test', 'get', '--help']], + ['test export', ['test', 'export', '--help']], + ['test import', ['test', 'import', '--help']], ['test code get', ['test', 'code', 'get', '--help']], ['test steps', ['test', 'steps', '--help']], ['test result', ['test', 'result', '--help']],