diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index da69607..0471de1 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -745,6 +745,8 @@ Trigger a run for a test. Without `--wait`, prints `{ runId, status: "queued", e `--all --project ` runs every test in the project in wave order. Add `--target-url ` to point its frontend tests at a run-time target. On the current unified engine that means **all tests, frontend and backend**; on the legacy backend-only engine, frontend tests can't run — they are skipped and enumerated in `skippedFrontend` with a stderr advisory. +With `--all`, `--filter ` narrows the selection to test names containing a non-empty substring (case-insensitive); omit the flag to run every project test. An explicitly empty value (`--filter ""` or `--filter=`) fails with `VALIDATION_ERROR` / exit **5** before any request, including on single-test and `--dry-run` paths. Non-empty values are used as written, including whitespace. + `testsprite test run list` — a literal `list` where a test id goes — is a usage error (exit 5), not a run against a test named "list": there is no `test run list` subcommand. It points you at `test result --history`, which lists a test's prior runs. The server controls whether a fresh run auto-heals drifted generated code by default. When healing is enabled, the agent re-authors stale code and runs the new version instead of replaying a script that can only fail. Healing saves the new code as the test's stored version. @@ -905,6 +907,7 @@ Batch `--report` flags apply only to batch `--wait` reruns (`--all` or multiple Flags: - `--all` — rerun every test in the resolved project; requires `--project `. +- `--filter ` — with `--all`, rerun only tests whose name contains this non-empty substring (case-insensitive); omit it to rerun every project test. An explicitly empty value fails with `VALIDATION_ERROR` / exit **5** before any request, including on single-test and `--dry-run` paths. Whitespace is preserved. - `--wait`, `--timeout ` — block until terminal; same exit matrix as `test run --wait`. - `--no-auto-heal` — ask for a verbatim replay instead of frontend AI heal-on-drift, which is on by default for frontend reruns. When healing engages, the agent re-authors the test and the new code replaces the stored version — a rerun asks for healing explicitly, so this includes code you wrote yourself; pass `--no-auto-heal` to replay it verbatim. Ignored for backend reruns. What a heal costs depends on which execution engine answers — on the current engine it is a full agent run, billed as the run with nothing extra; on legacy V2 accounts a verbatim-replay pass is free and only a heal engage costs credit. `testsprite usage` reports the figure that applies to your account. Unlike `test run --no-auto-heal`, a rerun gets no echo of the server's effective decision, so a dropped opt-out cannot be detected here — there is no advisory on this path. - `--skip-dependencies` — backend only: rerun just the named test without expanding the producer/teardown closure. @@ -1091,7 +1094,7 @@ testsprite ci run --project proj_xxxxxxxx --filter checkout --fail-on-skipped testsprite ci run --project proj_xxxxxxxx --summary-file testsprite-summary.json ``` -`--project` defaults to `TESTSPRITE_PROJECT_ID`. All project tests are selected unless `--filter ` narrows them (case-insensitive substring); `--all` is optional. `--env ` names the environment whose sign-in the run uses; the name must be one of the project's environments (`testsprite project env list `), and an unknown name is refused **before** anything dispatches or charges, even with `--allow-empty`. That check fails closed: an environment list that cannot be read is that error (a bad key exits **3**, a backend that is down **10**), never a run that goes on to dispatch unchecked; only a backend answering that the project has no environment surface (a V2 project, or an older backend without the route) leaves the name to the batch's own validation. `--target-url ` is the address to test — a pull request's preview deployment, or a local app through an automatic tunnel (see [`test run`](#testsprite-test-run-test-id)); `--skip-preflight` skips its reachability probe when the preview is still warming up. Backend tests use the URL in their own code, not `--target-url`. +`--project` defaults to `TESTSPRITE_PROJECT_ID`. All project tests are selected unless `--filter ` narrows them (case-insensitive, non-empty substring); `--all` is optional. An explicitly empty `--filter` fails with `VALIDATION_ERROR` / exit **5** before any request, including with `--dry-run`; omit the flag to select every test. Whitespace is preserved. `--env ` names the environment whose sign-in the run uses; the name must be one of the project's environments (`testsprite project env list `), and an unknown name is refused **before** anything dispatches or charges, even with `--allow-empty`. That check fails closed: an environment list that cannot be read is that error (a bad key exits **3**, a backend that is down **10**), never a run that goes on to dispatch unchecked; only a backend answering that the project has no environment surface (a V2 project, or an older backend without the route) leaves the name to the batch's own validation. `--target-url ` is the address to test — a pull request's preview deployment, or a local app through an automatic tunnel (see [`test run`](#testsprite-test-run-test-id)); `--skip-preflight` skips its reachability probe when the preview is still warming up. Backend tests use the URL in their own code, not `--target-url`. **One execution.** Every selected test is seeded into one execution and the server paces dispatch under the project's execution settings, exactly as for a GitHub App run — a CI run is one row in the Portal, never a first batch plus a rate-deferred retry. A backend that cannot honor this is refused with exit **7** (`admission-not-honored`) after the outputs are published; the batch is never silently cut to the per-key budget. diff --git a/src/commands/ci.ts b/src/commands/ci.ts index 9e3eec2..1b0f78e 100644 --- a/src/commands/ci.ts +++ b/src/commands/ci.ts @@ -24,7 +24,12 @@ import { } from '../lib/ci-linkage.js'; import { ciRefusalOutcome, recordTelemetryExtras } from '../lib/telemetry.js'; import type { CliProject } from './project.js'; -import { runTestRunAll, type RunTestRunAllOptions, type TestDeps } from './test.js'; +import { + parseNameFilter, + runTestRunAll, + type RunTestRunAllOptions, + type TestDeps, +} from './test.js'; import { emitCiStepOutputs, type CiSummary } from '../lib/gh-output.js'; import { previewEnvironmentName, readGithubRunContext } from '../lib/github-context.js'; import type { CiGithubTrigger } from '../lib/runs.types.js'; @@ -1427,6 +1432,7 @@ async function assertCiEnvironmentExists(opts: CiRunOptions, deps: TestDeps): Pr ); } +/** Build CI run and workflow scaffolding commands with injectable runtime dependencies. */ export function createCiCommand(deps: CiDeps = {}): Command { const ci = new Command('ci').description('Run tests in CI and scaffold CI integration'); @@ -1435,7 +1441,10 @@ export function createCiCommand(deps: CiDeps = {}): Command { .allowExcessArguments(false) .option('--project ', 'project id (defaults to TESTSPRITE_PROJECT_ID)') .option('--all', 'run all project tests (already the default)') - .option('--filter ', 'only run tests whose name contains this substring') + .option( + '--filter ', + 'only run tests whose name contains this non-empty substring (case-insensitive); omit to run every test', + ) .option( '--env ', 'the environment whose sign-in the run uses (unknown names fail). In a pull request, with --target-url, the run happens in a temporary clone of it at the preview address, named github-pr- like a GitHub App run', @@ -1490,7 +1499,7 @@ export function createCiCommand(deps: CiDeps = {}): Command { { ...resolveCommonOptions(command), projectId: cmdOpts.project, - nameFilter: cmdOpts.filter, + nameFilter: parseNameFilter(cmdOpts.filter), environment: cmdOpts.env, targetUrl: cmdOpts.targetUrl, skipPreflight: cmdOpts.skipPreflight === true, diff --git a/src/commands/test.filter.spec.ts b/src/commands/test.filter.spec.ts new file mode 100644 index 0000000..4dc0d56 --- /dev/null +++ b/src/commands/test.filter.spec.ts @@ -0,0 +1,182 @@ +import { Command } from 'commander'; +import { describe, expect, it, vi } from 'vitest'; +import { createCiCommand } from './ci.js'; +import { createTestCommand } from './test.js'; + +const projectTests = [ + { id: 'test-alpha', name: 'Alpha checkout', type: 'backend', status: 'passed' }, + { id: 'test-beta', name: 'Beta', type: 'backend', status: 'passed' }, +]; + +const batchCommands = [ + { name: 'test run', args: ['test', 'run', '--all', '--no-wait'], path: '/tests/batch/run' }, + { + name: 'test rerun', + args: ['test', 'rerun', '--all', '--no-wait'], + path: '/tests/batch/rerun', + }, + { name: 'ci run', args: ['ci', 'run', '--no-execution-report'], path: '/tests/batch/run' }, +]; + +/** All reads and dispatches are fixtures; no request reaches a real server. */ +function harness() { + const stdout: string[] = []; + const requests: { path: string; method: string; body: Record }[] = []; + const fetchImpl = vi.fn(async (input: string | URL | Request, init: RequestInit = {}) => { + const url = new URL( + typeof input === 'string' ? input : input instanceof URL ? input.href : input.url, + ); + const path = url.pathname.replace('/api/cli/v1', ''); + const method = init.method ?? 'GET'; + const body = init.body ? (JSON.parse(init.body as string) as Record) : {}; + requests.push({ path, method, body }); + let response: unknown; + if (path === '/tests' && method === 'GET') { + response = { items: projectTests, nextToken: null }; + } else if (path === '/projects/project-1/env' && method === 'GET') { + response = { + environments: [ + { + id: 'env-default', + name: 'default', + url: 'https://app.example.com', + isDefault: true, + authMode: 'account', + }, + ], + }; + } else if (method === 'POST' && ['/tests/batch/run', '/tests/batch/rerun'].includes(path)) { + const ids = (body.testIds as string[] | undefined) ?? projectTests.map(test => test.id); + response = { + accepted: ids.map(testId => ({ + testId, + runId: `run-${testId}`, + enqueuedAt: '2026-09-18T00:00:00Z', + })), + conflicts: [], + deferred: [], + skippedFrontend: [], + skippedIntegration: [], + closure: { byProject: [] }, + ...(body.admission !== undefined ? { admission: body.admission } : {}), + }; + } else if (path.startsWith('/runs/') && method === 'GET') { + const runId = path.slice('/runs/'.length); + response = { + runId, + testId: runId.slice('run-'.length), + projectId: 'project-1', + userId: 'user-1', + status: 'passed', + source: 'cli', + createdFrom: 'cli', + createdAt: '2026-09-18T00:00:00Z', + startedAt: '2026-09-18T00:00:01Z', + finishedAt: '2026-09-18T00:00:03Z', + codeVersion: 'v1', + targetUrl: 'https://app.example.com', + failedStepIndex: null, + failureKind: null, + error: null, + videoUrl: null, + stepSummary: { total: 1, completed: 1, passedCount: 1, failedCount: 0 }, + }; + } else { + // A non-retryable fixture error also bounds pre-fix single-test failures. + return new Response( + JSON.stringify({ + code: 'AUTH_INVALID', + message: `Unexpected request: ${method} ${path}`, + nextAction: 'Check the test fixture.', + requestId: 'fixture', + details: {}, + }), + { status: 401, headers: { 'content-type': 'application/json' } }, + ); + } + return new Response(JSON.stringify(response), { + status: method === 'POST' ? 202 : 200, + headers: { 'content-type': 'application/json' }, + }); + }); + const deps = { + env: { + TESTSPRITE_API_KEY: 'sk-user-test', + TESTSPRITE_API_URL: 'http://localhost:13599', + }, + fetchImpl, + stdout: (line: string) => stdout.push(line), + stderr: () => {}, + sleep: async () => {}, + }; + const root = new Command('testsprite').option('--output ', '', 'json').option('--dry-run'); + root.addCommand(createTestCommand(deps)); + root.addCommand(createCiCommand(deps)); + return { + fetchImpl, + stdout, + requests, + run: (args: string[]) => root.parseAsync(args, { from: 'user' }), + }; +} + +describe('CLI name-filter validation', () => { + const commands = [ + ...batchCommands.map(command => ({ + name: `${command.name} project`, + args: [...command.args, '--project', 'project-1'], + })), + { name: 'test run single', args: ['test', 'run', 'test-alpha', '--no-wait'] }, + { name: 'test rerun single', args: ['test', 'rerun', 'test-alpha', '--no-wait'] }, + ]; + + describe.each(commands)('$name', ({ args }) => { + it.each([false, true])( + 'rejects an empty filter before any request (dryRun=%s)', + async dryRun => { + const h = harness(); + await expect( + h.run([...(dryRun ? ['--dry-run'] : []), ...args, '--filter', '']), + ).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'filter' }, + nextAction: expect.stringMatching(/omit --filter.*non-empty substring/), + }); + expect(h.fetchImpl).not.toHaveBeenCalled(); + expect(h.stdout).toEqual([]); + }, + ); + + it('also rejects the --filter= spelling before any request', async () => { + const h = harness(); + await expect(h.run([...args, '--filter='])).rejects.toMatchObject({ + code: 'VALIDATION_ERROR', + exitCode: 5, + details: { field: 'filter' }, + }); + expect(h.fetchImpl).not.toHaveBeenCalled(); + }); + }); + + describe.each(batchCommands)('$name selection controls', ({ args, path }) => { + it.each([ + { name: 'omitted', filter: undefined, expectedIds: ['test-alpha', 'test-beta'] }, + { name: 'non-empty, case-insensitive', filter: 'aLPHa', expectedIds: ['test-alpha'] }, + { name: 'whitespace-only, not trimmed', filter: ' ', expectedIds: ['test-alpha'] }, + ])('preserves $name filtering', async ({ filter, expectedIds }) => { + const h = harness(); + await h.run([ + ...args, + '--project', + 'project-1', + ...(filter === undefined ? [] : ['--filter', filter]), + ]); + const dispatches = h.requests.filter(request => request.method === 'POST'); + expect(dispatches).toHaveLength(1); + expect(dispatches[0]!.path).toBe(path); + const selected = dispatches[0]!.body.testIds ?? projectTests.map(test => test.id); + expect(selected).toEqual(expectedIds); + }); + }); +}); diff --git a/src/commands/test.ts b/src/commands/test.ts index 8bf5ce5..25d0049 100644 --- a/src/commands/test.ts +++ b/src/commands/test.ts @@ -7683,6 +7683,17 @@ function normalizeEnvironmentName(raw: string | undefined): string | undefined { return name; } +/** Reject an explicitly empty name filter; omission alone selects every test. */ +export function parseNameFilter(raw: string | undefined): string | undefined { + if (raw === '') { + throw localValidationError( + 'filter', + 'must not be empty; omit --filter to run every project test, or pass a non-empty substring', + ); + } + return raw; +} + /** * Validate the `--timeout` flag value. Returns a clamped integer in * range [1, 3600], or the default when absent. @@ -15098,7 +15109,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { ) .option( '--filter ', - 'with --all: only run tests whose name contains this substring (case-insensitive)', + 'with --all: only run tests whose name contains this non-empty substring (case-insensitive); omit to run every test', ) .option( '--max-concurrency ', @@ -15186,6 +15197,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { ); } const isAll = cmdOpts.all === true; + const nameFilter = parseNameFilter(cmdOpts.filter); const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); // Mutual-exclusion: exactly one of positional vs --all must be set. @@ -15204,7 +15216,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { // --filter is an --all-only narrowing flag (mirrors `test rerun --filter`). // Without --all it would be SILENTLY ignored while the explicit // still runs — defeating the caller's narrowing intent. Reject early. - if (cmdOpts.filter !== undefined && cmdOpts.filter !== '' && !isAll) { + if (nameFilter !== undefined && !isAll) { throw localValidationError( 'filter', '--filter only applies with --all (it narrows which project tests run). Remove --filter, or add --all with --project or TESTSPRITE_PROJECT_ID.', @@ -15334,7 +15346,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { { ...resolveCommonOptions(command, deps.env), projectId, - nameFilter: cmdOpts.filter, + nameFilter, wait: effectiveWait, noWait: command.getOptionValueSource('wait') === 'cli' && cmdOpts.wait === false, concurrencyIsDefault: cmdOpts.maxConcurrency === undefined, @@ -15399,7 +15411,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { wait: effectiveWait, all: isAll, projectId: cmdOpts.project, - nameFilter: cmdOpts.filter, + nameFilter, localPort, localHost, targetUrl: cmdOpts.targetUrl, @@ -15557,7 +15569,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { ) .option( '--filter ', - 'with --all: only rerun tests whose name contains this substring (case-insensitive)', + 'with --all: only rerun tests whose name contains this non-empty substring (case-insensitive); omit to rerun every test', ) .option('--wait', 'block until terminal status or --timeout elapses', false) .option('--no-wait', 'detach without opening an automatic environment tunnel') @@ -15636,6 +15648,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { ) .addHelpText('after', GLOBAL_OPTS_HINT) .action(async (testIdsArg: string[], cmdOpts: RerunFlagOpts, command: Command) => { + const nameFilter = parseNameFilter(cmdOpts.filter); // Commander's `--no-auto-heal` pattern makes `cmdOpts.autoHeal` default // `true` (unchanged by the user) and `false` when the user passes // `--no-auto-heal`. There is no explicit `--auto-heal` flag, so @@ -15680,7 +15693,7 @@ export function createTestCommand(deps: TestDeps = {}): Command { projectId: cmdOpts.project, skipTerminal: cmdOpts.skipTerminal === true, statusFilter: cmdOpts.status, - nameFilter: cmdOpts.filter, + nameFilter, wait: cmdOpts.wait === true, noWait: command.getOptionValueSource('wait') === 'cli' && cmdOpts.wait === false, skipPreflight: cmdOpts.skipPreflight === true, diff --git a/test/__snapshots__/help.snapshot.test.ts.snap b/test/__snapshots__/help.snapshot.test.ts.snap index a619979..33572d2 100644 --- a/test/__snapshots__/help.snapshot.test.ts.snap +++ b/test/__snapshots__/help.snapshot.test.ts.snap @@ -186,7 +186,8 @@ Options: TESTSPRITE_PROJECT_ID) --all run all project tests (already the default) --filter only run tests whose name contains this - substring + non-empty substring (case-insensitive); omit + to run every test --env the environment whose sign-in the run uses (unknown names fail). In a pull request, with --target-url, the run happens in a temporary @@ -1180,7 +1181,8 @@ Options: accepted: draft|ready|queued|running|passed|failed|blocked|cancelled|unknown) --filter with --all: only rerun tests whose name contains - this substring (case-insensitive) + this non-empty substring (case-insensitive); omit + to rerun every test --wait block until terminal status or --timeout elapses (default: false) --no-wait detach without opening an automatic environment @@ -1382,7 +1384,8 @@ Options: TESTSPRITE_PROJECT_ID; returned by \`testsprite project list\`) --filter with --all: only run tests whose name contains - this substring (case-insensitive) + this non-empty substring (case-insensitive); omit + to run every test --max-concurrency with a tunnel, max runs in flight through one tunnel (1-10, default 5); with --all --wait without a tunnel, max polls (1-100, default 50)