fix(test): reject empty --filter before any requests

DOCUMENTATION.md

@@ -745,6 +745,8 @@ Trigger a run for a test. Without `--wait`, prints `{ runId, status: "queued", e
 
 `--all --project <id>` runs every test in the project in wave order. Add `--target-url <url>` to point its frontend tests at a run-time target. On the current unified engine that means **all tests, frontend and backend**; on the legacy backend-only engine, frontend tests can't run — they are skipped and enumerated in `skippedFrontend` with a stderr advisory.
 
+With `--all`, `--filter <substr>` narrows the selection to test names containing a non-empty substring (case-insensitive); omit the flag to run every project test. An explicitly empty value (`--filter ""` or `--filter=`) fails with `VALIDATION_ERROR` / exit **5** before any request, including on single-test and `--dry-run` paths. Non-empty values are used as written, including whitespace.
+
 `testsprite test run list` — a literal `list` where a test id goes — is a usage error (exit 5), not a run against a test named "list": there is no `test run list` subcommand. It points you at `test result <test-id> --history`, which lists a test's prior runs.
 
 The server controls whether a fresh run auto-heals drifted generated code by default. When healing is enabled, the agent re-authors stale code and runs the new version instead of replaying a script that can only fail. Healing saves the new code as the test's stored version.
@@ -905,6 +907,7 @@ Batch `--report` flags apply only to batch `--wait` reruns (`--all` or multiple
 Flags:
 
 - `--all` — rerun every test in the resolved project; requires `--project <id>`.
+- `--filter <substr>` — with `--all`, rerun only tests whose name contains this non-empty substring (case-insensitive); omit it to rerun every project test. An explicitly empty value fails with `VALIDATION_ERROR` / exit **5** before any request, including on single-test and `--dry-run` paths. Whitespace is preserved.
 - `--wait`, `--timeout <s>` — block until terminal; same exit matrix as `test run --wait`.
 - `--no-auto-heal` — ask for a verbatim replay instead of frontend AI heal-on-drift, which is on by default for frontend reruns. When healing engages, the agent re-authors the test and the new code replaces the stored version — a rerun asks for healing explicitly, so this includes code you wrote yourself; pass `--no-auto-heal` to replay it verbatim. Ignored for backend reruns. What a heal costs depends on which execution engine answers — on the current engine it is a full agent run, billed as the run with nothing extra; on legacy V2 accounts a verbatim-replay pass is free and only a heal engage costs credit. `testsprite usage` reports the figure that applies to your account. Unlike `test run --no-auto-heal`, a rerun gets no echo of the server's effective decision, so a dropped opt-out cannot be detected here — there is no advisory on this path.
 - `--skip-dependencies` — backend only: rerun just the named test without expanding the producer/teardown closure.
@@ -1091,7 +1094,7 @@ testsprite ci run --project proj_xxxxxxxx --filter checkout --fail-on-skipped
 testsprite ci run --project proj_xxxxxxxx --summary-file testsprite-summary.json
 ```
 
-`--project` defaults to `TESTSPRITE_PROJECT_ID`. All project tests are selected unless `--filter <substr>` narrows them (case-insensitive substring); `--all` is optional. `--env <name>` names the environment whose sign-in the run uses; the name must be one of the project's environments (`testsprite project env list <id>`), and an unknown name is refused **before** anything dispatches or charges, even with `--allow-empty`. That check fails closed: an environment list that cannot be read is that error (a bad key exits **3**, a backend that is down **10**), never a run that goes on to dispatch unchecked; only a backend answering that the project has no environment surface (a V2 project, or an older backend without the route) leaves the name to the batch's own validation. `--target-url <url>` is the address to test — a pull request's preview deployment, or a local app through an automatic tunnel (see [`test run`](#testsprite-test-run-test-id)); `--skip-preflight` skips its reachability probe when the preview is still warming up. Backend tests use the URL in their own code, not `--target-url`.
+`--project` defaults to `TESTSPRITE_PROJECT_ID`. All project tests are selected unless `--filter <substr>` narrows them (case-insensitive, non-empty substring); `--all` is optional. An explicitly empty `--filter` fails with `VALIDATION_ERROR` / exit **5** before any request, including with `--dry-run`; omit the flag to select every test. Whitespace is preserved. `--env <name>` names the environment whose sign-in the run uses; the name must be one of the project's environments (`testsprite project env list <id>`), and an unknown name is refused **before** anything dispatches or charges, even with `--allow-empty`. That check fails closed: an environment list that cannot be read is that error (a bad key exits **3**, a backend that is down **10**), never a run that goes on to dispatch unchecked; only a backend answering that the project has no environment surface (a V2 project, or an older backend without the route) leaves the name to the batch's own validation. `--target-url <url>` is the address to test — a pull request's preview deployment, or a local app through an automatic tunnel (see [`test run`](#testsprite-test-run-test-id)); `--skip-preflight` skips its reachability probe when the preview is still warming up. Backend tests use the URL in their own code, not `--target-url`.
 
 **One execution.** Every selected test is seeded into one execution and the server paces dispatch under the project's execution settings, exactly as for a GitHub App run — a CI run is one row in the Portal, never a first batch plus a rate-deferred retry. A backend that cannot honor this is refused with exit **7** (`admission-not-honored`) after the outputs are published; the batch is never silently cut to the per-key budget.
 

src/commands/ci.ts

@@ -24,7 +24,12 @@ import {
 } from '../lib/ci-linkage.js';
 import { ciRefusalOutcome, recordTelemetryExtras } from '../lib/telemetry.js';
 import type { CliProject } from './project.js';
-import { runTestRunAll, type RunTestRunAllOptions, type TestDeps } from './test.js';
+import {
+  parseNameFilter,
+  runTestRunAll,
+  type RunTestRunAllOptions,
+  type TestDeps,
+} from './test.js';
 import { emitCiStepOutputs, type CiSummary } from '../lib/gh-output.js';
 import { previewEnvironmentName, readGithubRunContext } from '../lib/github-context.js';
 import type { CiGithubTrigger } from '../lib/runs.types.js';
@@ -1427,6 +1432,7 @@ async function assertCiEnvironmentExists(opts: CiRunOptions, deps: TestDeps): Pr
   );
 }
 
+/** Build CI run and workflow scaffolding commands with injectable runtime dependencies. */
 export function createCiCommand(deps: CiDeps = {}): Command {
   const ci = new Command('ci').description('Run tests in CI and scaffold CI integration');
 
@@ -1435,7 +1441,10 @@ export function createCiCommand(deps: CiDeps = {}): Command {
     .allowExcessArguments(false)
     .option('--project <id>', 'project id (defaults to TESTSPRITE_PROJECT_ID)')
     .option('--all', 'run all project tests (already the default)')
-    .option('--filter <substr>', 'only run tests whose name contains this substring')
+    .option(
+      '--filter <substr>',
+      'only run tests whose name contains this non-empty substring (case-insensitive); omit to run every test',
+    )
     .option(
       '--env <name>',
       'the environment whose sign-in the run uses (unknown names fail). In a pull request, with --target-url, the run happens in a temporary clone of it at the preview address, named github-pr-<N> like a GitHub App run',
@@ -1490,7 +1499,7 @@ export function createCiCommand(deps: CiDeps = {}): Command {
         {
           ...resolveCommonOptions(command),
           projectId: cmdOpts.project,
-          nameFilter: cmdOpts.filter,
+          nameFilter: parseNameFilter(cmdOpts.filter),
           environment: cmdOpts.env,
           targetUrl: cmdOpts.targetUrl,
           skipPreflight: cmdOpts.skipPreflight === true,

src/commands/test.filter.spec.ts

@@ -0,0 +1,182 @@
+import { Command } from 'commander';
+import { describe, expect, it, vi } from 'vitest';
+import { createCiCommand } from './ci.js';
+import { createTestCommand } from './test.js';
+
+const projectTests = [
+  { id: 'test-alpha', name: 'Alpha checkout', type: 'backend', status: 'passed' },
+  { id: 'test-beta', name: 'Beta', type: 'backend', status: 'passed' },
+];
+
+const batchCommands = [
+  { name: 'test run', args: ['test', 'run', '--all', '--no-wait'], path: '/tests/batch/run' },
+  {
+    name: 'test rerun',
+    args: ['test', 'rerun', '--all', '--no-wait'],
+    path: '/tests/batch/rerun',
+  },
+  { name: 'ci run', args: ['ci', 'run', '--no-execution-report'], path: '/tests/batch/run' },
+];
+
+/** All reads and dispatches are fixtures; no request reaches a real server. */
+function harness() {
+  const stdout: string[] = [];
+  const requests: { path: string; method: string; body: Record<string, unknown> }[] = [];
+  const fetchImpl = vi.fn(async (input: string | URL | Request, init: RequestInit = {}) => {
+    const url = new URL(
+      typeof input === 'string' ? input : input instanceof URL ? input.href : input.url,
+    );
+    const path = url.pathname.replace('/api/cli/v1', '');
+    const method = init.method ?? 'GET';
+    const body = init.body ? (JSON.parse(init.body as string) as Record<string, unknown>) : {};
+    requests.push({ path, method, body });
+    let response: unknown;
+    if (path === '/tests' && method === 'GET') {
+      response = { items: projectTests, nextToken: null };
+    } else if (path === '/projects/project-1/env' && method === 'GET') {
+      response = {
+        environments: [
+          {
+            id: 'env-default',
+            name: 'default',
+            url: 'https://app.example.com',
+            isDefault: true,
+            authMode: 'account',
+          },
+        ],
+      };
+    } else if (method === 'POST' && ['/tests/batch/run', '/tests/batch/rerun'].includes(path)) {
+      const ids = (body.testIds as string[] | undefined) ?? projectTests.map(test => test.id);
+      response = {
+        accepted: ids.map(testId => ({
+          testId,
+          runId: `run-${testId}`,
+          enqueuedAt: '2026-09-18T00:00:00Z',
+        })),
+        conflicts: [],
+        deferred: [],
+        skippedFrontend: [],
+        skippedIntegration: [],
+        closure: { byProject: [] },
+        ...(body.admission !== undefined ? { admission: body.admission } : {}),
+      };
+    } else if (path.startsWith('/runs/') && method === 'GET') {
+      const runId = path.slice('/runs/'.length);
+      response = {
+        runId,
+        testId: runId.slice('run-'.length),
+        projectId: 'project-1',
+        userId: 'user-1',
+        status: 'passed',
+        source: 'cli',
+        createdFrom: 'cli',
+        createdAt: '2026-09-18T00:00:00Z',
+        startedAt: '2026-09-18T00:00:01Z',
+        finishedAt: '2026-09-18T00:00:03Z',
+        codeVersion: 'v1',
+        targetUrl: 'https://app.example.com',
+        failedStepIndex: null,
+        failureKind: null,
+        error: null,
+        videoUrl: null,
+        stepSummary: { total: 1, completed: 1, passedCount: 1, failedCount: 0 },
+      };
+    } else {
+      // A non-retryable fixture error also bounds pre-fix single-test failures.
+      return new Response(
+        JSON.stringify({
+          code: 'AUTH_INVALID',
+          message: `Unexpected request: ${method} ${path}`,
+          nextAction: 'Check the test fixture.',
+          requestId: 'fixture',
+          details: {},
+        }),
+        { status: 401, headers: { 'content-type': 'application/json' } },
+      );
+    }
+    return new Response(JSON.stringify(response), {
+      status: method === 'POST' ? 202 : 200,
+      headers: { 'content-type': 'application/json' },
+    });
+  });
+  const deps = {
+    env: {
+      TESTSPRITE_API_KEY: 'sk-user-test',
+      TESTSPRITE_API_URL: 'http://localhost:13599',
+    },
+    fetchImpl,
+    stdout: (line: string) => stdout.push(line),
+    stderr: () => {},
+    sleep: async () => {},
+  };
+  const root = new Command('testsprite').option('--output <mode>', '', 'json').option('--dry-run');
+  root.addCommand(createTestCommand(deps));
+  root.addCommand(createCiCommand(deps));
+  return {
+    fetchImpl,
+    stdout,
+    requests,
+    run: (args: string[]) => root.parseAsync(args, { from: 'user' }),
+  };
+}
+
+describe('CLI name-filter validation', () => {
+  const commands = [
+    ...batchCommands.map(command => ({
+      name: `${command.name} project`,
+      args: [...command.args, '--project', 'project-1'],
+    })),
+    { name: 'test run single', args: ['test', 'run', 'test-alpha', '--no-wait'] },
+    { name: 'test rerun single', args: ['test', 'rerun', 'test-alpha', '--no-wait'] },
+  ];
+
+  describe.each(commands)('$name', ({ args }) => {
+    it.each([false, true])(
+      'rejects an empty filter before any request (dryRun=%s)',
+      async dryRun => {
+        const h = harness();
+        await expect(
+          h.run([...(dryRun ? ['--dry-run'] : []), ...args, '--filter', '']),
+        ).rejects.toMatchObject({
+          code: 'VALIDATION_ERROR',
+          exitCode: 5,
+          details: { field: 'filter' },
+          nextAction: expect.stringMatching(/omit --filter.*non-empty substring/),
+        });
+        expect(h.fetchImpl).not.toHaveBeenCalled();
+        expect(h.stdout).toEqual([]);
+      },
+    );
+
+    it('also rejects the --filter= spelling before any request', async () => {
+      const h = harness();
+      await expect(h.run([...args, '--filter='])).rejects.toMatchObject({
+        code: 'VALIDATION_ERROR',
+        exitCode: 5,
+        details: { field: 'filter' },
+      });
+      expect(h.fetchImpl).not.toHaveBeenCalled();
+    });
+  });
+
+  describe.each(batchCommands)('$name selection controls', ({ args, path }) => {
+    it.each([
+      { name: 'omitted', filter: undefined, expectedIds: ['test-alpha', 'test-beta'] },
+      { name: 'non-empty, case-insensitive', filter: 'aLPHa', expectedIds: ['test-alpha'] },
+      { name: 'whitespace-only, not trimmed', filter: ' ', expectedIds: ['test-alpha'] },
+    ])('preserves $name filtering', async ({ filter, expectedIds }) => {
+      const h = harness();
+      await h.run([
+        ...args,
+        '--project',
+        'project-1',
+        ...(filter === undefined ? [] : ['--filter', filter]),
+      ]);
+      const dispatches = h.requests.filter(request => request.method === 'POST');
+      expect(dispatches).toHaveLength(1);
+      expect(dispatches[0]!.path).toBe(path);
+      const selected = dispatches[0]!.body.testIds ?? projectTests.map(test => test.id);
+      expect(selected).toEqual(expectedIds);
+    });
+  });
+});

src/commands/test.ts

@@ -7683,6 +7683,17 @@ function normalizeEnvironmentName(raw: string | undefined): string | undefined {
   return name;
 }
 
+/** Reject an explicitly empty name filter; omission alone selects every test. */
+export function parseNameFilter(raw: string | undefined): string | undefined {
+  if (raw === '') {
+    throw localValidationError(
+      'filter',
+      'must not be empty; omit --filter to run every project test, or pass a non-empty substring',
+    );
+  }
+  return raw;
+}
+
 /**
  * Validate the `--timeout` flag value. Returns a clamped integer in
  * range [1, 3600], or the default when absent.
@@ -15098,7 +15109,7 @@ export function createTestCommand(deps: TestDeps = {}): Command {
     )
     .option(
       '--filter <substr>',
-      'with --all: only run tests whose name contains this substring (case-insensitive)',
+      'with --all: only run tests whose name contains this non-empty substring (case-insensitive); omit to run every test',
     )
     .option(
       '--max-concurrency <n>',
@@ -15186,6 +15197,7 @@ export function createTestCommand(deps: TestDeps = {}): Command {
         );
       }
       const isAll = cmdOpts.all === true;
+      const nameFilter = parseNameFilter(cmdOpts.filter);
       const stderrFn = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`));
 
       // Mutual-exclusion: exactly one of positional <test-id> vs --all must be set.
@@ -15204,7 +15216,7 @@ export function createTestCommand(deps: TestDeps = {}): Command {
       // --filter is an --all-only narrowing flag (mirrors `test rerun --filter`).
       // Without --all it would be SILENTLY ignored while the explicit <test-id>
       // still runs — defeating the caller's narrowing intent. Reject early.
-      if (cmdOpts.filter !== undefined && cmdOpts.filter !== '' && !isAll) {
+      if (nameFilter !== undefined && !isAll) {
         throw localValidationError(
           'filter',
           '--filter only applies with --all (it narrows which project tests run). Remove --filter, or add --all with --project <id> or TESTSPRITE_PROJECT_ID.',
@@ -15334,7 +15346,7 @@ export function createTestCommand(deps: TestDeps = {}): Command {
           {
             ...resolveCommonOptions(command, deps.env),
             projectId,
-            nameFilter: cmdOpts.filter,
+            nameFilter,
             wait: effectiveWait,
             noWait: command.getOptionValueSource('wait') === 'cli' && cmdOpts.wait === false,
             concurrencyIsDefault: cmdOpts.maxConcurrency === undefined,
@@ -15399,7 +15411,7 @@ export function createTestCommand(deps: TestDeps = {}): Command {
             wait: effectiveWait,
             all: isAll,
             projectId: cmdOpts.project,
-            nameFilter: cmdOpts.filter,
+            nameFilter,
             localPort,
             localHost,
             targetUrl: cmdOpts.targetUrl,
@@ -15557,7 +15569,7 @@ export function createTestCommand(deps: TestDeps = {}): Command {
     )
     .option(
       '--filter <substr>',
-      'with --all: only rerun tests whose name contains this substring (case-insensitive)',
+      'with --all: only rerun tests whose name contains this non-empty substring (case-insensitive); omit to rerun every test',
     )
     .option('--wait', 'block until terminal status or --timeout elapses', false)
     .option('--no-wait', 'detach without opening an automatic environment tunnel')
@@ -15636,6 +15648,7 @@ export function createTestCommand(deps: TestDeps = {}): Command {
     )
     .addHelpText('after', GLOBAL_OPTS_HINT)
     .action(async (testIdsArg: string[], cmdOpts: RerunFlagOpts, command: Command) => {
+      const nameFilter = parseNameFilter(cmdOpts.filter);
       // Commander's `--no-auto-heal` pattern makes `cmdOpts.autoHeal` default
       // `true` (unchanged by the user) and `false` when the user passes
       // `--no-auto-heal`. There is no explicit `--auto-heal` flag, so
@@ -15680,7 +15693,7 @@ export function createTestCommand(deps: TestDeps = {}): Command {
           projectId: cmdOpts.project,
           skipTerminal: cmdOpts.skipTerminal === true,
           statusFilter: cmdOpts.status,
-          nameFilter: cmdOpts.filter,
+          nameFilter,
           wait: cmdOpts.wait === true,
           noWait: command.getOptionValueSource('wait') === 'cli' && cmdOpts.wait === false,
           skipPreflight: cmdOpts.skipPreflight === true,