diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index da696075..053fb8da 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -1095,7 +1095,7 @@ testsprite ci run --project proj_xxxxxxxx --summary-file testsprite-summary.json **One execution.** Every selected test is seeded into one execution and the server paces dispatch under the project's execution settings, exactly as for a GitHub App run — a CI run is one row in the Portal, never a first batch plus a rate-deferred retry. A backend that cannot honor this is refused with exit **7** (`admission-not-honored`) after the outputs are published; the batch is never silently cut to the per-key budget. -**In a pull request.** Under GitHub Actions, `ci run` reads the pull request from the job: a `pull_request` event names it, and a `deployment_status` event (the platform-deploys-every-PR shape) only names the commit, so the PR is looked up with the job's `GITHUB_TOKEN` (`pull-requests: read`). With `--env X --target-url U`, the run then happens in a temporary clone of X pointed at U, named `github-pr-` — the GitHub App's own mechanism, so the Portal shows `github-pr-` as the run's environment, links the run back to the pull request ("View on Github"), and releases the environment when the run finishes. X must sign in with a username and password or not at all: a recorded (SSO) session cannot be cloned and is refused before anything is charged. A draft pull request, or a deployment that belongs to no open pull request, is skipped with a `::notice::` and exit **0** — there is nothing to gate; the step outputs are still written, as zeros, so a later step reading them gets a number. Outside a pull request (a push, a schedule, a shell), `--env X --target-url U` keeps `test run`'s semantics: X's sign-in against U, recorded under X. +**In a pull request.** Under GitHub Actions, `ci run` reads the pull request from the job: a `pull_request` event names it, and a `deployment_status` event (the platform-deploys-every-PR shape) only names the commit, so the PR is looked up with the job's `GITHUB_TOKEN` (`pull-requests: read`). With `--env X --target-url U`, the run then happens in a temporary clone of X pointed at U, named `github-pr-` — the GitHub App's own mechanism, so the Portal shows `github-pr-` as the run's environment, links the run back to the pull request ("View on Github"), and releases the environment when the run finishes. X must sign in with a username and password or not at all: a recorded (SSO) session cannot be cloned and is refused before anything is charged. A draft pull request, or a deployment that belongs to no open pull request, is skipped with a `::notice::` and exit **0** — there is nothing to gate; the step outputs are still written, as zeros, so a later step reading them gets a number, and under `--output json` stdout carries `{"skipped": true, "reason": "", "summary": {...}}` with the same zeroed summary. Outside a pull request (a push, a schedule, a shell), `--env X --target-url U` keeps `test run`'s semantics: X's sign-in against U, recorded under X. **`ci run` does not filter deployments — the workflow does.** A `deployment_status` event fires for every state of every deployment (`pending`, `failure`, `success`, …) and for every environment (preview and production alike). `ci run` reads only the deployment's commit; it never looks at `state` or `environment`. A hand-written workflow therefore needs the job's `if:` to select the deployments worth gating, or a failed deployment is gated against a URL that never came up and a production deploy is gated like a preview: diff --git a/src/commands/ci-run.spec.ts b/src/commands/ci-run.spec.ts index a51d1761..82c04c3a 100644 --- a/src/commands/ci-run.spec.ts +++ b/src/commands/ci-run.spec.ts @@ -788,6 +788,54 @@ describe('ci run — pull request context and single-execution admission', () => expect(none.outputs()).toBe('total=0\npassed=0\nfailed=0\nskipped=0\ntimedOut=0\n'); }); + const skips = [ + [ + 'a draft pull request', + 'pull_request', + { pull_request: { number: 7, draft: true, head: { sha: SHA } } }, + 'Skipping draft pull request #7; the gate runs once it is ready for review.', + ], + [ + 'a deployment with no pull request', + 'deployment_status', + { deployment: { sha: SHA } }, + 'nothing to gate', + ], + ] as const; + + it.each(skips)( + 'under --output json, %s still prints a result on stdout', + async (_label, event, payload, notice) => { + const h = harness(undefined, 'passed', { pulls: [] }); + await runCiRun( + { ...h.opts, environment: 'staging', targetUrl: PREVIEW }, + githubEnv(h, event, payload, { GITHUB_TOKEN: 'ghs_x' }), + ); + expect(h.dispatches()).toEqual([]); + const result = JSON.parse(h.stdout.join('\n')) as { reason: string }; + // The zeroed CiSummary the step outputs carry, under the key a run's + // envelope uses, so `jq .summary.passed` reads on every path. + expect(result).toEqual({ + skipped: true, + reason: expect.stringContaining(notice) as unknown, + summary: { total: 0, passed: 0, failed: 0, skipped: 0, timedOut: 0, runs: [] }, + }); + expect(h.stderr).toContain(`::notice title=TestSprite::${result.reason}`); + // eslint-disable-next-line security/detect-non-literal-fs-filename -- Harness-owned summary path under this test's mkdtemp directory; no external path input. + expect(existsSync(h.opts.summaryFile!)).toBe(false); + }, + ); + + it.each(skips)('in text mode, %s prints nothing on stdout', async (_label, event, payload) => { + const h = harness(undefined, 'passed', { pulls: [] }); + await runCiRun( + { ...h.opts, output: 'text', environment: 'staging', targetUrl: PREVIEW }, + githubEnv(h, event, payload, { GITHUB_TOKEN: 'ghs_x' }), + ); + expect(h.stdout).toEqual([]); + expect(h.stderr.some(s => s.startsWith('::notice'))).toBe(true); + }); + it('a backend that does not echo the admission mode is refused with exit 7, after publishing', async () => { const h = harness(undefined, 'passed', { echo: false }); await expect(runCiRun(h.opts, h.deps)).rejects.toMatchObject({ diff --git a/src/commands/ci.ts b/src/commands/ci.ts index 9e3eec2e..68af99e2 100644 --- a/src/commands/ci.ts +++ b/src/commands/ci.ts @@ -10,7 +10,7 @@ import { import { ApiError, CLIError, localValidationError } from '../lib/errors.js'; import { loadConfig, resolveProfileName } from '../lib/config.js'; import { type FetchImpl, type HttpClient } from '../lib/http.js'; -import { GLOBAL_OPTS_HINT, Output, resolveOutputMode } from '../lib/output.js'; +import { GLOBAL_OPTS_HINT, Output, resolveOutputMode, type OutputMode } from '../lib/output.js'; import type { Page } from '../lib/pagination.js'; import { assertNotLocal } from '../lib/target-url.js'; import { @@ -1307,6 +1307,7 @@ export type CiRunOptions = Omit< */ export async function runCiRun(opts: CiRunOptions, deps: TestDeps = {}): Promise { const env = deps.env ?? process.env; + const stdout = deps.stdout; const stderr = deps.stderr ?? ((line: string) => process.stderr.write(`${line}\n`)); if (opts.targetUrl !== undefined && opts.targetUrl.trim() === '') { throw localValidationError('target-url', 'must be a non-empty URL'); @@ -1335,7 +1336,8 @@ export async function runCiRun(opts: CiRunOptions, deps: TestDeps = {}): Promise recordTelemetryExtras({ ciRunOutcome: 'skipped_draft' }); skipCiRun( `Skipping draft pull request #${github.trigger.prNumber}; the gate runs once it is ready for review.`, - { env, stderr }, + opts.output, + { env, stdout, stderr }, ); return; } @@ -1347,7 +1349,11 @@ export async function runCiRun(opts: CiRunOptions, deps: TestDeps = {}): Promise } else if (github.kind === 'no_pull_request' && github.event === 'deployment_status') { // A deployment that belongs to no pull request has nothing to gate. recordTelemetryExtras({ ciRunOutcome: 'skipped_no_pr' }); - skipCiRun(`Skipping: ${github.reason}; nothing to gate.`, { env, stderr }); + skipCiRun(`Skipping: ${github.reason}; nothing to gate.`, opts.output, { + env, + stdout, + stderr, + }); return; } @@ -1373,15 +1379,21 @@ export async function runCiRun(opts: CiRunOptions, deps: TestDeps = {}): Promise * A `ci run` that has nothing to gate (a draft PR, a deployment with no pull * request) exits 0 with a `::notice::` — and still writes its step outputs, as * zeros, so a later step reading `steps..outputs.passed` sees a number - * rather than an empty string. Nothing was dispatched, so nothing else (job + * rather than an empty string. Under `--output json` stdout carries the + * result too, `{skipped: true, reason, summary}`, so a script parsing it never + * reads an empty string either. Nothing was dispatched, so nothing else (job * summary, summary file, report) is written: those describe runs. */ function skipCiRun( notice: string, - io: { env: NodeJS.ProcessEnv; stderr: (line: string) => void }, + output: OutputMode, + io: { env: NodeJS.ProcessEnv; stdout?: (line: string) => void; stderr: (line: string) => void }, ): void { io.stderr(`::notice title=TestSprite::${notice}`); const nothing: CiSummary = { total: 0, passed: 0, failed: 0, skipped: 0, timedOut: 0, runs: [] }; + if (output === 'json') { + new Output(output, io).print({ skipped: true, reason: notice, summary: nothing }); + } emitCiStepOutputs(nothing, io); }