diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 6f4e1e3..0c65506 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "adept", - "version": "2.9.16", + "version": "2.9.17", "description": "Development-workflow skills: design, TDD, adversarial review, shipping, and campaign orchestration for Claude Code and Codex.", "author": { "name": "David Christensen" diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index ed5f168..09f5e5b 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -95,10 +95,53 @@ jobs: # version already installed leaves npm's tree satisfied and the binary # still missing, which is a different fault wearing the same symptom. claude --version + # GH_TOKEN is what puts `just actions-check` into zizmor's online mode, so + # the pin-provenance audits -- impostor-commit, ref-confusion, + # known-vulnerable-actions, stale-action-refs, ref-version-mismatch -- run + # here rather than nowhere. CI is where connectivity is assured and where + # the required check lives. + # + # `permissions:` above is deliberately NOT widened for this and stays + # `contents: read`: those audits resolve other repositories' refs and the + # advisory database, neither of which is a resource of this repository. + # What the line changes is that the token's value is visible to the whole + # `just ci` step; on a fork pull request GitHub makes it read-only, so the + # worst it grants is authenticated read of a public repository. Checkout's + # `persist-credentials: false` is likewise untouched -- that governs a + # credential in .git/config, and zizmor reads the environment. See ADR 0036. - name: Verify env: BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} - run: just ci + GH_TOKEN: ${{ github.token }} + # The guard asserts the one precondition CI's whole provenance coverage + # rests on. A workstation is offline by default and says so, so this + # runner is the only place the five online audits execute; if + # `github.token` ever resolves empty -- a repository or organization + # setting, a platform change, or this line lost in a conflict resolution + # -- run-zizmor.sh would report the offline condition and exit 0, and the + # required check would stay green with no provenance audited anywhere. + # That is the defect this change removes, relocated one level up, so it + # gets a red rather than a log line nobody reads. It re-types no gate + # command; `just ci` is still the recipe. + # + # Two arms, not one. `-z` alone cannot tell an unset variable from an + # empty one, and those call for opposite responses: unset means the + # `GH_TOKEN:` line above is gone and the fix is one line in this file, + # while set-and-empty means `github.token` itself resolved empty and the + # fix is in repository or organization settings. Reporting one cause for + # both would send the second reader on the first reader's investigation, + # on a red check that blocks every merge. `${VAR+set}` is the same idiom + # scripts/run-zizmor.sh uses for the same distinction (ADR 0025). + run: | + if [ -z "${GH_TOKEN+set}" ]; then + printf 'verify: GH_TOKEN is unset in this step; the GH_TOKEN: github.token wiring is missing from the env: block above, so the zizmor online audits would not run\n' >&2 + exit 1 + fi + if [ -z "$GH_TOKEN" ]; then + printf 'verify: GH_TOKEN is set but empty; github.token resolved empty, so the zizmor online audits would not run\n' >&2 + exit 1 + fi + just ci # A matrix renames its checks to `job (axis)`, so the bare `verify` context that # branch protection requires would stop reporting and every PR would sit BLOCKED diff --git a/Justfile b/Justfile index 87eb2d5..0bfd238 100644 --- a/Justfile +++ b/Justfile @@ -267,7 +267,10 @@ plugin-check: actions-check: actionlint - zizmor --offline .github/workflows/ + # Mode selection and mode reporting live in the script, not here: a Justfile + # recipe body is invisible to list-shell-sources.sh, so an inline branch would + # be unseen by shellcheck and shfmt and could carry no suite. See ADR 0036. + ./scripts/run-zizmor.sh .github/workflows/ commit-check: lint format-check public-safety diff --git a/docs/adr/0036-actions-check-names-its-zizmor-mode.md b/docs/adr/0036-actions-check-names-its-zizmor-mode.md new file mode 100644 index 0000000..940c120 --- /dev/null +++ b/docs/adr/0036-actions-check-names-its-zizmor-mode.md @@ -0,0 +1,333 @@ +# 0036 — `actions-check` names the zizmor mode it ran and the condition that chose it + +## Status + +Accepted (2026-08-25) + +## Context + +`actions-check` runs `zizmor --offline .github/workflows/`, so the audits that need the +GitHub API never run. Measured rather than taken from the published table: +`zizmor --offline -vv --no-progress .github/workflows/` on this workstation (arm64 macOS, +Darwin 25.6.0, zizmor 1.29.0) logs `skipping : can't run without a GitHub API +token` for `impostor-commit`, `known-vulnerable-actions`, `ref-confusion`, +`stale-action-refs`, and `ref-version-mismatch`, and schedules everything else. Those five +are what a token buys. `typosquat-uses` runs offline and is not among them, and + does not list `ref-version-mismatch` as online-only, so +the binary is cited here in preference to the page. + +Offline mode confirms a pin's *shape*; the five above are what confirm its *provenance*. +An impostor SHA borrowed from a fork is a well-formed pin. Five `uses:` pins are in the +tree; the next bump — a hand edit, or Dependabot once #236 lands — would be checked by +shape alone. + +Issue #239 proposes dropping `--offline` so zizmor uses the API when a token is present. +That leaves the mode unreported: zizmor announces a tokenless degrade only as a `WARN` +inside its own log stream, and the gate adds nothing, so a reader learns the mode by +knowing zizmor's log conventions and learns the condition that chose it not at all — the +discrimination [ADR 0025](0025-a-skip-reports-the-condition-not-the-cause.md) decision 2 +requires a degrade to carry. + +The operator settled the one design-changing question before this record was written: a +machine with no token degrades to the offline subset and stays green, because no other +gate here requires network or credentials. Failing without a token was considered and +explicitly rejected. + +Three measurements then decide the shape of the fix. All are from this workstation, with +`--cache-dir` outside the worktree. + +1. **A token is not connectivity.** `GH_TOKEN= GH_HOST=github.invalid.example + zizmor --no-progress .github/workflows/` gives `fatal: no audit was performed`, exit 1. + Online mode does not degrade when the API is unreachable; it fails. +2. **A token does not decide the mode.** `zizmor --help` documents `--offline + [env: ZIZMOR_OFFLINE=]` and `--no-online-audits [env: ZIZMOR_NO_ONLINE_AUDITS=]`; it + prints no value enumeration for either. With `ZIZMOR_OFFLINE=true` a run carrying a + valid token completes offline at exit 0 printing no `WARN`; with `ZIZMOR_OFFLINE=false` + the same run goes online. The *value* is the instruction, not the variable's presence. +3. **An empty or non-boolean value is rejected — but only where zizmor still parses the + variable.** `GH_TOKEN= zizmor --offline` exits 2 with `invalid value '' for + '--gh-token': GitHub token cannot be empty`, and the explicit `--offline` does not + rescue it. A malformed `ZIZMOR_NO_ONLINE_AUDITS` is likewise fatal with `--offline` on + argv. A malformed `ZIZMOR_OFFLINE` is **not**: an explicit `--offline` shadows the + variable, so `ZIZMOR_OFFLINE=0 zizmor --offline .github/workflows/` exits 0, while the + same value with no flag exits 2 with `invalid value '0' for '--offline'` and + `[possible values: true, false]`. That enumeration comes from clap's rejection path, + not from `--help`. +4. **A findings exit is distinct from a failure exit.** Measured: 0 clean, 1 the run + failed, 2 usage error, 3 a workflow that would not parse, and **14** for a run that + completes and reports findings — `zizmor --no-progress --offline` against a throwaway + workflow with an unpinned `uses:` and a `${{ }}` interpolation in `run:` reported + `7 findings ... 2 medium, 2 high` at exit 14. The list is "at least these", not an + enumeration of zizmor's whole status space, and only the 1-versus-14 split is + load-bearing: a gate that keys on "non-zero" cannot tell a finding from a failure. + Status 1 itself carries no cause — an unreachable API, a rejected token, + `invalid input: ` for a missing directory, and an unloadable `zizmor.yml` all + land there. + +## Decision + +**1. The gate reads every variable that decides the mode, in zizmor's own vocabulary, and +names the mode and the condition before the scan runs.** `actions-check` invokes +`scripts/run-zizmor.sh`, which resolves the mode as follows. + +- `ZIZMOR_OFFLINE`, then `ZIZMOR_NO_ONLINE_AUDITS`: a value of `true` selects offline and + is the reported condition. A value of `false` is not an offline instruction and falls + through. **Any other value, empty included, selects nothing**: the gate prints a warning + naming the variable and the values zizmor accepts, and falls through to the token check, + leaving zizmor's own parser to accept or reject it. The gate does not exit on it, + because whether the value is fatal depends on which flag ends up on argv (measurement 3) + — and a gate that refused where zizmor would have run is the second opinion this + decision exists to avoid. It is also what keeps `ZIZMOR_OFFLINE=1` green on the offline + path, where it is green today. + + Measurement 3's shadowing applies to `ZIZMOR_OFFLINE` alone. Nothing shadows + `--no-online-audits`, so a malformed `ZIZMOR_NO_ONLINE_AUDITS` still makes zizmor exit 2 + after the gate has printed its warning and its mode line. The warning therefore says the + value "selects no mode here, and zizmor may still reject it" rather than claiming the + gate ignored it — ignoring is not an outcome this gate controls for that variable, and + one wording true of both beats a per-variable branch for the sake of one word. +- Then `GH_TOKEN`, `GITHUB_TOKEN`, `ZIZMOR_GITHUB_TOKEN` — zizmor's documented order for + `--gh-token`. The first with a non-empty value selects online and is the reported source. +- A token variable that is set but **empty** is removed from the child's environment. An + empty string is not a token and carries no instruction, and leaving it in place makes + zizmor exit 2 on a usage error after the gate has already announced a mode. Removing it + is what makes "empty is not a token" true rather than merely asserted. +- Otherwise: offline, with the three token names reported as the condition. + +The gate then prints one line, or two: + +``` +run-zizmor: online mode; API token from GH_TOKEN +run-zizmor: offline mode (--offline); ZIZMOR_OFFLINE=true +run-zizmor: offline mode (--offline); no API token: GH_TOKEN, GITHUB_TOKEN and + ZIZMOR_GITHUB_TOKEN are all unset or empty +run-zizmor: pin provenance was NOT audited: ... +``` + +Three offline conditions, each with its own response — set `ZIZMOR_OFFLINE=false` or unset +it, likewise `ZIZMOR_NO_ONLINE_AUDITS`, or export a token — which is the discriminator ADR +0025 decision 2 requires where one observation has several causes calling for different +responses. The online line also names `GH_HOST` when set, since that decides which API an +online run talks to. + +The online path carries a response too, and it is the path that matters most for one: +online mode is the only mode that can fail, and `actions-check` is in `verify`, which the +managed pre-push hook re-runs — so a developer on hotel wifi is blocked from pushing. When +an online run exits with zizmor's **tool-failure** status the gate prints the remedy after +it: + +``` +run-zizmor: the audit run failed rather than reporting findings; zizmor own error is + above. If it is an API or token fault, a local offline run is ZIZMOR_OFFLINE=true +``` + +**Keyed on tool failure alone — status 1 — never on "non-zero".** Measurement 4 is why: +findings exit 14, and a hint offered there would tell a developer to switch off the audit +that just caught something, which is a documented route to a green gate over an unaudited +pin. A usage error (2) is not offered it either, because the offline subset does not fix a +malformed variable. If a future zizmor renumbers its statuses the hint stops appearing, +which is the safe direction to fail: a missing hint costs a reader nothing, a misapplied +one costs an audit. + +That is a printf on a path already being handled, not a retry or a reachability probe. +zizmor's exit status is captured into a variable and re-raised unchanged, never piped and +never `|| true`; the hint does not alter it. + +A `true` `ZIZMOR_NO_ONLINE_AUDITS` is executed as `--offline`, which is the *stronger* of +the two controls the operator asked for. The mode line discloses the substitution — it +names the variable that chose the mode next to the flag that was passed — and the two are +equivalent for this gate, because the sole capability `--no-online-audits` has over +`--offline` is auditing a remote `user/repo` slug and this gate's only input is a local +directory. Worth stating rather than leaving to a reader to notice, since a future change +that let the gate take a remote input would make the difference behavioural. + +**2. The offline path passes `--offline`, not `--no-online-audits`.** `--offline` forbids +all online operations; `--no-online-audits` is the documented weaker form that disables +connectivity-dependent audits while still permitting online operations. The requirement is +that the gate be green with no network at all, so the flag that states that as a property +of the run is the right one. The one thing `--no-online-audits` permits that `--offline` +does not — auditing a remote `user/repo` input — this gate never uses. `--offline` is also +the flag in the recipe today, so the offline invocation is unchanged and every audit that +passes now passes identically. + +**3. CI receives a token, at the permission it already has, and fails closed without one.** +The `Verify` step in `.github/workflows/verify.yml` gains `GH_TOKEN: ${{ github.token }}` +in its existing `env:` block, plus a guard ahead of `just ci` that exits 1 when that +variable is unset or empty. CI is where connectivity is assured and where the required +check lives, so it is where the online audits belong — and because a workstation is +offline by default, CI is the *only* place they run, which is why the precondition is +asserted rather than assumed. The guard has two arms because an unset variable and an +empty one call for opposite fixes: unset means the `env:` wiring is gone and the repair is +in this file, empty means `github.token` itself resolved empty and the repair is in +repository or organization settings. It re-types no gate command; `just ci` is still the +recipe. The job's `permissions: contents: read` is **not** +widened and checkout's `persist-credentials: false` is **not** relaxed. The +security-posture change is therefore that the token's *value* becomes visible to the +`just ci` step, not that any permission grows. On a fork pull request GitHub makes +`GITHUB_TOKEN` read-only, so the most a compromised gate script obtains is authenticated +read access to a public repository — content the same actor can already read +unauthenticated, at a lower rate limit. + +## Consequences + +- Every run states its mode and the condition that chose it, so a reader of a green gate + learns whether provenance was audited, and an operator who expected online mode learns + which variable to change. +- CI audits provenance on both runners, on every pull request. A workstation does so only + when someone exports a token; by default it runs the offline subset, says so, and stays + green with no network. Coverage is one gate on the merge path rather than two — the + price of keeping the local gate hermetic. +- **The required check now depends on state outside this repository.** + `stale-action-refs`, `ref-version-mismatch`, and `known-vulnerable-actions` resolve + against upstream repositories and the advisory database, so `verify` can newly fail on a + re-run of an unchanged commit — a tag moved, an advisory published. That is the point of + the audits, and it is also a way for an unrelated pull request to go red. Accepted. +- With a token exported the dependency is hard, not graceful: a failed API call gives + `fatal: no audit was performed` and exit 1, in CI as much as locally. +- **CI fails rather than degrading when its token is empty.** The `Verify` step asserts + `GH_TOKEN` is non-empty before `just ci`. Without it, a `github.token` that resolved + empty — a repository or organization setting, a platform change, or the `env:` line lost + in a conflict resolution — would put the runner on the offline path, print the + no-token condition, and exit 0: the required check green with the five audits running + nowhere, since a workstation is offline by default. That is this record's own subject one + level up, and a log line is too weak a mitigation for it. Deliberate removal of the + wiring stays visible in a reviewed diff; the guard covers the silent case, which is the + one a diff does not show. +- `GH_HOST` is ambient and zizmor honours it as `--gh-hostname`, so a developer configured + for a GitHub Enterprise instance gets the same hard `fatal`. The online line names the + host, so the destination is stated before the scan rather than only inside a failure's + causal chain — which does name it, contrary to a weaker claim an earlier draft made: the + headline names the audit, the chain names the URL. +- The online line reports the mode the run was launched in, not that every audit reached + the API; zizmor reports a per-audit online failure as a `WARN` in its own stream. Stated + rather than engineered against. +- **A token pulls more than the five online-only audits onto the network.** With + `GH_HOST` pointed at an unresolvable host, the audit that failed was `artipacked`, which + runs offline and is not among the five. So the five are what a token buys, not the limit + of what an unreachable API can break. +- The gate never reads a token's value — it tests each variable for emptiness and lets + zizmor read the values itself, so no credential passes through the script, its argv, or + its output. +- **A non-boolean mode variable is inert offline and fatal online.** `ZIZMOR_OFFLINE=1` is + green today, because the recipe's explicit `--offline` shadows it. It stays green on the + offline path here for the same reason. On the *online* path no flag is passed, so + nothing shadows it and zizmor exits 2 with `invalid value '1' for '--offline'` — after + the gate has announced online mode, and with no hint, since status 2 is excluded from + it. Reaching it takes a malformed mode variable and an exported token on the same + machine; the gate's warning names the variable and zizmor's error names the accepted + values, so it is diagnosable rather than mysterious. Accepted as a narrow regression + against "everything running today keeps running unchanged". +- **Every fact above about zizmor's CLI was measured on 1.29.0, and CI installs zizmor + unpinned** from Homebrew alongside the other gate tools. The variable names, the + `true`/`false` vocabulary, and the exit statuses are therefore a contract with one + observed version, not a version-independent one — and the two renames fail in opposite + directions. Renaming a **token** variable is self-consistent: the gate finds no token, + reports offline, and runs offline. Renaming a **mode** variable is the dangerous one and + it is silent: the gate would read the old name, find nothing, see a token, print "online + mode", and pass no flag, while an operator who set the *new* variable to `true` gets an + offline run at exit 0 with no `WARN` — unaudited and labelled audited, the state this + change exists to remove. Pinning the tool set is a separate decision (rejected below); + this residual is recorded rather than engineered against, and the suite pins the gate's + own behaviour against a stub, not against zizmor. +- Mode selection lives in a script rather than the recipe body, which brings it under + `shellcheck`, `shfmt`, and a suite. It is a gate script under `scripts/`, so anatomy + rules 1 and 2 do not bind it; rule 3 holds, as it runs and exits. + +## Considered & rejected + +- **Drop `--offline` and change nothing else** — the issue's literal proposal. verified: on + this host, `env -u GH_TOKEN -u GITHUB_TOKEN -u ZIZMOR_GITHUB_TOKEN zizmor --no-progress + .github/workflows/` exits 0 reporting `No findings to report`, preceded by ` WARN audit: + zizmor: zizmor is running in offline mode by default`; the same command with a token, and + `zizmor --offline`, print no such line. The `WARN` does distinguish an unaudited run, but + it carries no condition, and it appears only in the tokenless default — never when + `ZIZMOR_OFFLINE=true` sets the mode. A gate leaning on it would be silent in the case it + most needs to speak. +- **Infer the mode from the token variables alone**, or from their presence rather than + their value. verified: measurements 2 and 3 in Context. `ZIZMOR_OFFLINE=true` overrides a + present token silently at exit 0, so the gate would print "online mode" over an unaudited + run; `ZIZMOR_OFFLINE=false` is an *online* request that a presence test would report as + the operator's own offline instruction. Both are the second-opinion drift this decision + exists to avoid, and the second is the cause-not-carried-by-the-observation defect ADR + 0025 decision 2 forbids. +- **Unset `ZIZMOR_OFFLINE` and `ZIZMOR_NO_ONLINE_AUDITS` on the online path** (`env -u`) + rather than reporting them. judgment: it silently overrides an operator who asked to stay + offline — the same class of defect as the silent degrade this record closes. Removing an + *empty* token variable is not this: an empty string carries no instruction to override. + There is also no flag to assert online mode over them, since `zizmor --offline=false` + errors with `unexpected value 'false' for '--offline'`. +- **Fall back to `gh auth token` when no variable is set**, so a workstation audits + provenance without anyone exporting anything. verified: `gh auth token` needs no network + — behind a dead proxy (`HTTPS_PROXY=http://127.0.0.1:1 HTTP_PROXY=http://127.0.0.1:1 + ALL_PROXY=http://127.0.0.1:1 gh auth token`) it still exits 0 returning a 40-character + token — so it reports a token on a machine that cannot reach the API, and Context + measurement 1 shows what zizmor then does with one. Since every developer who can push here has an authenticated + `gh`, the fallback would redden `just verify` and the pre-push hook for all of them + whenever the API is unreachable — the outcome the fail-without-a-token alternative was + rejected for causing, on a strictly larger set of machines. It also inherits `gh`'s + configured host, which zizmor does not share. +- **Fail `actions-check` when no token is present.** verified: settled by the operator + before design, and no gate other than the one this change adds consults the network or a + credential. Scanning the gate scripts that predate this change — + `rg --no-config -e '\bgh\b|curl|wget|api\.github|GH_TOKEN|GITHUB_TOKEN|https?://' scripts/check-*.sh scripts/list-shell-sources.sh scripts/verify-push.sh scripts/test-fixture-helpers.sh .github/scripts/check-records.sh` + — returns exactly one line, the `gh[pousr]_` *pattern literal* at + `check-public-safety.sh:42`, which is a secret-detection pattern rather than a call. The + same result comes from `scripts/*.sh` on `main`; on this branch that wider glob also + matches `run-zizmor.sh` and its suite, which are this change's own token-emptiness tests, + so the narrower path list above is the form that reproduces. +- **Use `--no-online-audits` on the offline path.** judgment: see Decision 2, which states + the grounds and quotes `zizmor --help` for them. +- **Write the mode selection inline in the `actions-check` recipe.** verified: + `scripts/list-shell-sources.sh` classifies a shell source by a `.sh` name or a bash + shebang (`is_shell_source`, line 89), and `Justfile` is neither — + `scripts/check-ripgrep-config.sh` documents the same blind spot for its own scan — so an + inline branch would be unseen by `shellcheck` and `shfmt`. The `test` recipe discovers + suites as tracked `*-test.sh` paths (`Justfile:188`), so it would also be untestable, and + the reporting behaviour is the whole subject of this record. +- **Leave the CI posture knobs alone but reach for them if the audits complain** — widen + `permissions:` past `contents: read`, or relax `persist-credentials: false`. verified for + the first, as far as this host settles it: the five audits `zizmor --offline -vv` reports + as skipped resolve *other* repositories' refs and the advisory database, none of which is + a resource of this repository, so no permission on this repository is what gates them. + That is reasoning, not a green run — the pull request cites the CI run that exercises it, + and if CI shows the audits need more, that is reported as a finding rather than answered + by widening the grant. judgment for the second: `persist-credentials` governs a + credential written into `.git/config` for this repository's remote, and zizmor + authenticates to the API from the environment — nothing to gain, a stricter setting to + lose. +- **Scope `GH_TOKEN` to a dedicated `actions-check` step** instead of the `Verify` step + that runs the whole suite. judgment: the narrower blast radius is real, but buying it + means running `actions-check` twice or splitting the guardrail recipe so CI invokes it in + pieces — and `CLAUDE.md` requires CI to invoke the project's recipe rather than re-typed + command strings, which a per-gate step would reintroduce. The exposure is not narrow: the + token reaches every binary `just verify` runs, the unpinned npm-installed Claude CLI that + `plugin-check` invokes included. What bounds it is `permissions: contents: read` and the + token being read-only on a public repository, not the callees being first-party. +- **Run the online audits outside the required check** — a scheduled workflow, or a + separate non-required job — so the merge gate stays offline and deterministic and the + upstream-state residual above disappears. judgment: a provenance finding that arrives + after the merge is a bad pin already on `main`, which is the thing the audit exists to + prevent; catching it at the bump is the whole point of putting it on the gate. A + non-required job is also one people learn to ignore. The residual is preferred to that + latency, deliberately. +- **A `ZIZMOR_REQUIRE_ONLINE` switch honoured by `run-zizmor.sh`**, as a general assertion + that a CI run really went online. judgment: it would put a new flag in the gate's + contract for every caller to learn, and make the script responsible for a policy only CI + holds. Note precisely what is and is not covered without it. Decision 3's guard closes + the **empty-token** route to a silently-unaudited green check, and that is the route with + a mechanical trigger. It does **not** assert the run went online, so the + variable-rename route in the residual above passes straight through it: a renamed token + variable leaves `GH_TOKEN` non-empty, satisfies the guard, and still yields an offline + run at exit 0. That half stays recorded rather than engineered against, and this bullet + is not a claim of full coverage. An earlier draft also rejected the idea on the ground + that it "would only convert a silent degrade into a red required check on an unrelated + pull request"; that ground does not survive, because this record already accepts a red on + an unrelated pull request when a tag moves or an advisory lands. + Red-because-the-audit-worked and red-because-the-audit-stopped interrupt the same people, + and only the second means coverage is gone. +- **Probe reachability and retry** so a token plus a dead API degrades instead of failing. + judgment: it re-introduces the judgement call — how many retries, how long a timeout — + that a hard failure states plainly, and it grows the design past what this decision + governs. +- **Do nothing.** judgment: the five audits the threat model credits run nowhere today, and + the next pin bump is the one that would need them. diff --git a/docs/workflow/specs/2026-08-25-zizmor-operating-mode-design.md b/docs/workflow/specs/2026-08-25-zizmor-operating-mode-design.md new file mode 100644 index 0000000..95f87ff --- /dev/null +++ b/docs/workflow/specs/2026-08-25-zizmor-operating-mode-design.md @@ -0,0 +1,574 @@ +# `actions-check` selects and names zizmor's operating mode + +Issue [#239](https://github.com/randomparity/adept/issues/239). Decision record: +[ADR 0036](../../adr/0036-actions-check-names-its-zizmor-mode.md). + +## Problem + +`Justfile`'s `actions-check` recipe runs + +``` +actionlint +zizmor --offline .github/workflows/ +``` + +`--offline` disables every audit that needs the GitHub API. The audits it disables are +the ones that check a pin's *provenance* — whether the 40-character SHA a `uses:` names +is reachable in the repository it names, and whether the pinned revision is covered by a +known advisory. Offline mode confirms only that a pin has the right *shape*. An impostor +SHA borrowed from a fork is a well-formed pin offline. + +The repository's threat model credits those audits. Five action pins are in the tree +today; the next bump — a hand edit, or Dependabot once issue #236 lands it — would be +checked by shape alone. + +Measured on this workstation (arm64 macOS, Darwin 25.6.0, zizmor 1.29.0), dropping +`--offline` is not by itself a fix. With no token in the environment, +`zizmor .github/workflows/` exits 0 having audited nothing online, announcing it only as + +``` + WARN audit: zizmor: zizmor is running in offline mode by default; some audits and + auto-fixes will not be available. see https://docs.zizmor.sh/usage/#operating-modes +``` + +on stderr among zizmor's `INFO` lines. That `WARN` does appear in exactly the unaudited +case and in no other, so it is a real distinguisher — but it names the mode without the +condition that chose it, and the gate itself adds nothing. A reader learns the mode only +by knowing zizmor's log conventions, and learns why not at all, which is the +discrimination [ADR 0025](../../adr/0025-a-skip-reports-the-condition-not-the-cause.md) +decision 2 requires a degrade to carry. + +Two further measurements, neither of which the issue mentions, shape the design. + +**A token is not connectivity.** With a token present and the API unreachable, zizmor does +not fall back to the offline subset — it exits 1 with `fatal: no audit was performed`. + +**A token does not decide the mode by itself.** `ZIZMOR_OFFLINE=true` and +`ZIZMOR_NO_ONLINE_AUDITS=true` each override a present token: the run completes offline at +exit 0, and — unlike the tokenless default above — prints no `WARN` at all. Anything +reading only the token variables would announce "online mode" over that run. Their value, +not their presence, is what zizmor reads: `ZIZMOR_OFFLINE=false` selects online. + +The audits at stake are measured from the binary rather than taken from the published +table. `zizmor --offline -vv --no-progress .github/workflows/` on this host logs +`skipping : can't run without a GitHub API token` for `impostor-commit`, +`known-vulnerable-actions`, `ref-confusion`, `stale-action-refs`, and +`ref-version-mismatch`. `typosquat-uses` is scheduled offline and is *not* among them, and + does not list `ref-version-mismatch` as online-only — so +where the page and the binary disagree, this spec follows the binary the gate runs. + +## Requirements + +Sourced from the frozen scope charter on issue #239. + +- **R1.** With an API token available, `actions-check` runs zizmor's online audits. +- **R2.** With no API token, `actions-check` runs the offline subset, exits 0, and prints + a line naming both the mode it ran and the condition that chose it. A bare silent + fallback is a defect. +- **R3.** `just verify` stays green on a machine with no network and no credentials, as + every other gate in this repository does. Failing without a token was considered and + rejected by the operator before this design started. +- **R4.** The design states whether CI receives a token and, if so, the minimum + permission it needs. A change to the workflow's `permissions:` or to checkout's + `persist-credentials:` is a security-posture change and is justified in the record and + the pull-request body, never made silently. +- **R5.** `--offline` and `--no-online-audits` are distinguished deliberately, and the + choice is stated with its reason. +- **R6.** `actionlint` and every audit that runs today keep running unchanged. +- **R7.** The scan's exit status is captured explicitly. "zizmor found nothing" and + "zizmor could not run" are never collapsed, per `CLAUDE.md`. + +### Out of scope + +- `.github/dependabot.yml` (issue #236) and `scripts/check-skill-shape.sh` (issue #238), + both owned by other work in flight. +- The content of `.github/workflows/pages.yml`. The gate audits it; this change does not + edit it. +- What zizmor audits beyond its operating mode — configuration file, persona, severity + and confidence thresholds are all left as they are. + +## Design + +### Where the logic lives + +A new gate script, `scripts/run-zizmor.sh`, owns mode selection, mode reporting, and the +zizmor invocation. `actions-check` becomes + +``` +actions-check: + actionlint + ./scripts/run-zizmor.sh .github/workflows/ +``` + +`actionlint` keeps its own line and its own invocation, unchanged (R6). + +A script rather than an inline recipe body, for one reason that decides it: recipe bodies +are invisible to this repository's guardrails. `scripts/list-shell-sources.sh` discovers +tracked files named `*.sh` or opening with a bash shebang; `Justfile` is neither, which +is why `check-ripgrep-config.sh` documents the `Justfile` recipe bodies as out of its +reach. A mode-selection branch written inline would be unlinted by `shellcheck`, +unformatted by `shfmt`, and — decisively — untestable, because `just test` discovers +suites as tracked `*-test.sh` files. The behaviour this change is about *is* the +reporting, so reporting that no suite can exercise is the wrong shape for it. + +The script's own suite is `scripts/run-zizmor-test.sh`, beside it, as every gate script +in `scripts/` already has. + +### Contract of `scripts/run-zizmor.sh` + +``` +usage: run-zizmor.sh ... +``` + +Inputs are forwarded to zizmor unchanged. Zero arguments is a usage fault, exit 2 — the +status `list-shell-sources.sh` and `check-ripgrep-config.sh` already reserve for "this +script could not run at all", as against a verdict about what it scanned. + +**Mode discovery.** Five environment variables decide zizmor's mode, and the script reads +all five *in zizmor's own vocabulary* — the two mode controls are clap booleans whose +**value** is the instruction, not their presence. + +Resolution order: + +| # | variable | `true` | `false` | anything else, empty included | +|---|---|---|---|---| +| 1 | `ZIZMOR_OFFLINE` | offline; reported condition | falls through | warn, then fall through | +| 2 | `ZIZMOR_NO_ONLINE_AUDITS` | offline; reported condition | falls through | warn, then fall through | + +| # | variable | non-empty | empty | unset | +|---|---|---|---|---| +| 3 | `GH_TOKEN` | online; reported source | removed from the child env | falls through | +| 4 | `GITHUB_TOKEN` | online; reported source | removed from the child env | falls through | +| 5 | `ZIZMOR_GITHUB_TOKEN` | online; reported source | removed from the child env | falls through | + +Falling through all five gives offline with the three token names as the condition. + +Rows 3–5 are zizmor's documented discovery order for `--gh-token` (`zizmor --help`, +Network Options: `[env: GH_TOKEN or GITHUB_TOKEN or ZIZMOR_GITHUB_TOKEN]`). Rows 1–2 are +the mode controls under the same heading — `--offline [env: ZIZMOR_OFFLINE=]` and +`--no-online-audits [env: ZIZMOR_NO_ONLINE_AUDITS=]`. `--help` prints no value +enumeration for either; `[possible values: true, false]` comes from clap rejecting an +invalid one. + +Three measurements on this host fix these semantics, and getting any of them wrong is a +defect rather than a detail: + +- `ZIZMOR_OFFLINE=true` with a valid token completes **offline** at exit 0, printing no + `WARN`. A script inferring the mode from the token alone would announce "online mode" + over a run that audited no provenance — worse than the status quo, because the run would + then be unaudited *and labelled audited*, and the label is this change's whole product. +- `ZIZMOR_OFFLINE=false` with a token goes **online**. A presence test would force + `--offline` and report the operator's own variable as the cause of a mode they asked + against — the "cause the observation does not carry" defect ADR 0025 decision 2 forbids. +- An empty or non-boolean value is rejected by zizmor — but only where zizmor still parses + the variable. `GH_TOKEN=` exits 2 with `GitHub token cannot be empty` *even with + `--offline` on argv*. `ZIZMOR_OFFLINE=0` exits 2 with `invalid value '0'` when no flag is + on argv, but exits **0** with `--offline` present, because the explicit flag shadows the + variable. A malformed `ZIZMOR_NO_ONLINE_AUDITS` is fatal either way — no flag shadows it. +- The exit statuses are distinct: **0** clean, **1** tool failure, **2** usage error, and + **14** for a completed run reporting findings. + +Hence rows 1–2 **warn and fall through** rather than exiting. A malformed value is not a +recognised instruction, so it selects no mode; the gate names it and lets zizmor's own +parser accept or reject it. Exiting instead would refuse in a case where zizmor runs +happily — a second opinion drifting from zizmor's, which is the defect this design exists +to avoid — and it would turn `ZIZMOR_OFFLINE=1`, green on the offline path today, into a +hard failure, regressing R6. + +An empty *token* variable is removed from the child's environment, which is what makes "an +empty value is not a token" true by construction instead of asserted. Removing an empty +variable is not the override rejected below: an empty string carries no instruction. + +A `true` mode variable is reported, never unset. Silently overriding an operator who asked +to stay offline is the same class of defect as the silent degrade this change closes, and +there is no flag to assert online mode over them — `zizmor --offline=false` errors with +`unexpected value 'false' for '--offline'`. + +**There is deliberately no fallback to `gh auth token`.** A token is not connectivity: +`gh auth token` is a keyring read that contacts nothing, so it yields a token on a machine +with no network, and zizmor handed a token it cannot use exits 1 with `fatal: no audit was +performed` rather than degrading. Since every developer who can push here has an +authenticated `gh`, such a fallback would redden `just verify` and the pre-push hook for +all of them whenever the API is unreachable — violating R3 for a larger set of machines +than failing-without-a-token would have. Requiring an *export* makes online mode an +opt-in, so the person who takes on the network dependency is the person who asked for it. +[ADR 0036](../../adr/0036-actions-check-names-its-zizmor-mode.md) records this with the +reproduction. + +**The script never reads a token's value.** It tests only whether each variable is +non-empty, and lets zizmor read the value itself from the environment it already +inherits. No credential passes through the script, so none can reach its output or its +argv. + +**Online path.** The script prints one line, then runs zizmor with no mode flag: + +``` +run-zizmor: online mode; API token from GH_TOKEN +run-zizmor: online mode; API token from GH_TOKEN; GH_HOST=ghe.example.com +``` + +The second form is printed when `GH_HOST` is non-empty. That variable is ambient and +zizmor honours it as `--gh-hostname`, so it decides *which* API an online run talks to. +The clause states that destination *before* the scan. A failure does eventually name the +host — measured on this host, `GH_HOST=github.invalid.example` gives a `fatal` whose +headline names the audit (`'artipacked' audit failed`) and whose causal chain names +`https://github.invalid.example/...` — but only inside a stack a reader has to reach. +Naming it up front is what makes the mode line describe the run rather than the intent. +`GH_HOST` is a hostname, not a credential, so it is safe to print. + +An exported-but-empty `GH_HOST` is read the same way as the other variables, with +`${VAR+set}`: an empty hostname is a value zizmor uses and fails on, not an absent one, so +the clause prints `GH_HOST=` for it rather than staying silent about the pointing most +likely to confuse. + +That `artipacked` was the failing audit is worth recording: it is **not** one of the five +online-only audits, so with a token present zizmor takes audits outside that set onto the +network too. The five are what a token *buys*; they are not the full extent of what a dead +API costs. + +No flag is passed on this path because zizmor has no `--online`: a token in the +environment is what selects online mode, verified above. + +**Offline path.** The script prints two lines, then runs `zizmor --offline`: + +``` +run-zizmor: offline mode (--offline); ZIZMOR_OFFLINE=true +run-zizmor: offline mode (--offline); ZIZMOR_NO_ONLINE_AUDITS=true +run-zizmor: offline mode (--offline); no API token: GH_TOKEN, GITHUB_TOKEN and + ZIZMOR_GITHUB_TOKEN are all unset or empty +run-zizmor: pin provenance was NOT audited: a well-formed 40-character SHA that is + unreachable in the repository its `uses:` names, or that a known advisory covers, + passes this run +``` + +Three conditions, each with a different response — set `ZIZMOR_OFFLINE=false` or unset it, +likewise `ZIZMOR_NO_ONLINE_AUDITS`, or export a token. That is exactly the case ADR 0025 +decision 2 covers: one observation ("the run was offline") with several causes calling for +different responses, so the line carries what discriminates them. The condition reports +the variable and value observed and asserts nothing about why they were set. + +"or empty" in the third condition is accurate because the script has already removed any +empty token variable from the child's environment, so empty and unset genuinely reach +zizmor the same way. + +**Malformed mode value.** A mode variable holding neither `true` nor `false` is warned +about and then ignored for mode selection; the mode line follows as usual, and zizmor's +own parser has the last word: + +``` +run-zizmor: ZIZMOR_OFFLINE=0 is not a value zizmor accepts (true or false); it selects no mode + here, and zizmor may still reject it +``` + +**Online-failure hint.** Online mode is the only mode that can fail, and `actions-check` +is in `verify`, which the managed pre-push hook re-runs — so a failed online audit blocks +`git push`. When an online run exits with zizmor's **tool-failure** status the script +prints the response after it, so the red path carries a remedy as the offline conditions +do: + +``` +run-zizmor: the audit run failed rather than reporting findings; zizmor own error is + above. If it is an API or token fault, a local offline run is ZIZMOR_OFFLINE=true +``` + +**The condition is `status == 1`, never "non-zero".** Findings exit 14, and offering this +hint there would advise switching off the audit that just caught something — a documented +route to a green gate over an unaudited pin. A usage error (2) is not offered it either, +since the offline subset does not fix a malformed variable. A future zizmor that +renumbers its statuses simply stops printing the hint, which is the safe direction to +fail. + +It prints only in online mode, only on status 1, and does not alter that status. It is a +`printf` on a branch already being handled — not a retry, a timeout, or a reachability +probe, all of which the ADR rejects. + +The second line states the consequence. A gate that says only "offline" leaves a reader +to know which audits that costs; the whole issue is that the cost was invisible. + +Both lines print **before** zizmor runs, so a log of a failing run opens with the mode +that produced it. + +**Exit status.** zizmor's status is captured into a variable explicitly and re-raised as +the script's own: + +```sh +status=0 +zizmor --offline "$@" || status=$? +exit "$status" +``` + +No `|| true`, no pipe. A zizmor that is not installed reaches this as 127 and is +re-raised as 127 (R7). + +### `--offline` rather than `--no-online-audits` + +`zizmor --help`, Network Options, on zizmor 1.29.0: + +- `-o, --offline` — "Perform only offline operations. This disables all online audit + rules, and prevents zizmor from auditing remote repositories." +- `--no-online-audits` — "Perform only offline audits. This is a weaker version of + `--offline`: instead of completely forbidding all online operations, it only disables + audits that require connectivity." + +`--offline` is chosen. R3 is the deciding requirement: the tokenless path must be green +with no network at all, and `--offline` is the flag that states "no network" as a +property of the run rather than as a property of the audit set. `--no-online-audits` +leaves online operations permitted, and its one capability over `--offline` — auditing a +remote `user/repo` input — is a capability this gate never uses, because its only input +is the local `.github/workflows/` directory. + +`--offline` is also today's flag, so the offline path is byte-for-byte the invocation +that runs now (R6): the audits that pass today pass identically, and the change to that +path is the two lines printed above it. + +One consequence to state rather than leave implicit: when `ZIZMOR_NO_ONLINE_AUDITS=true` +selects the mode, the gate still passes `--offline`, so the operator's request for the +weaker control is executed as the stronger one. The mode line discloses it by naming the +variable beside the flag, and the two are equivalent here because +`--no-online-audits`'s only extra capability is auditing a remote `user/repo` slug, which +this gate never passes. A future change that let the gate take a remote input would make +the difference behavioural. + +### CI receives a token + +**Decision: yes.** `.github/workflows/verify.yml` gains one line in the `Verify` step's +existing `env:` block, and a guard ahead of `just ci` that fails the job when that +variable is unset or empty: + +```yaml + - name: Verify + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before }} + GH_TOKEN: ${{ github.token }} + run: | + if [ -z "${GH_TOKEN+set}" ]; then + printf 'verify: GH_TOKEN is unset in this step; ...\n' >&2 + exit 1 + fi + if [ -z "$GH_TOKEN" ]; then + printf 'verify: GH_TOKEN is set but empty; ...\n' >&2 + exit 1 + fi + just ci +``` + +The guard exists because CI is the *only* place the online audits run — a workstation is +offline by default — so a green required check is the sole signal that provenance was +audited at all. Without it, a `github.token` that resolved empty would put the runner on +the offline path at exit 0 and the check would stay green with no coverage anywhere. Two +arms rather than one because unset and empty call for opposite repairs: unset means the +`env:` wiring is missing and the fix is in this file; empty means `github.token` resolved +empty and the fix is in repository or organization settings. It re-types no gate command. + +**Minimum permission: the job's existing `permissions: contents: read`, unchanged.** The +online audits read public repositories' git refs and the public advisory database through +the GitHub API. `contents: read` on *this* repository is the floor a usable +`GITHUB_TOKEN` needs; nothing about reading another public repository's refs requires a +scope beyond it. So: + +- `permissions:` is **not** changed. It stays `contents: read`. +- checkout's `persist-credentials: false` is **not** changed. That setting governs whether + a credential is written into `.git/config` for later git commands; it is unrelated to an + environment variable on one step, and the workflow keeps the stricter setting. + +What does change is that the token's *value* becomes visible to the `just ci` step, which +runs this repository's whole gate chain. That is the security-posture change this design +makes, and the threat model below is where it is judged. + +CI sets `GH_TOKEN` and neither mode variable, so the runner takes the online branch by +the same rule every other machine does — no CI-specific path in the script. + +Adding the token to the environment rather than interpolating `${{ }}` inside the `run:` +block is the pattern zizmor's own `template-injection` audit exists to enforce; the gate +audits its own workflow, so the change has to satisfy it. + +## Threat model + +The change widens what a CI step can reach, so the security section applies. + +### Boundary inventory + +Boundaries this design **adds**: + +- **B1 — CI step environment.** A GitHub App installation token reaches the `just ci` + step's environment on every `pull_request` and every push to `main`. Everything `just + verify` runs — gate scripts, `prek` hooks, `shellcheck`, `shfmt`, `actionlint`, + `zizmor`, the npm-installed Claude CLI — inherits it. +- **B2 — token egress to a third-party binary.** With a token in the environment, zizmor + sends it as an API credential to **the host `GH_HOST` names, defaulting to + `github.com`** — the GitHub Server Hostname zizmor derives its API and git endpoints + from (`zizmor --help` 1.29.0: `--gh-hostname ... [env: GH_HOST=] [default: github.com]`), + not an API host given directly — zizmor honours that variable as `--gh-hostname`. This is the first + path on which this repository's gate sends a credential off the machine at all: the + recipe previously ran `zizmor --offline`, which forbids all online operations. The + destination is therefore chosen by ambient input, which is why it is named here rather + than left implicit. + +Boundaries this design **widens**: none. + +Boundaries this design does **not** touch: `permissions:`, `persist-credentials:`, the +set of triggers the workflow runs on, and what the gate scans. In particular the gate +reaches no credential store: dropping the `gh auth token` fallback (above) means a +workstation credential is never read, so the design adds no workstation boundary at all. + +### Actor model + +- **Anonymous internet, via a fork pull request.** Can propose arbitrary content in the + tree that `just ci` executes, including an edited `Justfile` and edited gate scripts. + This is the untrusted actor that matters for B1. +- **A repository collaborator.** Can push a branch whose `just ci` runs with a token that + is not read-only. Trusted with write access already. +- **The developer at a workstation.** Reaches online mode only by exporting a token + themselves. Trusted with their own credential by construction — the design never goes + looking for it. +- **zizmor and its transitive dependencies.** Trusted to send the token only to the + GitHub API. This is a stated trust assumption, not a verified one. + +### Control per boundary + +- **B1, fork pull request.** The control is GitHub's own: for a `pull_request` event from + a fork, `GITHUB_TOKEN` is read-only regardless of the `permissions:` block, and secrets + are not exposed. Combined with `permissions: contents: read` this design leaves in + place, the most a compromised gate script obtains is authenticated read access to a + **public** repository — content the same actor can already read unauthenticated. The + marginal capability gained is a higher API rate limit. This is an existing platform + control, preferred to a new one. +- **B1, same-repository branch.** `permissions: contents: read` bounds the token to + reading this public repository. A collaborator who can push a branch can already do + strictly more than that token permits. +- **B1, coverage assurance — a workflow-side control this design adds.** The `Verify` + step's guard fails the job when `GH_TOKEN` is unset or empty. This is *not* a mitigation + of B1's egress risk; it protects the opposite property — that a green required check + really does mean the provenance audits ran. It closes the empty-token route to a + silently-unaudited green. It does not assert the run went online, so the + variable-rename residual recorded in ADR 0036 passes through it; that half stays + accepted rather than engineered against. +- **B2, egress.** Two controls, for the two halves of the boundary. + + *The credential itself:* the script never holds the value. It tests the five mode + variables for emptiness and lets zizmor read the environment it already inherits, so the + token appears in no argv the script builds and in nothing the script prints. + `run-zizmor.sh` prints the *name* of the source variable only, and the suite asserts the + value is absent from the script's output. Beyond that, this design trusts zizmor with a + credential it is designed to receive — `--gh-token` is its documented interface. + + *The destination:* the only control is visibility. `run-zizmor.sh` names `GH_HOST` on the + online mode line whenever it is set, so a run pointed somewhere unexpected says so in the + log it fails in. There is deliberately no allowlist or validation: that would be a second + opinion on zizmor's own configuration — the drift this design exists to avoid — and it + would cost more than the risk it removes. Reachability is narrow. In CI `GH_HOST` is + unset and nothing in the diff sets it, and an actor who could set it there already + controls the tree `just ci` executes, which is B1's problem and judged there. On a + workstation it is the developer's own configuration, and online mode requires them to + export a token themselves, so the worst case is misdelivering one's own credential to + one's own configured host — with the symptom the design already predicts, a hard `fatal` + naming an audit. No untrusted actor gains a capability. + +### Explicitly out of scope + +- **A malicious zizmor release.** Not addressed. zizmor is installed from Homebrew, + unpinned, by the same step that installs every other gate tool; pinning the whole tool + set is a separate decision this change does not make. Noted rather than silently + omitted. +- **Token theft by a malicious pull request to a *public* repository.** Judged + uninteresting rather than mitigated, for the reason under B1: a read-only token against + public content is worth a rate-limit bump. +- **A workstation whose `gh` credential is already compromised.** Out of reach of a gate, + and out of this design's reach in particular — it never reads that credential. +- **An exported token sent to a `GH_HOST` the developer configured.** Accepted. The gate + names the host on the online line and does not validate it, per B2's control list above; + the residual is a developer misdelivering their own credential to their own configured + host, which fails hard rather than silently. +- **An exported token with an unreachable API.** Verified on this host: zizmor exits 1 + with `fatal: no audit was performed`, and the gate re-raises that status. Deliberately + not mitigated. A reachability probe or a retry wrapper would grow the design past what + it governs, and treating a failed online audit as a pass would reinstate exactly the + silent green this change removes. Requiring an *export* is what bounds the blast + radius: the machines that can hit this are the ones whose owner opted in. +- **A per-audit online failure inside an otherwise-successful run.** zizmor reports one as + a `WARN` in its own stream, so a run can announce online mode and still finish green + having not reached the API for some audit. The mode line reports the token found, not + that every audit completed. Stated rather than mitigated, for the same + proportionality reason. + +## Testing + +`scripts/run-zizmor-test.sh`, discovered by `just test`, using the `fixture_init` / +`fail` scaffold from `scripts/test-fixture-helpers.sh` like every other suite in +`scripts/`. `zizmor` is stubbed on `PATH`; the real binary is never invoked, so the suite +needs no network and no credentials. + +The `zizmor` stub records its argv and the token variables it inherited to files in +the scratch directory, and exits with a status the case chooses. + +| case | asserts | +|---|---| +| `GH_TOKEN` set | announces online mode naming `GH_TOKEN`; argv carries no `--offline` | +| `GITHUB_TOKEN` set, `GH_TOKEN` unset | announces online naming `GITHUB_TOKEN` | +| `ZIZMOR_GITHUB_TOKEN` set, other two unset | announces online naming `ZIZMOR_GITHUB_TOKEN` | +| all three set to different values | announces `GH_TOKEN` (precedence matches zizmor's) | +| `GH_TOKEN` set to the empty string, `GITHUB_TOKEN` set | empty is not a token; announces `GITHUB_TOKEN` | +| all three set to the empty string | announces offline, exit 0 | +| all three unset | announces offline, `--offline` in argv, condition names all three variables, exit 0 | +| `ZIZMOR_OFFLINE=true` **and** `GH_TOKEN` set | announces **offline**, condition names `ZIZMOR_OFFLINE=true`, `--offline` in argv | +| `ZIZMOR_NO_ONLINE_AUDITS=true` **and** `GH_TOKEN` set | announces offline, condition names `ZIZMOR_NO_ONLINE_AUDITS=true` | +| both mode variables `true` | condition names `ZIZMOR_OFFLINE` (first in order) | +| `ZIZMOR_OFFLINE=false`, `GH_TOKEN` set | falls through; announces **online** naming `GH_TOKEN` | +| `ZIZMOR_OFFLINE=false`, no token | falls through; announces offline with the no-token condition | +| `ZIZMOR_OFFLINE=0`, no token | warns naming the variable and `true or false`, then announces offline; zizmor still runs | +| `ZIZMOR_OFFLINE=0`, `GH_TOKEN` set | warns, then announces **online** — the malformed value selects nothing | +| `ZIZMOR_OFFLINE` set to the empty string | warns, then falls through | +| `ZIZMOR_NO_ONLINE_AUDITS=maybe` | warns, then falls through | +| any valid mode value | no warning is printed | +| `GH_TOKEN` empty, others unset | announces offline; the stub sees `GH_TOKEN` **absent** from its environment | +| `GH_TOKEN` empty, `GITHUB_TOKEN` set | announces online naming `GITHUB_TOKEN`; the stub sees `GH_TOKEN` absent | +| offline path | prints the "pin provenance was NOT audited" consequence line | +| online path | the stub inherits the token variable unchanged, so zizmor reads it itself | +| `GH_HOST` set on the online path | the online line names the host | +| `GH_HOST` unset on the online path | the online line omits the host clause entirely | +| no `gh` on `PATH` | irrelevant to the outcome — the script never invokes `gh` | +| zizmor stub exits 1 | the script exits 1 (a real finding still reddens the gate) | +| zizmor stub exits 2 | the script exits 2 (a zizmor fault is not collapsed into a finding) | +| zizmor stub exits 1, online mode | the online-failure hint is printed; status is still 1 | +| zizmor stub exits 1, offline mode | the online-failure hint is **not** printed | +| zizmor stub exits 0, online mode | the online-failure hint is **not** printed | +| zizmor stub exits **14**, online mode | the hint is **not** printed — findings must never draw "switch the audit off"; status is still 14 | +| zizmor stub exits 2, online mode | the hint is **not** printed; status is still 2 | +| any online case | the token value appears nowhere in the script's stdout or stderr | +| no arguments | usage message, exit 2 | +| inputs forwarded | argv ends with exactly the inputs given, in order | + +Each behaviour has a triggering case, including every error path, per `CLAUDE.md`. + +### Verification beyond the unit suite + +- `just verify` on this workstation with no token exported exercises the **offline** path + end to end against the real zizmor — which is now the default path everywhere except + CI. It must exit 0 and print the offline lines (R2, R3, criterion 3). +- The **online** path is exercised against the real zizmor and the real API by running the + recipe's script with `GH_TOKEN` exported (R1, criterion 1). +- Both real-zizmor runs are reported with the environment each was run in, per ADR 0025 + decision 3. +- CI exercises the online path on both runners once `GH_TOKEN` is wired, on this pull + request itself. + +## Consequences + +- Every `actions-check` run — local, pre-push hook, CI — now states its mode. A reader of + a green gate learns whether provenance was audited without knowing zizmor's defaults. +- CI audits provenance on every pull request, which is the coverage issue #239 asks for. +- A workstation audits provenance only when someone exports a token; by default it runs + the offline subset and says so, twice, and stays green with no network. Coverage is one + gate on the merge path rather than two, which is the price of keeping the local gate + hermetic. +- With a token exported, a failed API call fails the gate hard rather than degrading. That + applies to CI too: an API outage can redden the required `verify` check for a reason + unrelated to the pull request. Recorded in ADR 0036 as an accepted residual. +- The required check now depends on state outside this repository. `stale-action-refs`, + `ref-version-mismatch`, and `known-vulnerable-actions` resolve against upstream + repositories and the advisory database, so `verify` can newly fail on a re-run of an + unchanged commit when a tag moves or an advisory is published. That is what the audits + are for, and it is also a way for an unrelated pull request to go red. +- One new script and one new suite. By the repository's anatomy rules these are gate + scripts under `scripts/`, not skill files, so rules 1 and 2 do not bind them; rule 3 + holds — the script runs and exits. diff --git a/scripts/run-zizmor-test.sh b/scripts/run-zizmor-test.sh new file mode 100755 index 0000000..b554ff8 --- /dev/null +++ b/scripts/run-zizmor-test.sh @@ -0,0 +1,365 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Fixture suite for scripts/run-zizmor.sh. zizmor is stubbed on PATH, so the suite +# needs no network, no credentials, and no zizmor installation, and it pins the +# gate's own behaviour rather than the tool's -- which matters because CI installs +# zizmor unpinned (ADR 0036). +# +# The stub records the argv it was handed and which token variables reached it, +# and exits with whatever status a case asks for. Two of those recordings are the +# whole point: argv proves --offline is passed on exactly the offline path, and +# the token recording proves an exported-but-empty variable was removed rather +# than forwarded to a zizmor that would reject it. +# +# Every invocation sets all five mode variables explicitly -- including to unset -- +# because a developer with ZIZMOR_OFFLINE or GH_TOKEN exported would otherwise run +# a different suite than CI does. That is the same reason check-plugin-version-test +# always sets BASE_SHA. + +script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=SCRIPTDIR/test-fixture-helpers.sh +. "$script_dir/test-fixture-helpers.sh" + +fixture_init run-zizmor-test + +gate=$script_dir/run-zizmor.sh + +stub_dir=$SCRATCH/bin +mkdir -p "$stub_dir" + +# Written once; each case chooses the exit status through ZIZMOR_STUB_STATUS, +# which is read at run time rather than baked in. +cat >"$stub_dir/zizmor" <<'STUB' +#!/usr/bin/env bash +set -euo pipefail +printf '%s\n' "$*" >"$ZIZMOR_STUB_ARGV" +{ + printf 'GH_TOKEN=%s\n' "${GH_TOKEN+}" + printf 'GITHUB_TOKEN=%s\n' "${GITHUB_TOKEN+}" + printf 'ZIZMOR_GITHUB_TOKEN=%s\n' "${ZIZMOR_GITHUB_TOKEN+}" +} >"$ZIZMOR_STUB_ENV" +exit "${ZIZMOR_STUB_STATUS:-0}" +STUB +chmod +x "$stub_dir/zizmor" + +argv_file=$SCRATCH/argv +env_file=$SCRATCH/env + +# run [VAR=VALUE ...] -- sets RUN_OUTPUT, RUN_STATUS, RUN_ARGV, RUN_ENV. +# +# `env -u` on all five names first, then the case's own assignments, so a value +# the case does not mention is genuinely absent rather than inherited. +run() { + local status=$1 + shift + : >"$argv_file" + : >"$env_file" + RUN_STATUS=0 + RUN_OUTPUT=$( + env -u ZIZMOR_OFFLINE -u ZIZMOR_NO_ONLINE_AUDITS \ + -u GH_TOKEN -u GITHUB_TOKEN -u ZIZMOR_GITHUB_TOKEN -u GH_HOST \ + PATH="$stub_dir:$PATH" \ + ZIZMOR_STUB_ARGV="$argv_file" \ + ZIZMOR_STUB_ENV="$env_file" \ + ZIZMOR_STUB_STATUS="$status" \ + "$@" \ + "$gate" .github/workflows/ 2>&1 + ) || RUN_STATUS=$? + RUN_ARGV=$(cat "$argv_file") + RUN_ENV=$(cat "$env_file") +} + +assert_contains() { # label haystack needle + case $2 in + *"$3"*) ;; + *) fail "$1: expected output to contain '$3', got: $2" ;; + esac +} + +assert_lacks() { # label haystack needle + case $2 in + *"$3"*) fail "$1: expected output NOT to contain '$3', got: $2" ;; + esac +} + +assert_status() { # label expected actual + [[ $2 -eq $3 ]] || fail "$1: expected exit $2, got $3" +} + +ok() { # message + printf 'ok %s\n' "$1" +} + +# -- mode selection from the token variables -- + +# The online argv is pinned exactly, not merely checked for the absence of +# --offline. A negative assertion passes a mutant that adds --no-online-audits, +# --min-severity high, --no-exit-codes, or -o, each of which disables online +# auditing while the gate still announces "online mode" -- unaudited AND labelled +# audited, which the design calls worse than the status quo it replaces. +run 0 GH_TOKEN=t1 +assert_contains 'GH_TOKEN online' "$RUN_OUTPUT" 'online mode; API token from GH_TOKEN' +[[ $RUN_ARGV == '.github/workflows/' ]] || + fail "GH_TOKEN online: expected argv '.github/workflows/', got '$RUN_ARGV'" +ok 'GH_TOKEN selects online and zizmor is invoked with the inputs alone' + +run 0 GITHUB_TOKEN=t2 +assert_contains 'GITHUB_TOKEN online' "$RUN_OUTPUT" 'API token from GITHUB_TOKEN' +ok 'GITHUB_TOKEN selects online' + +run 0 ZIZMOR_GITHUB_TOKEN=t3 +assert_contains 'ZIZMOR_GITHUB_TOKEN online' "$RUN_OUTPUT" 'API token from ZIZMOR_GITHUB_TOKEN' +ok 'ZIZMOR_GITHUB_TOKEN selects online' + +run 0 GH_TOKEN=t1 GITHUB_TOKEN=t2 ZIZMOR_GITHUB_TOKEN=t3 +assert_contains 'precedence' "$RUN_OUTPUT" 'API token from GH_TOKEN' +ok 'GH_TOKEN wins, matching zizmor own documented order' + +run 0 +assert_status 'no token' 0 "$RUN_STATUS" +assert_contains 'no token' "$RUN_OUTPUT" 'offline mode (--offline); no API token' +assert_contains 'no token' "$RUN_OUTPUT" 'GH_TOKEN, GITHUB_TOKEN and ZIZMOR_GITHUB_TOKEN' +assert_contains 'no token' "$RUN_ARGV" '--offline' +ok 'no token gives offline, exit 0, and names all three variables' + +run 0 +assert_contains 'consequence' "$RUN_OUTPUT" 'pin provenance was NOT audited' +ok 'the offline path states the consequence' + +run 0 GH_TOKEN=t1 +assert_lacks 'online consequence' "$RUN_OUTPUT" 'pin provenance was NOT audited' +ok 'the online path does not state the offline consequence' + +# -- an empty token variable is removed, not forwarded -- + +run 0 GH_TOKEN= +assert_status 'empty GH_TOKEN' 0 "$RUN_STATUS" +assert_contains 'empty GH_TOKEN' "$RUN_OUTPUT" 'offline mode (--offline); no API token' +assert_contains 'empty GH_TOKEN' "$RUN_ENV" 'GH_TOKEN=' +assert_lacks 'empty GH_TOKEN' "$RUN_ENV" 'GH_TOKEN=' +ok 'an exported-but-empty GH_TOKEN is removed before zizmor sees it' + +run 0 GH_TOKEN= GITHUB_TOKEN=t2 +assert_contains 'empty then set' "$RUN_OUTPUT" 'API token from GITHUB_TOKEN' +assert_lacks 'empty then set' "$RUN_ENV" 'GH_TOKEN=' +assert_contains 'empty then set' "$RUN_ENV" 'GITHUB_TOKEN=' +ok 'an empty variable is skipped and the next one selects online' + +run 0 GH_TOKEN=t1 +assert_contains 'token forwarded' "$RUN_ENV" 'GH_TOKEN=' +ok 'a real token reaches zizmor unchanged, for zizmor to read itself' + +# All three names, not just GH_TOKEN. Each is separately fatal to zizmor when +# exported empty -- measured on 1.29.0, `GITHUB_TOKEN= zizmor --offline` exits 2 +# with `invalid value '' for '--github-token'`, and ZIZMOR_GITHUB_TOKEN gives the +# same against --zizmor-github-token -- so a removal dropped for one of them +# turns a green tokenless gate into exit 2 after the mode line has printed. +run 0 GITHUB_TOKEN= +assert_status 'empty GITHUB_TOKEN' 0 "$RUN_STATUS" +assert_contains 'empty GITHUB_TOKEN' "$RUN_OUTPUT" 'offline mode (--offline); no API token' +assert_lacks 'empty GITHUB_TOKEN' "$RUN_ENV" 'GITHUB_TOKEN=' +ok 'an exported-but-empty GITHUB_TOKEN is removed before zizmor sees it' + +run 0 ZIZMOR_GITHUB_TOKEN= +assert_status 'empty ZIZMOR_GITHUB_TOKEN' 0 "$RUN_STATUS" +assert_contains 'empty ZIZMOR_GITHUB_TOKEN' "$RUN_OUTPUT" 'offline mode (--offline); no API token' +assert_lacks 'empty ZIZMOR_GITHUB_TOKEN' "$RUN_ENV" 'ZIZMOR_GITHUB_TOKEN=' +ok 'an exported-but-empty ZIZMOR_GITHUB_TOKEN is removed before zizmor sees it' + +run 0 GH_TOKEN= GITHUB_TOKEN= ZIZMOR_GITHUB_TOKEN= +assert_status 'all three empty' 0 "$RUN_STATUS" +assert_contains 'all three empty' "$RUN_OUTPUT" 'offline mode (--offline); no API token' +assert_lacks 'all three empty' "$RUN_ENV" '' +ok 'all three empty at once gives offline at exit 0, with none forwarded' + +# -- the mode variables outrank a token, by value -- + +run 0 ZIZMOR_OFFLINE=true GH_TOKEN=t1 +assert_contains 'ZIZMOR_OFFLINE=true' "$RUN_OUTPUT" 'offline mode (--offline); ZIZMOR_OFFLINE=true' +assert_contains 'ZIZMOR_OFFLINE=true' "$RUN_ARGV" '--offline' +ok 'ZIZMOR_OFFLINE=true beats a present token and is the reported condition' + +run 0 ZIZMOR_NO_ONLINE_AUDITS=true GH_TOKEN=t1 +assert_contains 'ZIZMOR_NO_ONLINE_AUDITS=true' "$RUN_OUTPUT" 'ZIZMOR_NO_ONLINE_AUDITS=true' +ok 'ZIZMOR_NO_ONLINE_AUDITS=true beats a present token' + +# The two offline conditions compete here, and the reported one must be the +# variable rather than the missing token. Getting it backwards would tell an +# operator to export a token, which cannot change the mode while their own +# ZIZMOR_OFFLINE still wins -- a cause the observation does not carry, which is +# the failure ADR 0025 decision 2 forbids and this change exists to remove. +run 0 ZIZMOR_OFFLINE=true +assert_status 'mode var, no token' 0 "$RUN_STATUS" +assert_contains 'mode var, no token' "$RUN_OUTPUT" 'offline mode (--offline); ZIZMOR_OFFLINE=true' +assert_lacks 'mode var, no token' "$RUN_OUTPUT" 'no API token' +ok 'a mode variable is reported ahead of the missing token when both would apply' + +run 0 ZIZMOR_NO_ONLINE_AUDITS=true +assert_contains 'sibling, no token' "$RUN_OUTPUT" 'offline mode (--offline); ZIZMOR_NO_ONLINE_AUDITS=true' +assert_lacks 'sibling, no token' "$RUN_OUTPUT" 'no API token' +ok 'the sibling mode variable is likewise reported ahead of the missing token' + +run 0 ZIZMOR_OFFLINE=true ZIZMOR_NO_ONLINE_AUDITS=true GH_TOKEN=t1 +assert_contains 'both mode vars' "$RUN_OUTPUT" 'ZIZMOR_OFFLINE=true' +assert_lacks 'both mode vars' "$RUN_OUTPUT" 'ZIZMOR_NO_ONLINE_AUDITS=true' +ok 'ZIZMOR_OFFLINE is reported first when both are set' + +run 0 ZIZMOR_OFFLINE=false GH_TOKEN=t1 +assert_contains 'ZIZMOR_OFFLINE=false' "$RUN_OUTPUT" 'online mode; API token from GH_TOKEN' +assert_lacks 'ZIZMOR_OFFLINE=false' "$RUN_ARGV" '--offline' +ok 'ZIZMOR_OFFLINE=false is an online request, not an offline one' + +run 0 ZIZMOR_OFFLINE=false +assert_status 'false without a token' 0 "$RUN_STATUS" +assert_contains 'false without a token' "$RUN_OUTPUT" 'no API token' +ok 'ZIZMOR_OFFLINE=false without a token still reports the no-token condition' + +# -- a malformed mode value warns and selects nothing -- + +run 0 ZIZMOR_OFFLINE=0 +assert_status 'malformed, no token' 0 "$RUN_STATUS" +assert_contains 'malformed, no token' "$RUN_OUTPUT" 'ZIZMOR_OFFLINE=0 is not a value zizmor accepts (true or false); it selects no mode here' +assert_contains 'malformed, no token' "$RUN_OUTPUT" 'no API token' +ok 'a malformed ZIZMOR_OFFLINE warns, then falls through, and zizmor still runs' + +run 0 ZIZMOR_OFFLINE=0 GH_TOKEN=t1 +assert_contains 'malformed with token' "$RUN_OUTPUT" 'is not a value zizmor accepts' +assert_contains 'malformed with token' "$RUN_OUTPUT" 'online mode' +ok 'a malformed value selects nothing, so a token still gives online' + +run 0 ZIZMOR_OFFLINE= +assert_contains 'empty mode var' "$RUN_OUTPUT" 'ZIZMOR_OFFLINE= is not a value zizmor accepts' +ok 'an exported-but-empty mode variable is reported, not treated as absent' + +run 0 ZIZMOR_NO_ONLINE_AUDITS=maybe +assert_contains 'malformed sibling' "$RUN_OUTPUT" 'ZIZMOR_NO_ONLINE_AUDITS=maybe is not a value' +ok 'the sibling mode variable is validated the same way' + +run 0 ZIZMOR_OFFLINE=true +assert_lacks 'valid value quiet' "$RUN_OUTPUT" 'is not a value' +ok 'a valid mode value produces no warning' + +# -- GH_HOST is named on the online line -- + +run 0 GH_TOKEN=t1 GH_HOST=ghe.example.invalid +assert_contains 'GH_HOST named' "$RUN_OUTPUT" 'GH_HOST=ghe.example.invalid' +ok 'GH_HOST is named on the online line' + +run 0 GH_TOKEN=t1 +assert_lacks 'GH_HOST absent' "$RUN_OUTPUT" 'GH_HOST' +ok 'the online line omits the host clause when GH_HOST is unset' + +# An exported-but-empty GH_HOST is a value zizmor uses and fails on, not an +# absent one, so the clause must print for it -- otherwise the one pointing most +# likely to confuse is the one the line stays silent about. +run 0 GH_TOKEN=t1 GH_HOST= +assert_contains 'GH_HOST empty' "$RUN_OUTPUT" 'GH_HOST=' +ok 'an exported-but-empty GH_HOST is still named on the online line' + +# -- the status is re-raised, and the hint fires only on tool failure -- + +run 1 GH_TOKEN=t1 +assert_status 'tool failure' 1 "$RUN_STATUS" +assert_contains 'tool failure' "$RUN_OUTPUT" 'the audit run failed rather than reporting findings' +ok 'a tool failure is re-raised and draws the hint' + +# Status 1 covers an unreachable API, a rejected token, `invalid input: ` +# for a missing directory, and a zizmor.yml that fails to load. The hint must +# therefore not assert which of those happened -- that is the cause-not-carried +# defect ADR 0025 decision 2 forbids, and the one this change exists to remove. +case $RUN_OUTPUT in +*'If it is an API or token fault'*) ;; +*) fail 'hint names no cause: expected the remedy to be offered conditionally' ;; +esac +ok 'the hint offers its remedy conditionally and asserts no cause' + +run 14 GH_TOKEN=t1 +assert_status 'findings' 14 "$RUN_STATUS" +assert_lacks 'findings' "$RUN_OUTPUT" 'the audit run failed rather than reporting findings' +ok 'findings are re-raised with no hint -- never advise disabling the audit' + +run 2 GH_TOKEN=t1 +assert_status 'usage error' 2 "$RUN_STATUS" +assert_lacks 'usage error' "$RUN_OUTPUT" 'the audit run failed rather than reporting findings' +ok 'a usage error is re-raised with no hint' + +run 1 +assert_status 'offline failure' 1 "$RUN_STATUS" +assert_lacks 'offline failure' "$RUN_OUTPUT" 'the audit run failed rather than reporting findings' +ok 'the hint never fires on the offline path' + +run 0 GH_TOKEN=t1 +assert_status 'clean online' 0 "$RUN_STATUS" +assert_lacks 'clean online' "$RUN_OUTPUT" 'the audit run failed rather than reporting findings' +ok 'a clean online run draws no hint' + +# -- the token value never reaches the output -- + +run 1 GH_TOKEN=s3cret-token-value +assert_lacks 'no token leak' "$RUN_OUTPUT" 's3cret-token-value' +ok 'the token value appears nowhere in the output, even on a failure' + +# -- inputs and usage -- + +RUN_STATUS=0 +RUN_OUTPUT=$( + env -u ZIZMOR_OFFLINE -u ZIZMOR_NO_ONLINE_AUDITS \ + -u GH_TOKEN -u GITHUB_TOKEN -u ZIZMOR_GITHUB_TOKEN \ + PATH="$stub_dir:$PATH" ZIZMOR_STUB_ARGV="$argv_file" \ + ZIZMOR_STUB_ENV="$env_file" "$gate" 2>&1 +) || RUN_STATUS=$? +assert_status 'no arguments' 2 "$RUN_STATUS" +assert_contains 'no arguments' "$RUN_OUTPUT" 'usage: run-zizmor.sh' +ok 'no arguments is a usage fault at exit 2' + +# A mode flag on argv beats the mode this script resolved and announced, so the +# script refuses one rather than printing a line that the run then contradicts. +for flag in --offline --no-online-audits -o --min-severity; do + RUN_STATUS=0 + : >"$argv_file" + RUN_OUTPUT=$( + env -u ZIZMOR_OFFLINE -u ZIZMOR_NO_ONLINE_AUDITS \ + -u GH_TOKEN -u GITHUB_TOKEN -u ZIZMOR_GITHUB_TOKEN \ + PATH="$stub_dir:$PATH" ZIZMOR_STUB_ARGV="$argv_file" \ + ZIZMOR_STUB_ENV="$env_file" ZIZMOR_STUB_STATUS=0 \ + "$gate" "$flag" .github/workflows/ 2>&1 + ) || RUN_STATUS=$? + assert_status "flag $flag" 2 "$RUN_STATUS" + assert_contains "flag $flag" "$RUN_OUTPUT" 'usage: run-zizmor.sh' + [[ -z $(cat "$argv_file") ]] || + fail "flag $flag: zizmor was invoked despite the usage fault" + assert_lacks "flag $flag" "$RUN_OUTPUT" 'online mode' + assert_lacks "flag $flag" "$RUN_OUTPUT" 'offline mode' +done +ok 'a mode flag among the inputs is refused at exit 2, before any mode is announced' + +RUN_STATUS=0 +: >"$argv_file" +RUN_OUTPUT=$( + env -u ZIZMOR_OFFLINE -u ZIZMOR_NO_ONLINE_AUDITS \ + -u GH_TOKEN -u GITHUB_TOKEN -u ZIZMOR_GITHUB_TOKEN \ + PATH="$stub_dir:$PATH" ZIZMOR_STUB_ARGV="$argv_file" \ + ZIZMOR_STUB_ENV="$env_file" ZIZMOR_STUB_STATUS=0 \ + "$gate" one/ two/ 2>&1 +) || RUN_STATUS=$? +RUN_ARGV=$(cat "$argv_file") +[[ $RUN_ARGV == '--offline one/ two/' ]] || + fail "forwarding: expected '--offline one/ two/', got '$RUN_ARGV'" +ok 'every input is forwarded, in order, after the mode flag' + +RUN_STATUS=0 +: >"$argv_file" +RUN_OUTPUT=$( + env -u ZIZMOR_OFFLINE -u ZIZMOR_NO_ONLINE_AUDITS \ + -u GITHUB_TOKEN -u ZIZMOR_GITHUB_TOKEN -u GH_HOST \ + PATH="$stub_dir:$PATH" ZIZMOR_STUB_ARGV="$argv_file" \ + ZIZMOR_STUB_ENV="$env_file" ZIZMOR_STUB_STATUS=0 \ + GH_TOKEN=t1 "$gate" one/ two/ 2>&1 +) || RUN_STATUS=$? +RUN_ARGV=$(cat "$argv_file") +[[ $RUN_ARGV == 'one/ two/' ]] || + fail "online forwarding: expected 'one/ two/', got '$RUN_ARGV'" +ok 'the online path forwards the inputs with no flag of any kind' + +printf 'run-zizmor-test: all cases passed\n' diff --git a/scripts/run-zizmor.sh b/scripts/run-zizmor.sh new file mode 100755 index 0000000..9ccb8b4 --- /dev/null +++ b/scripts/run-zizmor.sh @@ -0,0 +1,193 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Runs zizmor over the inputs it is given, in the strongest mode this environment +# supports, and states which mode that was and the condition that chose it before +# the scan starts. +# +# The gate exists because zizmor's own answer is invisible. Without a token it +# degrades to the offline subset announcing only ` WARN audit: zizmor: zizmor is +# running in offline mode by default` among its INFO lines -- and that WARN does +# not appear at all when ZIZMOR_OFFLINE or ZIZMOR_NO_ONLINE_AUDITS set the mode +# explicitly. Offline mode confirms a pin's shape; the audits it disables are the +# ones that confirm its provenance, so a run that skipped them and a run that +# passed them looked the same from here. See ADR 0036, and ADR 0025 for the rule +# that a degrade reports the condition it observed. +# +# Five environment variables decide zizmor's mode and this script reads all five, +# so the mode it announces is the mode zizmor chooses rather than a second opinion +# that can drift from it. Reading only the token names would announce "online" over +# a run ZIZMOR_OFFLINE had already turned offline. +# +# The two mode controls are clap booleans -- the VALUE is the instruction, not the +# variable's presence, so ZIZMOR_OFFLINE=false is a request for online mode. A +# value that is neither `true` nor `false` selects nothing: it is reported and +# ignored, and zizmor's own parser has the last word. Exiting on it instead would +# refuse in cases where zizmor runs happily, because an explicit --offline on argv +# shadows ZIZMOR_OFFLINE entirely (measured on 1.29.0: `ZIZMOR_OFFLINE=0 zizmor +# --offline` exits 0, the same value with no flag exits 2). +# +# The token's value is never read. Emptiness is all this script tests, and zizmor +# reads the value itself from the environment it inherits, so no credential passes +# through here, through this script's argv, or into its output. An exported-but- +# empty token variable is removed instead, because zizmor rejects one as a usage +# error even with --offline on argv -- that removal is what makes "an empty value +# is not a token" true rather than merely asserted. +# +# Everything above about zizmor's CLI was measured on 1.29.0, and CI installs +# zizmor unpinned. A zizmor that renames a variable makes the mode line disagree +# with the run, which is loud; one that renumbers its exit statuses makes the +# online-failure hint stop appearing, which is silent and harmless. +# +# Exit 2 on usage, otherwise zizmor's own status, re-raised unchanged. + +LABEL='run-zizmor' + +say() { + printf '%s: %s\n' "$LABEL" "$*" +} + +if (($# == 0)); then + printf '%s: usage: run-zizmor.sh ...\n' "$LABEL" >&2 + exit 2 +fi + +# Inputs only. A zizmor mode flag passed here would reach zizmor on argv after the +# mode line had already been printed, and argv wins: measured on 1.29.0, +# `GH_TOKEN= run-zizmor.sh --offline .github/workflows/` announces "online +# mode" and then skips all five provenance audits -- unaudited and labelled +# audited, which is the state this script exists to prevent. The gate itself only +# ever passes local paths, so rejecting every `-*` costs nothing; zizmor's `-` +# stdin form is not used here either. +for argument in "$@"; do + case $argument in + -*) + printf '%s: usage: run-zizmor.sh ...; the mode flags are this script'"'"'s to choose, not the caller'"'"'s: %s\n' \ + "$LABEL" "$argument" >&2 + exit 2 + ;; + esac +done + +# Returns 0 when the value selects offline, 1 when it does not -- including the +# malformed case, which warns first. +# +# The warning says "selects no mode here" rather than "ignoring it" because +# ignoring is not something this gate can promise. An explicit --offline on argv +# shadows ZIZMOR_OFFLINE, so a malformed value there really is inert; nothing +# shadows --no-online-audits, so a malformed ZIZMOR_NO_ONLINE_AUDITS still makes +# zizmor exit 2 after this line has been printed. One wording that is true of both +# beats a per-variable branch for the sake of one word. +selects_offline() { # name value + case $2 in + true) return 0 ;; + false) return 1 ;; + *) + printf '%s: %s=%s is not a value zizmor accepts (true or false); it selects no mode here, and zizmor may still reject it\n' \ + "$LABEL" "$1" "$2" >&2 + return 1 + ;; + esac +} + +mode='online' +condition='' + +# ${VAR+set} rather than ${VAR:-}: an exported-but-empty mode variable is a value +# zizmor will reject, not an absent one, so it has to reach selects_offline to be +# reported. +if [[ -n ${ZIZMOR_OFFLINE+set} ]]; then + if selects_offline ZIZMOR_OFFLINE "$ZIZMOR_OFFLINE"; then + mode='offline' + condition="ZIZMOR_OFFLINE=$ZIZMOR_OFFLINE" + fi +fi + +if [[ $mode == online && -n ${ZIZMOR_NO_ONLINE_AUDITS+set} ]]; then + if selects_offline ZIZMOR_NO_ONLINE_AUDITS "$ZIZMOR_NO_ONLINE_AUDITS"; then + mode='offline' + condition="ZIZMOR_NO_ONLINE_AUDITS=$ZIZMOR_NO_ONLINE_AUDITS" + fi +fi + +# Spelled out three times rather than looped through indirect expansion: three +# names, and the explicit form is the one bash 3.2 cannot misread. +if [[ -n ${GH_TOKEN+set} && -z $GH_TOKEN ]]; then + unset GH_TOKEN +fi +if [[ -n ${GITHUB_TOKEN+set} && -z $GITHUB_TOKEN ]]; then + unset GITHUB_TOKEN +fi +if [[ -n ${ZIZMOR_GITHUB_TOKEN+set} && -z $ZIZMOR_GITHUB_TOKEN ]]; then + unset ZIZMOR_GITHUB_TOKEN +fi + +# zizmor's own order for --gh-token: [env: GH_TOKEN or GITHUB_TOKEN or +# ZIZMOR_GITHUB_TOKEN]. +token_source='' +if [[ -n ${GH_TOKEN:-} ]]; then + token_source='GH_TOKEN' +elif [[ -n ${GITHUB_TOKEN:-} ]]; then + token_source='GITHUB_TOKEN' +elif [[ -n ${ZIZMOR_GITHUB_TOKEN:-} ]]; then + token_source='ZIZMOR_GITHUB_TOKEN' +fi + +if [[ $mode == online && -z $token_source ]]; then + mode='offline' + condition='no API token: GH_TOKEN, GITHUB_TOKEN and ZIZMOR_GITHUB_TOKEN are all unset or empty' +fi + +if [[ $mode == online ]]; then + # GH_HOST decides which API an online run talks to, and zizmor honours it as + # --gh-hostname. Naming it here puts the destination in front of the operator + # before the scan, rather than only inside a failure's causal chain. + # + # ${VAR+set}, for the reason given above: an exported-but-empty GH_HOST is a + # value zizmor uses and fails on -- it builds https:///... -- not an absent + # one, so the clause has to print for it too. Reading it as ${GH_HOST:-} would + # stay silent about the one pointing most likely to confuse. + if [[ -n ${GH_HOST+set} ]]; then + say "online mode; API token from $token_source; GH_HOST=$GH_HOST" + else + say "online mode; API token from $token_source" + fi +else + say "offline mode (--offline); $condition" + say 'pin provenance was NOT audited: a well-formed 40-character SHA that is unreachable in the repository its uses: names, or that a known advisory covers, passes this run' +fi + +# Captured explicitly and re-raised. A pipe here would report the last command's +# status and hide zizmor's, and `|| true` would make a scan that could not run +# read as one that found nothing. +status=0 +if [[ $mode == online ]]; then + zizmor "$@" || status=$? +else + zizmor --offline "$@" || status=$? +fi + +# Only zizmor's tool-failure status, never "non-zero". Findings exit 14, and +# offering this there would advise switching off the audit that just caught +# something -- a documented route to a green gate over an unaudited pin. A usage +# error (2) is not offered it either: the offline subset does not fix a malformed +# variable. +# +# The line reports what status 1 carries and no cause it does not. All 1 tells us +# is that the run failed instead of reporting findings: an unreachable API and a +# rejected token both land here, but so does `invalid input: ` for a +# missing directory and a zizmor.yml that fails to load, and for those two +# ZIZMOR_OFFLINE=true fixes nothing. Naming a cause the observation does not carry +# is what ADR 0025 decision 2 forbids and what this whole change exists to remove, +# so the remedy is offered conditionally and zizmor's own error is pointed at. +# +# The remedy is scoped to a local run because CI is where a token is always set +# and so where status 1 is most likely. On a runner the only way to act on +# ZIZMOR_OFFLINE=true is to edit verify.yml or the Justfile, turning the five +# provenance audits off for good -- on a red required check, where the pressure to +# go green is highest. That is the status-14 mistake one step removed. +if [[ $mode == online ]] && ((status == 1)); then + say 'the audit run failed rather than reporting findings; zizmor own error is above. If it is an API or token fault, a local offline run is ZIZMOR_OFFLINE=true' >&2 +fi + +exit "$status"