Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 59 additions & 21 deletions .github/workflows/daily-release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,10 @@ jobs:
# its own RELEASE_OPEN_TOKEN and no required reviewers; before that the nightly
# fails loudly at the opener step instead of opening its pull request.
environment: release
timeout-minutes: 20
# #864: the decide step re-reads `ci-ok` for up to ten minutes when CI is
# still running, so the job keeps room for the waits plus the push, the
# opener, auto-merge and the comment that follow.
timeout-minutes: 30
permissions:
contents: write
pull-requests: read
Expand Down Expand Up @@ -198,40 +201,75 @@ jobs:
# Same rule as Plan: the drive reads the state and the plan files, so a
# failed collect step must be reported once, not again as a missing file.
if: ${{ steps.state.outcome == 'success' }}
# #864: the step may sit through the bounded `ci-ok` re-read (three reads
# five minutes apart), so it states a budget of its own rather than
# inheriting the job's.
timeout-minutes: 15
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
MAIN_SHA: ${{ steps.state.outputs.main_sha }}
run: |
set -euo pipefail
# Rule 4 and the dry-run reduction live in the tested drive module. The
# pull-request half of the drive is #187's, so this job reports it and
# acts only on `open-pr`, which is the branch push below.
node --input-type=module -e '
import { readFileSync, writeFileSync, appendFileSync } from "node:fs"
import { decideDrive } from "./scripts/lib/release-drive.mjs"
import { execFileSync } from "node:child_process"
import { decideWithCiRetry } from "./scripts/lib/release-drive.mjs"
import { latestConclusion, latestRun } from "./scripts/lib/release-state.mjs"
const TMP = process.env.RUNNER_TEMP ?? "/tmp"
const state = JSON.parse(readFileSync(TMP + "/state.json", "utf8"))
const plan = JSON.parse(readFileSync(TMP + "/plan.json", "utf8"))
const MAIN_SHA = String(process.env.MAIN_SHA ?? "")
const REPO = String(process.env.GITHUB_REPOSITORY ?? "")
// #864: the collect step reads `ci-ok` once, and a run that starts while
// CI is still in flight sees `missing`. The drive asks for a bounded
// re-read instead of treating that as a quiet night. The re-read names
// the same tip the plan was computed from — polling `commits/main`
// could judge a commit the release branch was never cut from — and
// asks for the same `check_name`, so a busy tip cannot page the
// verdict away (#567).
let last = { ciOk: state.ciOk, ciOkRun: state.ciOkRun }
const readCiOk = async () => {
const text = execFileSync("gh", ["api", `repos/${REPO}/commits/${MAIN_SHA}/check-runs?per_page=100&check_name=ci-ok`], { encoding: "utf8" })
const parsed = JSON.parse(text)
last = { ciOk: latestConclusion(parsed, "ci-ok"), ciOkRun: latestRun(parsed, "ci-ok") }
return last
}
const open = state.openPr
const decision = decideDrive({
plan: { shouldRelease: plan.shouldRelease, nextVersion: plan.nextVersion, reason: plan.reason },
openReleasePr: open === null || open === undefined
? null
: {
number: Number(open.number ?? 0),
headSha: String(open.head?.sha ?? ""),
branch: String(open.head?.ref ?? ""),
title: String(open.title ?? ""),
files: [],
blocked: false,
checks: [],
sameRepo: open.head?.repo?.full_name === open.base?.repo?.full_name,
},
mergedPrs: [],
tags: state.tags,
orphanBranches: state.orphanBranches,
ciOk: state.ciOk,
dryRun: state.dryRun,
const decision = await decideWithCiRetry({
state: {
plan: { shouldRelease: plan.shouldRelease, nextVersion: plan.nextVersion, reason: plan.reason },
openReleasePr: open === null || open === undefined
? null
: {
number: Number(open.number ?? 0),
headSha: String(open.head?.sha ?? ""),
branch: String(open.head?.ref ?? ""),
title: String(open.title ?? ""),
files: [],
blocked: false,
checks: [],
sameRepo: open.head?.repo?.full_name === open.base?.repo?.full_name,
},
mergedPrs: [],
tags: state.tags,
orphanBranches: state.orphanBranches,
ciOk: state.ciOk,
ciOkRun: state.ciOkRun,
dryRun: state.dryRun,
},
readCiOk,
wait: (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
})
// The re-read can leave the file behind the fact it established: the
// summary and the #80 comment read `.ciOk` from here, and reporting
// `missing` beside a released branch would be the old confusion again.
writeFileSync(TMP + "/state.json", `${JSON.stringify({ ...state, ciOk: last.ciOk, ciOkRun: last.ciOkRun })}\n`)
writeFileSync(TMP + "/drive.json", `${JSON.stringify(decision)}\n`)
appendFileSync(process.env.GITHUB_OUTPUT, `action=${decision.action}\nreason=${decision.reason}\nversion=${decision.version ?? ""}\ntag=${decision.tag ?? ""}\n`)
appendFileSync(process.env.GITHUB_STEP_SUMMARY, `- ci-ok on main at \`${MAIN_SHA}\`: **${String(last.ciOk)}** (run ${String(last.ciOkRun?.id ?? 0)}, ${String(last.ciOkRun?.status ?? "unknown")}) after ${String(decision.ciOkReads ?? 1)} read(s)\n`)
appendFileSync(process.env.GITHUB_STEP_SUMMARY, `- action: **${decision.action}** — ${decision.reason}\n`)
console.log(JSON.stringify(decision))
// An orphan branch is a wedged slot, not a quiet night: the run is red
Expand Down
1 change: 1 addition & 0 deletions changelog.d/864-release-waits-for-ci-ok.fixed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
The daily release waits for the `ci-ok` verdict instead of reading it once: a run whose aggregate is still in progress is re-read at the same tip up to three times, five minutes apart, so a green tip that was still running no longer loses the slot, while a red aggregate still blocks at once. The release summary names the `ci-ok` run id, its status and how many reads were made.
15 changes: 15 additions & 0 deletions docs/deploy.md
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,21 @@ release in the same week bumps the patch (`0.4.0 -> 0.4.1`). The planner reads
the last tag's commit time and the workflow's run time; if either is unreadable
it falls back to the minor, which is the first-release behaviour.

#### The `ci-ok` read waits for the verdict (#864)

Rule 4 releases only on a green `ci-ok`, and a check run that is still queued or
running reports no conclusion. The planner used to read the aggregate once, so a
run that started at the same moment as `ci.yml` saw `missing` and ended the slot
with `none`; `0.3.7` was refused that way. `scripts/lib/release-drive.mjs` now
keeps the two facts apart: `failure` (and any other concluded non-success) still
blocks at once, while `missing` returns `wait-ci` and the decide step re-reads
`ci-ok` at the *same* tip the plan was computed from — three reads, five minutes
apart, so at most ten minutes of waiting. The bound is in the module
(`CI_OK_READS`, `CI_OK_INTERVAL_MS`), the step summary prints the run id, the
status and how many reads were made, and a tip that never concludes ends the slot
with `none` and that run id in the reason. A dry run never waits: it reports what
a real run would have done.

#### Reliable slot trigger

GitHub's `schedule` event is best-effort: on 2026-11-14 the then-`14:30Z` slot
Expand Down
25 changes: 24 additions & 1 deletion scripts/lib/release-drive.d.mts
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,14 @@ export interface DriveState {
lastMergedPr?: { number: number; title: string; headRef: string } | null
tags?: string[]
ciOk?: string
/** The newest `ci-ok` run: the id tells "not yet" from "red" (#864). */
ciOkRun?: { id?: number; status?: string; conclusion?: string | null }
dryRun?: boolean
orphanBranches?: Array<{ branch?: string; version?: string; tag?: string; sha?: string }>
}

export interface DriveDecision {
action: 'open-pr' | 'enqueue' | 'report-blocked' | 'tag' | 'none'
action: 'open-pr' | 'enqueue' | 'report-blocked' | 'tag' | 'wait-ci' | 'none'
reason: string
pr?: number
headSha?: string
Expand All @@ -55,6 +57,27 @@ export interface DriveDecision {
branch?: string
dryRun?: boolean
wouldHave?: string
ciOkRunId?: number
ciOkReads?: number
ciOkWaitedMs?: number
}

export interface CiOkObservation {
ciOk?: string
ciOkRun?: { id?: number; status?: string; conclusion?: string | null }
}

/** Reads before the `ci-ok` re-read gives up, and the pause between them (#864). */
export declare const CI_OK_READS: number
export declare const CI_OK_INTERVAL_MS: number

export declare function decideDrive(state: DriveState): DriveDecision

export declare function decideWithCiRetry(input: {
state: DriveState
decide?: (state: DriveState) => DriveDecision
readCiOk: () => Promise<CiOkObservation | null | undefined>
wait: (ms: number) => Promise<void>
attempts?: number
intervalMs?: number
}): Promise<DriveDecision>
97 changes: 94 additions & 3 deletions scripts/lib/release-drive.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@
// branch is the same action with a different reason: it names
// the branch and its sha, and the workflow turns it red (#333).
// tag a merged release PR has no tag yet: tag its squash commit
// wait-ci the rule is met but `ci-ok` on main has not concluded yet: the
// caller re-reads it on a bounded schedule and decides again
// none no release tonight; the counts go to the summary
//
// A dry run mutates nothing: `workflow_dispatch` defaults `dry_run: true`, so a
Expand Down Expand Up @@ -41,6 +43,15 @@ function isMechanicalReleasePr(openReleasePr) {
return identity.mechanical ? { ok: true, reason: identity.reason, version: identity.version } : { ok: false, reason: identity.reason }
}

// A conclusion that means "not decided yet", never "red": a queued or running
// check run answers `null` for its conclusion, and `latestRun` reports that — and
// "no run of this name at all" — as `missing`.
const NOT_CONCLUDED = new Set(['missing', 'unknown', 'pending', ''])

function ciNotConcluded(conclusion) {
return NOT_CONCLUDED.has(conclusion)
}

function failingChecks(checks) {
const list = Array.isArray(checks) ? checks : []
return list.filter((check) => {
Expand Down Expand Up @@ -155,21 +166,54 @@ function decide(state) {
if (plan?.shouldRelease !== true) {
return { action: 'none', reason: `the rule is not met: ${String(plan?.reason ?? 'no plan')}` }
}
if (ciOk !== 'success') {
return { action: 'none', reason: `the latest ci-ok on main is ${String(ciOk === undefined || ciOk === null ? 'unknown' : ciOk)}` }

// Rule 4: only a green `ci-ok` releases. What is *not* green is two facts, not
// one (#864): a run that has not concluded yet is `missing` — CI was still
// running when the check-runs request was made — while `failure` is a red tip.
// Folding the first into `none` is how the 0.3.7 slot was refused; it comes
// back as `wait-ci`, and the caller re-reads on a bounded schedule.
const conclusion = String(ciOk ?? 'unknown')
const run = state?.ciOkRun ?? {}
const runId = Number(run?.id ?? 0)
const runNote = Number.isSafeInteger(runId) && runId > 0
? ` (run ${String(runId)}${typeof run?.status === 'string' && run.status !== '' ? `, ${run.status}` : ''})`
: ' (no ci-ok run found)'
if (ciNotConcluded(conclusion)) {
return {
action: 'wait-ci',
reason: `the latest ci-ok on main is ${conclusion === '' ? 'missing' : conclusion}${runNote}: CI has not concluded yet, so the release waits and re-reads`,
ciOkRunId: runId,
}
}
if (conclusion !== 'success') {
return {
action: 'none',
reason: `the latest ci-ok on main is ${conclusion}${runNote}: a concluded failure blocks the release`,
ciOkRunId: runId,
}
}
return { action: 'open-pr', reason: 'the rule is met: open the mechanical release PR', version: String(plan?.nextVersion ?? '') }
}

// A dry run reports, it does not act. `workflow_dispatch` defaults `dry_run` to
// true, so this is the default path for a manual dispatch: returning `enqueue`
// or `tag` from it would set auto-merge or publish on the first click.
// or `tag` from it would set auto-merge or publish on the first click. A dry run
// does not wait either: sitting on a runner for ten minutes to report a fact it
// already holds would only delay the summary.
const MUTATING = new Set(['open-pr', 'enqueue', 'tag'])

export function decideDrive(state) {
const decision = decide(state)
const dryRun = state?.dryRun === true || state?.dryRun === 'true'
if (!dryRun) return decision
if (decision.action === 'wait-ci') {
return {
action: 'none',
reason: `dry run: would have waited for ci-ok — ${String(decision.reason)}`,
dryRun: true,
wouldHave: 'wait-ci',
}
}
if (MUTATING.has(String(decision.action))) {
return {
action: 'none',
Expand All @@ -184,3 +228,50 @@ export function decideDrive(state) {
}
return { ...decision, dryRun: true }
}

// The bound on the re-read (#864): three reads in all — the first is the one the
// workflow's collect step already made — five minutes apart, so the decide step
// sleeps at most ten minutes of a twenty-minute slot. A reader that is injected
// and a wait that is injected keep the loop testable without a clock, which is
// how the `missing -> success` transition is asserted rather than approximated.
export const CI_OK_READS = 3
export const CI_OK_INTERVAL_MS = 5 * 60 * 1000

/**
* Decide, and when the only thing missing is a conclusion for `ci-ok`, wait and
* read it again until the bound. Only `wait-ci` re-reads: a `failure` is a
* decision, and no amount of waiting changes a red tip.
*
* `readCiOk` answers the same shape the state carries (`{ ciOk, ciOkRun }`), so
* the caller decides where the answer comes from — the checks API in the
* workflow, a fixture in the suite.
*/
export async function decideWithCiRetry({ state, decide = decideDrive, readCiOk, wait, attempts = CI_OK_READS, intervalMs = CI_OK_INTERVAL_MS }) {
if (typeof readCiOk !== 'function') throw new TypeError('decideWithCiRetry needs readCiOk()')
if (typeof wait !== 'function') throw new TypeError('decideWithCiRetry needs wait(ms)')
const total = Number.isSafeInteger(attempts) && attempts > 0 ? attempts : CI_OK_READS
const paused = Number.isFinite(intervalMs) && intervalMs >= 0 ? intervalMs : CI_OK_INTERVAL_MS
let current = state
let reads = 1
let decision = decide(current)
while (decision.action === 'wait-ci' && reads < total) {
await wait(paused)
const observed = await readCiOk()
reads += 1
current = { ...current, ciOk: String(observed?.ciOk ?? 'missing'), ...(observed?.ciOkRun === undefined ? {} : { ciOkRun: observed.ciOkRun }) }
decision = decide(current)
}
if (decision.action !== 'wait-ci') return { ...decision, ciOkReads: reads, ciOkWaitedMs: paused * (reads - 1) }
// The bound is a conclusion too: name what the last read saw and how many
// reads produced it, so an operator can tell "CI never finished" from "the
// reader never looked" without opening the run.
const minutes = Math.round((paused * (total - 1)) / 60_000)
const lastRunId = Number(current?.ciOkRun?.id ?? 0)
const lastNote = Number.isSafeInteger(lastRunId) && lastRunId > 0 ? ` (run ${String(lastRunId)})` : ''
return {
action: 'none',
reason: `the latest ci-ok on main is still ${String(current?.ciOk ?? 'missing')}${lastNote} after ${String(total)} reads over ${String(minutes)} minutes, so no release this slot`,
ciOkReads: reads,
ciOkWaitedMs: paused * (reads - 1),
}
}
9 changes: 9 additions & 0 deletions scripts/lib/release-state.d.mts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,12 @@ export interface ReleaseBranchEntry {
commit?: { sha?: string }
}

export interface CiOkObservation {
id: number
status: string
conclusion: string
}

export interface OrphanBranch {
branch: string
version: string
Expand All @@ -34,6 +40,8 @@ export interface ReleaseState {
tags: string[]
dryRun: boolean
ciOk: string
/** The newest `ci-ok` run, with the id and status the drive's wait needs (#864). */
ciOkRun: CiOkObservation
openPr: PullRequestPageEntry | null
openPrRejected: { number: number; title: string; branch: string; repo: string } | null
lastMerged: PullRequestPageEntry | null
Expand All @@ -44,6 +52,7 @@ export interface ReleaseState {

export declare function flattenPages(value: unknown): PullRequestPageEntry[]
export declare function checkRunsOf(value: unknown): CheckRunEntry[]
export declare function latestRun(value: unknown, name: string): CiOkObservation
export declare function latestConclusion(value: unknown, name: string): string
export declare function buildState(input: {
lastTag?: string
Expand Down
27 changes: 23 additions & 4 deletions scripts/lib/release-state.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,26 @@ export function checkRunsOf(value) {

// The newest run by id, not the first in the list: `first` read the oldest run of
// that name, which after a re-run is a stale conclusion.
export function latestConclusion(value, name) {
//
// The id and the status travel with the conclusion because the release drive has
// to tell "not finished yet" from "red" (#864): a conclusion of `missing` on a
// run that is still `in_progress` is a re-read, while a `failure` is a block.
// The run id is what makes that distinction checkable after the fact.
export function latestRun(value, name) {
const runs = checkRunsOf(value).filter((run) => String(run?.name ?? '') === name)
if (runs.length === 0) return 'missing'
if (runs.length === 0) return { id: 0, status: '', conclusion: 'missing' }
const newest = runs.reduce((best, run) => (Number(run?.id ?? 0) >= Number(best?.id ?? 0) ? run : best))
return String(newest?.conclusion ?? 'missing')
const conclusion = typeof newest?.conclusion === 'string' && newest.conclusion !== '' ? newest.conclusion : 'missing'
const id = Number(newest?.id ?? 0)
return {
id: Number.isSafeInteger(id) && id > 0 ? id : 0,
status: String(newest?.status ?? ''),
conclusion,
}
}

export function latestConclusion(value, name) {
return latestRun(value, name).conclusion
}

function mergedAt(pr) {
Expand Down Expand Up @@ -88,14 +103,18 @@ export function buildState({ lastTag, lastTagCommittedAt, packageVersion, fragme
})
.filter((entry) => entry !== null)
.filter((entry) => !pullRequestHeads.has(entry.branch) && !tagNames.includes(entry.tag))
// One read of the check-runs answer, shared by the summary line and the drive's
// wait decision, so the two can never disagree about which run they mean.
const ciOkRun = latestRun(checks, 'ci-ok')
return {
lastTag: String(lastTag ?? ''),
lastTagCommittedAt: String(lastTagCommittedAt ?? ''),
packageVersion: String(packageVersion ?? ''),
fragments: Number(fragments ?? 0),
tags: Array.isArray(tags) ? tags.map(String) : [],
dryRun: dryRun === true || dryRun === 'true',
ciOk: latestConclusion(checks, 'ci-ok'),
ciOk: ciOkRun.conclusion,
ciOkRun,
openPr: candidate,
openPrRejected: rejected === null
? null
Expand Down
Loading
Loading