From af84dd5996f5510627f640be18732aacf0be1510 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 28 Aug 2026 09:01:15 -0700 Subject: [PATCH] fix(release): reject rebuilds of existing versions --- .github/scripts/release-preflight.cjs | 27 +++++ .github/scripts/release-preflight.test.cjs | 81 +++++++++++++ .github/workflows/ci.yml | 1 + .github/workflows/release-unified.yml | 23 ++++ CHANGELOG.md | 1 + Makefile | 8 +- docs/publishing.md | 39 +++++-- docs/release-31838411168.md | 126 +++++++++++++++++++++ 8 files changed, 294 insertions(+), 12 deletions(-) create mode 100644 .github/scripts/release-preflight.cjs create mode 100644 .github/scripts/release-preflight.test.cjs create mode 100644 docs/release-31838411168.md diff --git a/.github/scripts/release-preflight.cjs b/.github/scripts/release-preflight.cjs new file mode 100644 index 0000000..02bf31d --- /dev/null +++ b/.github/scripts/release-preflight.cjs @@ -0,0 +1,27 @@ +module.exports = async function preflight({ github, context, version }) { + // Match the reusable workflow's version canonicalization before any lookup. + if (typeof version !== 'string' || version !== version.trim() || !/^v?[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$/.test(version)) { + throw new Error('version must be SemVer, with optional leading v'); + } + const tag = `v${version.replace(/^v/, '')}`; + const requireAbsent = async (lookup, kind) => { + try { + await lookup(); + } catch (error) { + if (error.status === 404) return; + throw error; + } + throw new Error(`${tag} already has a ${kind}; refusing to rebuild an existing release version. Use a new patch version, or reconcile the original run without rebuilding its verified payload.`); + }; + + // Tags are public Go module versions even without a published GitHub release. + // The shared workflow creates the tag before creating any draft or assets. + await requireAbsent(() => github.rest.git.getRef({ + ...context.repo, + ref: `tags/${tag}`, + }), 'Git tag'); + await requireAbsent(() => github.rest.repos.getReleaseByTag({ + ...context.repo, + tag, + }), 'GitHub release'); +}; diff --git a/.github/scripts/release-preflight.test.cjs b/.github/scripts/release-preflight.test.cjs new file mode 100644 index 0000000..31fe480 --- /dev/null +++ b/.github/scripts/release-preflight.test.cjs @@ -0,0 +1,81 @@ +const assert = require('node:assert/strict'); +const { readFileSync } = require('node:fs'); +const { test } = require('node:test'); +const preflight = require('./release-preflight.cjs'); + +const context = { repo: { owner: 'openclaw', repo: 'crawlkit' } }; +const missing = () => { throw Object.assign(new Error('Not Found'), { status: 404 }); }; + +function fixture({ tag = missing, release = missing } = {}) { + const calls = []; + return { + calls, + // Only read endpoints exist: mutations are never part of preflight. + github: { rest: { + git: { getRef: async (args) => { calls.push(['tag', args]); return tag(); } }, + repos: { getReleaseByTag: async (args) => { calls.push(['release', args]); return release(); } }, + } }, + }; +} + +for (const version of ['0.14.8', 'v0.14.8', '0.14.8-rc.1', 'v0.14.8+build.1']) { + test(`unused ${version} reaches the shared release workflow`, async () => { + const { github, calls } = fixture(); + await preflight({ github, context, version }); + const tag = `v${version.replace(/^v/, '')}`; + assert.deepEqual(calls, [ + ['tag', { ...context.repo, ref: `tags/${tag}` }], + ['release', { ...context.repo, tag }], + ]); + }); +} + +for (const kind of ['tag', 'commit']) { + test(`existing ${kind} stops before rebuilding, even without a GitHub release`, async () => { + const { github, calls } = fixture({ tag: () => ({ data: { object: { type: kind } } }) }); + await assert.rejects(preflight({ github, context, version: 'v0.14.7' }), /v0\.14\.7 already has a Git tag/); + assert.equal(calls.length, 1); + }); +} + +for (const draft of [false, true]) { + test(`existing ${draft ? 'draft' : 'public release'} blocks even if its tag is missing`, async () => { + const { github } = fixture({ release: () => ({ data: { id: 123, draft } }) }); + await assert.rejects(preflight({ github, context, version: '0.14.7' }), /already has a GitHub release/); + }); +} + +test('the halted v0.14.7 state cannot reach build or draft creation', async () => { + const { github, calls } = fixture({ + tag: () => ({ data: { object: { type: 'tag', sha: 'f0b0ee206d874feea0304d076246e2ae9277bb9c' } } }), + release: () => ({ data: { id: 370802867, draft: false } }), + }); + await assert.rejects(preflight({ github, context, version: '0.14.7' }), /refusing to rebuild/); + assert.equal(calls.length, 1); +}); + +for (const endpoint of ['tag', 'release']) { + for (const status of [401, 403, 429, 500]) { + test(`${endpoint} HTTP ${status} fails closed`, async () => { + const error = Object.assign(new Error('API unavailable'), { status }); + const { github } = fixture({ [endpoint]: () => { throw error; } }); + await assert.rejects(preflight({ github, context, version: '0.14.8' }), (observed) => observed === error); + }); + } +} + +for (const version of ['', 'latest', 'v1.2', ' v0.14.7', 'v0.14.7\n', 'v0.14.7/other', undefined]) { + test(`invalid input ${JSON.stringify(version)} does not call GitHub`, async () => { + const { github, calls } = fixture(); + await assert.rejects(preflight({ github, context, version }), /version must be SemVer/); + assert.equal(calls.length, 0); + }); +} + +test('workflow gates the reusable job and serializes preflight through publication', () => { + const workflow = readFileSync(`${__dirname}/../workflows/release-unified.yml`, 'utf8'); + assert.match(workflow, /^concurrency:\n group: crawlkit-release-\$\{\{ github.repository \}\}\n cancel-in-progress: false$/m); + assert.match(workflow, /^ release:\n needs: preflight\n/m); + assert.match(workflow, /contents: read/); + assert.match(workflow, /await preflight\(\{ github, context, version: process.env.RELEASE_VERSION \}\)/); +}); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aca17b5..d2680dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - run: node --test .github/scripts/*.test.cjs - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: go.mod diff --git a/.github/workflows/release-unified.yml b/.github/workflows/release-unified.yml index 07e9819..f780ce4 100644 --- a/.github/workflows/release-unified.yml +++ b/.github/workflows/release-unified.yml @@ -10,8 +10,31 @@ on: permissions: {} +# Hold the lock across preflight and publication, including v-prefixed inputs. +concurrency: + group: crawlkit-release-${{ github.repository }} + cancel-in-progress: false + jobs: + preflight: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + - name: Require an unused release version + uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 + env: + RELEASE_VERSION: ${{ inputs.version }} + with: + script: | + const preflight = require('./.github/scripts/release-preflight.cjs'); + await preflight({ github, context, version: process.env.RELEASE_VERSION }); + release: + needs: preflight permissions: actions: read checks: read diff --git a/CHANGELOG.md b/CHANGELOG.md index bf32494..e0acfe6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Reject release dispatches for existing tags or releases before rebuilding artifacts, and serialize dispatch checks through publication. - Require Go 1.27.0, update SQLite to v1.57.0 and supporting Go dependencies, and refresh deadcode, govulncheck, CodeQL, and TruffleHog validation tools. ## v0.14.7 - 2026-08-14 diff --git a/Makefile b/Makefile index e48aabd..06cd956 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ BINARY := crawlctl .DEFAULT_GOAL := help -.PHONY: help build test test-race vet tidy tidy-check fmt lint check clean +.PHONY: help build test test-race test-release vet tidy tidy-check fmt lint check clean help: @printf '%s\n' \ @@ -14,6 +14,7 @@ help: ' lint Run vet, dead-code, and vulnerability checks.' \ ' check Run every local gate enforced by CI.' \ ' test-race Run the Go test suite with the race detector.' \ + ' test-release Test the release dispatch guard (Node.js required).' \ ' vet Run go vet (compatibility target).' \ ' tidy Apply go.mod and go.sum tidying.' \ ' tidy-check Verify go.mod and go.sum are tidy.' \ @@ -29,6 +30,9 @@ test: test-race: GOWORK=off go test -race ./... +test-release: + node --test .github/scripts/*.test.cjs + vet: GOWORK=off go vet ./... @@ -51,7 +55,7 @@ lint: vet if [ -s "$$output_file" ]; then cat "$$output_file"; exit 1; fi GOWORK=off go run golang.org/x/vuln/cmd/govulncheck@v1.7.0 ./... -check: tidy-check fmt lint test test-race +check: tidy-check fmt lint test test-race test-release clean: rm -rf bin diff --git a/docs/publishing.md b/docs/publishing.md index f025557..55d8c70 100644 --- a/docs/publishing.md +++ b/docs/publishing.md @@ -6,9 +6,10 @@ owns tag creation, Developer ID signing, notarization, independent artifact verification, and GitHub Release publication. Do not create release tags or handle signing credentials locally. -v0.14.5 is an SSH-signed Go module tag without a GitHub Release or attached -artifacts. v0.14.4 remains the latest historical release with binary assets. -The unified pipeline applies to future releases. +v0.14.7 is published with signed CLI assets and is available from the Go module +proxy. A second dispatch for that version rebuilt the payload and correctly +failed its comparison with the existing public release; it left a separate +unpublished draft. See the [incident evidence](release-31838411168.md). ## Release assets @@ -36,8 +37,11 @@ Homebrew handoff. `MACOS_SIGNING_P12`, `MACOS_SIGNING_P12_PASSWORD`, `ASC_KEY_ID`, `ASC_ISSUER_ID`, and `ASC_PRIVATE_KEY_P8`. The shared workflow validates them before creating a tag. -2. Prepare a release PR from the current protected `main` head. Date the - versioned changelog section and run: +2. Choose an unused version and prepare a release PR from the current protected + `main` head. For the next release, use **v0.14.8**, not v0.14.7. It will carry + the post-v0.14.7 release-check HTTP timeout fix, dependency/toolchain updates, + and dispatch guard. Date the versioned changelog section and run (Node.js is + required for the dispatch regression tests): ```bash make check @@ -51,8 +55,11 @@ Homebrew handoff. gh workflow run release-unified.yml --repo openclaw/crawlkit -f version=X.Y.Z ``` -5. Watch the exact workflow run through publication. It creates or reuses an - immutable annotated version tag, builds all four native archives, signs and +5. Watch the exact workflow run through publication. A read-only preflight + rejects existing tags and releases before the shared workflow runs. A + repository-wide workflow lock spans preflight through publication, so queued + dispatches (including `X.Y.Z` and `vX.Y.Z`) recheck state after the previous + run finishes. The shared workflow creates an annotated version tag, builds all four native archives, signs and notarizes the two macOS binaries, verifies the immutable draft independently on Apple Silicon and Intel, and publishes only the verified bytes. 6. Confirm the GitHub Release notes match the dated changelog section. Download @@ -73,9 +80,21 @@ Homebrew handoff. 8. Merge the workflow's closeout PR, or otherwise add the next patch-version Unreleased changelog section. -Never delete or force-update a release tag. If a run fails after tag creation, -fix the blocker and rerun the same version; the shared workflow requires the -exact annotated tag object and target observed during validation. +Never delete or force-update a release tag or replace published assets. A Git +tag is already a public Go module version, even when GitHub publication fails. +Fresh dispatches require a new version once a tag exists; this also blocks +leftover drafts because the shared pipeline freezes the tag before drafting. +The release lookup additionally rejects a visible release whose tag is missing. +API failures other than an explicit 404 stop preflight. + +After a failure, inspect the exact run, tag, release IDs, and retained artifacts +before taking action. Do not rerun all jobs or redispatch a published version: +rebuilding/re-signing does not reproduce the original verified archive bytes. +Any recovery of an unpublished draft must retain its original payload and +attestations, and requires a separately reviewed recovery decision. If those +bytes cannot be recovered, use a new patch version. Old workflow runs retain +their original workflow definition; this guard does not retrofit historical +reruns. Keep the shared workflow's final byte-binding checks intact. Use a patch version for narrow fixes on the existing API. Use a minor version for broad crawler infrastructure changes. If the module reaches v2, Go requires diff --git a/docs/release-31838411168.md b/docs/release-31838411168.md new file mode 100644 index 0000000..f4565ae --- /dev/null +++ b/docs/release-31838411168.md @@ -0,0 +1,126 @@ +# v0.14.7 duplicate release dispatch + +Read-only reconciliation on 2026-08-28. No release, tag, asset, or registry +entry was changed during this investigation. + +## State left behind + +The [first run, 31836986812](https://github.com/openclaw/crawlkit/actions/runs/31836986812), +started on 2026-08-14 at 20:14:31 UTC and successfully published v0.14.7 at +20:19:00 UTC. The [failed run, 31838411168](https://github.com/openclaw/crawlkit/actions/runs/31838411168), +started at 20:32:59 UTC for the same source commit, +`00a94648f6f27441303aa3d2ce57271caa23afdb`. + +| Object | Observed state | +| --- | --- | +| `refs/tags/v0.14.7` | Annotated, unsigned tag object `f0b0ee206d874feea0304d076246e2ae9277bb9c`, created by the first run at 20:15:43 UTC; peels to `00a94648f6f27441303aa3d2ce57271caa23afdb`. The second run reused it. | +| [Public release](https://github.com/openclaw/crawlkit/releases/tag/v0.14.7), ID `370802867` | Published, not a prerelease, eight complete assets. Asset IDs `514873552`–`514873624`; upload/update times remain 20:18:13–20:18:16 UTC. GitHub reports `immutable: false`; this investigation did not modify it. | +| [Retry draft](https://api.github.com/repos/openclaw/crawlkit/releases/370810895), ID `370810895` | `draft: true`, `published_at: null`, empty body, eight complete assets uploaded at 20:35:28–20:35:32 UTC. Not partially published; left untouched. Draft API access requires authorization. | +| [Go module proxy](https://proxy.golang.org/github.com/openclaw/crawlkit/@v/v0.14.7.info) | v0.14.7 already available, resolving to the same source commit. The Go checksum database also records the module. There is no npm, PyPI, or Homebrew publication for crawlkit. | + +Each release contains four native archives, `checksums.txt`, +`ASSET-INVENTORY.json`, `SIGNING-MANIFEST.json`, and `RELEASE-NOTES.md`. +The GitHub release API's `created_at` is 20:15:43 for both records; the retry's +actual creation/upload sequence is established by its job log and asset times, +not that field. + +## Why the checksums differed + +The original public checksums match **both original verifier attestations**. +The failed run's payload matches **both retry verifier attestations**, and every +file in each set passes its own checksum manifest. The manifest file hashes are: + +| Manifest | SHA-256 | +| --- | --- | +| Public v0.14.7 | `fef970341337e9bc1be96cfb376ed8bc12173ed6939444d8a54f6b8a1e71454d` | +| Retry draft and attestations | `5e25ef92474548bd60f43a49ae0a9ec0695b8f50665819c3344fa7932520a634` | + +All four archive hashes changed. `RELEASE-NOTES.md` and +`SIGNING-MANIFEST.json` stayed identical. `ASSET-INVENTORY.json` changed because +it includes archive sizes and hashes. The exact archive differences are: + +- Both Linux executable files are byte-identical. The amd64 binary SHA-256 is + `b04914aae9117241230252602621d11fb3b30fa70c84a4b81e9b6382d92dec4f` and arm64 is + `b5baf66ddf5a9212b56f81a7a9faac4d39c9f58fe38f0afb04c61fe7b55a3a3e` in both + runs. Their tar member timestamps changed from `1786738614` to `1786739645`; + gzip header timestamps changed from `1786738673` to `1786739703`. +- Both macOS executables differ only within their `LC_CODE_SIGNATURE` data. + Comparing the bytes before and after those regions proves the executable + content is unchanged. The arm64 code directory hash is + `e716d4aa0f282ee2169bd721810cdcfce797cc4c` in both runs, but the trusted signing + timestamp changed from 20:17:09 to 20:34:20 UTC. Tar/gzip timestamps also changed. + +This is a fresh build, package, and signing pass over the same source, not a +stale cache, swapped verifier artifact, changed source, or concurrent publish. +The first release was public before the second dispatch even started. + +The failed run used +[`release-go-cli.yml@v1`](https://github.com/openclaw/release-workflows/blob/30512dfd1defd2902b9700e1cc55148834b7c320/.github/workflows/release-go-cli.yml). +Its validation accepts an existing annotated tag; its draft stage always +creates a new draft and uploads newly built assets. Packaging uses +`tar -czf` without timestamp normalization, and signing uses a fresh trusted +timestamp. The publisher's public-release retry branch was introduced in +[`65ca35e`](https://github.com/openclaw/release-workflows/commit/65ca35e692409ebdaedcd53fca558dff2597e052) +on 2026-08-02. That branch only works if a second payload is byte-identical to +the published one, an assumption this packaging/signing path does not satisfy. + +At 20:36:03 UTC the publisher correctly raised +`existing public release checksums.txt bytes differ from verified attestation`. +Its own draft had already passed the same binding checks. The error occurs +before deleting the redundant draft or publishing anything, explaining the +intact public release and leftover draft. + +## Fix and next release + +The crawlkit caller now rejects existing version tags and releases **before** +invoking the shared pipeline, and holds one concurrency lock across preflight +and publication. Checking tags also protects already discoverable Go module +versions and catches pipeline drafts even when a read-only release query cannot +see drafts. GitHub's [release API contract](https://docs.github.com/en/rest/releases/releases#list-releases) +limits draft visibility to users with push access. No write permission or +signing secrets are needed by preflight. Final independent verification and +public-byte binding remain unchanged. + +This is a caller dispatch fix, not a replacement release implementation. The +generic shared workflow still permits rebuild-based retries for other callers; +changing that fleet-wide contract is separate work. Historical workflow runs +also retain the old definition and must not be rerun to publish v0.14.7. + +The next unused patch version is **v0.14.8**. Prepare and date its changelog from +green protected `main` after merging this fix. Relative to v0.14.7 it will carry +the 30-second default release-check HTTP timeout (#92), dependency refreshes +(#96 and #97, including Go 1.27.0 and SQLite 1.57.0), and this dispatch guard. +It will produce the same four-platform `crawlctl` asset set, with newly signed +and notarized macOS binaries. Do not republish v0.14.7 or move its tag. The old +draft can remain while a fresh version is released; any cleanup is a separate +authorized action. No release has been dispatched as part of this fix. + +## Evidence locations and repeatable reads + +The failed run's immutable verification payload is artifact +[`9233408987`](https://github.com/openclaw/crawlkit/actions/runs/31838411168/artifacts/9233408987). +Its verifier artifacts are `9233415421` (arm64) and `9233419063` (x86_64). +The successful run's verifier artifacts are `9232946731` (arm64) and +`9232951858` (x86_64). These were still available during reconciliation; +Actions artifact retention is finite. Raw build/signing artifacts had expired. + +Read-only commands used to establish state: + +```bash +gh run view 31838411168 -R openclaw/crawlkit --json headSha,conclusion,jobs,url +gh api repos/openclaw/crawlkit/git/ref/tags/v0.14.7 +gh api repos/openclaw/crawlkit/git/tags/f0b0ee206d874feea0304d076246e2ae9277bb9c +gh api repos/openclaw/crawlkit/releases/370802867 +gh api repos/openclaw/crawlkit/releases/370810895 +gh api repos/openclaw/crawlkit/actions/runs/31838411168/artifacts +gh release download v0.14.7 -R openclaw/crawlkit -D /tmp/crawlkit-v0.14.7-public +(cd /tmp/crawlkit-v0.14.7-public && shasum -a 256 -c checksums.txt) +GOPROXY=https://proxy.golang.org GONOSUMDB= go list -m github.com/openclaw/crawlkit@v0.14.7 +GOPROXY=https://proxy.golang.org go list -m github.com/openclaw/crawlkit@v0.14.7 +``` + +Attestation JSON `sha256sums` fields were compared byte-for-byte to the matching +manifest, then every manifest entry was hashed independently. Python `tarfile` +and `struct` inspection compared gzip headers, tar members, and Mach-O signature +regions without executing any downloaded binary. `codesign -dv --verbose=4` +confirmed matching code directory hashes and differing signing timestamps.