Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 27 additions & 0 deletions .github/scripts/release-preflight.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
module.exports = async function preflight({ github, context, version }) {
// Match the reusable workflow's version canonicalization before any lookup.
if (typeof version !== 'string' || version !== version.trim() || !/^v?[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?(\+[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$/.test(version)) {
throw new Error('version must be SemVer, with optional leading v');
}
const tag = `v${version.replace(/^v/, '')}`;
const requireAbsent = async (lookup, kind) => {
try {
await lookup();
} catch (error) {
if (error.status === 404) return;
throw error;
}
throw new Error(`${tag} already has a ${kind}; refusing to rebuild an existing release version. Use a new patch version, or reconcile the original run without rebuilding its verified payload.`);
};

// Tags are public Go module versions even without a published GitHub release.
// The shared workflow creates the tag before creating any draft or assets.
await requireAbsent(() => github.rest.git.getRef({
...context.repo,
ref: `tags/${tag}`,
}), 'Git tag');
await requireAbsent(() => github.rest.repos.getReleaseByTag({
...context.repo,
tag,
}), 'GitHub release');
};
81 changes: 81 additions & 0 deletions .github/scripts/release-preflight.test.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
const assert = require('node:assert/strict');
const { readFileSync } = require('node:fs');
const { test } = require('node:test');
const preflight = require('./release-preflight.cjs');

const context = { repo: { owner: 'openclaw', repo: 'crawlkit' } };
const missing = () => { throw Object.assign(new Error('Not Found'), { status: 404 }); };

function fixture({ tag = missing, release = missing } = {}) {
const calls = [];
return {
calls,
// Only read endpoints exist: mutations are never part of preflight.
github: { rest: {
git: { getRef: async (args) => { calls.push(['tag', args]); return tag(); } },
repos: { getReleaseByTag: async (args) => { calls.push(['release', args]); return release(); } },
} },
};
}

for (const version of ['0.14.8', 'v0.14.8', '0.14.8-rc.1', 'v0.14.8+build.1']) {
test(`unused ${version} reaches the shared release workflow`, async () => {
const { github, calls } = fixture();
await preflight({ github, context, version });
const tag = `v${version.replace(/^v/, '')}`;
assert.deepEqual(calls, [
['tag', { ...context.repo, ref: `tags/${tag}` }],
['release', { ...context.repo, tag }],
]);
});
}

for (const kind of ['tag', 'commit']) {
test(`existing ${kind} stops before rebuilding, even without a GitHub release`, async () => {
const { github, calls } = fixture({ tag: () => ({ data: { object: { type: kind } } }) });
await assert.rejects(preflight({ github, context, version: 'v0.14.7' }), /v0\.14\.7 already has a Git tag/);
assert.equal(calls.length, 1);
});
}

for (const draft of [false, true]) {
test(`existing ${draft ? 'draft' : 'public release'} blocks even if its tag is missing`, async () => {
const { github } = fixture({ release: () => ({ data: { id: 123, draft } }) });
await assert.rejects(preflight({ github, context, version: '0.14.7' }), /already has a GitHub release/);
});
}

test('the halted v0.14.7 state cannot reach build or draft creation', async () => {
const { github, calls } = fixture({
tag: () => ({ data: { object: { type: 'tag', sha: 'f0b0ee206d874feea0304d076246e2ae9277bb9c' } } }),
release: () => ({ data: { id: 370802867, draft: false } }),
});
await assert.rejects(preflight({ github, context, version: '0.14.7' }), /refusing to rebuild/);
assert.equal(calls.length, 1);
});

for (const endpoint of ['tag', 'release']) {
for (const status of [401, 403, 429, 500]) {
test(`${endpoint} HTTP ${status} fails closed`, async () => {
const error = Object.assign(new Error('API unavailable'), { status });
const { github } = fixture({ [endpoint]: () => { throw error; } });
await assert.rejects(preflight({ github, context, version: '0.14.8' }), (observed) => observed === error);
});
}
}

for (const version of ['', 'latest', 'v1.2', ' v0.14.7', 'v0.14.7\n', 'v0.14.7/other', undefined]) {
test(`invalid input ${JSON.stringify(version)} does not call GitHub`, async () => {
const { github, calls } = fixture();
await assert.rejects(preflight({ github, context, version }), /version must be SemVer/);
assert.equal(calls.length, 0);
});
}

test('workflow gates the reusable job and serializes preflight through publication', () => {
const workflow = readFileSync(`${__dirname}/../workflows/release-unified.yml`, 'utf8');
assert.match(workflow, /^concurrency:\n group: crawlkit-release-\$\{\{ github.repository \}\}\n cancel-in-progress: false$/m);
assert.match(workflow, /^ release:\n needs: preflight\n/m);
assert.match(workflow, /contents: read/);
assert.match(workflow, /await preflight\(\{ github, context, version: process.env.RELEASE_VERSION \}\)/);
});
1 change: 1 addition & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- run: node --test .github/scripts/*.test.cjs
- uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
with:
go-version-file: go.mod
Expand Down
23 changes: 23 additions & 0 deletions .github/workflows/release-unified.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,31 @@ on:

permissions: {}

# Hold the lock across preflight and publication, including v-prefixed inputs.
concurrency:
group: crawlkit-release-${{ github.repository }}
cancel-in-progress: false

jobs:
preflight:
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- name: Require an unused release version
uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8
env:
RELEASE_VERSION: ${{ inputs.version }}
with:
script: |
const preflight = require('./.github/scripts/release-preflight.cjs');
await preflight({ github, context, version: process.env.RELEASE_VERSION });

release:
needs: preflight
permissions:
actions: read
checks: read
Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

## Unreleased

- Reject release dispatches for existing tags or releases before rebuilding artifacts, and serialize dispatch checks through publication.
- Require Go 1.27.0, update SQLite to v1.57.0 and supporting Go dependencies, and refresh deadcode, govulncheck, CodeQL, and TruffleHog validation tools.

## v0.14.7 - 2026-08-14
Expand Down
8 changes: 6 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ BINARY := crawlctl

.DEFAULT_GOAL := help

.PHONY: help build test test-race vet tidy tidy-check fmt lint check clean
.PHONY: help build test test-race test-release vet tidy tidy-check fmt lint check clean

help:
@printf '%s\n' \
Expand All @@ -14,6 +14,7 @@ help:
' lint Run vet, dead-code, and vulnerability checks.' \
' check Run every local gate enforced by CI.' \
' test-race Run the Go test suite with the race detector.' \
' test-release Test the release dispatch guard (Node.js required).' \
' vet Run go vet (compatibility target).' \
' tidy Apply go.mod and go.sum tidying.' \
' tidy-check Verify go.mod and go.sum are tidy.' \
Expand All @@ -29,6 +30,9 @@ test:
test-race:
GOWORK=off go test -race ./...

test-release:
node --test .github/scripts/*.test.cjs

vet:
GOWORK=off go vet ./...

Expand All @@ -51,7 +55,7 @@ lint: vet
if [ -s "$$output_file" ]; then cat "$$output_file"; exit 1; fi
GOWORK=off go run golang.org/x/vuln/cmd/govulncheck@v1.7.0 ./...

check: tidy-check fmt lint test test-race
check: tidy-check fmt lint test test-race test-release

clean:
rm -rf bin
39 changes: 29 additions & 10 deletions docs/publishing.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,10 @@ owns tag creation, Developer ID signing, notarization, independent artifact
verification, and GitHub Release publication. Do not create release tags or
handle signing credentials locally.

v0.14.5 is an SSH-signed Go module tag without a GitHub Release or attached
artifacts. v0.14.4 remains the latest historical release with binary assets.
The unified pipeline applies to future releases.
v0.14.7 is published with signed CLI assets and is available from the Go module
proxy. A second dispatch for that version rebuilt the payload and correctly
failed its comparison with the existing public release; it left a separate
unpublished draft. See the [incident evidence](release-31838411168.md).

## Release assets

Expand Down Expand Up @@ -36,8 +37,11 @@ Homebrew handoff.
`MACOS_SIGNING_P12`, `MACOS_SIGNING_P12_PASSWORD`, `ASC_KEY_ID`,
`ASC_ISSUER_ID`, and `ASC_PRIVATE_KEY_P8`. The shared workflow validates
them before creating a tag.
2. Prepare a release PR from the current protected `main` head. Date the
versioned changelog section and run:
2. Choose an unused version and prepare a release PR from the current protected
`main` head. For the next release, use **v0.14.8**, not v0.14.7. It will carry
the post-v0.14.7 release-check HTTP timeout fix, dependency/toolchain updates,
and dispatch guard. Date the versioned changelog section and run (Node.js is
required for the dispatch regression tests):

```bash
make check
Expand All @@ -51,8 +55,11 @@ Homebrew handoff.
gh workflow run release-unified.yml --repo openclaw/crawlkit -f version=X.Y.Z
```

5. Watch the exact workflow run through publication. It creates or reuses an
immutable annotated version tag, builds all four native archives, signs and
5. Watch the exact workflow run through publication. A read-only preflight
rejects existing tags and releases before the shared workflow runs. A
repository-wide workflow lock spans preflight through publication, so queued
dispatches (including `X.Y.Z` and `vX.Y.Z`) recheck state after the previous
run finishes. The shared workflow creates an annotated version tag, builds all four native archives, signs and
notarizes the two macOS binaries, verifies the immutable draft independently
on Apple Silicon and Intel, and publishes only the verified bytes.
6. Confirm the GitHub Release notes match the dated changelog section. Download
Expand All @@ -73,9 +80,21 @@ Homebrew handoff.
8. Merge the workflow's closeout PR, or otherwise add the next patch-version
Unreleased changelog section.

Never delete or force-update a release tag. If a run fails after tag creation,
fix the blocker and rerun the same version; the shared workflow requires the
exact annotated tag object and target observed during validation.
Never delete or force-update a release tag or replace published assets. A Git
tag is already a public Go module version, even when GitHub publication fails.
Fresh dispatches require a new version once a tag exists; this also blocks
leftover drafts because the shared pipeline freezes the tag before drafting.
The release lookup additionally rejects a visible release whose tag is missing.
API failures other than an explicit 404 stop preflight.

After a failure, inspect the exact run, tag, release IDs, and retained artifacts
before taking action. Do not rerun all jobs or redispatch a published version:
rebuilding/re-signing does not reproduce the original verified archive bytes.
Any recovery of an unpublished draft must retain its original payload and
attestations, and requires a separately reviewed recovery decision. If those
bytes cannot be recovered, use a new patch version. Old workflow runs retain
their original workflow definition; this guard does not retrofit historical
reruns. Keep the shared workflow's final byte-binding checks intact.

Use a patch version for narrow fixes on the existing API. Use a minor version
for broad crawler infrastructure changes. If the module reaches v2, Go requires
Expand Down
126 changes: 126 additions & 0 deletions docs/release-31838411168.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
# v0.14.7 duplicate release dispatch

Read-only reconciliation on 2026-08-28. No release, tag, asset, or registry
entry was changed during this investigation.

## State left behind

The [first run, 31836986812](https://github.com/openclaw/crawlkit/actions/runs/31836986812),
started on 2026-08-14 at 20:14:31 UTC and successfully published v0.14.7 at
20:19:00 UTC. The [failed run, 31838411168](https://github.com/openclaw/crawlkit/actions/runs/31838411168),
started at 20:32:59 UTC for the same source commit,
`00a94648f6f27441303aa3d2ce57271caa23afdb`.

| Object | Observed state |
| --- | --- |
| `refs/tags/v0.14.7` | Annotated, unsigned tag object `f0b0ee206d874feea0304d076246e2ae9277bb9c`, created by the first run at 20:15:43 UTC; peels to `00a94648f6f27441303aa3d2ce57271caa23afdb`. The second run reused it. |
| [Public release](https://github.com/openclaw/crawlkit/releases/tag/v0.14.7), ID `370802867` | Published, not a prerelease, eight complete assets. Asset IDs `514873552`–`514873624`; upload/update times remain 20:18:13–20:18:16 UTC. GitHub reports `immutable: false`; this investigation did not modify it. |
| [Retry draft](https://api.github.com/repos/openclaw/crawlkit/releases/370810895), ID `370810895` | `draft: true`, `published_at: null`, empty body, eight complete assets uploaded at 20:35:28–20:35:32 UTC. Not partially published; left untouched. Draft API access requires authorization. |
| [Go module proxy](https://proxy.golang.org/github.com/openclaw/crawlkit/@v/v0.14.7.info) | v0.14.7 already available, resolving to the same source commit. The Go checksum database also records the module. There is no npm, PyPI, or Homebrew publication for crawlkit. |

Each release contains four native archives, `checksums.txt`,
`ASSET-INVENTORY.json`, `SIGNING-MANIFEST.json`, and `RELEASE-NOTES.md`.
The GitHub release API's `created_at` is 20:15:43 for both records; the retry's
actual creation/upload sequence is established by its job log and asset times,
not that field.

## Why the checksums differed

The original public checksums match **both original verifier attestations**.
The failed run's payload matches **both retry verifier attestations**, and every
file in each set passes its own checksum manifest. The manifest file hashes are:

| Manifest | SHA-256 |
| --- | --- |
| Public v0.14.7 | `fef970341337e9bc1be96cfb376ed8bc12173ed6939444d8a54f6b8a1e71454d` |
| Retry draft and attestations | `5e25ef92474548bd60f43a49ae0a9ec0695b8f50665819c3344fa7932520a634` |

All four archive hashes changed. `RELEASE-NOTES.md` and
`SIGNING-MANIFEST.json` stayed identical. `ASSET-INVENTORY.json` changed because
it includes archive sizes and hashes. The exact archive differences are:

- Both Linux executable files are byte-identical. The amd64 binary SHA-256 is
`b04914aae9117241230252602621d11fb3b30fa70c84a4b81e9b6382d92dec4f` and arm64 is
`b5baf66ddf5a9212b56f81a7a9faac4d39c9f58fe38f0afb04c61fe7b55a3a3e` in both
runs. Their tar member timestamps changed from `1786738614` to `1786739645`;
gzip header timestamps changed from `1786738673` to `1786739703`.
- Both macOS executables differ only within their `LC_CODE_SIGNATURE` data.
Comparing the bytes before and after those regions proves the executable
content is unchanged. The arm64 code directory hash is
`e716d4aa0f282ee2169bd721810cdcfce797cc4c` in both runs, but the trusted signing
timestamp changed from 20:17:09 to 20:34:20 UTC. Tar/gzip timestamps also changed.

This is a fresh build, package, and signing pass over the same source, not a
stale cache, swapped verifier artifact, changed source, or concurrent publish.
The first release was public before the second dispatch even started.

The failed run used
[`release-go-cli.yml@v1`](https://github.com/openclaw/release-workflows/blob/30512dfd1defd2902b9700e1cc55148834b7c320/.github/workflows/release-go-cli.yml).
Its validation accepts an existing annotated tag; its draft stage always
creates a new draft and uploads newly built assets. Packaging uses
`tar -czf` without timestamp normalization, and signing uses a fresh trusted
timestamp. The publisher's public-release retry branch was introduced in
[`65ca35e`](https://github.com/openclaw/release-workflows/commit/65ca35e692409ebdaedcd53fca558dff2597e052)
on 2026-08-02. That branch only works if a second payload is byte-identical to
the published one, an assumption this packaging/signing path does not satisfy.

At 20:36:03 UTC the publisher correctly raised
`existing public release checksums.txt bytes differ from verified attestation`.
Its own draft had already passed the same binding checks. The error occurs
before deleting the redundant draft or publishing anything, explaining the
intact public release and leftover draft.

## Fix and next release

The crawlkit caller now rejects existing version tags and releases **before**
invoking the shared pipeline, and holds one concurrency lock across preflight
and publication. Checking tags also protects already discoverable Go module
versions and catches pipeline drafts even when a read-only release query cannot
see drafts. GitHub's [release API contract](https://docs.github.com/en/rest/releases/releases#list-releases)
limits draft visibility to users with push access. No write permission or
signing secrets are needed by preflight. Final independent verification and
public-byte binding remain unchanged.

This is a caller dispatch fix, not a replacement release implementation. The
generic shared workflow still permits rebuild-based retries for other callers;
changing that fleet-wide contract is separate work. Historical workflow runs
also retain the old definition and must not be rerun to publish v0.14.7.

The next unused patch version is **v0.14.8**. Prepare and date its changelog from
green protected `main` after merging this fix. Relative to v0.14.7 it will carry
the 30-second default release-check HTTP timeout (#92), dependency refreshes
(#96 and #97, including Go 1.27.0 and SQLite 1.57.0), and this dispatch guard.
It will produce the same four-platform `crawlctl` asset set, with newly signed
and notarized macOS binaries. Do not republish v0.14.7 or move its tag. The old
draft can remain while a fresh version is released; any cleanup is a separate
authorized action. No release has been dispatched as part of this fix.

## Evidence locations and repeatable reads

The failed run's immutable verification payload is artifact
[`9233408987`](https://github.com/openclaw/crawlkit/actions/runs/31838411168/artifacts/9233408987).
Its verifier artifacts are `9233415421` (arm64) and `9233419063` (x86_64).
The successful run's verifier artifacts are `9232946731` (arm64) and
`9232951858` (x86_64). These were still available during reconciliation;
Actions artifact retention is finite. Raw build/signing artifacts had expired.

Read-only commands used to establish state:

```bash
gh run view 31838411168 -R openclaw/crawlkit --json headSha,conclusion,jobs,url
gh api repos/openclaw/crawlkit/git/ref/tags/v0.14.7
gh api repos/openclaw/crawlkit/git/tags/f0b0ee206d874feea0304d076246e2ae9277bb9c
gh api repos/openclaw/crawlkit/releases/370802867
gh api repos/openclaw/crawlkit/releases/370810895
gh api repos/openclaw/crawlkit/actions/runs/31838411168/artifacts
gh release download v0.14.7 -R openclaw/crawlkit -D /tmp/crawlkit-v0.14.7-public
(cd /tmp/crawlkit-v0.14.7-public && shasum -a 256 -c checksums.txt)
GOPROXY=https://proxy.golang.org GONOSUMDB= go list -m github.com/openclaw/crawlkit@v0.14.7
GOPROXY=https://proxy.golang.org go list -m github.com/openclaw/crawlkit@v0.14.7
```

Attestation JSON `sha256sums` fields were compared byte-for-byte to the matching
manifest, then every manifest entry was hashed independently. Python `tarfile`
and `struct` inspection compared gzip headers, tar members, and Mach-O signature
regions without executing any downloaded binary. `codesign -dv --verbose=4`
confirmed matching code directory hashes and differing signing timestamps.