diff --git a/.github/workflows/platform-ci.yml b/.github/workflows/platform-ci.yml index 85a063fdf..7d40dd58b 100644 --- a/.github/workflows/platform-ci.yml +++ b/.github/workflows/platform-ci.yml @@ -12,6 +12,11 @@ on: - 'vite.config.ts' - '.github/workflows/platform-*.yml' workflow_call: + inputs: + require-schema-compat: + description: Run the production schema-compatibility guard. The production deploy sets this for every trigger. + type: boolean + default: false permissions: {} @@ -75,11 +80,89 @@ jobs: - run: vp run --filter './platform/packages/**' build - run: vp test run --project ${{ matrix.project }} - # The former repository compared migrations against its `prod` branch. The merged repository's - # production ref has intentionally not been chosen yet, so this job remains visible but inert - # until the deployment-cutover change supplies the authoritative ref. + # The production guard. `platform-deploy.yml` migrates before it deploys the api worker, so a + # push to `prod` applies a new schema while the previous release is still serving traffic. This + # migrates a real Postgres with the migrations under test, then runs the `prod` branch's + # integration tests against that schema: if the code that is live cannot run on the new schema, + # the push must not ship. Accepted limits: the baseline is the `prod` branch head rather than the + # deployed revision, and released migrations are assumed append-only (drizzle never re-applies an + # applied entry and skips entries not newer than the last applied). + # + # It has teeth on PRs, where the code under test is a different commit from the one that is + # live. The production deploy sets `require-schema-compat` for every trigger — including a + # manual dispatch, which can be started from any ref and would otherwise skip the job. Staging + # deploys skip it: they cannot break production, and they run on nearly every push to `main`. schema-compat: - if: ${{ false }} + if: github.event_name == 'pull_request' || inputs.require-schema-compat runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 20 + permissions: + contents: read + env: + POSTGRES_USER: void_ci + POSTGRES_PASSWORD: void_ci_pass + POSTGRES_DB: void_schema_compat + COMPAT_DATABASE_URL: postgresql://void_ci:void_ci_pass@localhost:5432/void_schema_compat + # Pinned: the suite compares timestamps written by Postgres with local time, so a + # runner whose locale is not UTC shifts them by the offset. + TZ: UTC + services: + postgres: + image: postgres:18 + env: + POSTGRES_USER: ${{ env.POSTGRES_USER }} + POSTGRES_PASSWORD: ${{ env.POSTGRES_PASSWORD }} + POSTGRES_DB: ${{ env.POSTGRES_DB }} + ports: + - 5432:5432 + options: >- + --health-cmd="pg_isready -U $POSTGRES_USER" + --health-interval=5s + --health-timeout=5s + --health-retries=5 steps: - - run: echo 'Enable after choosing the merged repository production ref.' + # Phase 1 — apply the migrations under test to a real Postgres. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: voidzero-dev/setup-vp@1b32467adbe183473499fd9d5d372c3ed9641754 # v1.18.0 + with: + cache: true + node-version-file: .node-version + - run: vp run build + - run: vp run --filter @voidcloud/api migrate + env: + DATABASE_URL: ${{ env.COMPAT_DATABASE_URL }} + + # Phase 2 — the release that is live must still run on that schema. `prod` is that release. + # PRs are checked against it; a push to `prod` checks itself, as in the old repository; a + # manual dispatch from another ref is checked against `prod`. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: prod + persist-credentials: false + - name: Does the baseline carry the platform? + id: production + run: | + if [ -d platform/packages/api ]; then + echo 'usable=true' >> "$GITHUB_OUTPUT" + else + # Production still runs a pre-monorepo revision, so there is no platform code to test + # against: the schema is checked, the code is not. That gap closes with the first + # release from this repository, after which this branch stops being taken. + echo 'usable=false' >> "$GITHUB_OUTPUT" + echo '::warning::The compatibility baseline predates the platform, so the production code was not exercised against the migrated schema. This stops once prod carries the platform.' + fi + - if: steps.production.outputs.usable == 'true' + uses: voidzero-dev/setup-vp@1b32467adbe183473499fd9d5d372c3ed9641754 # v1.18.0 + with: + cache: true + node-version-file: .node-version + - name: Run the production ref's integration tests on the migrated schema + if: steps.production.outputs.usable == 'true' + run: | + vp run build + vp run --filter './platform/packages/**' build + vp test run --project platform-api-integration + env: + VOID_TEST_DATABASE_URL: ${{ env.COMPAT_DATABASE_URL }} diff --git a/.github/workflows/platform-deploy.yml b/.github/workflows/platform-deploy.yml index 44ecf40d0..085e404f5 100644 --- a/.github/workflows/platform-deploy.yml +++ b/.github/workflows/platform-deploy.yml @@ -1,14 +1,28 @@ name: Deploy Platform Production on: + push: + # `prod` is the production ref: it holds the commit that is live, and + # advancing it is the release. No path filter — a push here is deliberate. + branches: [prod] workflow_dispatch: +concurrency: + # One running deploy plus at most one pending. Never interrupt a deploy + # between the api migration and the workers that depend on it. + group: platform-deploy-production + cancel-in-progress: false + permissions: {} jobs: ci: if: ${{ vars.VOID_PLATFORM_DEPLOY_ENABLED == 'true' }} uses: ./.github/workflows/platform-ci.yml + with: + # Every production trigger runs the schema-compatibility guard, whichever ref the + # dispatch came from. + require-schema-compat: true permissions: contents: read diff --git a/platform/README.md b/platform/README.md index f7ac71832..bce547677 100644 --- a/platform/README.md +++ b/platform/README.md @@ -73,7 +73,7 @@ Paths in this file are relative to `platform/` unless they begin with `../`. For The `@voidcloud/platform` workspace package retains the dashboard-only `dev` and `build` scripts. Dashboard deploys run after the platform workers in the platform deploy pipeline. -**Production deploys are manual only; staging deploys on every push to `main`.** See [`meta/operations.md`](./meta/operations.md#github-actions) for the workflow triggers. Do not ask to push or deploy. +**Production deploys on push to `prod`; staging deploys on every push to `main`.** See [`meta/operations.md`](./meta/operations.md#github-actions) for the workflow triggers. Do not push `prod` or ask to deploy. ## Local Development diff --git a/platform/meta/operations.md b/platform/meta/operations.md index 964cfa20e..76a08907b 100644 --- a/platform/meta/operations.md +++ b/platform/meta/operations.md @@ -27,10 +27,10 @@ Five Cloudflare Workers: Two fully isolated environments. Each has its own CF resources (KV, R2, dispatch namespace, analytics engine) and a separate Hyperdrive configuration pointing to its own PlanetScale PostgreSQL database. -| Environment | API URL | Dashboard | User Sites | Trigger | -| -------------- | ------------------------ | ------------------------- | -------------------- | ----------------------------------------------------------------------------- | -| **Production** | `api.void.cloud` | `dash.void.cloud` | `*.void.app` | Manual: "Deploy Platform Production" workflow | -| **Staging** | `api.staging.void.cloud` | `dash.staging.void.cloud` | `*.staging.void.app` | Automatic: "Deploy Platform Staging" on push to `main` (also manual dispatch) | +| Environment | API URL | Dashboard | User Sites | Trigger | +| -------------- | ------------------------ | ------------------------- | -------------------- | -------------------------------------------------------------------------------- | +| **Production** | `api.void.cloud` | `dash.void.cloud` | `*.void.app` | Automatic: "Deploy Platform Production" on push to `prod` (also manual dispatch) | +| **Staging** | `api.staging.void.cloud` | `dash.staging.void.cloud` | `*.staging.void.app` | Automatic: "Deploy Platform Staging" on push to `main` (also manual dispatch) | ## Cloudflare Resources @@ -231,7 +231,7 @@ Two GitHub environments (`Prod` and `Staging`) share the same repo-level secrets | `CF_ACCESS_CLIENT_ID_STAGING` | Staging | CF Access service-token client id for reaching staging (see Staging Access Control) | | `CF_ACCESS_CLIENT_SECRET_STAGING` | Staging | CF Access service-token client secret for reaching staging | -`platform-deploy.yml` uses the `Prod` environment and runs only on manual dispatch; `platform-deploy-staging.yml` uses `Staging` and also runs on every push to `main` that matches the `platform-ci.yml` path filter. Both run only when the repository variable `VOID_PLATFORM_DEPLOY_ENABLED` is `true`. +`platform-deploy.yml` uses the `Prod` environment and runs on every push to `prod` or on manual dispatch; `platform-deploy-staging.yml` uses `Staging` and runs on every push to `main` that matches the `platform-ci.yml` path filter, or on manual dispatch. Both run only when the repository variable `VOID_PLATFORM_DEPLOY_ENABLED` is `true`. **GitHub Actions `CF_API_TOKEN` permissions** — narrower-scope token used only for wrangler deploys. Separate from the API worker runtime token. @@ -513,13 +513,16 @@ Four platform workflows live in the repository-level `.github/workflows/` direct 5. `vp run check:platform` and `vp run check:sdk` (typecheck the platform packages and the SDK packages; the playgrounds are typechecked by `ci.yml`) 6. `vp test run --project platform-unit` and `--project platform-api-integration` 7. Builds the API build container image (PRs only) +8. The production schema-compatibility guard (`schema-compat`; PRs and production deploys): migrates a fresh Postgres 18 with the candidate's migrations and runs the `prod` branch's `platform-api-integration` suite on it -**`platform-deploy.yml`** — manual dispatch (production, environment: `Prod`): +**`platform-deploy.yml`** — push to `prod`, or manual dispatch (production, environment: `Prod`): 1. Runs `platform-ci.yml` first 2. `vp run build` (SDK packages the dashboard bundles) 3. `vp run --filter @voidcloud/platform deploy` (api with drizzle-kit migrations, proxy, tail, dispatch, dashboard) +`prod` is the production ref: it points at the commit that is live, so advancing it is the release (`git push origin main:prod` deploys whatever `main` is at). There is no path filter — a push to `prod` is deliberate. The branch carries a `prod` ruleset that blocks deletion and force-pushes and allows updates only from its bypass actors (organization admins, repository admins, and the `Void` team), so a push to `prod` is a maintainer action. A `concurrency` group (`platform-deploy-production`, `cancel-in-progress: false`) keeps one deploy running and at most one pending, so a deploy is never interrupted between the API migration and the workers. + **`platform-deploy-staging.yml`** — push to `main` that matches the `platform-ci.yml` path filter, or manual dispatch (staging, environment: `Staging`): 1. Runs `platform-ci.yml` first diff --git a/platform/packages/api/test/integration/api-migrations.test.ts b/platform/packages/api/test/integration/api-migrations.test.ts index 5026d8e63..d4038bcc1 100644 --- a/platform/packages/api/test/integration/api-migrations.test.ts +++ b/platform/packages/api/test/integration/api-migrations.test.ts @@ -4476,18 +4476,19 @@ describe('POST /projects/:id/migrations/rename', () => { expect(rollbackBody.migrationDiff).toEqual(['m3.sql']); }); - it('rollback-vs-rollback serializes on the row lock and keeps pg and ROUTING in sync', async () => { - // Concurrent rollbacks must fully serialize. The row lock covers the - // whole cutover (pg flip + ROUTING.put + cron + queue + ISR), so when - // both finish the final state is internally consistent: whatever pg - // says is active, ROUTING points at the matching worker, and the - // diffs returned by both rollbacks are computed against the correct - // pre-flip snapshots. + it('two rollbacks in sequence keep pg and ROUTING in sync, the second after the first has finished', async () => { + // Two rollbacks run one after the other must leave a consistent state. + // The row lock covers the whole cutover (pg flip + ROUTING.put + cron + + // queue + ISR), so when both finish the final state is internally + // consistent: whatever pg says is active, ROUTING points at the + // matching worker, and the diffs returned by both rollbacks are + // computed against the correct pre-flip snapshots. // - // This test pins that invariant by holding rollback A mid-cutover + // This test pins that invariant by holding rollback A mid-rollback // (inside its locked tx, AFTER the pg flip but during its // `ROUTING.put(slug, …)` external write) and kicking rollback B while - // A is paused. B must block on the row lock until A commits. + // A is paused. B starts only after A has fully finished, post-commit + // lease cleanup included. const user = await createTestUser(env.db); const project = await createTestProject(env.db, user.id, 'r13f1-rb-rb'); await insertD1Resource(env.db, project.id); @@ -4552,9 +4553,8 @@ describe('POST /projects/:id/migrations/rename', () => { // // Both rollbacks in this scenario carry non-empty diffs (A→B drops m3; // B→C drops m2) so the destructive guard would reject them without an - // opt-in. The invariant here (lock serialization between concurrent - // rollbacks) is orthogonal, so echo each rollback's exact diff via - // `allowDestructive`. + // opt-in. The invariant here (a second rollback after the first) is + // orthogonal, so echo each rollback's exact diff via `allowDestructive`. const rollbackAPromise = app.fetch( authedReq(`/projects/${project.id}/rollback`, user.jwt, { method: 'POST', @@ -4564,10 +4564,48 @@ describe('POST /projects/:id/migrations/rename', () => { ); await aReachedPut; - // Kick rollback B to C. It enters the tx callback and blocks on the - // per-project row lock — A still holds it. (On PGlite this is also - // enforced by the underlying single-connection serialization, a safe - // upper bound for this test.) + // B is held at the waypoint between its pre-lock reads and its row-lock + // acquisition, so it can only run once A's rollback — including the + // post-commit lease cleanup — has finished. The wrapper is installed after + // A's transaction started, so the first transaction it sees is B's. + // + // Without the hold the driver decides the order: PGlite queues statements, + // so B's in-lock read runs after A's cleanup, while a pooled Postgres grants + // the queued row lock at A's commit, so B resumes before that cleanup, reads + // A's rollback lease and answers 409 `deploy in progress`. That fence is + // real, but it is driver order rather than the sequence under test. + // + // This test therefore does not exercise row-lock contention between + // rollbacks: on Postgres a rollback that wins the lock at the other's + // commit still sees its rollback lease and answers 409, which is intended. + // The row lock itself is covered by the `/apply` contention tests in this + // file. + const realDb = env.db; + const realTransaction = realDb.transaction.bind(realDb); + let releaseB: () => void = () => {}; + const bHold = new Promise((r) => { + releaseB = r; + }); + let bHoldFired = false; + const wrappedDb: Db = new Proxy(realDb, { + get(target, prop, receiver) { + if (prop === 'transaction') { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return (async (fn: any) => { + if (!bHoldFired) { + bHoldFired = true; + await bHold; + } + return realTransaction(fn); + }) as unknown as typeof realDb.transaction; + } + return Reflect.get(target, prop, receiver); + }, + }) as Db; + setTestDb(wrappedDb); + + // Kick rollback B to C. It does its pre-lock reads, then waits at the + // waypoint until A's rollback has fully completed. const rollbackBPromise = app.fetch( authedReq(`/projects/${project.id}/rollback`, user.jwt, { method: 'POST', @@ -4576,18 +4614,14 @@ describe('POST /projects/:id/migrations/rename', () => { env, ); - // Give B's request time to reach its tx/lock acquisition and - // actually be blocked. We cannot poll pg to confirm this — a - // `SELECT` through env.db would itself block on the single - // PGlite connection that A's tx still holds — so we use a small - // sleep. Release A afterward. - await new Promise((r) => setTimeout(r, 30)); - - // Release A. It commits the tx (pg active=B, ROUTING=B, etc), - // then B acquires the lock and flips pg active=C, ROUTING=C. + // Release A. It commits the tx (pg active=B, ROUTING=B, etc) and clears its + // rollback lease, then B acquires the lock and flips pg active=C, ROUTING=C. releaseA(); - const [resA, resB] = await Promise.all([rollbackAPromise, rollbackBPromise]); + const resA = await rollbackAPromise; expect(resA.status).toBe(200); + releaseB(); + const resB = await rollbackBPromise; + setTestDb(realDb); expect(resB.status).toBe(200); const bodyA = (await resA.json()) as { @@ -4609,8 +4643,8 @@ describe('POST /projects/:id/migrations/rename', () => { // Final consistency: pg says active=C AND ROUTING.workerName matches C. // The pg flip happens FIRST inside the lock and external writes follow, - // so the serialized order guarantees pg and ROUTING converge on the - // same terminal state. + // so running B after A guarantees pg and ROUTING converge on the same + // terminal state. const finalPg = await env.db.select().from(projects).where(eq(projects.id, project.id)); expect(finalPg[0]!.active_deployment_id).toBe(depC.id); diff --git a/platform/packages/api/test/integration/api-sandbox.test.ts b/platform/packages/api/test/integration/api-sandbox.test.ts index a591b0f69..2dafd560a 100644 --- a/platform/packages/api/test/integration/api-sandbox.test.ts +++ b/platform/packages/api/test/integration/api-sandbox.test.ts @@ -503,25 +503,26 @@ describe('per-deploy sandbox integration', () => { const originalPut = env.ROUTING.put.bind(env.ROUTING); const originalDelete = env.ROUTING.delete.bind(env.ROUTING); let teardownTriggered = false; - let teardownPromise: Promise | undefined; const putSpy = vi.spyOn(env.ROUTING, 'put').mockImplementation((async ( key: string, value: string | ArrayBuffer | ArrayBufferView | ReadableStream, options?: KVNamespacePutOptions, ) => { const result = await originalPut(key, value as never, options as never); - if (key === project.slug && !teardownTriggered) { + // The `cutover-ready:` key is written after the routing-switch transaction + // commits and before the deploy's finalize transaction opens, so the + // teardown writes can be awaited here: no transaction is open, nothing + // deadlocks, and the flip is guaranteed to land before the finalize read. + // Queuing them instead (the earlier shape) only ordered them in PGlite's + // favour — on a pooled Postgres the flip opened a new session and lost the + // race against the deploy's warm connection, which committed live. + if (key.startsWith('cutover-ready:') && !teardownTriggered) { teardownTriggered = true; - // Queue teardown behind the cutover's project-row lock. Do not await - // it here: PGlite uses one connection, so awaiting a second query from - // inside the transaction-backed ROUTING mock would deadlock the test. - // The queued query runs as soon as the cutover transaction releases - // the lock, before deploy's following database-finalize transaction. - teardownPromise = env.db + await env.db .update(projects) .set({ status: 'deleting' }) - .where(eq(projects.id, project.id)) - .then(() => originalDelete(project.slug)); + .where(eq(projects.id, project.id)); + await originalDelete(project.slug); } return result; }) as typeof env.ROUTING.put); @@ -538,7 +539,6 @@ describe('per-deploy sandbox integration', () => { typeof e === 'object' && e !== null && (e as { event?: unknown }).event === 'error', ); expect(errorEvent).toBeDefined(); - await teardownPromise; // Teardown's deletion remains authoritative. Before the compensation // fence, restoreTrackedRouting() put `previousRouting` back here and a