feat(api): support active-active PostgreSQL panels and fenced recovery

.github/actions/build-e2e-images/action.yml

@@ -1,9 +1,9 @@
 name: build e2e images
 description: >
   Build the operator/api/agent/sentinel/capture-sidecar/telemetry-receiver/fakeoidc/mcp-server/audit-syslog-bridge/tunnel-frp
-  (+ gameprobe and the postgres-tagged api) images the kind e2e suites load into the cluster, tagged
+  (+ gameprobe) images the kind e2e suites load into the cluster, tagged
   gameplane-test/<component>:e2e exactly like make e2e-images, via buildx bake
-  — then `docker save` all twelve into a single tar uploaded as an artifact
+  — then `docker save` all eleven into a single tar uploaded as an artifact
   (e2e-images-<arch>). The heavy e2e jobs load that tar instead of each
   recompiling the same binaries. This is the ONLY builder, so it always writes
   the layer cache back (the old single-writer export dance is gone).
@@ -52,15 +52,6 @@ runs:
         set: |
           e2e-gameprobe.cache-from=type=gha,scope=e2e-gameprobe-${{ inputs.arch }}
           e2e-gameprobe.cache-to=type=gha,mode=max,scope=e2e-gameprobe-${{ inputs.arch }}
-    - name: bake postgres-tagged api image
-      uses: docker/bake-action@d3418bd7d0e9324001bca92fa8ba175ea7e6dc9b # v7.3.0
-      with:
-        files: containers/docker-bake.hcl
-        targets: e2e-api-postgres
-        load: true
-        set: |
-          e2e-api-postgres.cache-from=type=gha,scope=e2e-api-postgres-${{ inputs.arch }}
-          e2e-api-postgres.cache-to=type=gha,mode=max,scope=e2e-api-postgres-${{ inputs.arch }}
     - name: save images to a single tar
       shell: bash
       run: |
@@ -76,7 +67,6 @@ runs:
           gameplane-test/audit-syslog-bridge:e2e \
           gameplane-test/tunnel-frp:e2e \
           gameplane-test/gameprobe:e2e \
-          gameplane-test/api:e2e-postgres \
           -o "$RUNNER_TEMP/e2e-images.tar"
     - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
       with:

.github/workflows/ci.yaml

@@ -1783,7 +1783,7 @@ jobs:
         working-directory: api
         env:
           GAMEPLANE_TEST_POSTGRES_DSN: postgres://postgres@localhost:5432/gameplane_test?sslmode=disable
-        run: go test -tags postgres -count=1 ./internal/db/... ./internal/controlplane/...
+        run: go test -tags postgres -count=1 ./internal/db/... ./internal/controlplane/... ./cmd
 
   # ---------------------------------------------------------------------
   # kind e2e — one job per suite, each with its own throwaway cluster on
@@ -2473,17 +2473,16 @@ jobs:
 
   # Re-runs the api-auth, api-roles and api-rbac buckets against a
   # Postgres-backed API (spec 013 US5, OD-3). amd64 only. The API is restarted
-  # between buckets: its login rate limiter lives in memory and each bucket is
-  # sized to one fresh limiter's budget (see test/e2e/buckets.sh). The restart
-  # also means each later bucket runs on data the earlier ones left in Postgres.
+  # between buckets to check persistence. PostgreSQL owns login budgets,
+  # so independent suites wait for natural refill using the database clock.
   # A final serial bucket migrates the Deployment strategy and exercises a
   # PostgreSQL outage/recovery against that populated installation.
   e2e-postgres:
     name: e2e postgres / amd64 (kind)
     needs: [changes, build-images]
     if: (needs.changes.outputs.e2e == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'e2e-postgres')
     runs-on: ubuntu-latest
-    timeout-minutes: 75 # Boots PostgreSQL and runs auth, roles, RBAC, then availability.
+    timeout-minutes: 105 # Sequential singleton/PITR and isolated multi-worker failover drills.
     permissions:
       contents: read
     env:
@@ -2518,44 +2517,71 @@ jobs:
             GAMEPLANE_E2E_REUSE_CLUSTER=1 \
             GAMEPLANE_E2E_CLUSTER="$CLUSTER" \
             go test -tags=e2e -timeout 15m -v -parallel 4 -run "$RUN_REGEX" ./...
-      - name: restart the API (fresh login limiter)
+      - name: restart the API between suites
         run: |
           kubectl --context "$CTX" --namespace gameplane-system rollout restart deployment/gameplane-api
           kubectl --context "$CTX" --namespace gameplane-system rollout status deployment/gameplane-api --timeout=180s
       - name: run go e2e (api-roles on postgres)
         run: |
+          bash deploy/kind/wait-postgres-login-budget.sh "$CTX"
           RUN_REGEX="$(./test/e2e/buckets.sh regex api-roles)"
           cd test/e2e && \
             GAMEPLANE_E2E_REUSE_CLUSTER=1 \
             GAMEPLANE_E2E_CLUSTER="$CLUSTER" \
             go test -tags=e2e -timeout 10m -v -parallel 3 -run "$RUN_REGEX" ./...
-      - name: restart the API again (fresh login limiter)
+      - name: restart the API again between suites
         run: |
           kubectl --context "$CTX" --namespace gameplane-system rollout restart deployment/gameplane-api
           kubectl --context "$CTX" --namespace gameplane-system rollout status deployment/gameplane-api --timeout=180s
       - name: run go e2e (api-rbac on postgres)
         run: |
+          bash deploy/kind/wait-postgres-login-budget.sh "$CTX"
           RUN_REGEX="$(./test/e2e/buckets.sh regex api-rbac)"
           cd test/e2e && \
             GAMEPLANE_E2E_REUSE_CLUSTER=1 \
             GAMEPLANE_E2E_CLUSTER="$CLUSTER" \
             go test -tags=e2e -timeout 10m -v -parallel 4 -run "$RUN_REGEX" ./...
       - name: run go e2e (PostgreSQL singleton upgrade and outage recovery)
         run: |
+          bash deploy/kind/wait-postgres-login-budget.sh "$CTX"
           RUN_REGEX="$(./test/e2e/buckets.sh regex postgres-availability)"
           cd test/e2e && \
             GAMEPLANE_E2E_REUSE_CLUSTER=1 \
             GAMEPLANE_E2E_CLUSTER="$CLUSTER" \
             go test -tags=e2e -timeout 15m -v -parallel 1 -run "$RUN_REGEX" ./...
       - name: verify Secret-backed PostgreSQL configuration after upgrade
         run: bash deploy/kind/check-postgres-secret.sh "$CTX"
+      - name: run go e2e (postgres-pitr)
+        run: |
+          bash deploy/kind/wait-postgres-login-budget.sh "$CTX"
+          RUN_REGEX="$(./test/e2e/buckets.sh regex postgres-pitr)"
+          cd test/e2e
+          GAMEPLANE_E2E_REUSE_CLUSTER=1 GAMEPLANE_E2E_CLUSTER="$CLUSTER" \
+            go test -tags=e2e -timeout 15m -v -parallel 1 -run "$RUN_REGEX" ./...
       - name: dump cluster state on failure (e2e-postgres)
         if: failure()
         uses: ./.github/actions/dump-cluster-state
         with:
           context: ${{ env.CTX }}
           namespaces: "gameplane-system,gameplane-games"
           include-previous-logs: "true"
+      - name: replace the PITR cluster with an isolated HA cluster
+        run: |
+          ./deploy/kind/e2e.sh down "$CLUSTER"
+          ./deploy/kind/e2e.sh up gameplane-postgres-ha e2e postgres-ha
+      - name: run go e2e (postgres-ha)
+        run: |
+          RUN_REGEX="$(./test/e2e/buckets.sh regex postgres-ha)"
+          cd test/e2e
+          GAMEPLANE_E2E_REUSE_CLUSTER=1 GAMEPLANE_E2E_CLUSTER=gameplane-postgres-ha \
+            go test -tags=e2e -timeout 20m -v -parallel 1 -run "$RUN_REGEX" ./...
+      - name: dump the HA cluster on failure
+        if: failure()
+        uses: ./.github/actions/dump-cluster-state
+        with:
+          context: kind-gameplane-postgres-ha
+          namespaces: "gameplane-system,gameplane-games"
+          include-previous-logs: "true"
 
   # Smoke-tests the dev ingress-nginx manifest pin (deploy/kind/up.sh,
   # INGRESS_NGINX_VERSION) on the same Kind node config `make dev-up` uses
@@ -2969,6 +2995,10 @@ jobs:
               if (/^e2e postgres \//.test(n) && Array.isArray(j.steps)) {
                 const availability = j.steps.find((s) => s.name === 'run go e2e (PostgreSQL singleton upgrade and outage recovery)');
                 if (availability && availability.conclusion === 'success') bucketSet.add('postgres-availability');
+                const pitr = j.steps.find((s) => s.name === 'run go e2e (postgres-pitr)');
+                if (pitr && pitr.conclusion === 'success') bucketSet.add('postgres-pitr');
+                const ha = j.steps.find((s) => s.name === 'run go e2e (postgres-ha)');
+                if (ha && ha.conclusion === 'success') bucketSet.add('postgres-ha');
               }
               // The ratelimit bucket runs as a trailing step inside the
               // api-auth job leg (deliberately, so it drains the shared

api/Dockerfile

@@ -23,8 +23,8 @@ ARG VERSION=dev
 # QEMU emulation: multi-arch release builds of the arm64 image were slow
 # enough to hit the release job timeout.
 ARG TARGETARCH
-# Build tags: "postgres" for the e2e-postgres image; empty by default so the release build is unchanged.
-ARG GO_TAGS=""
+# Both drivers ship in the multi-architecture API image. SQLite remains the runtime default.
+ARG GO_TAGS="postgres"
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build \
     -tags "${GO_TAGS}" \
     -trimpath \

api/cmd/bootstrap.go

@@ -84,14 +84,19 @@ func bootstrapAdmin(ctx context.Context, args []string, stdin io.Reader, stderr
 		display = bf.username
 	}
 
+	tx, err := store.BeginUserManagement(ctx)
+	if err != nil {
+		return fmt.Errorf("begin admin update: %w", err)
+	}
+	defer func() { _ = tx.Rollback() }()
 	var existingID int64
-	row := store.DB.QueryRowContext(ctx, `SELECT id FROM users WHERE username = ?`, bf.username)
+	row := tx.QueryRowContext(ctx, `SELECT id FROM users WHERE username = ?`, bf.username)
 	switch err := row.Scan(&existingID); {
 	case errors.Is(err, sql.ErrNoRows):
 		// RETURNING id instead of LastInsertId: pgx's database/sql driver
 		// has no LastInsertId, and both SQLite and Postgres support RETURNING.
 		var newID int64
-		if err := store.DB.QueryRowContext(ctx,
+		if err := tx.QueryRowContext(ctx,
 			`INSERT INTO users(username, display_name, email, role, pw_hash) VALUES (?, ?, ?, 'admin', ?) RETURNING id`,
 			bf.username, display, bf.email, hash,
 		).Scan(&newID); err != nil {
@@ -100,9 +105,12 @@ func bootstrapAdmin(ctx context.Context, args []string, stdin io.Reader, stderr
 		// Mirror the admin role into a cluster-wide role binding so RBAC
 		// resolves the user's permissions (this runs after Migrate, so the
 		// migration's backfill never saw this row).
-		if err := store.SetClusterRoleBinding(ctx, nil, newID, scope.DefaultCluster, "admin"); err != nil {
+		if err := store.SetClusterRoleBinding(ctx, tx, newID, scope.DefaultCluster, "admin"); err != nil {
 			return fmt.Errorf("bind admin role: %w", err)
 		}
+		if err := tx.Commit(); err != nil {
+			return fmt.Errorf("commit admin creation: %w", err)
+		}
 		_, _ = fmt.Fprintf(stderr, "bootstrap-admin: created user %q\n", bf.username)
 		return nil
 	case err != nil:
@@ -113,15 +121,6 @@ func bootstrapAdmin(ctx context.Context, args []string, stdin io.Reader, stderr
 		return fmt.Errorf("user %q already exists; pass --force to overwrite password and promote to admin", bf.username)
 	}
 
-	// Wrap password, role, and session eviction in a transaction so they
-	// commit together: no session outlives a reset that succeeded and
-	// nothing changes if any step fails.
-	tx, err := store.DB.BeginTx(ctx, nil)
-	if err != nil {
-		return fmt.Errorf("begin reset: %w", err)
-	}
-	defer func() { _ = tx.Rollback() }()
-
 	if _, err := tx.ExecContext(ctx,
 		`UPDATE users
 		   SET pw_hash = ?, session_epoch = session_epoch + 1, role = 'admin', display_name = ?, email = ?, updated_at = ?
@@ -220,16 +219,20 @@ func enableLocalLogin(ctx context.Context, store *db.Store, stderr io.Writer) er
 	if err != nil {
 		return fmt.Errorf("encode auth config: %w", err)
 	}
-	if _, err := store.DB.ExecContext(ctx,
-		`INSERT INTO config(key, value, updated_at)
-		 VALUES ('auth', ?, ?)
-		 ON CONFLICT(key) DO UPDATE SET
-		     value      = excluded.value,
-		     updated_at = excluded.updated_at`,
-		string(canon), db.NowTimestamp(),
-	); err != nil {
+	// Refuse to overwrite an identity-provider change made on another API
+	// replica while this command was decoding the old configuration.
+	res, err := store.DB.ExecContext(ctx, `UPDATE config SET value = ?, updated_at = ? WHERE key = 'auth' AND value = ?`,
+		string(canon), db.NowTimestamp(), raw)
+	if err != nil {
 		return fmt.Errorf("write auth config: %w", err)
 	}
+	n, err := res.RowsAffected()
+	if err != nil {
+		return fmt.Errorf("confirm auth config write: %w", err)
+	}
+	if n != 1 {
+		return errors.New("auth config changed concurrently; retry enabling local login")
+	}
 	_, _ = fmt.Fprintln(stderr, "bootstrap-admin: local login re-enabled in the auth config")
 	return nil
 }

api/cmd/bootstrap_test.go

@@ -476,3 +476,24 @@ func TestBootstrap_ForceResetIsAllOrNothing(t *testing.T) {
 		t.Fatalf("session count=%d want 1 (unchanged)", sessionCount)
 	}
 }
+
+func TestBootstrap_NewAdminAndRoleBindingCommitTogether(t *testing.T) {
+	dsn := dsnIn(t)
+	auth.SetFastHashParams(t)
+	s := mustOpen(t, dsn)
+	ctx := t.Context()
+	if err := s.Migrate(ctx); err != nil {
+		t.Fatal(err)
+	}
+	if _, err := s.DB.ExecContext(ctx, `CREATE TRIGGER fail_admin_grant BEFORE INSERT ON user_role_bindings BEGIN SELECT RAISE(ABORT, 'blocked'); END`); err != nil {
+		t.Fatal(err)
+	}
+	t.Setenv("GAMEPLANE_ADMIN_PASSWORD", "")
+	if _, err := runBootstrap(t, dsn, "", "--username=recovery-admin", "--password=new-password"); err == nil {
+		t.Fatal("bootstrap accepted a failed administrator role binding")
+	}
+	var count int
+	if err := s.DB.QueryRowContext(ctx, "SELECT COUNT(*) FROM users WHERE username='recovery-admin'").Scan(&count); err != nil || count != 0 {
+		t.Fatalf("partial bootstrap left %d admin accounts: %v", count, err)
+	}
+}

api/cmd/health.go

@@ -20,7 +20,7 @@ func mountHealthRoutes(r chi.Router, store *db.Store) {
 		_, _ = w.Write([]byte("ok"))
 	})
 	pingSlot := make(chan struct{}, 1)
-	r.Get("/readyz", func(w http.ResponseWriter, req *http.Request) {
+	ready := func(w http.ResponseWriter, req *http.Request) {
 		select {
 		case pingSlot <- struct{}{}:
 			defer func() { <-pingSlot }()
@@ -35,5 +35,11 @@ func mountHealthRoutes(r chi.Router, store *db.Store) {
 			return
 		}
 		_, _ = w.Write([]byte("ok"))
-	})
+	}
+	r.Get("/readyz", ready)
+	// Used to stage a PostgreSQL single writer before enabling rolling
+	// replicas. Older API images cannot pass this capability probe.
+	if store.Driver == "postgres" {
+		r.Get("/readyz/postgres-ha", ready)
+	}
 }

api/cmd/health_test.go

@@ -30,6 +30,7 @@ func TestHealthRoutesDatabaseUnavailable(t *testing.T) {
 		}
 	}
 	check("/readyz", http.StatusOK, "ok")
+	check("/readyz/postgres-ha", http.StatusNotFound, "404 page not found\n")
 	if err := store.Close(); err != nil {
 		t.Fatal(err)
 	}