diff --git a/apps/cli/src/command-internal/stack-config.ts b/apps/cli/src/command-internal/stack-config.ts index 3d5e438188..ed7fed08b4 100644 --- a/apps/cli/src/command-internal/stack-config.ts +++ b/apps/cli/src/command-internal/stack-config.ts @@ -1,5 +1,5 @@ import { getDefaultCliConfig, type CliConfig } from "@supabase/config"; -import { resolveCliConfigSubtree } from "@supabase/config/internal"; +import { ENV_CAPTURE_REGEX, resolveCliConfigSubtree } from "@supabase/config/internal"; import { validateCliConfig } from "@supabase/config/effect"; import { DEFAULT_LOCAL_JWT_SECRET, @@ -717,6 +717,14 @@ const unsupportedConfigPaths = [ "experimental.s3_secret_key", ] as const; +/** An unset `env(NAME)` stays as that literal, which is not a configured value. */ +const unresolvedEnvLiteral = (value: unknown): boolean => { + if (typeof value === "string") return ENV_CAPTURE_REGEX.test(value); + if (!Redacted.isRedacted(value)) return false; + const inner = Redacted.value(value); + return typeof inner === "string" && ENV_CAPTURE_REGEX.test(inner); +}; + const pathValue = (value: unknown, path: string): unknown => { let current = value; for (const segment of path.split(".")) { @@ -779,6 +787,7 @@ const configValidationError = (config: CliConfig): string | undefined => { for (const path of unsupportedConfigPaths) { if (path.startsWith("auth.") && !config.auth.enabled) continue; const value = pathValue(config, path); + if (path.startsWith("experimental.s3_") && unresolvedEnvLiteral(value)) continue; const baseline = pathValue(defaults, path); const difference = firstDifference(value, baseline, path); if (difference !== undefined) return `${difference} is unsupported by the experimental stack`; diff --git a/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts index 6637d1398c..3edd257096 100644 --- a/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts @@ -1,5 +1,5 @@ import { BunServices } from "@effect/platform-bun"; -import { describe, expect, it } from "@effect/vitest"; +import { afterEach, describe, expect, it, vi } from "@effect/vitest"; import { DEFAULT_LOCAL_JWT_SECRET, DEFAULT_POSTGRES_ROOT_KEY } from "@supabase/stack"; import { Effect, Exit, Layer, Redacted, Schema } from "effect"; import { ServiceCreation } from "../../../../../../packages/stack/src/services/Catalog.ts"; @@ -23,6 +23,10 @@ const byService = (services: ReadonlyArray [service.service, service])); describe("loadStackConfig", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + it.live("decodes the default recipe and leaves listeners automatic", () => Effect.gen(function* () { const root = yield* project(`project_id = "stack-config-defaults" @@ -172,6 +176,35 @@ major_version = 14 }).pipe(Effect.provide(BunServices.layer)), ); + it.live("ignores unresolved experimental S3 env placeholders", () => { + for (const name of ["s3_host", "S3_REGION", "S3_ACCESS_KEY", "S3_SECRET_KEY"]) + vi.stubEnv(name, ""); + return Effect.gen(function* () { + const root = yield* project(`project_id = "stack-config-s3-placeholder" +[experimental] +s3_host = "env(s3_host)" +s3_region = "env(S3_REGION)" +s3_access_key = "env(S3_ACCESS_KEY)" +s3_secret_key = "env(S3_SECRET_KEY)" +`); + const config = yield* load(root); + const services = yield* config.creations("stack-s3-placeholder"); + expect(services.some((service) => service.service === "database")).toBe(true); + }).pipe(Effect.provide(BunServices.layer)); + }); + + it.live("still rejects an unresolved auth service role key", () => + Effect.gen(function* () { + const root = yield* project(`project_id = "stack-config-auth-key" +[auth] +service_role_key = "env(SUPABASE_AUTH_SERVICE_ROLE_KEY)" +`); + const exit = yield* load(root).pipe(Effect.exit); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isFailure(exit)) expect(String(exit.cause)).toContain("auth.service_role_key"); + }).pipe(Effect.provide(BunServices.layer)), + ); + it.live("rejects unsupported OrioleDB and experimental S3 settings", () => Effect.gen(function* () { const orioledb = yield* project(`project_id = "stack-config-orioledb" diff --git a/apps/cli/src/commands/experimental/stack/start/start.handler.ts b/apps/cli/src/commands/experimental/stack/start/start.handler.ts index 33c2e3164b..1fba17b5df 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.handler.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.handler.ts @@ -2,7 +2,7 @@ import { endpointReports } from "../stack-endpoints.format.ts"; import { readStackFunctionsEnv } from "../../../../command-internal/stack-functions-env.ts"; import { defaultStackRuntime } from "../../../../command-internal/stack-runtime.ts"; import { Effect, Equal, FileSystem, Fiber, Option, Path, Redacted, Ref } from "effect"; -import type { ServiceCreation, Stack } from "@supabase/stack/effect"; +import { nativePostgresRootError, type ServiceCreation, type Stack } from "@supabase/stack/effect"; import { Output } from "../../../../shared/output/output.service.ts"; import { OutputFlag, @@ -207,6 +207,10 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags }), ), ); + const selectedRuntime = target.runtime ?? defaultStackRuntime(runtime); + const rootError = nativePostgresRootError(selectedRuntime, process.getuid?.()); + if (rootError !== undefined) + return yield* new StackCommandStartError({ reason: "lifecycle", message: rootError }); const stateRoot = path.join(settings.supabaseHome, "stacks"); const cacheRoot = path.join(settings.supabaseHome, "cache", "stack"); const stack = @@ -216,7 +220,7 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags projectRoot: target.projectRoot, stateRoot, cacheRoot, - runtime: target.runtime ?? defaultStackRuntime(runtime), + runtime: selectedRuntime, ...(target.name === undefined ? {} : { name: target.name }), }) .pipe(Effect.mapError(stackError)) diff --git a/docs/adr/0026-slim-artifact-mirrors.md b/docs/adr/0026-slim-artifact-mirrors.md index 004e40294e..31b429dcc1 100644 --- a/docs/adr/0026-slim-artifact-mirrors.md +++ b/docs/adr/0026-slim-artifact-mirrors.md @@ -7,7 +7,7 @@ Slim service images publish to `ghcr.io/supabase/cli/:`. Native archives publish to GitHub Releases. A broken build must be replaceable under the **same** upstream version string (`force=true`); bumping the tag is not an option. -Some consumers cannot reach GitHub release assets or a single registry blob CDN. Publishing one digest to more than one host is the prerequisite for later client fail-through; this ADR covers the publish path only. +Some consumers cannot reach GitHub release assets or a single registry blob CDN. This ADR covers publishing those copies and the local stack's fail-through. ## Decision @@ -15,14 +15,14 @@ Publish each slim **image** to GHCR and AWS ECR Public (`public.ecr.aws/supabase `force=true` always requests the ECR copy, including when the image digest is unchanged (natives may have moved). The **image** destination digest gates `publish-release` once the dispatch token exists. Native ECR copy is best-effort (`continue-on-error`) and must not fail that release. Native tags ride in a separate `natives[]` payload field. This mirror does not open a pull request to pin `packages/stack/src/Artifacts.ts`. A Dependabot Dockerfile bump commits that catalog pin onto the same pull request, so the docker.io tag, the slim image, and the native archive stay on one version. A PostgreSQL release line that is not the Dockerfile `pg` tag, and a same-version digest republish, stay put until that Dockerfile line changes. Do not prune untagged GHCR or ECR manifests: already-shipped CLIs still pin old image digests until that pull request merges. Natives follow the moved tag immediately. -ECR Public tags are always mutable; `ecr-public create-repository` has no immutability flag. Reuse the existing `PROD_AWS_ROLE` in `supabase/cli`. Do not add a public S3 bucket or a second cloud vendor for this cut. +ECR Public tags are always mutable; `ecr-public create-repository` has no immutability flag. Reuse the existing `PROD_AWS_ROLE` in `supabase/cli`. -Client fail-through (ordered GHCR / ECR / GitHub candidates in the CLI and stack) is out of scope here and lands in a follow-up PR. +A public S3 bucket on `*.amazonaws.com` holds native archives for hosts that cannot reach GitHub Releases ([infra/cli-artifacts](../../infra/cli-artifacts/README.md)). It is not a checksum authority. The stack tries the GitHub Release first and falls back to that bucket. The expected checksum comes from the release `SHA256SUMS` or, when that is blocked, from the archive layer of the `:version-native-` artifact on GHCR. An archive from either host is accepted only when it matches. + +The container runtime pulls a catalog image from GHCR first and falls back to the same reference on ECR Public. A failing fallback, for an image or a native archive, still reports the primary's original error. ## Follow-up -- Consume the mirrored image tags and `:version-native-` artifacts from the CLI and local stack, with host-aware order and digest pins preserved ([ADR 0017](0017-simplified-managed-stack-architecture.md)). -- Claude Trusted lists `public.ecr.aws` and `ghcr.io` but 403s blob CDNs (`*.cloudfront.net`, `pkg-containers.githubusercontent.com`) and GitHub release assets for unattached repos ([claude-code#71629](https://github.com/anthropics/claude-code/issues/71629)). Later options: S3 HTTPS on `*.amazonaws.com`, or Anthropic adding blob hosts. - Native ECR copy is best-effort; a daily mirror audit should report native drift. That audit is not defined in this repository yet. ## Rationale @@ -32,14 +32,14 @@ ECR Public mirroring already exists for other CLI images and needs no new vendor ## Consequences - Same-version overwrite works on GHCR, GitHub Releases, and ECR Public. -- Until the follow-up client PR, local stack and CLI still pull the GHCR catalog pin and GitHub Releases for natives. +- Images come from the GHCR catalog pin with ECR Public as fallback, and natives from GitHub Releases with the S3 bucket as fallback. - A GitHub native can be live while the ECR native is stale. - Old CLI releases keep pulling the previous image digest. ## Alternatives considered 1. **Google Artifact Registry / GCS** — blob host `storage.googleapis.com` is on at least one major sandbox Trusted list. Rejected for this cut: new company-wide vendor. -2. **Public S3 bucket** — HTTPS GET on `*.amazonaws.com` can work without CloudFront. Rejected for this cut: new public bucket, IAM, and security review on `PROD_AWS_ROLE`. +2. **Public S3 bucket** — HTTPS GET on `*.amazonaws.com` can work without CloudFront. Adopted for native archives. The bucket is not a checksum authority, and image pulls stay on GHCR and ECR Public. 3. **npm packages** — allowlisted widely, but a published version cannot be replaced. 4. **Docker Hub `supabase/cli-*`** — deferred; blob CDN is also off some default allowlists, and names collide with upstream `supabase/`. 5. **Unpin slim images** — would make old CLIs see a moved tag. Rejected: conflicts with ADR 0017. diff --git a/packages/stack/src/Artifacts.ts b/packages/stack/src/Artifacts.ts index 1f7ed92735..50ba106eaf 100644 --- a/packages/stack/src/Artifacts.ts +++ b/packages/stack/src/Artifacts.ts @@ -157,6 +157,27 @@ const platformText = (platform: { readonly os: string; readonly arch: string }): const errorMessage = (cause: unknown): string => cause instanceof Error ? cause.message : typeof cause === "string" ? cause : String(cause); +const SLIM_NATIVE_GITHUB_RELEASES = "https://github.com/supabase/slim-services/releases/download"; + +/** + * Public S3 copy of the release assets for hosts that block GitHub release downloads, such as + * agent sandboxes that allow `*.amazonaws.com`. + * @see ../../../infra/cli-artifacts/README.md + */ +const SLIM_NATIVE_SUPABASE_S3_MIRROR = "https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com"; + +const SLIM_NATIVE_GHCR_REGISTRY = "ghcr.io"; +const SLIM_NATIVE_GHCR_REPOSITORY = "supabase/cli"; + +const SLIM_IMAGE_GHCR_REGISTRY = "ghcr.io/supabase/cli/"; +const SLIM_IMAGE_SUPABASE_ECR_MIRROR = "public.ecr.aws/supabase/cli/"; + +/** Mirrors carrying a catalog slim image under the same tag and digest, in fallback order. */ +export const slimImageMirrors = (image: string): ReadonlyArray => + image.startsWith(SLIM_IMAGE_GHCR_REGISTRY) + ? [`${SLIM_IMAGE_SUPABASE_ECR_MIRROR}${image.slice(SLIM_IMAGE_GHCR_REGISTRY.length)}`] + : []; + const artifactFor = ( service: ServiceKind, resolved: ArtifactResolution, @@ -165,7 +186,8 @@ const artifactFor = ( const sourceService = definitions[service].sourceService; const releaseTag = `${sourceService}-${resolved.version}`; const assetName = `${releaseTag}-${target}`; - const base = `https://github.com/supabase/slim-services/releases/download/${releaseTag}`; + const githubRelease = `${SLIM_NATIVE_GITHUB_RELEASES}/${releaseTag}`; + const supabaseS3 = `${SLIM_NATIVE_SUPABASE_S3_MIRROR}/${sourceService}/${resolved.version}`; return { provider: "supabase/slim-services", service: sourceService, @@ -174,9 +196,25 @@ const artifactFor = ( target, archive: "tar.zst", assetName, - downloadUrl: `${base}/${assetName}.tar.zst`, - manifestUrl: `${base}/${assetName}.manifest.json`, - checksumUrl: `${base}/SHA256SUMS`, + checksums: [ + { kind: "sha256sums", url: `${githubRelease}/SHA256SUMS` }, + { + kind: "oci", + registry: SLIM_NATIVE_GHCR_REGISTRY, + repository: `${SLIM_NATIVE_GHCR_REPOSITORY}/${sourceService}`, + tag: `${resolved.version}-native-${target}`, + }, + ], + mirrors: [ + { + downloadUrl: `${githubRelease}/${assetName}.tar.zst`, + manifestUrl: `${githubRelease}/${assetName}.manifest.json`, + }, + { + downloadUrl: `${supabaseS3}/${assetName}.tar.zst`, + manifestUrl: `${supabaseS3}/${assetName}.manifest.json`, + }, + ], requiredRuntimePaths: resolved.requiredRuntimePaths, executablePath: resolved.executablePath, }; diff --git a/packages/stack/src/Artifacts.unit.test.ts b/packages/stack/src/Artifacts.unit.test.ts new file mode 100644 index 0000000000..0e0ecf510d --- /dev/null +++ b/packages/stack/src/Artifacts.unit.test.ts @@ -0,0 +1,16 @@ +import { expect, it } from "@effect/vitest"; +import { slimImageMirrors } from "./Artifacts.ts"; + +it("rewrites a GHCR catalog image onto ECR Public and keeps the tag and digest", () => { + expect( + slimImageMirrors( + "ghcr.io/supabase/cli/postgres:17.6.1.173@sha256:1581c433d71a48a81e356a3ed2d4aa5ecfc8fc0465ea98661da7a88023317dcf", + ), + ).toEqual([ + "public.ecr.aws/supabase/cli/postgres:17.6.1.173@sha256:1581c433d71a48a81e356a3ed2d4aa5ecfc8fc0465ea98661da7a88023317dcf", + ]); +}); + +it("returns no mirror for an image outside the slim catalog registry", () => { + expect(slimImageMirrors("public.ecr.aws/supabase/postgres:17.6.1.173")).toEqual([]); +}); diff --git a/packages/stack/src/Service.ts b/packages/stack/src/Service.ts index 7e64997c7e..22671eb590 100644 --- a/packages/stack/src/Service.ts +++ b/packages/stack/src/Service.ts @@ -56,6 +56,7 @@ class ServiceNotStopped extends Data.TaggedError("ServiceNotStopped")<{ export class ServiceStaleLaunch extends Data.TaggedError("ServiceStaleLaunch")<{ readonly id: string; readonly launchId: number; + readonly message: string; }> {} /** The exact runtime resources owned by one service launch. */ @@ -181,6 +182,10 @@ export const makeService = ( const revision = yield* Ref.make(0); const launchCounter = yield* Ref.make(0); const current = yield* Ref.make(undefined); + // stopNow clears observation.launchId while keeping the error, so readiness must not read that field. + const launchError = yield* Ref.make< + { readonly launchId: number; readonly error: ServiceError } | undefined + >(undefined); const config = yield* Ref.make(options.config); const observations = yield* SubscriptionRef.make>({ id: options.id, @@ -296,15 +301,19 @@ export const makeService = ( "start", guard.pipe( Effect.andThen( - update({ - lifecycle: "starting", - health: "starting", - error: undefined, - cleanupError: undefined, - exit: undefined, - wakeEnabled: observation.wakeEnabled, - launchId, - }), + Ref.set(launchError, undefined).pipe( + Effect.andThen( + update({ + lifecycle: "starting", + health: "starting", + error: undefined, + cleanupError: undefined, + exit: undefined, + wakeEnabled: observation.wakeEnabled, + launchId, + }), + ), + ), ), ), ); @@ -319,11 +328,13 @@ export const makeService = ( ); if (Exit.isFailure(launchExit)) { yield* Scope.close(runtimeScope, launchExit); + const error = exitError("launch", launchExit); + if (error !== undefined) yield* Ref.set(launchError, { launchId, error }); yield* update({ lifecycle: "stopped", health: undefined, wakeEnabled: observation.wakeEnabled, - error: exitError("launch", launchExit), + error, launchId: undefined, }); return yield* Effect.failCause(launchExit.cause); @@ -340,6 +351,8 @@ export const makeService = ( removed: yield* Ref.make(false), }; yield* Ref.set(current, record); + if (launchFailure !== undefined) + yield* Ref.set(launchError, { launchId, error: launchFailure }); yield* update({ lifecycle: launchFailure === undefined ? "running" : "stopping", health: launchFailure === undefined ? "starting" : undefined, @@ -351,12 +364,19 @@ export const makeService = ( const observeHealth = runtime.health.pipe( Effect.onExit((exit) => Effect.gen(function* () { + const error = exitError("health", exit); + const currentObservation = yield* SubscriptionRef.get(observations); + if ( + error !== undefined && + currentObservation.launchId === launchId && + currentObservation.lifecycle === "running" + ) + yield* Ref.set(launchError, { launchId, error }); yield* SubscriptionRef.update( observations, (observation): ServiceObservation => { if (observation.launchId !== launchId || observation.lifecycle !== "running") return observation; - const error = exitError("health", exit); return { ...observation, health: Exit.isSuccess(exit) ? "healthy" : "unhealthy", @@ -381,6 +401,8 @@ export const makeService = ( operation: "exit", message: "Runtime exited unexpectedly", })); + if (error !== undefined) + yield* Ref.set(launchError, { launchId: record.launchId, error }); yield* update({ lifecycle: "stopping", health: undefined, @@ -551,6 +573,14 @@ export const makeService = ( ); }); + const staleLaunch = (launchId: number, message: string) => + new ServiceStaleLaunch({ id: options.id, launchId, message }); + const diedBeforeReady = (launchId: number) => + Effect.gen(function* () { + const owned = yield* Ref.get(launchError); + if (owned !== undefined && owned.launchId === launchId) return yield* owned.error; + return yield* staleLaunch(launchId, `Service ${options.id} stopped before it was ready`); + }); const ready = Effect.fn("Service.ready")(function* () { const initial = yield* SubscriptionRef.get(observations); if (!initial.registered) return yield* new ServiceDestroyed({ id: options.id }); @@ -563,26 +593,26 @@ export const makeService = ( } if (record === undefined) { if ((yield* Ref.get(revision)) !== expectedRevision) - return yield* new ServiceStaleLaunch({ - id: options.id, - launchId: yield* Ref.get(launchCounter), - }); + return yield* staleLaunch( + yield* Ref.get(launchCounter), + `Service ${options.id} changed before it was ready`, + ); return yield* new ServiceNotRunning({ id: options.id }); } const launchId = record.launchId; if ((yield* Ref.get(revision)) !== expectedRevision) - return yield* new ServiceStaleLaunch({ id: options.id, launchId }); + return yield* staleLaunch(launchId, `Service ${options.id} changed before it was ready`); if (expectedLaunchId !== undefined && record.launchId !== expectedLaunchId) - return yield* new ServiceStaleLaunch({ id: options.id, launchId }); + return yield* staleLaunch(launchId, `Service ${options.id} changed before it was ready`); if ((yield* SubscriptionRef.get(observations)).lifecycle === "stopping") { - return yield* new ServiceStaleLaunch({ id: options.id, launchId }); + return yield* diedBeforeReady(launchId); } const healthResult = yield* Deferred.await(record.health); if ( (yield* Ref.get(current)) !== record || (yield* SubscriptionRef.get(observations)).lifecycle !== "running" ) { - return yield* new ServiceStaleLaunch({ id: options.id, launchId }); + return yield* diedBeforeReady(launchId); } yield* healthResult; }); diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index 42a2d8a967..2e0c9ce88d 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -38,6 +38,7 @@ import { DEFAULT_LOCAL_JWT_SECRET, DEFAULT_POSTGRES_ROOT_KEY } from "./Defaults. export { DEFAULT_LOCAL_JWT_SECRET, DEFAULT_POSTGRES_ROOT_KEY } from "./Defaults.ts"; export { postgres } from "./Tools.ts"; +export { nativePostgresRootError } from "./services/Database.ts"; export { StackError } from "./Rpc.ts"; export type { ServiceCreation } from "./services/Catalog.ts"; type DatabaseCreation = Extract; diff --git a/packages/stack/src/host/ToolRunner.ts b/packages/stack/src/host/ToolRunner.ts index 017c93677f..b98cb4fdda 100644 --- a/packages/stack/src/host/ToolRunner.ts +++ b/packages/stack/src/host/ToolRunner.ts @@ -14,7 +14,12 @@ import { import { rmdir } from "node:fs/promises"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { HttpClient } from "effect/unstable/http"; -import { prepareNativeArtifact, postgresVersion, resolveArtifact } from "../Artifacts.ts"; +import { + slimImageMirrors, + prepareNativeArtifact, + postgresVersion, + resolveArtifact, +} from "../Artifacts.ts"; import { makeContainerRuntime } from "../runtime/Container.ts"; import { PostgresTool, type PgProveOptions } from "../Tools.ts"; @@ -67,7 +72,7 @@ const makeToolRunner = (options: { const container = options.runtime === "native" ? undefined - : yield* makeContainerRuntime({ engine: options.runtime }); + : yield* makeContainerRuntime({ engine: options.runtime, imageMirrors: slimImageMirrors }); const jobsRoot = path.join(options.root, "jobs"); yield* fs .makeDirectory(jobsRoot, { recursive: true, mode: 0o700 }) diff --git a/packages/stack/src/preparation/SlimServicesSource.ts b/packages/stack/src/preparation/SlimServicesSource.ts index faddbd0e3f..90b57330a5 100644 --- a/packages/stack/src/preparation/SlimServicesSource.ts +++ b/packages/stack/src/preparation/SlimServicesSource.ts @@ -7,6 +7,25 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import type { ArtifactRequest, ArtifactSource } from "./ArtifactStore.ts"; import { PreparationError } from "./Errors.ts"; +/** One host serving the archive and manifest of a slim-services release asset. */ +interface SlimServicesMirror { + readonly downloadUrl: string; + readonly manifestUrl: string; +} + +/** + * Where the expected archive digest comes from: a release `SHA256SUMS` file, or the archive layer + * of the native OCI artifact in a registry that allows anonymous pulls. + */ +type SlimServicesChecksumSource = + | { readonly kind: "sha256sums"; readonly url: string } + | { + readonly kind: "oci"; + readonly registry: string; + readonly repository: string; + readonly tag: string; + }; + export interface SlimServicesArtifact { readonly provider: "supabase/slim-services"; readonly service: string; @@ -15,9 +34,13 @@ export interface SlimServicesArtifact { readonly target: "darwin-arm64" | "linux-amd64" | "linux-arm64"; readonly archive: "tar.zst"; readonly assetName: string; - readonly downloadUrl: string; - readonly manifestUrl: string; - readonly checksumUrl: string; + /** Checksum authorities, tried in order. Download mirrors never supply their own checksum. */ + readonly checksums: readonly [ + SlimServicesChecksumSource, + ...ReadonlyArray, + ]; + /** Hosts carrying the same release assets, tried in order until one serves them. */ + readonly mirrors: readonly [SlimServicesMirror, ...ReadonlyArray]; readonly requiredRuntimePaths: ReadonlyArray; readonly executablePath: string; } @@ -73,9 +96,9 @@ const systemTarBoundary: TarBoundary = { ); }), }; -const responseFor = (url: string) => +const responseFor = (url: string, headers?: Readonly>) => Effect.flatMap(HttpClient.HttpClient, (client) => - HttpClient.followRedirects(client).get(url), + HttpClient.followRedirects(client).get(url, { headers }), ).pipe( Effect.flatMap((response) => Effect.gen(function* () { @@ -121,8 +144,9 @@ const withTransferRetry = const fetchBytes = Effect.fn("SlimServicesSource.fetchBytes")(function* ( url: string, backoff: Schedule.Schedule, + headers?: Readonly>, ) { - return yield* responseFor(url).pipe( + return yield* responseFor(url, headers).pipe( Effect.flatMap((response) => response.arrayBuffer), Effect.map((bytes) => new Uint8Array(bytes)), withTransferRetry(url, backoff), @@ -200,33 +224,172 @@ const downloadToFile = Effect.fn("SlimServicesSource.downloadToFile")(function* }); }); +const fallbackDetail = (error: PreparationError): string => { + const nested = error.cause instanceof PreparationError ? error.cause.message : undefined; + return nested === undefined || nested.length === 0 + ? error.message + : `${error.message} (${nested})`; +}; + +/** + * Runs `attempt` against each candidate until one succeeds. When every candidate fails, the + * returned error is still the primary's. A fallback failure, including a checksum mismatch, is + * logged as a warning, and a fallback that succeeds names the host it used. + */ +const firstSuccess = ( + candidates: readonly [T, ...ReadonlyArray], + describe: (candidate: T) => string, + attempt: (candidate: T) => Effect.Effect, +): Effect.Effect => { + const [primary, ...fallbacks] = candidates; + const fallback = ( + primaryError: PreparationError, + remaining: ReadonlyArray, + ): Effect.Effect => { + const [candidate, ...rest] = remaining; + if (candidate === undefined) return Effect.fail(primaryError); + return attempt(candidate).pipe( + Effect.tap(() => Effect.logInfo(`Slim-services used ${describe(candidate)}`)), + Effect.tapError((cause) => + Effect.logWarning( + `Slim-services fallback ${describe(candidate)} failed: ${fallbackDetail(cause)}`, + ), + ), + Effect.catch(() => fallback(primaryError, rest)), + ); + }; + return attempt(primary).pipe(Effect.catch((primaryError) => fallback(primaryError, fallbacks))); +}; + const checksumFor = (contents: string, archiveName: string): string | undefined => contents .split(/\r?\n/u) .map((line) => line.trim().match(/^([a-f0-9]{64})\s+[* ]?(.+)$/iu)) .find((match) => match?.[2] === archiveName || match?.[2]?.endsWith(`/${archiveName}`))?.[1]; +const ARCHIVE_MEDIA_TYPE = "application/vnd.supabase.slim.archive.v1.tar+zstd"; + +const RegistryToken = Schema.Struct({ token: Schema.String }); + +const NativeOciManifest = Schema.Struct({ + layers: Schema.Array(Schema.Struct({ mediaType: Schema.String, digest: Schema.String })), +}); + +const decodeJson = (schema: S, bytes: Uint8Array, message: string) => + Schema.decodeEffect(Schema.fromJsonString(schema))(new TextDecoder().decode(bytes)).pipe( + Effect.mapError((cause) => new PreparationError({ message, cause })), + ); + +/** Reads the archive layer digest of a native OCI artifact through an anonymous pull token. */ +const ociArchiveDigest = Effect.fn("SlimServicesSource.ociArchiveDigest")(function* ( + source: Extract, + backoff: Schedule.Schedule, +) { + const tokenUrl = `https://${source.registry}/token?scope=repository:${source.repository}:pull&service=${source.registry}`; + const { token } = yield* decodeJson( + RegistryToken, + yield* fetchBytes(tokenUrl, backoff), + "Slim-services registry token is invalid", + ); + const manifest = yield* decodeJson( + NativeOciManifest, + yield* fetchBytes( + `https://${source.registry}/v2/${source.repository}/manifests/${source.tag}`, + backoff, + { + accept: "application/vnd.oci.image.manifest.v1+json", + authorization: `Bearer ${token}`, + }, + ), + "Slim-services native OCI manifest is invalid", + ); + const digest = manifest.layers + .find((layer) => layer.mediaType === ARCHIVE_MEDIA_TYPE) + ?.digest.match(/^sha256:([a-f0-9]{64})$/u)?.[1]; + if (digest === undefined) + return yield* new PreparationError({ + message: "Slim-services native OCI manifest has no archive layer", + }); + return digest; +}); + +const describeChecksumSource = (source: SlimServicesChecksumSource): string => + source.kind === "sha256sums" + ? source.url + : `${source.registry}/${source.repository}:${source.tag}`; + export const slimServicesChecksum = Effect.fn("SlimServicesSource.checksum")(function* ( artifact: SlimServicesArtifact, backoff: Schedule.Schedule = transferBackoff, ) { - return yield* fetchBytes(artifact.checksumUrl, backoff).pipe( - Effect.map((bytes) => new TextDecoder().decode(bytes)), - Effect.flatMap((contents) => { - const checksum = checksumFor(contents, `${artifact.assetName}.tar.zst`); - return checksum === undefined || !/^[a-f0-9]{64}$/iu.test(checksum) - ? Effect.fail( - new PreparationError({ - message: "Slim-services checksum is missing", - service: artifact.service, - version: artifact.version, - }), - ) - : Effect.succeed(checksum.toLowerCase()); - }), + return yield* firstSuccess(artifact.checksums, describeChecksumSource, (source) => + source.kind === "oci" + ? ociArchiveDigest(source, backoff) + : fetchBytes(source.url, backoff).pipe( + Effect.map((bytes) => new TextDecoder().decode(bytes)), + Effect.flatMap((contents) => { + const checksum = checksumFor(contents, `${artifact.assetName}.tar.zst`); + return checksum === undefined || !/^[a-f0-9]{64}$/iu.test(checksum) + ? Effect.fail( + new PreparationError({ + message: "Slim-services checksum is missing", + service: artifact.service, + version: artifact.version, + }), + ) + : Effect.succeed(checksum.toLowerCase()); + }), + ), ); }); +const manifestSchema = Schema.Struct({ + service: Schema.String, + version: Schema.String, + target: Schema.Literals(["darwin-arm64", "linux-amd64", "linux-arm64"]), + entrypoint: Schema.optionalKey(Schema.Array(Schema.String)), + cmd: Schema.optionalKey(Schema.Array(Schema.String)), +}); + +const verifiedManifest = Effect.fn("SlimServicesSource.verifiedManifest")(function* ( + artifact: SlimServicesArtifact, + mirror: SlimServicesMirror, + backoff: Schedule.Schedule, +) { + const manifestBytes = yield* fetchBytes(mirror.manifestUrl, backoff); + const manifest = yield* Schema.decodeEffect(Schema.fromJsonString(manifestSchema))( + new TextDecoder().decode(manifestBytes), + ).pipe( + Effect.mapError( + (cause) => + new PreparationError({ + message: "Slim-services manifest is invalid", + cause, + }), + ), + ); + if ( + manifest.service !== artifact.service || + manifest.version !== artifact.version || + manifest.target !== artifact.target + ) + return yield* new PreparationError({ + message: "Slim-services manifest does not match the catalog artifact", + service: artifact.service, + version: artifact.version, + target: artifact.target, + }); + if ( + (manifest.entrypoint !== undefined && manifest.entrypoint.some(unsafeManifestCommand)) || + (manifest.cmd !== undefined && manifest.cmd.some(unsafeManifestCommand)) + ) + return yield* new PreparationError({ + message: "Slim-services manifest command is invalid", + service: artifact.service, + version: artifact.version, + }); +}); + const unsafeArchivePath = (value: string): boolean => { const normalized = value.trim(); if (normalized.length === 0) return false; @@ -332,61 +495,28 @@ export const makeSlimServicesSource = ( ]); return yield* Effect.gen(function* () { const artifact = yield* resolveArtifact(request); - const manifestBytes = yield* fetchBytes(artifact.manifestUrl, backoff); - const manifestText = new TextDecoder().decode(manifestBytes); - const manifestSchema = Schema.Struct({ - service: Schema.String, - version: Schema.String, - target: Schema.Literals(["darwin-arm64", "linux-amd64", "linux-arm64"]), - entrypoint: Schema.optionalKey(Schema.Array(Schema.String)), - cmd: Schema.optionalKey(Schema.Array(Schema.String)), - }); - const manifest = yield* Schema.decodeEffect(Schema.fromJsonString(manifestSchema))( - manifestText, - ).pipe( - Effect.mapError( - (cause) => - new PreparationError({ - message: "Slim-services manifest is invalid", - cause, - }), - ), - ); - if ( - manifest.service !== artifact.service || - manifest.version !== artifact.version || - manifest.target !== artifact.target - ) - return yield* new PreparationError({ - message: "Slim-services manifest does not match the catalog artifact", - service: artifact.service, - version: artifact.version, - target: artifact.target, - }); - const entrypoint = manifest.entrypoint; - const command = manifest.cmd; - if ( - (entrypoint !== undefined && entrypoint.some(unsafeManifestCommand)) || - (command !== undefined && command.some(unsafeManifestCommand)) - ) - return yield* new PreparationError({ - message: "Slim-services manifest command is invalid", - service: artifact.service, - version: artifact.version, - }); - yield* Effect.sync(() => onProgress?.("downloading")).pipe( - Effect.andThen( - downloadToFile(artifact.downloadUrl, compressedPath, expectedSha256, backoff), - ), - Effect.mapError( - (cause) => - new PreparationError({ - message: "Unable to download slim-services archive", - service: artifact.service, - version: artifact.version, - cause, - }), - ), + yield* firstSuccess( + artifact.mirrors, + (mirror) => mirror.downloadUrl, + (mirror) => + verifiedManifest(artifact, mirror, backoff).pipe( + Effect.andThen( + Effect.sync(() => onProgress?.("downloading")).pipe( + Effect.andThen( + downloadToFile(mirror.downloadUrl, compressedPath, expectedSha256, backoff), + ), + Effect.mapError( + (cause) => + new PreparationError({ + message: "Unable to download slim-services archive", + service: artifact.service, + version: artifact.version, + cause, + }), + ), + ), + ), + ), ); yield* Effect.sync(() => onProgress?.("preparing")); yield* decompressor.decompress(compressedPath, archivePath); diff --git a/packages/stack/src/preparation/slim-services.integration.test.ts b/packages/stack/src/preparation/slim-services.integration.test.ts index 919588ebba..467c31b253 100644 --- a/packages/stack/src/preparation/slim-services.integration.test.ts +++ b/packages/stack/src/preparation/slim-services.integration.test.ts @@ -17,6 +17,7 @@ import { import { createServer, type Server } from "node:http"; import { zstdCompress } from "node:zlib"; import { FetchHttpClient } from "effect/unstable/http"; +import { prepareNativeArtifact, resolveArtifact } from "../Artifacts.ts"; import { makeArtifactStore, type ArtifactRequest } from "./ArtifactStore.ts"; import { digestHex } from "./Integrity.ts"; import { @@ -41,9 +42,13 @@ const artifact: SlimServicesArtifact = { target: "linux-amd64", archive: "tar.zst", assetName: "demo-v1.0.0-linux-amd64", - downloadUrl: "https://example.test/demo.tar.zst", - manifestUrl: "https://example.test/demo.manifest.json", - checksumUrl: "https://example.test/SHA256SUMS", + checksums: [{ kind: "sha256sums", url: "https://example.test/SHA256SUMS" }], + mirrors: [ + { + downloadUrl: "https://example.test/demo.tar.zst", + manifestUrl: "https://example.test/demo.manifest.json", + }, + ], requiredRuntimePaths: ["bin/demo"], executablePath: "bin/demo", }; @@ -133,6 +138,44 @@ const withFetch = (fetcher: FetchLike, effect: Effect.Effect) Effect.provideService(FetchHttpClient.Fetch, fetcher), ); +const registryChecksum = { + kind: "oci", + registry: "registry.test", + repository: "supabase/cli/demo", + tag: "v1.0.0-native-linux-amd64", +} as const; + +/** Anonymous registry token and native manifest; the manifest requires the issued token. */ +const registryResponse = ( + input: Parameters[0], + init: Parameters[1], + archiveSha256: string, + registry = "registry.test", + repository = "supabase/cli/demo", + tag = "v1.0.0-native-linux-amd64", +): Response | undefined => { + const url = requestUrl(input); + if (url === `https://${registry}/token?scope=repository:${repository}:pull&service=${registry}`) + return new Response(JSON.stringify({ token: "anonymous" })); + if (url !== `https://${registry}/v2/${repository}/manifests/${tag}`) return undefined; + if (new Headers(init?.headers).get("authorization") !== "Bearer anonymous") + return new Response("", { status: 401 }); + return new Response( + JSON.stringify({ + layers: [ + { + mediaType: "application/vnd.supabase.slim.archive.v1.tar+zstd", + digest: `sha256:${archiveSha256}`, + }, + { + mediaType: "application/vnd.supabase.slim.checksum.v1", + digest: `sha256:${"f".repeat(64)}`, + }, + ], + }), + ); +}; + describe("slim-services artifact source", () => { it.live("follows release redirects through the supplied Node HTTP client", () => Effect.scoped( @@ -164,9 +207,9 @@ describe("slim-services artifact source", () => { const address = server.address(); if (address === null || typeof address === "string") return yield* Effect.die("redirect server did not expose a port"); - const redirected = { + const redirected: SlimServicesArtifact = { ...artifact, - checksumUrl: `http://127.0.0.1:${address.port}/redirect`, + checksums: [{ kind: "sha256sums", url: `http://127.0.0.1:${address.port}/redirect` }], }; const checksum = yield* slimServicesChecksum(redirected); expect(checksum).toBe("a".repeat(64)); @@ -243,10 +286,10 @@ describe("slim-services artifact source", () => { const destination = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-retry-" }); const source = makeSlimServicesSource(() => artifact, { backoff: immediate }); expect(yield* withFetch(flaky, source.checksum(request))).toBe(expected); - expect(attempts.get(artifact.checksumUrl)).toBe(2); + expect(attempts.get("https://example.test/SHA256SUMS")).toBe(2); yield* withFetch(flaky, source.materialize(request, destination, expected)); expect(yield* fs.readFileString(`${destination}/bin/demo`)).toBe("demo"); - expect(attempts.get(artifact.downloadUrl)).toBe(2); + expect(attempts.get(artifact.mirrors[0].downloadUrl)).toBe(2); }).pipe(Effect.provide(NodeServices.layer)), ), ); @@ -271,6 +314,143 @@ describe("slim-services artifact source", () => { }), ); + it.live("prepares the artifact from the next mirror when the release host is blocked", () => + Effect.scoped( + Effect.gen(function* () { + const archive = yield* compress(tar("bin/demo", "demo")); + const crypto = yield* Crypto.Crypto; + const expected = digestHex(yield* crypto.digest("SHA-256", archive)); + const mirrored: SlimServicesArtifact = { + ...artifact, + checksums: [ + { kind: "sha256sums", url: "https://release.test/SHA256SUMS" }, + registryChecksum, + ], + mirrors: [ + { + downloadUrl: "https://release.test/demo-v1.0.0-linux-amd64.tar.zst", + manifestUrl: "https://release.test/demo-v1.0.0-linux-amd64.manifest.json", + }, + { + downloadUrl: "https://bucket.test/demo-v1.0.0-linux-amd64.tar.zst", + manifestUrl: "https://bucket.test/demo-v1.0.0-linux-amd64.manifest.json", + }, + ], + }; + const blocked = new Set(); + const bucket: string[] = []; + const fetcher: FetchLike = (input, init) => { + const url = requestUrl(input); + if (url.startsWith("https://release.test/")) { + blocked.add(url); + return Promise.resolve(new Response("", { status: 403 })); + } + const registry = registryResponse(input, init, expected); + if (registry !== undefined) return Promise.resolve(registry); + bucket.push(url); + if (url.endsWith("manifest.json")) + return Promise.resolve( + new Response( + JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), + ), + ); + return Promise.resolve(new Response(archive)); + }; + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-fallback-" }); + const store = yield* makeArtifactStore({ + cacheRoot: root, + source: makeSlimServicesSource(() => mirrored, { backoff: immediate }), + }); + const prepared = yield* withFetch(fetcher, store.prepare(request)); + expect(prepared.outcome).toBe("downloaded"); + expect(yield* fs.readFileString(`${prepared.path}/bin/demo`)).toBe("demo"); + expect([...blocked]).toEqual([ + "https://release.test/SHA256SUMS", + "https://release.test/demo-v1.0.0-linux-amd64.manifest.json", + ]); + expect(bucket).toEqual([ + "https://bucket.test/demo-v1.0.0-linux-amd64.manifest.json", + "https://bucket.test/demo-v1.0.0-linux-amd64.tar.zst", + ]); + }).pipe(Effect.provide(NodeServices.layer)), + ), + ); + + it.live("reports the primary checksum failure when every checksum source fails", () => + Effect.gen(function* () { + const mirrored: SlimServicesArtifact = { + ...artifact, + checksums: [ + { kind: "sha256sums", url: "https://release.test/SHA256SUMS" }, + registryChecksum, + ], + }; + const requested: string[] = []; + const failed = yield* withFetch((input) => { + requested.push(requestUrl(input)); + return Promise.resolve(new Response("", { status: 403 })); + }, slimServicesChecksum(mirrored, immediate).pipe(Effect.exit)); + expect(errorOf(failed)?.message).toBe("Unable to download https://release.test/SHA256SUMS"); + expect(requested).toEqual([ + "https://release.test/SHA256SUMS", + "https://registry.test/token?scope=repository:supabase/cli/demo:pull&service=registry.test", + ]); + }), + ); + + it.live("accepts a fallback archive only when it matches the checksum", () => + Effect.scoped( + Effect.gen(function* () { + const archive = yield* compress(tar("bin/demo", "demo")); + const tampered = yield* compress(tar("bin/demo", "evil")); + const crypto = yield* Crypto.Crypto; + const expected = digestHex(yield* crypto.digest("SHA-256", archive)); + const mirrored = (fallback: Uint8Array): SlimServicesArtifact => ({ + ...artifact, + mirrors: [ + { + downloadUrl: "https://release.test/demo.tar.zst", + manifestUrl: "https://release.test/demo.manifest.json", + }, + { + downloadUrl: `https://bucket.test/${fallback === archive ? "good" : "bad"}.tar.zst`, + manifestUrl: "https://bucket.test/demo.manifest.json", + }, + ], + }); + const fetcher: FetchLike = (input) => { + const url = requestUrl(input); + if (url.endsWith("manifest.json")) + return Promise.resolve( + new Response( + JSON.stringify({ service: "demo", version: "v1.0.0", target: "linux-amd64" }), + ), + ); + return Promise.resolve(new Response(url.endsWith("good.tar.zst") ? archive : tampered)); + }; + const fs = yield* FileSystem.FileSystem; + + const recovered = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-recover-" }); + yield* withFetch( + fetcher, + makeSlimServicesSource(() => mirrored(archive)).materialize(request, recovered, expected), + ); + expect(yield* fs.readFileString(`${recovered}/bin/demo`)).toBe("demo"); + + const rejected = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-reject-" }); + const failed = yield* withFetch( + fetcher, + makeSlimServicesSource(() => mirrored(tampered)) + .materialize(request, rejected, expected) + .pipe(Effect.exit), + ); + expect(errorOf(failed)?.message).toBe("Unable to download slim-services archive"); + expect(yield* fs.exists(`${rejected}/bin/demo`)).toBe(false); + }).pipe(Effect.provide(NodeServices.layer)), + ), + ); + it.live("rejects archive members that escape the artifact root", () => Effect.scoped( Effect.gen(function* () { @@ -658,3 +838,54 @@ describe("slim-services artifact source", () => { ), ); }); + +describe("native artifact catalog", () => { + it.live("verifies an S3 download against GHCR when GitHub release assets are blocked", () => + Effect.scoped( + Effect.gen(function* () { + const { version } = yield* resolveArtifact({ service: "rest" }); + const asset = `postgrest-${version}-linux-amd64`; + const archive = yield* compress(tar("bin/postgrest", "postgrest")); + const crypto = yield* Crypto.Crypto; + const expected = digestHex(yield* crypto.digest("SHA-256", archive)); + const served: string[] = []; + const fetcher: FetchLike = (input, init) => { + const url = requestUrl(input); + if (url.startsWith("https://github.com/")) + return Promise.resolve(new Response("", { status: 403 })); + served.push(url); + const registry = registryResponse( + input, + init, + expected, + "ghcr.io", + "supabase/cli/postgrest", + `${version}-native-linux-amd64`, + ); + if (registry !== undefined) return Promise.resolve(registry); + if (url.endsWith(".manifest.json")) + return Promise.resolve( + new Response( + JSON.stringify({ service: "postgrest", version, target: "linux-amd64" }), + ), + ); + return Promise.resolve(new Response(archive)); + }; + const fs = yield* FileSystem.FileSystem; + const cacheRoot = yield* fs.makeTempDirectoryScoped({ prefix: "slim-services-catalog-" }); + const prepared = yield* withFetch( + fetcher, + prepareNativeArtifact({ service: "rest" }, cacheRoot, { os: "linux", arch: "x64" }), + ); + expect(yield* fs.readFileString(prepared.executable)).toBe("postgrest"); + const bucket = `https://supabase-cli-artifacts.s3.us-east-1.amazonaws.com/postgrest/${version}`; + expect(served).toEqual([ + "https://ghcr.io/token?scope=repository:supabase/cli/postgrest:pull&service=ghcr.io", + `https://ghcr.io/v2/supabase/cli/postgrest/manifests/${version}-native-linux-amd64`, + `${bucket}/${asset}.manifest.json`, + `${bucket}/${asset}.tar.zst`, + ]); + }).pipe(Effect.provide(NodeServices.layer)), + ), + ); +}); diff --git a/packages/stack/src/runtime/Container.ts b/packages/stack/src/runtime/Container.ts index 82c62475b0..7cf8c1ae24 100644 --- a/packages/stack/src/runtime/Container.ts +++ b/packages/stack/src/runtime/Container.ts @@ -93,9 +93,14 @@ const mountField = (key: string, value: string) => { return /[,"\n\r]/u.test(field) ? `"${field.replaceAll('"', '""')}"` : field; }; -/** Captures the selected local engine; each launch owns one exact container. */ +/** + * Captures the selected local engine; each launch owns one exact container. An image whose pull + * fails is pulled from the first of its `imageMirrors` that succeeds, and launches of it then use + * that mirror reference. When every mirror fails, the primary pull error is reported. + */ export const makeContainerRuntime = (options: { readonly engine: "docker" | "podman"; + readonly imageMirrors?: (image: string) => ReadonlyArray; }): Effect.Effect< ContainerRuntime, never, @@ -144,15 +149,49 @@ export const makeContainerRuntime = (options: { ); }); - const prepare = Effect.fn("Container.prepare")(function* (image: string) { - const inspected = yield* run(["image", "inspect", "--format", "{{.Id}}", image]).pipe( + const mirrored = yield* Ref.make>(new Map()); + const present = (image: string) => + run(["image", "inspect", "--format", "{{.Id}}", image]).pipe( Effect.catchTag("ContainerError", (error) => /no such image|image .*not known/iu.test(error.message) ? Effect.succeed("") : Effect.fail(error), ), + Effect.map((id) => id.length > 0), + ); + const pull = (image: string) => run(["pull", image], { timeout: "5 minutes" }); + + const fromMirror = ( + image: string, + mirrors: ReadonlyArray, + primaryError: ContainerError, + ): Effect.Effect => { + const [mirror, ...rest] = mirrors; + if (mirror === undefined) return Effect.fail(primaryError); + return Effect.gen(function* () { + if (!(yield* present(mirror))) yield* pull(mirror); + yield* Ref.update(mirrored, (map) => new Map(map).set(image, mirror)); + }).pipe( + Effect.tap(() => Effect.logInfo(`Pulled image from mirror ${mirror}`)), + Effect.tapError((cause) => Effect.logWarning(`Image mirror ${mirror} failed`, cause)), + Effect.catch(() => fromMirror(image, rest, primaryError)), + ); + }; + + const prepare = Effect.fn("Container.prepare")(function* (image: string) { + // A mirror chosen earlier may have been pruned since; launches follow the primary again. + const usePrimary = Ref.update(mirrored, (map) => { + if (!map.has(image)) return map; + const next = new Map(map); + next.delete(image); + return next; + }); + if (yield* present(image)) return yield* usePrimary; + const mirrors = options.imageMirrors?.(image) ?? []; + yield* pull(image).pipe( + Effect.andThen(usePrimary), + Effect.catch((primaryError) => fromMirror(image, mirrors, primaryError)), ); - if (inspected.length === 0) yield* run(["pull", image], { timeout: "5 minutes" }); }); const launch = Effect.fn("Container.launch")(function* ( @@ -160,6 +199,7 @@ export const makeContainerRuntime = (options: { interactive = false, ) { const owner = yield* Scope.Scope; + const image = (yield* Ref.get(mirrored)).get(spec.image) ?? spec.image; for (const [key, value] of Object.entries(spec.env)) { if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key) || /[\0\r\n]/u.test(value)) { return yield* errorFor( @@ -222,7 +262,7 @@ export const makeContainerRuntime = (options: { ...(spec.workingDir === undefined ? [] : ["--workdir", spec.workingDir]), ...(spec.ports ?? []).flatMap((port) => ["--publish", `127.0.0.1::${port}`]), ...(spec.entrypoint === undefined ? [] : ["--entrypoint", spec.entrypoint]), - spec.image, + image, ...(spec.args ?? []), ]; diff --git a/packages/stack/src/runtime/container-image-mirror.integration.test.ts b/packages/stack/src/runtime/container-image-mirror.integration.test.ts new file mode 100644 index 0000000000..0680053bd4 --- /dev/null +++ b/packages/stack/src/runtime/container-image-mirror.integration.test.ts @@ -0,0 +1,153 @@ +import { NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Cause, Effect, Exit, Layer, Option, Sink, Stream } from "effect"; +import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; +import { makeContainerRuntime } from "./Container.ts"; + +const primary = "ghcr.io/supabase/cli/postgrest:v16.2"; +const mirror = "public.ecr.aws/supabase/cli/postgrest:v16.2"; +const secondMirror = "registry.test/supabase/cli/postgrest:v16.2"; + +/** Docker stand-in: `image inspect` reports `local`, `pull` succeeds for `pullable`, `create` refuses. */ +const fakeEngine = (options: { readonly pullable: ReadonlyArray }) => { + const pullable = new Set(options.pullable); + const local = new Set(); + const commands: string[][] = []; + const handle = (exitCode: number, stdout = "", stderr = "") => + ChildProcessSpawner.makeHandle({ + pid: ChildProcessSpawner.ProcessId(4242), + exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(exitCode)), + isRunning: Effect.succeed(false), + kill: () => Effect.void, + stdin: Sink.drain, + stdout: Stream.succeed(new TextEncoder().encode(stdout)), + stderr: Stream.succeed(new TextEncoder().encode(stderr)), + all: Stream.empty, + getInputFd: () => Sink.drain, + getOutputFd: () => Stream.empty, + unref: Effect.succeed(Effect.void), + }); + const spawner = ChildProcessSpawner.make((command) => { + if (!ChildProcess.isStandardCommand(command)) return Effect.die("unexpected piped command"); + const args = [...command.args]; + commands.push(args); + const ref = args.at(-1) ?? ""; + if (args[0] === "image") return Effect.succeed(handle(0, local.has(ref) ? "sha256:1" : "")); + if (args[0] === "pull") { + if (!pullable.has(ref)) return Effect.succeed(handle(1, "", `denied: ${ref}`)); + local.add(ref); + return Effect.succeed(handle(0)); + } + return Effect.succeed(handle(1, "", "create refused")); + }); + return { + commands, + pullable, + local, + layer: Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, spawner), + }; +}; + +const launchImage = (runtime: Effect.Success>) => + Effect.scoped( + runtime.launch({ image: primary, stackId: "a".repeat(64), instanceId: "mirror", env: {} }), + ).pipe(Effect.exit); + +describe("container image mirror", () => { + it.live("pulls and launches the mirror image when the primary pull fails", () => { + const engine = fakeEngine({ pullable: [mirror] }); + return Effect.gen(function* () { + const runtime = yield* makeContainerRuntime({ + engine: "docker", + imageMirrors: (image) => (image === primary ? [mirror] : []), + }); + yield* runtime.prepare(primary); + yield* launchImage(runtime); + expect(engine.commands.filter((args) => args[0] === "pull")).toEqual([ + ["pull", primary], + ["pull", mirror], + ]); + expect(engine.commands.find((args) => args[0] === "create")?.at(-1)).toBe(mirror); + }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); + }); + + it.live("tries each mirror in order until one pull succeeds", () => { + const engine = fakeEngine({ pullable: [secondMirror] }); + return Effect.gen(function* () { + const runtime = yield* makeContainerRuntime({ + engine: "docker", + imageMirrors: (image) => (image === primary ? [mirror, secondMirror] : []), + }); + yield* runtime.prepare(primary); + yield* launchImage(runtime); + expect(engine.commands.filter((args) => args[0] === "pull")).toEqual([ + ["pull", primary], + ["pull", mirror], + ["pull", secondMirror], + ]); + expect(engine.commands.find((args) => args[0] === "create")?.at(-1)).toBe(secondMirror); + }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); + }); + + it.live("launches the primary again once it pulls after an earlier mirror rescue", () => { + const engine = fakeEngine({ pullable: [mirror] }); + return Effect.gen(function* () { + const runtime = yield* makeContainerRuntime({ + engine: "docker", + imageMirrors: (image) => (image === primary ? [mirror] : []), + }); + yield* runtime.prepare(primary); + engine.local.delete(mirror); + engine.pullable.add(primary); + yield* runtime.prepare(primary); + yield* launchImage(runtime); + expect(engine.commands.find((args) => args[0] === "create")?.at(-1)).toBe(primary); + }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); + }); + + it.live("launches the primary again once it is present after an earlier mirror rescue", () => { + const engine = fakeEngine({ pullable: [mirror] }); + return Effect.gen(function* () { + const runtime = yield* makeContainerRuntime({ + engine: "docker", + imageMirrors: (image) => (image === primary ? [mirror] : []), + }); + yield* runtime.prepare(primary); + engine.local.delete(mirror); + engine.local.add(primary); + yield* runtime.prepare(primary); + yield* launchImage(runtime); + expect(engine.commands.find((args) => args[0] === "create")?.at(-1)).toBe(primary); + }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); + }); + + it.live("reports the primary pull failure when the mirror also fails", () => { + const engine = fakeEngine({ pullable: [] }); + return Effect.gen(function* () { + const runtime = yield* makeContainerRuntime({ + engine: "docker", + imageMirrors: (image) => (image === primary ? [mirror] : []), + }); + const failed = yield* runtime.prepare(primary).pipe(Effect.exit); + const error = Exit.isFailure(failed) + ? Option.getOrUndefined(Cause.findErrorOption(failed.cause)) + : undefined; + expect(error?.message).toContain(`denied: ${primary}`); + yield* launchImage(runtime); + expect(engine.commands.find((args) => args[0] === "create")?.at(-1)).toBe(primary); + }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); + }); + + it.live("never contacts the mirror when the primary pull succeeds", () => { + const engine = fakeEngine({ pullable: [primary, mirror] }); + return Effect.gen(function* () { + const runtime = yield* makeContainerRuntime({ + engine: "docker", + imageMirrors: (image) => (image === primary ? [mirror] : []), + }); + yield* runtime.prepare(primary); + yield* launchImage(runtime); + expect(engine.commands.flat()).not.toContain(mirror); + }).pipe(Effect.provide(Layer.merge(NodeServices.layer, engine.layer))); + }); +}); diff --git a/packages/stack/src/services/Catalog.ts b/packages/stack/src/services/Catalog.ts index d300e627d3..bb86838747 100644 --- a/packages/stack/src/services/Catalog.ts +++ b/packages/stack/src/services/Catalog.ts @@ -29,7 +29,7 @@ import { type ProcessRecipeResult, } from "./Recipe.ts"; import { makeProcessRecipe, type ProcessDependencies } from "./ProcessRecipe.ts"; -import type { ServiceKind } from "../Artifacts.ts"; +import { slimImageMirrors, type ServiceKind } from "../Artifacts.ts"; import type { ServiceInstanceContext } from "../Service.ts"; export type { CatalogLog } from "./Recipe.ts"; @@ -289,7 +289,10 @@ export const makeServiceRecipe = Effect.fn("Catalog.makeServiceRecipe")( const container = options.runtime === "native" ? undefined - : yield* makeContainerRuntime({ engine: options.runtime }); + : yield* makeContainerRuntime({ + engine: options.runtime, + imageMirrors: slimImageMirrors, + }); const deps: ProcessDependencies = { fs, path, crypto, client, spawner, container }; switch (creation.service) { case "rest": diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index 3190aa058d..f6e89b365c 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -5,6 +5,7 @@ import { Data, Effect, Exit, + Fiber, FileSystem, Layer, Path, @@ -20,6 +21,7 @@ import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import type { ChildProcessSpawner as ChildProcessSpawnerService } from "effect/unstable/process/ChildProcessSpawner"; import { HttpClient } from "effect/unstable/http"; import { + slimImageMirrors, prepareNativeArtifact, postgresVersion, resolveArtifact, @@ -123,6 +125,13 @@ export interface DatabaseComponent { readonly logs: Stream.Stream; } +/** initdb and the server both refuse uid 0, so native startup fails before any process is spawned. */ +export const nativePostgresRootError = ( + runtime: string, + uid: number | undefined, +): string | undefined => + runtime === "native" && uid === 0 ? "PostgreSQL cannot be run as root" : undefined; + const errorFor = (operation: string, cause: unknown): ServiceError => cause instanceof ServiceError ? cause @@ -139,17 +148,41 @@ const databaseError = (operation: string, cause: unknown): DatabaseError => cause, }); -const processExit = ( +/** A descendant outside the process group can keep stderr open after the launcher exits. */ +const stderrTailReady = (drained: Fiber.Fiber) => + Fiber.await(drained).pipe( + Effect.asVoid, + Effect.raceFirst(Effect.sleep("1 second")), + Effect.ignore, + ); + +/** Settles a native PostgreSQL exit, waiting briefly after it for the stderr tail to drain. */ +export const processExit = ( exitCode: Effect.Effect, + stderr?: { + readonly tail: Ref.Ref; + readonly drained: Fiber.Fiber; + }, ): Effect.Effect> => - exitCode.pipe( + (stderr === undefined + ? exitCode + : exitCode.pipe(Effect.tap(() => stderrTailReady(stderr.drained))) + ).pipe( Effect.flatMap((code) => Number(code) === 0 ? Effect.void - : Effect.fail( - new ServiceError({ - operation: "exit", - message: `PostgreSQL exited with code ${String(code)}`, + : (stderr === undefined ? Effect.succeed("") : Ref.get(stderr.tail)).pipe( + Effect.flatMap((text) => { + const detail = text.trim(); + return Effect.fail( + new ServiceError({ + operation: "exit", + message: + detail.length === 0 + ? `PostgreSQL exited with code ${String(code)}` + : `PostgreSQL exited with code ${String(code)}: ${detail}`, + }), + ); }), ), ), @@ -319,19 +352,33 @@ const publishLogs = Effect.fn("Database.publishLogs")(( }, logs: PubSub.PubSub, scope: Scope.Closeable, -): Effect.Effect => { - const drain = (stream: Stream.Stream, name: DatabaseLog["stream"]) => - stream.pipe( - Stream.runForEach((bytes) => PubSub.publish(logs, { stream: name, bytes })), + stderrTail?: Ref.Ref, +) => { + const drain = (stream: Stream.Stream, name: DatabaseLog["stream"]) => { + const decoder = name === "stderr" && stderrTail !== undefined ? new TextDecoder() : undefined; + const appendTail = (text: string) => + text.length === 0 || stderrTail === undefined + ? Effect.void + : Ref.update(stderrTail, (current) => (current + text).slice(-4096)); + return stream.pipe( + Stream.runForEach((bytes) => + Effect.gen(function* () { + if (decoder !== undefined) yield* appendTail(decoder.decode(bytes, { stream: true })); + yield* PubSub.publish(logs, { stream: name, bytes }); + }), + ), + Effect.andThen( + decoder === undefined + ? Effect.void + : Effect.sync(() => decoder.decode()).pipe(Effect.flatMap(appendTail)), + ), Effect.catch((cause) => Effect.logError(cause)), ); - return Effect.all( - [ - Effect.forkIn(drain(process.stdout, "stdout"), scope), - Effect.forkIn(drain(process.stderr, "stderr"), scope), - ], - { concurrency: "unbounded", discard: true }, - ); + }; + return Effect.gen(function* () { + yield* Effect.forkIn(drain(process.stdout, "stdout"), scope); + return yield* Effect.forkIn(drain(process.stderr, "stderr"), scope); + }); }); const runtimeFromContainer = (process: ContainerProcess): RuntimeSession => ({ @@ -482,7 +529,7 @@ export const makeDatabase = ( const container: ContainerRuntime | undefined = options.runtime === "native" ? undefined - : yield* makeContainerRuntime({ engine: options.runtime }); + : yield* makeContainerRuntime({ engine: options.runtime, imageMirrors: slimImageMirrors }); const storage: DockerDatabaseStorage | undefined = options.runtime === "native" ? undefined @@ -633,6 +680,8 @@ export const makeDatabase = ( const prepare = Effect.fn("Database.prepare")( function* (input: DatabaseConfig) { + const rootError = nativePostgresRootError(options.runtime, process.getuid?.()); + if (rootError !== undefined) return yield* errorFor("prepare", rootError); if (storage !== undefined) yield* storage.prepare(postgresVersion(input.version)); const markerPath = path.join(instanceRoot, ".supabase-database-ready.json"); const hasMarker = yield* fs.exists(markerPath); @@ -668,6 +717,8 @@ export const makeDatabase = ( context: ServiceInstanceContext, ): Effect.Effect => Effect.gen(function* () { + const rootError = nativePostgresRootError(options.runtime, process.getuid?.()); + if (rootError !== undefined) return yield* errorFor("launch", rootError); const config = { ...context.config, version: postgresVersion(context.config.version) }; const dataPath = path.join(instanceRoot, "data"); const dataMount = @@ -727,7 +778,8 @@ export const makeDatabase = ( port: 5432, }; yield* Ref.set(endpoint, selectedEndpoint); - yield* publishLogs(process, logs, context.scope); + const stderrTail = yield* Ref.make(""); + const stderrDrained = yield* publishLogs(process, logs, context.scope, stderrTail); return { health: health(selectedEndpoint, config, Effect.void, { fs, @@ -741,7 +793,7 @@ export const makeDatabase = ( .markInitialized(config.version) .pipe(Effect.mapError((cause) => errorFor("health", cause))), }), - exit: processExit(process.exitCode), + exit: processExit(process.exitCode, { tail: stderrTail, drained: stderrDrained }), stop: process.kill.pipe(Effect.mapError((cause) => errorFor("stop", cause))), remove: fs.remove(socketPath, { recursive: true, force: true }).pipe( Effect.mapError((cause) => errorFor("remove", cause)), diff --git a/packages/stack/src/services/Database.unit.test.ts b/packages/stack/src/services/Database.unit.test.ts new file mode 100644 index 0000000000..106c559758 --- /dev/null +++ b/packages/stack/src/services/Database.unit.test.ts @@ -0,0 +1,30 @@ +import { expect, it } from "@effect/vitest"; +import { Cause, Effect, Exit, Fiber, Option, Ref } from "effect"; +import * as TestClock from "effect/testing/TestClock"; +import { nativePostgresRootError, processExit } from "./Database.ts"; + +it("refuses native PostgreSQL for uid 0 and allows every other runtime", () => { + expect(nativePostgresRootError("native", 0)).toBe("PostgreSQL cannot be run as root"); + expect(nativePostgresRootError("native", 501)).toBeUndefined(); + expect(nativePostgresRootError("native", undefined)).toBeUndefined(); + expect(nativePostgresRootError("docker", 0)).toBeUndefined(); + expect(nativePostgresRootError("podman", 0)).toBeUndefined(); +}); + +it.effect("waits for the stderr tail after a long-running native PostgreSQL exits", () => + Effect.gen(function* () { + const tail = yield* Ref.make(""); + const drained = yield* Effect.forkChild( + Effect.sleep("2500 millis").pipe(Effect.andThen(Ref.set(tail, "FATAL: data directory"))), + ); + const settled = yield* Effect.forkChild( + processExit(Effect.sleep("2 seconds").pipe(Effect.as(1)), { tail, drained }), + ); + yield* TestClock.adjust("3 seconds"); + const exit = yield* Fiber.join(settled); + const error = Exit.isFailure(exit) + ? Option.getOrUndefined(Cause.findErrorOption(exit.cause)) + : undefined; + expect(error?.message).toBe("PostgreSQL exited with code 1: FATAL: data directory"); + }), +);