Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion apps/cli/docs/stack-commands.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ when its engine answers, and otherwise native on Linux x64/arm64 and macOS arm64
platforms without a reachable engine, the command fails and asks you to start Docker or Podman. The
selected runtime is saved with the stack and reused without probing; when auto selection skips
Docker, the command prints a notice saying so. To switch, destroy the stack or choose a different
`--stack` name. Project stacks created by database commands, and shadow stacks
`--stack` name. When an explicit `--runtime docker` or a saved Docker stack cannot reach Docker,
the failure asks you to install or start it, and also suggests `--runtime native` for a new stack
on platforms that support native. Project stacks created by database commands, and shadow stacks
created without a project stack, use the same selection.

`supabase stack prepare` downloads or pulls artifacts for the selected stack without starting
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,11 @@ probe is bounded by 10 seconds. Without a reachable engine on other platforms, t
asks the user to start Docker or Podman. When auto selection skips Docker, an info line names the
saved Podman or native runtime and how to switch to Docker. An existing stack keeps its saved
runtime and runs no probe. Explicit `--runtime docker`, `podman`, or `native` has no fallback.
When an explicit or saved Docker runtime is unreachable, the reported failure suggests starting
Docker, and `--runtime native` for a new stack on platforms that support native.

Native startup refuses root because PostgreSQL `initdb` cannot run as root, unless a Claude Code
sandbox is detected or `SUPABASE_NATIVE_POSTGRES_USER` names a non-root user. PostgreSQL then runs
or Modal Sandbox is detected or `SUPABASE_NATIVE_POSTGRES_USER` names a non-root user. PostgreSQL then runs
as that user: the CLI chowns the instance data, root key, socket directory, and the cached bundle's
`pgsodium_getkey.sh` to it, and adds traverse-only `o+x` to their parent directories, including
root's home directory. Later commands that restrict the artifact cache and stack state roots to
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import { defaultRuntime } from "@supabase/stack/internal/artifacts";
import { endpointReports } from "../stack-endpoints.format.ts";
import { withProjectFunctionsEnv } from "../../../../command-internal/stack-functions-env.ts";
import {
automaticRuntimeNotice,
selectStackRuntime,
} from "../../../../command-internal/stack-runtime.ts";
import { RuntimeInfo } from "../../../../shared/runtime/runtime-info.service.ts";
import { Effect, FileSystem, Fiber, Option, Path, Redacted, Ref } from "effect";
import {
resolveNativePostgresUser,
Expand Down Expand Up @@ -95,6 +97,35 @@ const stackError = (
});
};

// A saved stack keeps its runtime, so only a new stack can switch to native.
const dockerUnavailableSuggestion = (
runtimeInfo: { readonly platform: string; readonly arch: string },
creating: boolean,
) =>
creating && defaultRuntime({ os: runtimeInfo.platform, arch: runtimeInfo.arch }) === "native"
? "Docker CLI or daemon isn't reachable. Install or start Docker, or run with --runtime native."
: "Docker CLI or daemon isn't reachable. Install or start Docker.";

const stackAcquireError = (
cause: StackError,
runtimeContext: {
readonly selectedRuntime: "native" | "docker" | "podman";
readonly runtime: { readonly platform: string; readonly arch: string };
readonly creating: boolean;
},
) => {
const base = stackError(cause);
if (cause.reason !== "runtime-unavailable" || runtimeContext.selectedRuntime !== "docker")
return base;
return new StackCommandStartError({
reason: "runtime",
message: base.message,
...(base.detail === undefined ? {} : { detail: base.detail }),
suggestion: dockerUnavailableSuggestion(runtimeContext.runtime, runtimeContext.creating),
cause: base.cause,
});
};

const loadStartConfig = (projectRoot: string, fs: FileSystem.FileSystem, path: Path.Path) =>
Effect.gen(function* () {
const config = yield* loadStackConfig(projectRoot);
Expand Down Expand Up @@ -193,6 +224,7 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags
runtime: flags.runtime,
})
.pipe(Effect.mapError(mapTargetError));
const runtime = yield* RuntimeInfo;
const selectedRuntime = yield* selectStackRuntime(target.runtime).pipe(
Effect.mapError(
(error) =>
Expand Down Expand Up @@ -247,7 +279,11 @@ export const stackStart = Effect.fn("experimental.stack.start")(function* (flags
),
),
),
).pipe(Effect.mapError(stackError));
).pipe(
Effect.mapError((cause) =>
stackAcquireError(cause, { selectedRuntime, runtime, creating: target.id === undefined }),
),
);
const runtimeNotice =
target.id === undefined ? automaticRuntimeNotice(target.runtime, selectedRuntime) : undefined;
if (runtimeNotice !== undefined) yield* output.info(runtimeNotice);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1107,6 +1107,11 @@ describe("experimental stack start", () => {
Effect.provide(Layer.mergeAll(layers(root, fakeStack(), output, false), target, api)),
);
expect(error.message).toContain("Cannot connect to the Docker daemon");
expect(error).toBeInstanceOf(StackCommandStartError);
if (error instanceof StackCommandStartError) {
expect(error.reason).toBe("runtime");
expect(error.suggestion).toContain("Docker CLI or daemon isn't reachable");
}
expect(output.stderrText).not.toContain("Failed to stop");

const stacks = yield* StackApi.pipe(
Expand Down
2 changes: 1 addition & 1 deletion docs/adr/0026-slim-artifact-mirrors.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ ECR Public tags are always mutable; `ecr-public create-repository` has no immuta

A public S3 bucket on `*.amazonaws.com` holds native archives for hosts that cannot reach GitHub Releases ([infra/cli-artifacts](../../infra/cli-artifacts/README.md)). It is not a checksum authority. The stack tries the GitHub Release first and falls back to that bucket. The expected checksum comes from the release `SHA256SUMS` or, when that is blocked, from the archive layer of the `:version-native-<target>` artifact on GHCR. An archive from either host is accepted only when it matches.

The container runtime pulls a catalog image from GHCR first and falls back to the same reference on ECR Public. A failing fallback, for an image or a native archive, still reports the primary's original error.
The container runtime pulls a catalog image from GHCR first and falls back to the same reference on ECR Public. A failing image fallback still reports the primary's original error. When every native checksum source or archive mirror fails, the error names each source with its failure.

## Follow-up

Expand Down
2 changes: 1 addition & 1 deletion packages/stack/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ Omitted database `jwtSecret` and `rootKey` inputs use the shared local-developme
as `DEFAULT_LOCAL_JWT_SECRET` and `DEFAULT_POSTGRES_ROOT_KEY`. Explicit values override these defaults.
The effective root key is supplied through a stack-owned file for both native and container runtimes.

Native PostgreSQL refuses to run as uid 0. When the stack runs as root inside a detected agent sandbox (Claude Code), or `SUPABASE_NATIVE_POSTGRES_USER=<name>` names a non-root system user, only the PostgreSQL process runs as that user: the instance data directory, root key file, socket directory, and the bundle's `pgsodium_getkey.sh` are chowned to it, and the instance directory, the PostgreSQL bundle directory, and their ancestors receive traverse-only (`o+x`) permission, which the artifact cache and stack state keep when they restrict their roots to the owner. Running as root elsewhere fails before PostgreSQL launches.
Native PostgreSQL refuses to run as uid 0. When the stack runs as root inside a detected agent sandbox (Claude Code, Modal Sandbox), or `SUPABASE_NATIVE_POSTGRES_USER=<name>` names a non-root system user, only the PostgreSQL process runs as that user: the instance data directory, root key file, socket directory, and the bundle's `pgsodium_getkey.sh` are chowned to it, and the instance directory, the PostgreSQL bundle directory, and their ancestors receive traverse-only (`o+x`) permission, which the artifact cache and stack state keep when they restrict their roots to the owner. Running as root elsewhere fails before PostgreSQL launches.

Native PostgreSQL listens only on its socket and reads a stack-generated HBA file, written to the socket directory on every launch, instead of `PGDATA/pg_hba.conf`. It trusts `supabase_admin`, including through the proxied loopback database port, and requires `scram-sha-256` passwords from every other role.

Expand Down
112 changes: 78 additions & 34 deletions packages/stack/src/HostProcess.ts
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ type HostFailureReason =
| "runtime-unavailable"
| "invalid-owner-pid"
| "owner-exit-pending"
| "owner-exit-zombie"
| "owner-exit-probe";
const HostIdentity = Schema.Struct({
projectRoot: Schema.String,
Expand Down Expand Up @@ -572,37 +573,62 @@ export const launchHost = Effect.fn("HostProcess.launchHost")(function* (
export type OwnerExitProbeResult =
| { readonly state: "absent" }
| { readonly state: "present" }
| { readonly state: "zombie" }
| { readonly state: "inconclusive"; readonly code: "EPERM" };
export type OwnerExitProbe = (pid: number) => Effect.Effect<OwnerExitProbeResult, HostProcessError>;

/** Probes the captured owner PID with signal 0. */
const probeOwnerExit: OwnerExitProbe = Effect.fn("HostProcess.probeOwnerExit")(function* (pid) {
return yield* Effect.try({
try: () => {
process.kill(pid, 0);
return { state: "present" } as const;
},
catch: (cause) =>
new HostProcessError({
operation: "shutdown-exit",
message: `Shutdown acknowledged, but probing owner process ${pid} failed: ${String(cause)}`,
reason: "owner-exit-probe",
cause,
const zombieProcStates = new Set(["Z", "X"]);

/** The state field follows the last `)`, because `comm` may itself contain spaces and parens. */
const procStatState = (stat: string): string | undefined =>
stat
.slice(stat.lastIndexOf(")") + 1)
.trim()
.split(/\s+/u)[0];

/** Probes the owner PID with signal 0 and, on Linux, reports a `Z` or `X` owner as a zombie. */
export const ownerExitProbe = (
fs: FileSystem.FileSystem,
platform: string = process.platform,
): OwnerExitProbe =>
Effect.fn("HostProcess.probeOwnerExit")(function* (pid) {
const signalResult = yield* Effect.try({
try: () => {
process.kill(pid, 0);
return { state: "present" } as const;
},
catch: (cause) =>
new HostProcessError({
operation: "shutdown-exit",
message: `Shutdown acknowledged, but probing owner process ${pid} failed: ${String(cause)}`,
reason: "owner-exit-probe",
cause,
}),
}).pipe(
Effect.catch((failure): Effect.Effect<OwnerExitProbeResult, HostProcessError> => {
const code = causeCode(failure.cause);
if (code === "ESRCH") return Effect.succeed({ state: "absent" } as const);
if (code === "EPERM") return Effect.succeed({ state: "inconclusive", code } as const);
return Effect.fail(failure);
}),
}).pipe(
Effect.catch((failure): Effect.Effect<OwnerExitProbeResult, HostProcessError> => {
const code = causeCode(failure.cause);
if (code === "ESRCH") return Effect.succeed({ state: "absent" } as const);
if (code === "EPERM") return Effect.succeed({ state: "inconclusive", code } as const);
return Effect.fail(failure);
}),
);
});
);
if (signalResult.state !== "present" || platform !== "linux") return signalResult;
const state = yield* fs.readFileString(`/proc/${pid}/stat`).pipe(
Effect.map(procStatState),
Effect.orElseSucceed(() => undefined),
);
return state !== undefined && zombieProcStates.has(state)
? ({ state: "zombie" } as const)
: signalResult;
});

/** Waits until the captured owner PID is absent from the process table. */
/**
* Waits until the captured owner PID is absent from the process table. An owner still a zombie when
* the wait ends has exited but was never reaped, as under a sandbox PID 1 that does not reap orphans.
*/
export const waitForOwnerExit = Effect.fn("HostProcess.waitForOwnerExit")(function* (
pid: number,
probe: OwnerExitProbe = probeOwnerExit,
probe: OwnerExitProbe,
) {
if (!Number.isSafeInteger(pid) || pid <= 0)
return yield* error(
Expand All @@ -611,25 +637,43 @@ export const waitForOwnerExit = Effect.fn("HostProcess.waitForOwnerExit")(functi
"invalid-owner-pid",
);
const check = probe(pid).pipe(
Effect.flatMap((result) =>
result.state === "absent"
? Effect.void
: Effect.fail(
Effect.flatMap((result) => {
switch (result.state) {
case "absent":
return Effect.void;
case "zombie":
return Effect.fail(
error("shutdown-exit", `Owner process ${pid} is not reaped yet`, "owner-exit-zombie"),
);
case "present":
return Effect.fail(
error(
"shutdown-exit",
result.state === "present"
? `Owner shutdown acknowledgement completed, but process ${pid} is still running`
: `Owner shutdown acknowledgement completed, but process ${pid} is inaccessible (${result.code}); exit is inconclusive`,
`Owner shutdown acknowledgement completed, but process ${pid} is still running`,
"owner-exit-pending",
),
),
),
);
case "inconclusive":
return Effect.fail(
error(
"shutdown-exit",
`Owner shutdown acknowledgement completed, but process ${pid} is inaccessible (${result.code}); exit is inconclusive`,
"owner-exit-pending",
),
);
}
}),
);
return yield* check.pipe(
Effect.retry({
schedule: Schedule.spaced("25 millis").pipe(Schedule.upTo({ duration: "5 seconds" })),
while: (failure) => failure.reason === "owner-exit-pending",
while: (failure) =>
failure.reason === "owner-exit-pending" || failure.reason === "owner-exit-zombie",
}),
Effect.catchIf(
(failure) => failure.reason === "owner-exit-zombie",
() => Effect.void,
),
);
});

Expand Down
66 changes: 65 additions & 1 deletion packages/stack/src/HostProcess.unit.test.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import { expect, it } from "@effect/vitest";
import { Cause, Deferred, Effect, Exit, Fiber, Ref } from "effect";
import { Cause, Deferred, Effect, Exit, Fiber, FileSystem, PlatformError, Ref } from "effect";
import * as TestClock from "effect/testing/TestClock";
import {
HostProcessError,
type OwnerExitProbe,
type OwnerExitProbeResult,
ownerExitProbe,
waitForOwnerExit,
} from "./HostProcess.ts";

Expand Down Expand Up @@ -119,3 +120,66 @@ it.effect("rejects an invalid owner PID before probing", () =>
expect(yield* Ref.get(attempts)).toBe(0);
}),
);

// The test process itself answers signal 0, so /proc content alone decides the result.
const statOf = (stat: string) =>
FileSystem.makeNoop({ readFileString: () => Effect.succeed(stat) });

it.effect("reports a zombie or dead owner on Linux", () =>
Effect.gen(function* () {
const probe = (stat: string) => ownerExitProbe(statOf(stat), "linux")(process.pid);
const pid = process.pid;
expect(yield* probe(`${pid} (node) Z 1 1 1 0 -1 4194560`)).toEqual({ state: "zombie" });
expect(yield* probe(`${pid} (node) X 1 1 1 0 -1 4194560`)).toEqual({ state: "zombie" });
expect(yield* probe(`${pid} (my ) weird) proc) Z 1 1 1 0 -1 4194304`)).toEqual({
state: "zombie",
});
expect(yield* probe(`${pid} (node) S 1 1 1 0 -1 4194560`)).toEqual({ state: "present" });
}),
);

it.effect("waits for a zombie owner to be reaped", () =>
Effect.gen(function* () {
const attempts = yield* Ref.make(0);
const probe: OwnerExitProbe = () =>
Ref.getAndUpdate(attempts, (count) => count + 1).pipe(
Effect.map((count): OwnerExitProbeResult =>
count < 2 ? { state: "zombie" } : { state: "absent" },
),
);
const fiber = yield* waitForOwnerExit(123, probe).pipe(Effect.forkChild);
yield* Effect.yieldNow;
yield* TestClock.adjust("100 millis");
yield* Fiber.join(fiber);
expect(yield* Ref.get(attempts)).toBe(3);
}).pipe(Effect.provide(TestClock.layer())),
);

it.effect("accepts an owner that is still an unreaped zombie when the wait ends", () =>
Effect.gen(function* () {
const probe: OwnerExitProbe = () => Effect.succeed({ state: "zombie" });
const fiber = yield* waitForOwnerExit(123, probe).pipe(Effect.exit, Effect.forkChild);
yield* Effect.yieldNow;
yield* TestClock.adjust("6 seconds");
expect(Exit.isSuccess(yield* Fiber.join(fiber))).toBe(true);
}).pipe(Effect.provide(TestClock.layer())),
);

it.effect("keeps the signal result when /proc is unreadable or the platform is not Linux", () =>
Effect.gen(function* () {
const unreadable = FileSystem.makeNoop({
readFileString: (path) =>
Effect.fail(
PlatformError.systemError({
_tag: "NotFound",
module: "FileSystem",
method: "readFileString",
pathOrDescriptor: path,
}),
),
});
expect(yield* ownerExitProbe(unreadable, "linux")(process.pid)).toEqual({ state: "present" });
const zombie = statOf(`${process.pid} (node) Z 1 1 1 0 -1 4194560`);
expect(yield* ownerExitProbe(zombie, "darwin")(process.pid)).toEqual({ state: "present" });
}),
);
17 changes: 2 additions & 15 deletions packages/stack/src/Orchestrator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import {
type ServiceObservation,
} from "./Service.ts";
import type { ServiceAdmission } from "./Service.ts";
import { errorChainMessage } from "./internal/error-message.ts";

export type OrchestratorOperation =
| "start"
Expand All @@ -47,21 +48,7 @@ export class OrchestratorError extends Data.TaggedError("OrchestratorError")<{

/** Summarizes a failure cause as one line per error, including its nested error causes. */
export const causeMessage = (cause: Cause.Cause<unknown>): string =>
Cause.prettyErrors(cause)
.map((error) => {
const parts: Array<string> = [];
const visited = new Set<Error>();
let current: unknown = error;
while (current instanceof Error && !visited.has(current)) {
visited.add(current);
const text = current.message || current.name;
// Wrappers often copy their cause's message; keep the chain but not the repeat.
if (parts.at(-1) !== text) parts.push(text);
current = current.cause;
}
return parts.join(": ");
})
.join("; ") || "interrupted";
Cause.prettyErrors(cause).map(errorChainMessage).join("; ") || "interrupted";

type CoreObservation = ServiceObservation<unknown>;

Expand Down
9 changes: 6 additions & 3 deletions packages/stack/src/Rpc.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,13 @@ export class StackError extends Schema.TaggedError<StackError>()("StackError", {
message: Schema.String,
outcomes: Schema.optionalKey(Schema.Array(Outcome)),
/**
* Why the client could not use an owner: none serves the stack (`owner-unavailable`), or one of
* another release does (`release-mismatch`).
* Why the client could not use an owner: none serves the stack (`owner-unavailable`), one of
* another release does (`release-mismatch`), or its container engine is unreachable
* (`runtime-unavailable`).
*/
reason: Schema.optionalKey(Schema.Literals(["owner-unavailable", "release-mismatch"])),
reason: Schema.optionalKey(
Schema.Literals(["owner-unavailable", "release-mismatch", "runtime-unavailable"]),
),
}) {}

/** Maps an owner failure to the RPC error, preserving per-member composition outcomes. */
Expand Down
Loading
Loading