diff --git a/apps/cli/docs/stack-commands.md b/apps/cli/docs/stack-commands.md index 24e4fe6453..e1e925c274 100644 --- a/apps/cli/docs/stack-commands.md +++ b/apps/cli/docs/stack-commands.md @@ -9,6 +9,11 @@ Native PostgreSQL requires passwords for every role except `supabase_admin`. Its and password-reconciliation connection uses that administrative role, so a native `supabase_admin` connection is not password-checked. +Native pg_cron runs jobs in background workers, so a job body cannot contain its own `BEGIN` or +`COMMIT`. Jobs enforce the same `supautils` role policies as client sessions. Native databases +default `max_worker_processes` to 17 to leave room for those workers; `postgresql.conf`, +`ALTER SYSTEM`, and `[db.settings]` values take precedence. + | Command | Purpose | | ------------------------ | --------------------------------------------------------------------------------- | | `supabase stack destroy` | Permanently delete one stack and its data. | diff --git a/packages/stack/README.md b/packages/stack/README.md index d396f6cb3a..4efc77320a 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -87,7 +87,7 @@ The effective root key is supplied through a stack-owned file for both native an Native PostgreSQL refuses to run as uid 0. When the stack runs as root inside a detected agent sandbox (Claude Code, Modal Sandbox), or `SUPABASE_NATIVE_POSTGRES_USER=` names a non-root system user, only the PostgreSQL process runs as that user: the instance data directory, root key file, socket directory, and the bundle's `pgsodium_getkey.sh` are chowned to it, and the instance directory, the PostgreSQL bundle directory, and their ancestors receive traverse-only (`o+x`) permission, which the artifact cache and stack state keep when they restrict their roots to the owner. Running as root elsewhere fails before PostgreSQL launches. -Native PostgreSQL listens only on its socket and reads a stack-generated HBA file, written to the socket directory on every launch, instead of `PGDATA/pg_hba.conf`. It trusts `supabase_admin`, including through the proxied loopback database port, and requires `scram-sha-256` passwords from every other role. +Native PostgreSQL listens only on its socket and reads a stack-generated HBA file, written to the socket directory on every launch, instead of `PGDATA/pg_hba.conf`. It trusts `supabase_admin`, including through the proxied loopback database port, and requires `scram-sha-256` passwords from every other role. pg_cron runs jobs in background workers instead of loopback TCP sessions, so a job body cannot contain its own `BEGIN` or `COMMIT`. `max_worker_processes` defaults to 17 for those workers; `PGDATA/postgresql.conf`, its includes, `ALTER SYSTEM`, and caller settings override it. Because workers skip session preload libraries, `supautils` is appended to the effective `shared_preload_libraries`; disposable shadow databases (`stopGraceSeconds: 0`) skip all of this and keep pg_cron's default mode. Native PostgreSQL receives the owner's `SSL_CERT_FILE` and `SSL_CERT_DIR` when they are non-empty. When `SSL_CERT_FILE` is unset or empty, it receives the first of `/etc/ssl/certs/ca-certificates.crt`, `/etc/pki/tls/certs/ca-bundle.crt`, `/etc/ssl/ca-bundle.pem`, and `/etc/ssl/cert.pem` that exists, if any. The `http` and `pg_net` extensions verify HTTPS certificates against these. On macOS, `/etc/ssl/cert.pem` does not include roots added to the Keychain; to trust those, set `SSL_CERT_FILE` to a bundle that also holds the public roots, such as a copy of `/etc/ssl/cert.pem` with those roots appended. When PostgreSQL runs as a separate user, that user must be able to read the owner's `SSL_CERT_FILE` and `SSL_CERT_DIR` paths, which are not chowned to it. The detached owner inherits the environment of the client that starts it and keeps it until it exits, through restarts and repeated starts, so set these before the owner starts, and stop the stack before starting it again after changing them. diff --git a/packages/stack/src/Artifacts.ts b/packages/stack/src/Artifacts.ts index a52945d102..c38c5ba21b 100644 --- a/packages/stack/src/Artifacts.ts +++ b/packages/stack/src/Artifacts.ts @@ -131,7 +131,14 @@ const definitions: Readonly> = { }, }, "bin/supabase-postgres-start", - ["bin/supabase-postgres-start", "bin/pg_dump", "bin/pg_dumpall", "bin/pg_prove", "bin/psql"], + [ + "bin/supabase-postgres-start", + "bin/postgres", + "bin/pg_dump", + "bin/pg_dumpall", + "bin/pg_prove", + "bin/psql", + ], { "15.19.0.004": { upstreamVersion: "15.19.0.004", diff --git a/packages/stack/src/services/Database.integration.test.ts b/packages/stack/src/services/Database.integration.test.ts index 9190ebf44f..553f105ed0 100644 --- a/packages/stack/src/services/Database.integration.test.ts +++ b/packages/stack/src/services/Database.integration.test.ts @@ -6,8 +6,10 @@ import { Context, Deferred, Effect, + Fiber, FileSystem, Layer, + Option, Path, Predicate, Redacted, @@ -31,7 +33,7 @@ const config: DatabaseConfig = { const artifactCacheRoot = `${tmpdir()}/supabase-stack-artifacts`; -const query = ( +const query = ( endpoint: BackendEndpoint, password: Redacted.Redacted, statement: string, @@ -50,7 +52,7 @@ const query = ( password, }), ); - return yield* Context.get(services, PgClient.PgClient).unsafe(statement); + return yield* Context.get(services, PgClient.PgClient).unsafe(statement); }), ); @@ -350,6 +352,122 @@ describe("database component", { timeout: 180_000 }, () => { ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), ); + for (const version of ["15", "17"]) + it.live(`runs native PostgreSQL ${version} cron jobs in supautils-guarded workers`, () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-database-cron-" }); + const database = yield* makeDatabase({ + stackId: "stack-integration", + instanceId: "cron", + root, + cacheRoot: artifactCacheRoot, + runtime: "native", + }); + const databaseConfig = { ...config, version }; + const service = yield* makeService(database.definition, { + id: "database:cron", + config: databaseConfig, + }); + yield* service.start; + yield* service.ready; + const endpoint = yield* database.endpoint; + const asPostgres = (statement: string) => + query(endpoint, config.databasePassword, statement, "postgres", "postgres"); + yield* asPostgres("CREATE EXTENSION pg_cron"); + yield* query( + endpoint, + config.databasePassword, + `CREATE FUNCTION report_cron_run() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + PERFORM pg_notify('cron_runs', (SELECT jobname FROM cron.job WHERE jobid = NEW.jobid) + || ': ' || NEW.status || ': ' || coalesce(NEW.return_message, '')); + RETURN NEW; + END $$; + CREATE TRIGGER report_cron_run AFTER INSERT OR UPDATE ON cron.job_run_details + FOR EACH ROW WHEN (NEW.status IN ('succeeded', 'failed')) + EXECUTE FUNCTION report_cron_run();`, + ); + const sql = Context.get( + yield* Layer.build( + PgClient.layer({ + host: endpoint.kind === "unix" ? endpoint.path : endpoint.host, + port: endpoint.port, + database: "postgres", + username: "supabase_admin", + password: config.databasePassword, + }), + ), + PgClient.PgClient, + ); + const cronRun = (name: string, command: string) => + Effect.gen(function* () { + /* Jobs repeat every second, so a run reported before LISTEN is ready is not lost. */ + const run = yield* sql.listen("cron_runs").pipe( + Stream.filter((message) => message.startsWith(`${name}: `)), + Stream.runHead, + Effect.forkScoped({ startImmediately: true }), + ); + yield* asPostgres(`SELECT cron.schedule('${name}', '1 seconds', $$${command}$$)`); + const message = yield* Fiber.join(run).pipe(Effect.timeout("60 seconds")); + yield* asPostgres(`SELECT cron.unschedule('${name}')`); + return Option.getOrElse(message, () => ""); + }); + expect(yield* cronRun("own_job", "SELECT 1")).toContain("own_job: succeeded"); + expect(yield* cronRun("reserved_role", "ALTER ROLE anon LOGIN")).toContain( + 'reserved_role: failed: ERROR: "anon" is a reserved role', + ); + const settings = Effect.flatMap(database.endpoint, (current) => + query<{ workers: string; cron: string; preload: string }>( + current, + config.databasePassword, + "SELECT current_setting('max_worker_processes') AS workers, current_setting('cron.use_background_workers') AS cron, current_setting('shared_preload_libraries') AS preload", + ), + ); + const [initial] = yield* settings; + expect(initial).toMatchObject({ workers: "17", cron: "on" }); + yield* service.restart({ ...databaseConfig, stopGraceSeconds: 0 }); + yield* service.ready; + const [shadow] = yield* settings; + expect(shadow).toMatchObject({ workers: "8", cron: "off" }); + expect(initial?.preload).toBe(`${shadow?.preload},supautils`); + yield* query( + yield* database.endpoint, + config.databasePassword, + "ALTER SYSTEM SET max_worker_processes = 32", + ); + yield* service.restart(databaseConfig); + yield* service.ready; + expect(yield* settings).toMatchObject([{ workers: "32", cron: "on" }]); + yield* service.destroy; + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); + + it.live("bounds the native configuration probe before first boot", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-database-probe-" }); + const database = yield* makeDatabase({ + stackId: "stack-integration", + instanceId: "probe", + root, + cacheRoot: artifactCacheRoot, + runtime: "native", + }); + const service = yield* makeService(database.definition, { + id: "database:probe", + config: { ...config, healthTimeoutMs: 0 }, + }); + const failure = yield* service.start.pipe(Effect.flip); + expect(String(failure)).toContain("PostgreSQL configuration probe timed out"); + expect(yield* fs.readDirectory(`${root}/probe/data`)).toEqual([]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + ); + it.live( "persists SQL data across exact-session stop and reopen, isolates instances, and validates restart before stopping", () => diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index a2848b9b51..db7417d5a8 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -61,6 +61,8 @@ import { defaultNativeProcessLauncher, spawnNativeProcess, type NativeProcess, + type NativeProcessIdentity, + type NativeProcessSpec, } from "../runtime/NativeProcess.ts"; import type { StackId } from "../identity/StackId.ts"; import { @@ -94,7 +96,7 @@ export const DatabaseConfig = Schema.Struct({ jwtSecret: Schema.Redacted(Schema.String), jwtExpiry: Schema.Finite, healthTimeoutMs: Schema.optionalKey(Schema.Finite), - /** When 0, the database is disposable and can use reduced-durability settings. */ + /** When 0, the database is disposable: reduced durability and pg_cron's default mode. */ stopGraceSeconds: Schema.optionalKey(Schema.Finite), rootKey: Schema.optionalKey(Schema.Redacted(Schema.String)), settings: Schema.optionalKey( @@ -449,6 +451,60 @@ const nativeTrustStore = Effect.gen(function* () { return env; }); +/* Background workers skip session_preload_libraries, so supautils must be shared-preloaded. */ +const sharedPreload = Effect.fn("Database.sharedPreload")(function* ( + artifact: PreparedNativeArtifact, + spec: NativeProcessSpec & { readonly args: ReadonlyArray }, + dataPath: string, + settings: ReadonlyArray, + identity: NativeProcessIdentity, +) { + const fs = yield* FileSystem.FileSystem; + const probe = yield* spawnNativeProcess( + { + ...spec, + executable: `${artifact.root}/bin/postgres`, + args: [ + ...((yield* fs.exists(`${dataPath}/PG_VERSION`)) + ? spec.args + : [ + "-D", + dataPath, + "-c", + `config_file=${artifact.root}/share/supabase-cli/config/postgresql.conf.template`, + ...settings, + ]), + "-C", + "shared_preload_libraries", + ], + }, + defaultNativeProcessLauncher(), + identity, + ); + const [stdout, stderr, code] = yield* Effect.all( + [ + probe.stdout.pipe(Stream.decodeText, Stream.mkString), + probe.stderr.pipe( + Stream.decodeText, + Stream.runFold( + () => "", + (text, chunk) => (text + chunk).slice(-4096), + ), + ), + probe.exitCode, + ], + { concurrency: "unbounded" }, + ); + yield* Effect.annotateCurrentSpan("process.exit_code", Number(code)); + if (code !== 0) + return yield* errorFor( + "exit", + `${describePostgresExit(Number(code))}: ${stderr.trim() || stdout.trim() || "PostgreSQL configuration probe failed"}`, + ); + const libraries = [stdout.trim(), "supautils"].filter((library) => library !== "").join(","); + return ["-c", `shared_preload_libraries=${libraries}`]; +}, Effect.scoped); + const nativeProcess = ( artifact: PreparedNativeArtifact, config: DatabaseConfig, @@ -457,6 +513,7 @@ const nativeProcess = ( readonly socketPath: string; readonly hbaPath: string; readonly rootKeyPath: string; + readonly configPath: string | undefined; }, settings: ReadonlyArray, context: ServiceInstanceContext, @@ -465,43 +522,63 @@ const nativeProcess = ( spawner: ChildProcessSpawnerService["Service"], user: PasswdEntry | undefined, trustStore: Readonly>, -): Effect.Effect => - spawnNativeProcess( - { - executable: artifact.executable, - ...(user === undefined ? {} : { uid: user.uid, gid: user.gid, cwd: "/" }), - args: [ - "-D", - paths.dataPath, - "-p", - "5432", - "-c", - "listen_addresses=", - "-c", - `unix_socket_directories=${paths.socketPath}`, - "-c", - `hba_file=${paths.hbaPath}`, - ...settings, - ], - env: { - ...trustStore, - ...(user === undefined ? {} : { HOME: user.home }), - PGDATA: paths.dataPath, - PGSODIUM_KEY_FILE: paths.rootKeyPath, - POSTGRES_USER: "supabase_admin", - POSTGRES_DB: "postgres", - POSTGRES_PASSWORD: Redacted.value(config.databasePassword), - }, - gracefulStopSignal: "SIGINT", - gracefulStopTimeout: "15 seconds", + fs: FileSystem.FileSystem, +): Effect.Effect => { + const identity = { stackId, workloadId: instanceId }; + const spec = { + executable: artifact.executable, + ...(user === undefined ? {} : { uid: user.uid, gid: user.gid, cwd: "/" }), + args: [ + "-D", + paths.dataPath, + "-p", + "5432", + "-c", + "listen_addresses=", + "-c", + `unix_socket_directories=${paths.socketPath}`, + "-c", + `hba_file=${paths.hbaPath}`, + ...(paths.configPath === undefined + ? [] + : ["-c", "cron.use_background_workers=on", "-c", `config_file=${paths.configPath}`]), + ...settings, + ], + env: { + ...trustStore, + ...(user === undefined ? {} : { HOME: user.home }), + PGDATA: paths.dataPath, + PGSODIUM_KEY_FILE: paths.rootKeyPath, + POSTGRES_USER: "supabase_admin", + POSTGRES_DB: "postgres", + POSTGRES_PASSWORD: Redacted.value(config.databasePassword), }, - defaultNativeProcessLauncher(), - { stackId, workloadId: instanceId }, - ).pipe( + gracefulStopSignal: "SIGINT", + gracefulStopTimeout: "15 seconds", + } satisfies NativeProcessSpec; + return Effect.gen(function* () { + const preload = + paths.configPath === undefined + ? [] + : yield* sharedPreload(artifact, spec, paths.dataPath, settings, identity).pipe( + Effect.timeoutOrElse({ + duration: config.healthTimeoutMs ?? 60_000, + orElse: () => + Effect.fail(errorFor("launch", "PostgreSQL configuration probe timed out")), + }), + ); + return yield* spawnNativeProcess( + { ...spec, args: [...spec.args, ...preload] }, + defaultNativeProcessLauncher(), + identity, + ); + }).pipe( Scope.provide(context.scope), Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, spawner), + Effect.provideService(FileSystem.FileSystem, fs), Effect.mapError((cause) => errorFor("launch", cause)), ); +}; /** Creates a database component backed by one native or container PostgreSQL session. */ export const makeDatabase = ( @@ -874,6 +951,23 @@ export const makeDatabase = ( yield* fs .writeFileString(hbaPath, NATIVE_HBA_RULES, { mode: 0o600 }) .pipe(Effect.mapError((cause) => errorFor("launch", cause))); + /* pg_cron's default TCP jobs can't reach this socket-only server; shadows keep that mode. */ + const configPath = + config.stopGraceSeconds === 0 ? undefined : path.join(socketPath, "postgresql.conf"); + if (configPath !== undefined) { + const original = path + .resolve(dataPath, "postgresql.conf") + .replaceAll("\\", "\\\\") + .replaceAll("'", "''"); + /* A wrapper default stays beneath postgresql.conf, ALTER SYSTEM, and caller settings. */ + yield* fs + .writeFileString( + configPath, + `max_worker_processes = 17\ninclude = '${original}'\n`, + { mode: 0o600 }, + ) + .pipe(Effect.mapError((cause) => errorFor("launch", cause))); + } const artifact = (yield* Ref.get(prepared)).get(config.version); if (artifact === undefined) return yield* errorFor("launch", `Artifact ${config.version} was not prepared`); @@ -891,7 +985,7 @@ export const makeDatabase = ( const process = yield* nativeProcess( artifact, config, - { dataPath, socketPath, hbaPath, rootKeyPath }, + { dataPath, socketPath, hbaPath, rootKeyPath, configPath }, settings, context, String(options.stackId), @@ -902,6 +996,7 @@ export const makeDatabase = ( Effect.provideService(FileSystem.FileSystem, fs), Effect.provideService(Path.Path, path), ), + fs, ); const selectedEndpoint: BackendEndpoint = { kind: "unix",