From 35cc23689ea8b315fc998a9f1ce55c19394eaf6c Mon Sep 17 00:00:00 2001 From: avallete Date: Tue, 29 Sep 2026 17:37:47 +0200 Subject: [PATCH 01/35] feat(stack): persist service logs to files Service output lived only in a per-recipe in-memory buffer, so the stack kept no log history, lost it when the owner exited, and native mode had nothing to replay. The owner now writes every instance's output to `logs///.log` under the stack's state directory. Output chunks are tagged with their launch, process part, sequence and publish time at the source, so lines are split once, never glued across launches, and upstream loss is recorded as `lost`. Segments rotate at 5 MiB with byte and count retention, and are removed on instance and stack destroy. A single position-based reader serves history, follow and offline reads. It is exposed as a `readLogs` RPC and `readStackLogs`; the existing `logs` RPC keeps its live-only behaviour for `supabase stack logs`. Co-Authored-By: Claude Opus 5.5 --- .../experimental/stack/start/SIDE_EFFECTS.md | 3 + packages/stack/ARCHITECTURE.md | 18 + packages/stack/README.md | 2 +- .../stack/src/Owner.logs.integration.test.ts | 249 +++++ packages/stack/src/Owner.ts | 43 +- packages/stack/src/Rpc.ts | 12 + packages/stack/src/Service.ts | 22 +- packages/stack/src/State.integration.test.ts | 41 + packages/stack/src/State.ts | 41 +- packages/stack/src/effect.ts | 28 + packages/stack/src/host/LogRecord.ts | 334 +++++++ .../stack/src/host/LogRecord.unit.test.ts | 290 ++++++ .../src/host/LogStore.integration.test.ts | 709 +++++++++++++ packages/stack/src/host/LogStore.ts | 945 ++++++++++++++++++ packages/stack/src/index.ts | 9 + .../stack/src/internal/sharing-violation.ts | 30 + packages/stack/src/runtime/Session.ts | 63 +- .../stack/src/runtime/Session.unit.test.ts | 34 +- packages/stack/src/services/Catalog.ts | 33 +- .../src/services/Database.integration.test.ts | 2 +- packages/stack/src/services/Database.ts | 22 +- .../services/Functions.integration.test.ts | 4 +- packages/stack/src/services/Functions.ts | 2 - .../ProcessRecipe.integration.test.ts | 35 +- packages/stack/src/services/ProcessRecipe.ts | 49 +- packages/stack/src/services/Recipe.ts | 16 +- packages/stack/src/services/Vector.ts | 3 +- 27 files changed, 2924 insertions(+), 115 deletions(-) create mode 100644 packages/stack/src/Owner.logs.integration.test.ts create mode 100644 packages/stack/src/host/LogRecord.ts create mode 100644 packages/stack/src/host/LogRecord.unit.test.ts create mode 100644 packages/stack/src/host/LogStore.integration.test.ts create mode 100644 packages/stack/src/host/LogStore.ts create mode 100644 packages/stack/src/internal/sharing-violation.ts diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index 59b0bf2b38..5251441504 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -36,6 +36,9 @@ Secrets needed by enabled services are passed to the runtime. State and service `$SUPABASE_HOME/stacks//` (`~/.supabase/stacks//` by default); native artifacts use `$SUPABASE_HOME/cache/stack`. Storage files use the caller-owned project directory `supabase/.temp/stack-uploads//`. Functions preparation may build the project's source. +The owner persists each service's output under `$SUPABASE_HOME/stacks//logs/`, keeping at +most about 10 MiB (plus the segment being written) per service instance. Destroying an instance or +the stack deletes those logs; stopping the stack and resetting database data keep them. For a new stack, `--runtime auto` selects Docker when `docker version` reaches its daemon, then Podman when `podman info` reaches its engine, then native on Linux x64/arm64 and macOS arm64. Each diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 51017714da..df434a462f 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -639,8 +639,26 @@ The state root is the stack registry root. Each stack keeps one state document a //owner.lock lease; opened only by SQLite //owner.json endpoint of the lease holder //owner.log owner stdout and stderr, truncated at each owner start +//logs///.log persisted service output ``` +The owner is the only subscriber of each instance's output. It writes one record per line, ` [ truncated] | `, into immutable generation-named +segments: each owner start opens a new generation on the instance's first output, and a record that +would take a segment past 5 MiB starts the next one. The oldest closed segments except the newest +are deleted while an instance holds more than 10 MiB or 64 segments. Records carry the publish time +of their first byte. Line state is kept per launch, process and stream; chunk sequence numbers span +a launch's processes, so a process whose output was all dropped still shows as `lost`. A late +partial line of an ended launch waits for its newline for two seconds of quiet, or until the store +closes. Lines are cut at 32 KiB; chunks the in-memory output buffer dropped, that failed to write, +or that were still queued when a bounded drain at close ran out are recorded as `lost`. The legacy +`logs` tail reads split lines from memory, so it keeps flowing while appends fail. Every reader, +live or offline, reads segments by position, so history and following share one path; a reader that +finds its segment deleted reports a `lost` gap marker, which carries `resumeAt` instead of a record +position. Destroying an instance deletes its segments, and an owner start removes directories of +instances no longer saved, so a failed deletion is retried; destroying the stack removes `logs/`; +resetting database data keeps them. + Registry updates use an OS-backed lock through a private `node:sqlite` connection to `/.registry-lock.sqlite`. Each `withLock` call opens its own connection, disables SQLite busy waiting, and retries `BEGIN IMMEDIATE` contention for up to five seconds through diff --git a/packages/stack/README.md b/packages/stack/README.md index 5363725a53..e32a995d9a 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -157,7 +157,7 @@ Bindings supply ordinary configuration values; a URL alone never creates a depen Exit confirmation is bounded. If cleanup is acknowledged but owner exit cannot be confirmed, the operation fails with `operation: "shutdown-exit"` and the owner PID in the message. A failed or cancelled call does not guarantee that teardown has completed. Do not start or restart the same stack concurrently with whole-stack shutdown; separate stacks remain independent. -Each service exposes `status`, `followStatus`, `logs`, and `credentials`. Observations include the currently bound public endpoints, including listeners for sleeping services. Credentials default to host addressing. Use `from: "runtime"` for a URL passed to a service or command container. +Each service exposes `status`, `followStatus`, `logs`, and `credentials`. The owner persists service output under `//logs`, and `readStackLogs({ stateRoot, stackId, instances?, since?, tail? })` reads it without an owner, ordered by timestamp, service, instance and position. Observations include the currently bound public endpoints, including listeners for sleeping services. Credentials default to host addressing. Use `from: "runtime"` for a URL passed to a service or command container. ## Testing diff --git a/packages/stack/src/Owner.logs.integration.test.ts b/packages/stack/src/Owner.logs.integration.test.ts new file mode 100644 index 0000000000..30a49f078e --- /dev/null +++ b/packages/stack/src/Owner.logs.integration.test.ts @@ -0,0 +1,249 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { + Context, + Crypto, + Effect, + Exit, + FileSystem, + Layer, + Option, + Path, + Queue, + Redacted, + Scope, + Stream, +} from "effect"; +import { tmpdir } from "node:os"; +import { ownerFor } from "../tests/owner-rpc.ts"; +import type { LogRecord } from "./host/LogRecord.ts"; +import { readStackLogs, StackError } from "./index.ts"; +import * as LogStore from "./host/LogStore.ts"; +import * as State from "./State.ts"; +import type { SavedStack } from "./State.ts"; + +const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; +const services = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); + +const stackFor = (id: string, runtime: SavedStack["runtime"]): SavedStack => ({ + id, + identity: { projectRoot: "/tmp/project", branchContext: "owner-logs-test", stackName: id }, + runtime, + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + ports: [], +}); + +/** Opens an owner that destroys its stack on scope exit unless `kept` preserves it for a later read. */ +const openOwner = ( + prefix: string, + runtime: SavedStack["runtime"], + kept?: { readonly root: string }, +) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const crypto = yield* Crypto.Crypto; + const root = kept?.root ?? (yield* fs.makeTempDirectoryScoped({ prefix })); + const stack = stackFor( + `owner-logs-${(yield* crypto.randomUUIDv4).replaceAll("-", "")}`, + runtime, + ); + const state = Context.get( + yield* Layer.build(State.layer({ root: `${root}/state` })), + State.Service, + ); + yield* state.save(stack); + const owner = yield* ownerFor({ saved: stack, state, root: `${root}/data`, cacheRoot }); + if (kept === undefined) + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + return { + ...owner, + state, + stack, + stateRoot: `${root}/state`, + logsRoot: state.logsRoot(stack.id), + }; + }); + +const isOutput = (record: LogRecord) => record.kind === "stdout" || record.kind === "stderr"; + +const firstOutput = (records: Stream.Stream) => + records.pipe(Stream.filter(isOutput), Stream.take(1), Stream.runCollect); + +describe("owner persisted logs", () => { + it.live("persists native output, serves history and follow, and deletes it on destroy", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const owner = yield* openOwner("owner-logs-native-", "native"); + const mail = yield* owner.rpc.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + yield* owner.rpc.startService({ id: mail.id }); + yield* owner.rpc.readyService({ id: mail.id }); + + const [followed] = yield* firstOutput(owner.rpc.readLogs({ id: mail.id, follow: true })); + const history = Array.from( + yield* owner.rpc.readLogs({ id: mail.id, follow: false }).pipe(Stream.runCollect), + ); + + expect(history[0]).toMatchObject({ kind: "launch", launchId: 1 }); + expect(history).toContainEqual(followed); + const directory = path.join(owner.logsRoot, "mail", mail.id); + expect(yield* fs.readDirectory(directory)).toEqual(["0000000001.log"]); + const offline = yield* LogStore.readStackLogs({ root: owner.logsRoot }); + expect(offline.slice(0, history.length).map(({ position }) => position)).toEqual( + history.map(({ position }) => position), + ); + + yield* owner.rpc.destroyService({ id: mail.id }); + expect(yield* fs.exists(directory)).toBe(false); + }), + ).pipe(Effect.provide(services)), + ); + + it.live("reads history through the public API after the owner stops", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "owner-logs-offline-" }); + const ownerScope = yield* Scope.make(); + const owner = yield* openOwner("owner-logs-offline-", "native", { root }).pipe( + Scope.provide(ownerScope), + ); + const mail = yield* owner.rpc.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + yield* owner.rpc.startService({ id: mail.id }); + yield* owner.rpc.readyService({ id: mail.id }); + const [followed] = yield* firstOutput(owner.rpc.readLogs({ id: mail.id, follow: true })); + yield* owner.rpc.stopService({ id: mail.id }); + yield* Scope.close(ownerScope, Exit.void); + + const records = yield* Effect.promise(() => + readStackLogs({ stateRoot: owner.stateRoot, stackId: owner.stack.id }), + ); + const invalid = yield* Effect.promise(() => + readStackLogs({ + stateRoot: owner.stateRoot, + stackId: owner.stack.id, + since: "soon", + }).then( + () => undefined, + (error: unknown) => error, + ), + ); + + expect(records[0]).toMatchObject({ kind: "launch", service: "mail", instanceId: mail.id }); + expect(records.map(({ text }) => text)).toContain(followed?.text); + expect(invalid).toBeInstanceOf(StackError); + }), + ).pipe(Effect.provide(services)), + ); + + it.live("keeps the logs RPC to new output lines of the tailed instance", () => + Effect.scoped( + Effect.gen(function* () { + const owner = yield* openOwner("owner-logs-tail-", "native"); + const mail = yield* owner.rpc.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + yield* owner.rpc.startService({ id: mail.id }); + yield* owner.rpc.readyService({ id: mail.id }); + const before = Array.from( + yield* owner.rpc.readLogs({ id: mail.id, follow: false }).pipe(Stream.runCollect), + ).filter(isOutput).length; + const received = yield* Queue.unbounded<{ + readonly stream: "stdout" | "stderr"; + readonly bytes: Uint8Array; + }>(); + yield* owner.rpc.logs({ id: mail.id }).pipe( + Stream.runForEach((entry) => Queue.offer(received, entry)), + Effect.forkScoped, + ); + + // Each restart writes new output; repeat until the tail has subscribed and seen some. + const tailed = yield* Effect.gen(function* () { + yield* owner.rpc.restartService({ id: mail.id }); + yield* owner.rpc.readyService({ id: mail.id }); + return yield* Queue.poll(received); + }).pipe(Effect.repeat({ until: Option.isSome })); + + if (Option.isNone(tailed)) return yield* Effect.die("logs RPC emitted nothing"); + const text = new TextDecoder().decode(tailed.value.bytes); + expect(text.endsWith("\n")).toBe(true); + expect(text.slice(0, -1)).not.toContain("\n"); + const history = Array.from( + yield* owner.rpc.readLogs({ id: mail.id, follow: false }).pipe(Stream.runCollect), + ).filter(isOutput); + expect(history.slice(before).map((record) => `${record.text}\n`)).toContain(text); + }), + ).pipe(Effect.provide(services)), + ); + + it.live("keeps database logs across a data reset and removes all logs with the stack", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const owner = yield* openOwner("owner-logs-reset-", "native"); + const database = yield* owner.rpc.createService({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("owner-logs-reset-password"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + yield* owner.rpc.startService({ id: database.id }); + yield* owner.rpc.readyService({ id: database.id }); + yield* owner.rpc.stopService({ id: database.id }); + const before = Array.from( + yield* owner.rpc.readLogs({ id: database.id, follow: false }).pipe(Stream.runCollect), + ); + + yield* owner.rpc.resetData({ id: database.id }); + const after = Array.from( + yield* owner.rpc.readLogs({ id: database.id, follow: false }).pipe(Stream.runCollect), + ); + + expect(before.some(isOutput)).toBe(true); + expect(after.slice(0, before.length)).toEqual(before); + yield* owner.namespace.destroy; + expect(yield* fs.exists(owner.logsRoot)).toBe(false); + }), + ).pipe(Effect.provide(services)), + ); + + it.live.skipIf(process.platform === "win32")("persists container output", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const owner = yield* openOwner("owner-logs-docker-", "docker"); + const mail = yield* owner.rpc.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + yield* owner.rpc.startService({ id: mail.id }); + yield* owner.rpc.readyService({ id: mail.id }); + + const [followed] = yield* firstOutput(owner.rpc.readLogs({ id: mail.id, follow: true })); + + expect(followed).toMatchObject({ launchId: 1 }); + expect(yield* fs.readDirectory(path.join(owner.logsRoot, "mail", mail.id))).toEqual([ + "0000000001.log", + ]); + }), + ).pipe(Effect.provide(services)), + ); +}); diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index b36d9fcfa1..9e42fdecd8 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -67,6 +67,7 @@ import { stackError, type OwnerRpc } from "./Rpc.ts"; import * as State from "./State.ts"; import type { SavedStack, StackCredentials, StackKeysInput } from "./State.ts"; import { makeDockerHelperRegistry } from "./storage/DockerHelperRegistry.ts"; +import * as LogStore from "./host/LogStore.ts"; export interface OwnerOptions { readonly saved: SavedStack; @@ -169,6 +170,8 @@ const withoutInstance = (current: SavedStack, id: string): SavedStack => const drainingBlocks: ReadonlyArray = ["start", "arm", "restart", "storage"]; +const encoder = new TextEncoder(); + const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { const services = yield* Effect.context< | FileSystem.FileSystem @@ -184,6 +187,9 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { const network = yield* Network.Service; const orchestrator = yield* Orchestrator.make(); const helpers = yield* makeDockerHelperRegistry(yield* crypto.randomUUIDv4); + const logStore = yield* LogStore.make({ root: options.state.logsRoot(options.saved.id) }).pipe( + Effect.provideContext(services), + ); const definitionGate = yield* Semaphore.make(1); const draining = yield* Ref.make(false); const { id: stackId, runtime } = options.saved; @@ -337,6 +343,18 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { Effect.mapError(serviceError("state")), ), ), + Effect.andThen( + logStore + .remove({ service: initial.service, instanceId: id }) + .pipe( + Effect.catch((cause) => + Effect.logWarning( + `${cause.message}; the next owner start or stack destroy retries`, + cause, + ), + ), + ), + ), ), }, { @@ -416,10 +434,19 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { yield* orchestrator .register(entry) .pipe(Effect.catch((cause) => namespace.release.pipe(Effect.andThen(Effect.fail(cause))))); + yield* logStore.attach({ + service: initial.service, + instanceId: id, + logs: recipe.logs, + observation: core.observation, + }); }); for (const saved of options.saved.instances) yield* register(saved.id, yield* recipeFor(saved.creation, saved.id)); + yield* logStore.removeOrphans.pipe( + Effect.catch((cause) => Effect.logWarning("Orphaned instance logs were not removed", cause)), + ); yield* orchestrator.configure(options.saved.composition); const removeSaved = (id: string) => updateState((current) => withoutInstance(current, id)); @@ -631,10 +658,21 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { ), ).pipe(Stream.mapError((cause) => stackError("followStatus", cause))), logs: ({ id }) => - Stream.unwrap(orchestrator.get(id).pipe(Effect.map((entry) => entry.recipe.logs))).pipe( - Stream.map(({ stream, bytes }) => ({ stream, bytes })), + Stream.unwrap(logStore.tail(id)).pipe( + Stream.map(({ stream, text }) => ({ stream, bytes: encoder.encode(`${text}\n`) })), Stream.mapError((cause) => stackError("logs", cause)), ), + readLogs: ({ id, since, tail, follow }) => + Stream.unwrap( + Effect.gen(function* () { + return yield* logStore.read(id, { + from: "oldest", + follow, + ...(since === undefined ? {} : { since: yield* LogStore.sinceMillis(since) }), + ...(tail === undefined ? {} : { tail }), + }); + }), + ).pipe(Stream.mapError((cause) => stackError("readLogs", cause))), credentials: ({ id, from }) => orchestrator.get(id).pipe( Effect.flatMap((entry) => @@ -695,6 +733,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { destroy: orchestrator.destroyNamespace.pipe( Effect.andThen(network.release), Effect.andThen(sweep), + Effect.andThen(logStore.close), Effect.andThen(options.state.remove(stackId)), definitionGate.withPermits(1), Effect.withSpan("Owner.destroyNamespace"), diff --git a/packages/stack/src/Rpc.ts b/packages/stack/src/Rpc.ts index 9b296509c4..6c8b7cf6d1 100644 --- a/packages/stack/src/Rpc.ts +++ b/packages/stack/src/Rpc.ts @@ -6,6 +6,7 @@ import { causeMessage, CompositionConfig, OrchestratorError } from "./Orchestrat import { CommandInvocation } from "./Commands.ts"; import { StackKeysInput } from "./State.ts"; import { failureMessage } from "./internal/failure-message.ts"; +import { LogRecord } from "./host/LogRecord.ts"; const Outcome = Schema.Struct({ id: Schema.String, @@ -121,6 +122,17 @@ export const OwnerRpc = RpcGroup.make( stream: true, }), Rpc.make("logs", { payload: Instance, success: Log, error: StackError, stream: true }), + Rpc.make("readLogs", { + payload: { + ...Instance, + since: Schema.optionalKey(Schema.String), + tail: Schema.optionalKey(Schema.Int.check(Schema.isGreaterThanOrEqualTo(0))), + follow: Schema.Boolean, + }, + success: LogRecord, + error: StackError, + stream: true, + }), Rpc.make("credentials", { payload: { ...Instance, from: Schema.Literals(["host", "runtime"]) }, success: Schema.Record(Schema.String, Schema.String), diff --git a/packages/stack/src/Service.ts b/packages/stack/src/Service.ts index d711c7faa6..462cb0c74b 100644 --- a/packages/stack/src/Service.ts +++ b/packages/stack/src/Service.ts @@ -84,7 +84,7 @@ export interface ServiceDefinition { readonly prepare?: (config: Config) => Effect.Effect; /** Failures after resource acquisition carry the session for ordinary cleanup. */ readonly launch: ( - context: ServiceInstanceContext, + context: ServiceLaunchContext, ) => Effect.Effect; readonly removeData: ( context: ServiceInstanceContext, @@ -98,6 +98,11 @@ export interface ServiceInstanceContext { readonly scope: Scope.Closeable; } +/** One launch of an instance; every output chunk the launch publishes carries its `launchId`. */ +export interface ServiceLaunchContext extends ServiceInstanceContext { + readonly launchId: number; +} + export interface ServiceInstance { readonly id: string; readonly observation: Stream.Stream>; @@ -380,12 +385,17 @@ export const makeService = ( ); const runtimeScope = yield* Scope.fork(owner, "parallel"); const launchExit = yield* Effect.exit( - definition.launch(contextFor(options.id, yield* Ref.get(config), runtimeScope)).pipe( - Effect.map((runtime) => ({ runtime, failure: undefined })), - Effect.catchTag("ServiceLaunchError", ({ runtime, failure }) => - Effect.succeed({ runtime, failure }), + definition + .launch({ + ...contextFor(options.id, yield* Ref.get(config), runtimeScope), + launchId, + }) + .pipe( + Effect.map((runtime) => ({ runtime, failure: undefined })), + Effect.catchTag("ServiceLaunchError", ({ runtime, failure }) => + Effect.succeed({ runtime, failure }), + ), ), - ), ); if (Exit.isFailure(launchExit)) { yield* Scope.close(runtimeScope, launchExit); diff --git a/packages/stack/src/State.integration.test.ts b/packages/stack/src/State.integration.test.ts index 19a1548de3..d4f8fa5e70 100644 --- a/packages/stack/src/State.integration.test.ts +++ b/packages/stack/src/State.integration.test.ts @@ -611,6 +611,47 @@ describe("durable stack state", () => { ), ); + it.live("removes persisted logs with the stack, retrying a Windows sharing violation", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-remove-logs-" }); + const held = yield* Ref.make(2); + const injectedFs = Layer.succeed(FileSystem.FileSystem, { + ...fs, + remove: (target: string, options?: Parameters[1]) => + Effect.gen(function* () { + if ( + target.endsWith(`${path.sep}logs`) && + (yield* Ref.getAndUpdate(held, (n) => n - 1)) > 0 + ) + return yield* PlatformError.systemError({ + _tag: "Busy", + module: "FileSystem", + method: "remove", + pathOrDescriptor: target, + cause: Object.assign(new Error("open by a reader"), { code: "EBUSY" }), + }); + return yield* fs.remove(target, options); + }), + }); + const store = yield* Layer.build( + State.layer({ root, platform: "win32" }).pipe(Layer.provide(injectedFs)), + ).pipe(Effect.map((context) => Context.get(context, State.Service))); + yield* store.save(initial); + const segment = path.join(store.logsRoot(initial.id), "auth", "instance", "0000000001.log"); + yield* fs.makeDirectory(path.dirname(segment), { recursive: true }); + yield* fs.writeFileString(segment, "record\n"); + + yield* store.remove(initial.id); + + expect(yield* Ref.get(held)).toBeLessThan(0); + expect(yield* fs.exists(path.join(root, initial.id))).toBe(false); + }), + ), + ); + it.live.skipIf(process.platform === "win32")( "restricts the state root to its owner while keeping a traverse-only grant", () => diff --git a/packages/stack/src/State.ts b/packages/stack/src/State.ts index 7b738ac908..f388ba3bef 100644 --- a/packages/stack/src/State.ts +++ b/packages/stack/src/State.ts @@ -1,7 +1,6 @@ import { Clock, Data, - Duration, Effect, Exit, FileSystem, @@ -20,6 +19,7 @@ import { rmdir } from "node:fs/promises"; import { DatabaseSync } from "node:sqlite"; import { pathToFileURL } from "node:url"; import { CompositionConfig } from "./Orchestrator.ts"; +import { errorCode, retrySharingViolation } from "./internal/sharing-violation.ts"; import { restrictDirectoryToOwner } from "./runtime/postgres-user.ts"; import { ServiceCreation } from "./services/Catalog.ts"; @@ -153,6 +153,8 @@ export interface Interface { readonly retractHolder: (id: string) => Effect.Effect; /** The file that receives the stack owner's stdout and stderr. */ readonly ownerLog: (id: string) => string; + /** The directory that holds the stack's persisted service logs. */ + readonly logsRoot: (id: string) => string; } export class Service extends Context.Service()("@supabase/stack/State") {} @@ -193,6 +195,16 @@ interface Options { readonly onLeaseContended?: (id: string) => Effect.Effect; } +const logsDirectory = "logs"; + +/** Resolves the persisted service logs directory of a stack without opening the state root. */ +export const stackLogsRoot = ( + path: Path.Path, + stateRoot: string, + id: string, +): Effect.Effect => + checkId(id).pipe(Effect.as(path.join(path.normalize(stateRoot), id, logsDirectory))); + const stateWritePrefix = ".state-write-"; /** A state write takes milliseconds, so a temporary directory this old belongs to a dead writer. */ const staleWriteAgeMillis = 24 * 60 * 60 * 1000; @@ -242,30 +254,16 @@ const makeState = ( const leasePath = (id: string) => path.join(stackRoot(id), "owner.lock"); const ownerPath = (id: string) => path.join(stackRoot(id), "owner.json"); const ownerLog = (id: string) => path.join(stackRoot(id), "owner.log"); - const publishRetrySchedule = Schedule.exponential("10 millis", 2).pipe( - Schedule.modifyDelay(({ duration }) => - Effect.succeed(Duration.min(duration, Duration.millis(100))), - ), - Schedule.upTo({ times: 12 }), - ); - const errorCode = (error: unknown): string | undefined => { - if (!Predicate.hasProperty(error, "cause")) return undefined; - return Predicate.hasProperty(error.cause, "code") && typeof error.cause.code === "string" - ? error.cause.code - : undefined; - }; - /** Windows reports a file that another process is replacing as a transient sharing violation. */ - const sharingViolation = (error: unknown) => - (options.platform ?? process.platform) === "win32" && - ["EPERM", "EACCES", "EBUSY"].includes(errorCode(error) ?? ""); + const logsRoot = (id: string) => path.join(stackRoot(id), logsDirectory); + const retryShared = retrySharingViolation(options.platform); const retryTransientRead = (effect: Effect.Effect) => effect.pipe( - Effect.retry({ schedule: publishRetrySchedule, while: sharingViolation }), + retryShared, Effect.mapError((cause) => stateError("read", cause)), ); const publish = Effect.fn("State.publish")(function* (temporary: string, target: string) { yield* fs.rename(temporary, target).pipe( - Effect.retry({ schedule: publishRetrySchedule, while: sharingViolation }), + retryShared, Effect.mapError( (cause) => new StateError({ @@ -370,6 +368,10 @@ const makeState = ( yield* fs .remove(file, { force: true }) .pipe(Effect.mapError((cause) => stateError("remove", cause))); + yield* fs.remove(logsRoot(id), { recursive: true, force: true }).pipe( + retryShared, + Effect.mapError((cause) => stateError("remove", cause)), + ); yield* removeEmptyDirectory(path.join(stackRoot(id), "data")); yield* removeEmptyDirectory(stackRoot(id)); }); @@ -590,6 +592,7 @@ const makeState = ( publishHolder, retractHolder, ownerLog, + logsRoot, }; }); diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index 40e09273f6..c8678f5287 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -45,6 +45,7 @@ import { failureMessage } from "./internal/failure-message.ts"; import * as State from "./State.ts"; import type { SavedStack, StackCredentials, StackKeysInput } from "./State.ts"; import { StackError, type Definition, type Observation } from "./Rpc.ts"; +import { readStackLogs as readPersistedLogs, sinceMillis } from "./host/LogStore.ts"; import { reclaimStack } from "./Sweep.ts"; import { ServiceCreationInput as ServiceCreationInputSchema, @@ -77,6 +78,7 @@ export { StackIdSchema as StackId } from "./identity/StackId.ts"; export type { SavedStack } from "./State.ts"; export type { StackCredentials, StackKeysInput }; export type { Observation } from "./Rpc.ts"; +export type { LogPosition, LogRecord, StackLogRecord } from "./host/LogRecord.ts"; export type { Command, InitializationCommand, @@ -1014,3 +1016,29 @@ export const find = Effect.fn("Stack.find")( }, Effect.mapError((cause) => failure("find", cause)), ); + +/** Selects the persisted logs of a stack; its owner does not need to run. */ +export interface ReadStackLogsOptions extends Pick { + readonly stackId: string; + /** Instance ids to read; every instance with persisted logs by default. */ + readonly instances?: ReadonlyArray; + /** An ISO-8601 timestamp; older records are skipped. */ + readonly since?: string; + /** Returns only this many of the latest records across the selected instances. */ + readonly tail?: number; +} + +/** Reads a stack's persisted records ordered by timestamp, service, instance and position. */ +export const readStackLogs = Effect.fn("Stack.readStackLogs")( + function* (options: ReadStackLogsOptions) { + const path = yield* Path.Path; + const root = yield* State.stackLogsRoot(path, options.stateRoot, options.stackId); + return yield* readPersistedLogs({ + root, + ...(options.instances === undefined ? {} : { instances: options.instances }), + ...(options.since === undefined ? {} : { since: yield* sinceMillis(options.since) }), + ...(options.tail === undefined ? {} : { tail: options.tail }), + }); + }, + Effect.mapError((cause) => failure("readStackLogs", cause)), +); diff --git a/packages/stack/src/host/LogRecord.ts b/packages/stack/src/host/LogRecord.ts new file mode 100644 index 0000000000..bb7597f0ec --- /dev/null +++ b/packages/stack/src/host/LogRecord.ts @@ -0,0 +1,334 @@ +import { DateTime, Schema } from "effect"; +import type { LaunchOutput } from "../runtime/Session.ts"; + +const OutputStream = Schema.Literals(["stdout", "stderr"]); +type OutputStream = Schema.Schema.Type; + +/** A byte position in an instance's log segments. */ +export const LogPosition = Schema.Struct({ + generation: Schema.Int, + byteOffset: Schema.Int, +}); +export interface LogPosition extends Schema.Schema.Type {} + +/** + * One log record: an output line, the first output of a launch, or chunks lost before they were + * written, each identified by its `position`. A reader reports segments that retention deleted + * before it reached them as a `lost` gap marker, which is not persisted and has `resumeAt`, the + * position reading continues from, instead of a position. + */ +export const LogRecord = Schema.Struct({ + kind: Schema.Literals(["stdout", "stderr", "launch", "lost"]), + timestamp: Schema.String, + launchId: Schema.optionalKey(Schema.Int), + text: Schema.optionalKey(Schema.String), + truncated: Schema.optionalKey(Schema.Boolean), + stream: Schema.optionalKey(OutputStream), + count: Schema.optionalKey(Schema.Int), + position: Schema.optionalKey(LogPosition), + resumeAt: Schema.optionalKey(LogPosition), +}); +export interface LogRecord extends Schema.Schema.Type {} + +/** A log record of one service instance in a stack. */ +export const StackLogRecord = LogRecord.pipe( + Schema.fieldsAssign({ service: Schema.String, instanceId: Schema.String }), +); +export interface StackLogRecord extends Schema.Schema.Type {} + +/** A record before it is written; `timestamp` is epoch milliseconds. */ +export type LogEntry = + | { + readonly kind: OutputStream; + readonly timestamp: number; + readonly launchId: number; + readonly text: string; + readonly truncated: boolean; + } + | { readonly kind: "launch"; readonly timestamp: number; readonly launchId: number } + | { + readonly kind: "lost"; + readonly timestamp: number; + readonly launchId: number; + readonly stream: OutputStream; + readonly count: number; + }; + +/** Lines are cut at this many UTF-8 bytes. */ +export const maxLineBytes = 32 * 1024; + +const encoder = new TextEncoder(); + +/** Encodes one record as ` [ truncated] | \n`. */ +export const encodeEntry = (entry: LogEntry): string => { + const time = DateTime.formatIso(DateTime.makeUnsafe(entry.timestamp)); + switch (entry.kind) { + case "launch": + return `${time} launch ${entry.launchId} | \n`; + case "lost": + return `${time} lost ${entry.launchId} | ${entry.count} ${entry.stream}\n`; + default: + return `${time} ${entry.kind} ${entry.launchId}${entry.truncated ? " truncated" : ""} | ${entry.text}\n`; + } +}; + +const recordLine = /^(\S+) (stdout|stderr|launch|lost) (\d+)( truncated)? \| (.*)$/su; +const lostText = /^(\d+) (stdout|stderr)$/u; + +/** Parses one record line without its trailing newline; unrecognized lines yield `undefined`. */ +export const parseRecord = (line: string, position: LogPosition): LogRecord | undefined => { + const match = recordLine.exec(line); + if (match === null) return undefined; + const [, timestamp = "", kind, launch = "", truncated, text = ""] = match; + const launchId = Number(launch); + if (kind === "stdout" || kind === "stderr") + return { + kind, + timestamp, + launchId, + text, + ...(truncated === undefined ? {} : { truncated: true }), + position, + }; + if (kind === "launch") return { kind, timestamp, launchId, position }; + const lost = lostText.exec(text); + const stream = lost?.[2]; + return { + kind: "lost", + timestamp, + launchId, + ...(lost === null || (stream !== "stdout" && stream !== "stderr") + ? {} + : { count: Number(lost[1]), stream }), + position, + }; +}; + +/** The file name of a segment generation. */ +export const segmentName = (generation: number) => `${String(generation).padStart(10, "0")}.log`; + +/** The generation a segment file name encodes, or `undefined` for other files. */ +export const segmentGeneration = (name: string): number | undefined => { + const match = /^(\d+)\.log$/u.exec(name); + return match === null ? undefined : Number(match[1]); +}; + +interface LineState { + readonly launchId: number; + readonly part: number; + readonly stream: OutputStream; + decoder: TextDecoder; + text: string; + bytes: number; + since: number | undefined; + /** Publish time of bytes the decoder holds until their character completes. */ + heldSince: number | undefined; + /** Publish time of the latest chunk. */ + lastChunk: number; + midCarriageReturn: boolean; + discarding: boolean; +} + +/** Converts tagged output chunks into records, keeping line state per launch, part and stream. */ +export interface Splitter { + /** Records take the publish time of their first byte. */ + readonly push: (chunk: LaunchOutput) => ReadonlyArray; + /** Flushes the partial lines of a launch that ended. */ + readonly endLaunch: (launchId: number, now: number) => ReadonlyArray; + /** Flushes partial lines late chunks of ended launches left once they are idle for a grace. */ + readonly flushEnded: (now: number) => ReadonlyArray; + readonly flush: (now: number) => ReadonlyArray; +} + +/** Launches older than this many launches behind the newest lose their line state. */ +const retainedLaunches = 4; +/** A launch first seen this far behind the newest may get a second `launch` record. */ +const seenLaunches = 64; +/** A late partial line of an ended launch waits this long for its newline. */ +export const endedLineGraceMillis = 2_000; + +/** Reports whether `bytes` ends inside a UTF-8 sequence the next chunk completes. */ +const endsMidCharacter = (bytes: Uint8Array) => { + for (let index = bytes.length - 1; index >= Math.max(0, bytes.length - 4); index--) { + const byte = bytes[index] ?? 0; + if ((byte & 0xc0) === 0x80) continue; + const width = byte >= 0xf0 ? 4 : byte >= 0xe0 ? 3 : byte >= 0xc0 ? 2 : 1; + return bytes.length - index < width; + } + return false; +}; + +/** Splits lines like `Stream.decodeText` plus `Stream.splitLines`, per launch, part and stream. */ +export const makeSplitter = (limit = maxLineBytes): Splitter => { + const states = new Map(); + const ended = new Set(); + const latestParts = new Map(); + /** Next expected `seq` per launch and stream, which spans the launch's parts. */ + const expected = new Map(); + const seen = new Set(); + let latest = 0; + + const resetLine = (state: LineState) => { + state.text = ""; + state.bytes = 0; + state.since = undefined; + state.discarding = false; + }; + const append = (state: LineState, piece: string, time: number, out: Array) => { + if (piece.length === 0 || state.discarding) return; + state.since ??= time; + const size = encoder.encode(piece).length; + if (state.bytes + size <= limit) { + state.text += piece; + state.bytes += size; + return; + } + const kept = piece.slice( + 0, + encoder.encodeInto(piece, new Uint8Array(limit - state.bytes)).read, + ); + out.push({ + kind: state.stream, + timestamp: state.since, + launchId: state.launchId, + text: state.text + kept, + truncated: true, + }); + resetLine(state); + state.discarding = true; + }; + const terminate = (state: LineState, now: number, out: Array) => { + if (!state.discarding) + out.push({ + kind: state.stream, + timestamp: state.since ?? now, + launchId: state.launchId, + text: state.text, + truncated: false, + }); + resetLine(state); + }; + const flushLine = (state: LineState, now: number, out: Array) => { + append(state, state.decoder.decode(), state.heldSince ?? now, out); + state.decoder = new TextDecoder(); + state.heldSince = undefined; + state.midCarriageReturn = false; + if (state.since !== undefined && !state.discarding) terminate(state, now, out); + else resetLine(state); + }; + const pending = (state: LineState) => state.since !== undefined || state.heldSince !== undefined; + + const push = (chunk: LaunchOutput) => { + const now = chunk.time; + const out: Array = []; + if (chunk.launchId > latest) { + for (const state of states.values()) + if (state.launchId < chunk.launchId) flushLine(state, now, out); + latest = chunk.launchId; + const stale = (launchId: number) => launchId <= latest - retainedLaunches; + for (const [key, state] of states) if (stale(state.launchId)) states.delete(key); + for (const launchId of ended) if (stale(launchId)) ended.delete(launchId); + for (const launchId of latestParts.keys()) if (stale(launchId)) latestParts.delete(launchId); + for (const key of expected.keys()) + if (stale(Number(key.slice(0, key.indexOf(":"))))) expected.delete(key); + for (const launchId of seen) if (launchId <= latest - seenLaunches) seen.delete(launchId); + } + if (!seen.has(chunk.launchId)) { + seen.add(chunk.launchId); + out.push({ kind: "launch", timestamp: now, launchId: chunk.launchId }); + } + const latestPart = latestParts.get(chunk.launchId); + if (latestPart === undefined || chunk.part > latestPart) { + for (const state of states.values()) + if (state.launchId === chunk.launchId && state.part < chunk.part) + flushLine(state, now, out); + latestParts.set(chunk.launchId, chunk.part); + } + const key = `${chunk.launchId}:${chunk.part}:${chunk.stream}`; + let state = states.get(key); + if (state === undefined) { + state = { + launchId: chunk.launchId, + part: chunk.part, + stream: chunk.stream, + decoder: new TextDecoder(), + text: "", + bytes: 0, + since: undefined, + heldSince: undefined, + lastChunk: now, + midCarriageReturn: false, + discarding: false, + }; + states.set(key, state); + } + state.lastChunk = now; + const sequence = `${chunk.launchId}:${chunk.stream}`; + const next = expected.get(sequence) ?? 0; + if (chunk.seq > next) { + out.push({ + kind: "lost", + timestamp: now, + launchId: chunk.launchId, + stream: chunk.stream, + count: chunk.seq - next, + }); + resetLine(state); + state.decoder = new TextDecoder(); + state.heldSince = undefined; + state.midCarriageReturn = false; + } + expected.set(sequence, Math.max(next, chunk.seq + 1)); + const carried = state.heldSince; + const text = state.decoder.decode(chunk.bytes, { stream: true }); + state.heldSince = endsMidCharacter(chunk.bytes) + ? text.length === 0 + ? (carried ?? now) + : now + : undefined; + if (text.length > 0) { + let from = state.midCarriageReturn && text.startsWith("\n") ? 1 : 0; + state.midCarriageReturn = false; + let time = carried ?? now; + const terminator = /\r\n|\r|\n/gu; + terminator.lastIndex = from; + for (let match = terminator.exec(text); match !== null; match = terminator.exec(text)) { + append(state, text.slice(from, match.index), time, out); + terminate(state, now, out); + time = now; + from = match.index + match[0].length; + } + append(state, text.slice(from), time, out); + state.midCarriageReturn = text.endsWith("\r"); + } + return out; + }; + + const endLaunch = (launchId: number, now: number) => { + ended.add(launchId); + const out: Array = []; + for (const state of states.values()) + if (state.launchId === launchId) flushLine(state, now, out); + return out; + }; + + const flushEnded = (now: number) => { + const out: Array = []; + for (const state of states.values()) + if ( + ended.has(state.launchId) && + pending(state) && + now - state.lastChunk >= endedLineGraceMillis + ) + flushLine(state, now, out); + return out; + }; + + const flush = (now: number) => { + const out: Array = []; + for (const state of states.values()) flushLine(state, now, out); + return out; + }; + + return { push, endLaunch, flushEnded, flush }; +}; diff --git a/packages/stack/src/host/LogRecord.unit.test.ts b/packages/stack/src/host/LogRecord.unit.test.ts new file mode 100644 index 0000000000..15781fecff --- /dev/null +++ b/packages/stack/src/host/LogRecord.unit.test.ts @@ -0,0 +1,290 @@ +import { describe, expect, it } from "@effect/vitest"; +import type { LaunchOutput } from "../runtime/Session.ts"; +import { + encodeEntry, + makeSplitter, + endedLineGraceMillis, + maxLineBytes, + parseRecord, + segmentGeneration, + segmentName, + type LogEntry, + type Splitter, +} from "./LogRecord.ts"; + +const bytes = (text: string) => new TextEncoder().encode(text); +const chunk = ( + launchId: number, + seq: number, + text: string, + stream: "stdout" | "stderr" = "stdout", + part = 0, +): LaunchOutput => ({ stream, bytes: bytes(text), launchId, part, seq, time: 0 }); +/** Pushes a chunk as if it had been published at `time`. */ +const push = (splitter: Splitter, value: LaunchOutput, time: number) => + splitter.push({ ...value, time }); +const lines = (entries: ReadonlyArray) => + entries.flatMap((entry) => + entry.kind === "stdout" || entry.kind === "stderr" ? [entry.text] : [], + ); +const position = { generation: 3, byteOffset: 42 }; + +describe("record format", () => { + it("round-trips every record kind with its launch id", () => { + const entries: ReadonlyArray = [ + { kind: "stdout", timestamp: 0, launchId: 2, text: "ready | with a pipe", truncated: false }, + { kind: "stderr", timestamp: 1_000, launchId: 2, text: "cut", truncated: true }, + { kind: "launch", timestamp: 2_000, launchId: 3 }, + { kind: "lost", timestamp: 3_000, launchId: 3, stream: "stderr", count: 7 }, + ]; + + const encoded = entries.map(encodeEntry); + const parsed = encoded.map((line) => parseRecord(line.slice(0, -1), position)); + + expect(encoded[0]).toBe("1970-01-01T00:00:00.000Z stdout 2 | ready | with a pipe\n"); + expect(encoded[1]).toBe("1970-01-01T00:00:01.000Z stderr 2 truncated | cut\n"); + expect(parsed).toEqual([ + { + kind: "stdout", + timestamp: "1970-01-01T00:00:00.000Z", + launchId: 2, + text: "ready | with a pipe", + position, + }, + { + kind: "stderr", + timestamp: "1970-01-01T00:00:01.000Z", + launchId: 2, + text: "cut", + truncated: true, + position, + }, + { kind: "launch", timestamp: "1970-01-01T00:00:02.000Z", launchId: 3, position }, + { + kind: "lost", + timestamp: "1970-01-01T00:00:03.000Z", + launchId: 3, + stream: "stderr", + count: 7, + position, + }, + ]); + }); + + it("ignores lines that are not records", () => { + expect(parseRecord("half a rec", position)).toBeUndefined(); + expect(parseRecord("2026-09-29T10:00:00.000Z info 1 | text", position)).toBeUndefined(); + }); + + it("names segments so only generation files are listed", () => { + expect(segmentName(12)).toBe("0000000012.log"); + expect(segmentGeneration(segmentName(12))).toBe(12); + expect(segmentGeneration("cursor.json")).toBeUndefined(); + expect(segmentGeneration("0000000012.log.tmp")).toBeUndefined(); + }); +}); + +describe("splitter", () => { + it("joins lines across chunk boundaries, including split CRLF and multibyte text", () => { + const splitter = makeSplitter(); + const euro = bytes("€"); + const first = push(splitter, chunk(1, 0, "alpha\r"), 10); + const second = push(splitter, chunk(1, 1, "\nbe"), 20); + const third = push( + splitter, + { + stream: "stdout", + bytes: new Uint8Array([...bytes("ta "), ...euro.subarray(0, 1)]), + launchId: 1, + part: 0, + seq: 2, + time: 0, + }, + 30, + ); + const fourth = push( + splitter, + { + stream: "stdout", + bytes: new Uint8Array([...euro.subarray(1), 0x0a]), + launchId: 1, + part: 0, + seq: 3, + time: 0, + }, + 40, + ); + + expect(first[0]).toEqual({ kind: "launch", timestamp: 10, launchId: 1 }); + expect(lines(first)).toEqual(["alpha"]); + expect(lines(second)).toEqual([]); + expect(third).toEqual([]); + expect(fourth).toEqual([ + { kind: "stdout", timestamp: 20, launchId: 1, text: "beta €", truncated: false }, + ]); + }); + + it("treats a lone carriage return and empty lines as line ends", () => { + const splitter = makeSplitter(); + + expect(lines(push(splitter, chunk(1, 0, "10%\r20%\r\n\nnext\n"), 0))).toEqual([ + "10%", + "20%", + "", + "next", + ]); + }); + + it("keeps a late chunk of an earlier launch in that launch after a later launch starts", () => { + const splitter = makeSplitter(); + push(splitter, chunk(1, 0, "failed par"), 10); + + const relaunched = push(splitter, chunk(2, 0, "second\n"), 20); + const late = push(splitter, chunk(1, 1, "late\n"), 30); + + expect(relaunched).toEqual([ + { kind: "stdout", timestamp: 10, launchId: 1, text: "failed par", truncated: false }, + { kind: "launch", timestamp: 20, launchId: 2 }, + { kind: "stdout", timestamp: 20, launchId: 2, text: "second", truncated: false }, + ]); + expect(late).toEqual([ + { kind: "stdout", timestamp: 30, launchId: 1, text: "late", truncated: false }, + ]); + }); + + it("flushes a launch's partial line when it ends and keeps a late line whole across batches", () => { + const splitter = makeSplitter(); + push(splitter, chunk(1, 0, "no newline"), 10); + + const ended = splitter.endLaunch(1, 20); + const early = [...push(splitter, chunk(1, 1, "late "), 30), ...splitter.flushEnded(31)]; + const completed = [...push(splitter, chunk(1, 2, "tail\n"), 40), ...splitter.flushEnded(41)]; + + expect(ended).toEqual([ + { kind: "stdout", timestamp: 10, launchId: 1, text: "no newline", truncated: false }, + ]); + expect(early).toEqual([]); + expect(completed).toEqual([ + { kind: "stdout", timestamp: 30, launchId: 1, text: "late tail", truncated: false }, + ]); + }); + + it("flushes a late partial of an ended launch once it stays idle for the grace", () => { + const splitter = makeSplitter(); + splitter.endLaunch(1, 0); + push(splitter, chunk(1, 0, "idle partial"), 50); + + const waiting = splitter.flushEnded(50 + endedLineGraceMillis - 1); + const flushed = splitter.flushEnded(50 + endedLineGraceMillis); + + expect(waiting).toEqual([]); + expect(lines(flushed)).toEqual(["idle partial"]); + }); + + it("reports a whole process of a launch whose chunks were dropped", () => { + const splitter = makeSplitter(); + push(splitter, chunk(1, 0, "startup\n", "stdout", 0), 10); + + const main = push(splitter, chunk(1, 3, "main\n", "stdout", 2), 20); + + expect(main).toEqual([ + { kind: "lost", timestamp: 20, launchId: 1, stream: "stdout", count: 2 }, + { kind: "stdout", timestamp: 20, launchId: 1, text: "main", truncated: false }, + ]); + }); + + it("stamps a line with the publish time of its first byte when a character spans chunks", () => { + const splitter = makeSplitter(); + const euro = bytes("€"); + + const first = push(splitter, { ...chunk(1, 0, ""), bytes: euro.subarray(0, 1) }, 10); + const second = push( + splitter, + { ...chunk(1, 1, ""), bytes: new Uint8Array([...euro.subarray(1), 0x0a]) }, + 20, + ); + + expect(first).toEqual([{ kind: "launch", timestamp: 10, launchId: 1 }]); + expect(second).toEqual([ + { kind: "stdout", timestamp: 10, launchId: 1, text: "€", truncated: false }, + ]); + }); + + it("ends a startup command's unterminated line when the main process of the launch starts", () => { + const splitter = makeSplitter(); + push(splitter, chunk(1, 0, "migrations applied", "stdout", 0), 10); + + const main = push(splitter, chunk(1, 1, "listening\n", "stdout", 1), 20); + + expect(main).toEqual([ + { kind: "stdout", timestamp: 10, launchId: 1, text: "migrations applied", truncated: false }, + { kind: "stdout", timestamp: 20, launchId: 1, text: "listening", truncated: false }, + ]); + }); + + it("reports a sequence gap as lost chunks and drops the interrupted partial line", () => { + const splitter = makeSplitter(); + push(splitter, chunk(1, 0, "interrupted"), 10); + + const resumed = push(splitter, chunk(1, 4, "resumed\n", "stdout"), 20); + + expect(resumed).toEqual([ + { kind: "lost", timestamp: 20, launchId: 1, stream: "stdout", count: 3 }, + { kind: "stdout", timestamp: 20, launchId: 1, text: "resumed", truncated: false }, + ]); + }); + + it("marks the first chunk of an earlier launch that arrives after a later launch", () => { + const splitter = makeSplitter(); + push(splitter, chunk(2, 0, "second\n"), 10); + + const late = push(splitter, chunk(1, 0, "failed early\n"), 20); + + expect(late).toEqual([ + { kind: "launch", timestamp: 20, launchId: 1 }, + { kind: "stdout", timestamp: 20, launchId: 1, text: "failed early", truncated: false }, + ]); + }); + + it("reports chunks lost before the first one it sees", () => { + const splitter = makeSplitter(); + + expect(push(splitter, chunk(5, 2, "x\n", "stderr"), 0).slice(0, 2)).toEqual([ + { kind: "launch", timestamp: 0, launchId: 5 }, + { kind: "lost", timestamp: 0, launchId: 5, stream: "stderr", count: 2 }, + ]); + }); + + it("truncates an over-long line at a character boundary and drops the rest of it", () => { + const splitter = makeSplitter(8); + + const first = push(splitter, chunk(1, 0, "abcdefg€"), 10); + const rest = push(splitter, chunk(1, 1, "more\nnext\n", "stdout"), 20); + + expect(first.slice(1)).toEqual([ + { kind: "stdout", timestamp: 10, launchId: 1, text: "abcdefg", truncated: true }, + ]); + expect(rest).toEqual([ + { kind: "stdout", timestamp: 20, launchId: 1, text: "next", truncated: false }, + ]); + }); + + it("caps lines at 32 KiB by default", () => { + const splitter = makeSplitter(); + + const [, cut] = push(splitter, chunk(1, 0, `${"x".repeat(maxLineBytes + 10)}\n`), 0); + + expect(cut).toMatchObject({ kind: "stdout", truncated: true }); + expect(cut?.kind === "stdout" ? cut.text.length : 0).toBe(maxLineBytes); + }); + + it("keeps stdout and stderr lines apart", () => { + const splitter = makeSplitter(); + push(splitter, chunk(1, 0, "out "), 0); + push(splitter, chunk(1, 0, "err\n", "stderr"), 0); + + expect(splitter.flush(1)).toEqual([ + { kind: "stdout", timestamp: 0, launchId: 1, text: "out ", truncated: false }, + ]); + }); +}); diff --git a/packages/stack/src/host/LogStore.integration.test.ts b/packages/stack/src/host/LogStore.integration.test.ts new file mode 100644 index 0000000000..1abf095396 --- /dev/null +++ b/packages/stack/src/host/LogStore.integration.test.ts @@ -0,0 +1,709 @@ +import { NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { + Clock, + Effect, + Exit, + Fiber, + FileSystem, + Latch, + Path, + PlatformError, + PubSub, + Queue, + Scope, + Stream, + SubscriptionRef, +} from "effect"; +import { TestClock } from "effect/testing"; +import type { LaunchOutput } from "../runtime/Session.ts"; +import { segmentName, type LogRecord } from "./LogRecord.ts"; +import * as LogStore from "./LogStore.ts"; + +const encoder = new TextEncoder(); + +const fakeInstance = ( + instanceId: string, + options: { readonly service?: string; readonly capacity?: number } = {}, +) => + Effect.gen(function* () { + const logs = + options.capacity === undefined + ? yield* PubSub.unbounded() + : yield* PubSub.sliding(options.capacity); + const launch = yield* SubscriptionRef.make<{ readonly launchId: number | undefined }>({ + launchId: undefined, + }); + const seq = { stdout: 0, stderr: 0 }; + const chunk = (launchId: number, text: string, stream: "stdout" | "stderr" = "stdout") => { + const next: LaunchOutput = { + stream, + bytes: encoder.encode(text), + launchId, + part: 0, + seq: seq[stream], + time: 0, + }; + seq[stream] += 1; + return next; + }; + return { + service: options.service ?? "auth", + instanceId, + logs: PubSub.subscribe(logs), + observation: SubscriptionRef.changes(launch), + publish: (...chunks: ReadonlyArray) => + Clock.currentTimeMillis.pipe( + Effect.flatMap((time) => + PubSub.publishAll( + logs, + chunks.map((chunk) => ({ ...chunk, time })), + ), + ), + ), + chunk, + setLaunch: (launchId: number | undefined) => SubscriptionRef.set(launch, { launchId }), + }; + }); + +/** Opens a store in its own scope so a test can close it like an owner stop. */ +const openStore = (root: string, options: Omit = {}) => + Effect.gen(function* () { + const scope = yield* Scope.make(); + const store = yield* LogStore.make({ root, ...options }).pipe(Scope.provide(scope)); + return { store, close: Scope.close(scope, Exit.void) }; + }); + +/** Takes `count` items; `Queue.takeN` returns fewer once it has waited for the first. */ +const takeExactly = (queue: Queue.Dequeue, count: number) => + Effect.gen(function* () { + const taken: Array = []; + while (taken.length < count) + taken.push(...(yield* Queue.takeBetween(queue, 1, count - taken.length))); + return taken; + }); + +/** Collects records from a stream into a queue for incremental assertions. */ +const collect = ( + opened: Effect.Effect< + Stream.Stream, + LogStore.LogStoreError, + Scope.Scope + >, +) => + Effect.gen(function* () { + const stream = yield* opened; + const records = yield* Queue.unbounded(); + const fiber = yield* stream.pipe( + Stream.runForEach((record) => Queue.offer(records, record)), + Effect.forkScoped, + ); + return { records, fiber, take: (count: number) => takeExactly(records, count) }; + }); + +/** Takes records until one carries `text`, returning every record taken. */ +const untilLast = ( + reader: { readonly take: (count: number) => Effect.Effect> }, + text: string, +) => + Effect.gen(function* () { + const taken: Array = []; + while (taken.at(-1)?.text !== text) taken.push(...(yield* reader.take(1))); + return taken; + }); + +const texts = (records: Iterable) => + Array.from(records).flatMap((record) => + record.kind === "stdout" || record.kind === "stderr" ? [record.text ?? ""] : [], + ); + +const tempRoot = (prefix: string) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + return yield* fs.makeTempDirectoryScoped({ prefix }); + }); + +describe("LogStore", () => { + it.effect("follows a burst larger than the live buffer and sees every record exactly once", () => + Effect.gen(function* () { + const root = yield* tempRoot("log-store-burst-"); + const { store } = yield* openStore(root); + const instance = yield* fakeInstance("burst"); + yield* store.attach(instance); + const release = yield* Latch.make(); + const seen: Array = []; + const follower = yield* (yield* store.read("burst", { from: "oldest", follow: true })).pipe( + Stream.tap((record) => (record.kind === "launch" ? release.await : Effect.void)), + Stream.take(1_001), + Stream.runForEach((record) => Effect.sync(() => seen.push(record))), + Effect.forkScoped, + ); + + yield* instance.publish( + ...Array.from({ length: 1_000 }, (_, index) => instance.chunk(1, `line ${index}\n`)), + ); + yield* release.open; + yield* Fiber.join(follower); + + expect(texts(seen)).toEqual(Array.from({ length: 1_000 }, (_, index) => `line ${index}`)); + const offline = yield* LogStore.readStackLogs({ root }); + expect(offline.map(({ position }) => position)).toEqual(seen.map(({ position }) => position)); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("records chunks the upstream buffer dropped as lost", () => + Effect.gen(function* () { + const root = yield* tempRoot("log-store-overflow-"); + const { store } = yield* openStore(root); + const instance = yield* fakeInstance("overflow", { capacity: 4 }); + yield* store.attach(instance); + const reader = yield* collect(store.read("overflow", { from: "oldest", follow: true })); + + yield* instance.publish( + ...Array.from({ length: 10 }, (_, index) => instance.chunk(1, `line ${index}\n`)), + ); + const [launch, lost] = yield* reader.take(2); + const lines = yield* reader.take(10 - (lost?.count ?? 0)); + + expect(launch?.kind).toBe("launch"); + expect(lost).toMatchObject({ kind: "lost", launchId: 1, stream: "stdout" }); + expect(lost?.count).toBeGreaterThan(0); + expect(texts(lines).at(-1)).toBe("line 9"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("attributes a late chunk of a failed launch to that launch", () => + Effect.gen(function* () { + const root = yield* tempRoot("log-store-late-"); + const { store } = yield* openStore(root); + const instance = yield* fakeInstance("late"); + yield* store.attach(instance); + const reader = yield* collect(store.read("late", { from: "oldest", follow: true })); + + yield* instance.publish(instance.chunk(1, "starting\nbo")); + yield* instance.publish({ ...instance.chunk(2, "second\n"), seq: 0 }); + yield* instance.publish({ ...instance.chunk(1, "om\n"), seq: 1 }); + const records = yield* reader.take(6); + + expect(Array.from(records, ({ kind, launchId, text }) => ({ kind, launchId, text }))).toEqual( + [ + { kind: "launch", launchId: 1, text: undefined }, + { kind: "stdout", launchId: 1, text: "starting" }, + { kind: "stdout", launchId: 1, text: "bo" }, + { kind: "launch", launchId: 2, text: undefined }, + { kind: "stdout", launchId: 2, text: "second" }, + { kind: "stdout", launchId: 1, text: "om" }, + ], + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("writes a launch's partial line when its observation leaves the launch", () => + Effect.gen(function* () { + const root = yield* tempRoot("log-store-partial-"); + const { store } = yield* openStore(root); + const instance = yield* fakeInstance("partial"); + yield* store.attach(instance); + const reader = yield* collect(store.read("partial", { from: "oldest", follow: true })); + yield* instance.setLaunch(1); + + yield* instance.publish(instance.chunk(1, "no trailing newline")); + yield* reader.take(1); + yield* instance.setLaunch(undefined); + const [flushed] = yield* reader.take(1); + + expect(flushed).toMatchObject({ kind: "stdout", launchId: 1, text: "no trailing newline" }); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("rotates segments while a follower reads and keeps its order", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* tempRoot("log-store-rotate-"); + const { store } = yield* openStore(root, { rotateBytes: 300 }); + const instance = yield* fakeInstance("rotate"); + yield* store.attach(instance); + const reader = yield* collect(store.read("rotate", { from: "oldest", follow: true })); + + for (let index = 0; index < 40; index++) { + yield* instance.publish(instance.chunk(1, `rotating line ${index}\n`)); + if (index % 10 === 0) yield* reader.take(index === 0 ? 2 : 10); + } + const rest = yield* reader.take(9); + + const segments = yield* fs.readDirectory(path.join(root, "auth", "rotate")); + expect(segments.length).toBeGreaterThan(3); + expect(texts(rest).at(-1)).toBe("rotating line 39"); + const all = yield* LogStore.readStackLogs({ root }); + expect(texts(all)).toEqual( + Array.from({ length: 40 }, (_, index) => `rotating line ${index}`), + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("deletes the oldest segments beyond retention and reports the gap to a reader", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* tempRoot("log-store-retention-"); + const { store } = yield* openStore(root, { rotateBytes: 200, retainBytes: 400 }); + const instance = yield* fakeInstance("retention"); + yield* store.attach(instance); + const release = yield* Latch.make(); + const reader = yield* collect( + store + .read("retention", { from: "oldest", follow: true }) + .pipe( + Effect.map( + Stream.tap((record) => (record.kind === "launch" ? release.await : Effect.void)), + ), + ), + ); + const progress = yield* collect(store.read("retention", { from: "oldest", follow: true })); + + for (let index = 0; index < 60; index++) { + yield* instance.publish(instance.chunk(1, `retained line ${index}\n`)); + yield* untilLast(progress, `retained line ${index}`); + } + // Opening an end reader takes the writer lock, so the last rotation's retention finished. + yield* store.read("retention", { from: "end", follow: false }).pipe(Effect.asVoid); + + const directory = path.join(root, "auth", "retention"); + const sizes = yield* Effect.forEach(yield* fs.readDirectory(directory), (name) => + fs.stat(path.join(directory, name)).pipe(Effect.map((info) => Number(info.size))), + ); + expect(sizes.reduce((sum, size) => sum + size, 0)).toBeLessThanOrEqual(400 + 200); + yield* release.open; + const records = yield* untilLast(reader, "retained line 59"); + const gap = records.find((record) => record.kind === "lost"); + expect(gap).toBeDefined(); + expect(gap?.launchId).toBeUndefined(); + expect(gap?.position).toBeUndefined(); + expect(gap?.resumeAt?.generation).toBeGreaterThan(1); + expect(texts(records).length).toBeLessThan(60); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("caps the number of retained segments", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* tempRoot("log-store-count-"); + const { store } = yield* openStore(root, { rotateBytes: 1, retainSegments: 3 }); + const instance = yield* fakeInstance("count"); + yield* store.attach(instance); + const progress = yield* collect(store.read("count", { from: "oldest", follow: true })); + + for (let index = 0; index < 10; index++) { + yield* instance.publish(instance.chunk(1, `counted line ${index}\n`)); + yield* untilLast(progress, `counted line ${index}`); + } + // Opening an end reader takes the writer lock, so the last rotation's retention finished. + yield* store.read("count", { from: "end", follow: false }).pipe(Effect.asVoid); + + const segments = yield* fs.readDirectory(path.join(root, "auth", "count")); + expect(segments.length).toBeLessThanOrEqual(3); + expect(texts(yield* LogStore.readStackLogs({ root })).at(-1)).toBe("counted line 9"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("creates no segment for an instance that wrote nothing", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* tempRoot("log-store-idle-"); + const opened = yield* openStore(root); + yield* opened.store.attach(yield* fakeInstance("idle")); + yield* opened.close; + + expect(yield* fs.exists(path.join(root, "auth", "idle"))).toBe(false); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("tails the newest records across segments and then follows", () => + Effect.gen(function* () { + const root = yield* tempRoot("log-store-tail-"); + const { store } = yield* openStore(root, { rotateBytes: 120 }); + const instance = yield* fakeInstance("tail"); + yield* store.attach(instance); + const progress = yield* collect(store.read("tail", { from: "oldest", follow: true })); + for (let index = 0; index < 20; index++) { + yield* instance.publish(instance.chunk(1, `tailed line ${index}\n`)); + yield* untilLast(progress, `tailed line ${index}`); + } + + const tailed = yield* collect(store.read("tail", { from: "oldest", tail: 3, follow: true })); + const history = yield* tailed.take(3); + yield* instance.publish(instance.chunk(1, "after tail\n")); + const [followed] = yield* tailed.take(1); + + expect(texts(history)).toEqual(["tailed line 17", "tailed line 18", "tailed line 19"]); + expect(followed?.text).toBe("after tail"); + expect(texts(yield* LogStore.readStackLogs({ root, tail: 2 }))).toEqual([ + "tailed line 19", + "after tail", + ]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("skips files that are not service or instance directories when reading offline", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* tempRoot("log-store-stray-"); + const opened = yield* openStore(root); + const instance = yield* fakeInstance("kept"); + yield* opened.store.attach(instance); + const reader = yield* collect(opened.store.read("kept", { from: "oldest", follow: true })); + yield* instance.publish(instance.chunk(1, "kept line\n")); + yield* reader.take(2); + yield* opened.close; + yield* fs.writeFileString(path.join(root, ".DS_Store"), ""); + yield* fs.writeFileString(path.join(root, "auth", ".DS_Store"), ""); + + const records = yield* LogStore.readStackLogs({ root }); + + expect(texts(records)).toEqual(["kept line"]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("keeps the newest records of one batch larger than the retention limit", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* tempRoot("log-store-big-batch-"); + const { store } = yield* openStore(root, { rotateBytes: 200, retainBytes: 400 }); + const instance = yield* fakeInstance("big"); + yield* store.attach(instance); + const progress = yield* collect(store.read("big", { from: "oldest", follow: true })); + + yield* instance.publish( + ...Array.from({ length: 60 }, (_, index) => instance.chunk(1, `batched line ${index}\n`)), + ); + yield* untilLast(progress, "batched line 59"); + // Opening an end reader takes the writer lock, so the last rotation's retention finished. + yield* store.read("big", { from: "end", follow: false }).pipe(Effect.asVoid); + + const directory = path.join(root, "auth", "big"); + const sizes = yield* Effect.forEach(yield* fs.readDirectory(directory), (name) => + fs.stat(path.join(directory, name)).pipe(Effect.map((info) => Number(info.size))), + ); + expect(Math.max(...sizes)).toBeLessThanOrEqual(200); + const kept = texts(yield* LogStore.readStackLogs({ root })); + expect(kept.length).toBeGreaterThan(0); + expect(kept.at(-1)).toBe("batched line 59"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("writes output still queued when the store closes", () => + Effect.gen(function* () { + const root = yield* tempRoot("log-store-drain-"); + const opened = yield* openStore(root); + const instance = yield* fakeInstance("drain"); + yield* opened.store.attach(instance); + + yield* instance.publish( + ...Array.from({ length: 1_000 }, (_, index) => instance.chunk(1, `queued line ${index}\n`)), + ); + yield* opened.close; + + const records = yield* LogStore.readStackLogs({ root }); + expect(texts(records)).toEqual( + Array.from({ length: 1_000 }, (_, index) => `queued line ${index}`), + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("keeps the live tail flowing while appends fail", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const base = yield* tempRoot("log-store-unwritable-"); + const root = path.join(base, "not-a-directory"); + yield* fs.writeFileString(root, ""); + const { store } = yield* openStore(root); + const instance = yield* fakeInstance("unwritable"); + yield* store.attach(instance); + const lines = yield* Queue.unbounded(); + yield* (yield* store.tail("unwritable")).pipe( + Stream.runForEach(({ text }) => Queue.offer(lines, text)), + Effect.forkScoped, + ); + + yield* instance.publish(instance.chunk(1, "first live line\n")); + yield* instance.publish(instance.chunk(1, "second live line\n")); + const received = yield* takeExactly(lines, 2); + + expect(Array.from(received)).toEqual(["first live line", "second live line"]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("tails with since past a newer segment that holds an older flushed line", () => + Effect.gen(function* () { + const root = yield* tempRoot("log-store-late-flush-"); + const { store } = yield* openStore(root, { rotateBytes: 1 }); + const instance = yield* fakeInstance("flushed"); + yield* store.attach(instance); + const progress = yield* collect(store.read("flushed", { from: "oldest", follow: true })); + yield* instance.setLaunch(1); + + yield* TestClock.setTime(1_000); + yield* instance.publish(instance.chunk(1, "early partial", "stderr")); + yield* TestClock.setTime(2_000); + yield* instance.publish(instance.chunk(1, "later line\n")); + yield* untilLast(progress, "later line"); + yield* TestClock.setTime(3_000); + yield* instance.setLaunch(undefined); + yield* untilLast(progress, "early partial"); + + const since = { since: 1_500, tail: 10 }; + const offline = yield* LogStore.readStackLogs({ root, ...since }); + const live = yield* (yield* store.read("flushed", { + from: "oldest", + follow: false, + ...since, + })).pipe(Stream.runCollect); + + expect(texts(offline)).toEqual(["later line"]); + expect(texts(live)).toEqual(["later line"]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("delivers one batch larger than any buffer to a waiting live tail", () => + Effect.gen(function* () { + const root = yield* tempRoot("log-store-live-batch-"); + const { store } = yield* openStore(root); + const instance = yield* fakeInstance("live-batch"); + yield* store.attach(instance); + const lines = yield* Queue.unbounded(); + yield* (yield* store.tail("live-batch")).pipe( + Stream.runForEach(({ text }) => Queue.offer(lines, text)), + Effect.forkScoped, + ); + const expected = Array.from({ length: 2_000 }, (_, index) => `burst line ${index}`); + + yield* instance.publish(instance.chunk(1, `${expected.join("\n")}\n`)); + const received = yield* takeExactly(lines, expected.length); + + expect(received).toEqual(expected); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("keeps sweeping older segments past one it cannot delete", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* tempRoot("log-store-stuck-segment-"); + const held = segmentName(1); + const injected: FileSystem.FileSystem = { + ...fs, + remove: (target, options) => + target.endsWith(held) + ? Effect.fail( + PlatformError.systemError({ + _tag: "Busy", + module: "FileSystem", + method: "remove", + pathOrDescriptor: target, + }), + ) + : fs.remove(target, options), + }; + const { store } = yield* openStore(root, { rotateBytes: 1, retainSegments: 3 }).pipe( + Effect.provideService(FileSystem.FileSystem, injected), + ); + const instance = yield* fakeInstance("stuck"); + yield* store.attach(instance); + const progress = yield* collect(store.read("stuck", { from: "oldest", follow: true })); + + for (let index = 0; index < 10; index++) { + yield* instance.publish(instance.chunk(1, `stuck line ${index}\n`)); + yield* untilLast(progress, `stuck line ${index}`); + } + // Opening an end reader takes the writer lock, so the last rotation's retention finished. + yield* store.read("stuck", { from: "end", follow: false }).pipe(Effect.asVoid); + + const segments = yield* fs.readDirectory(path.join(root, "auth", "stuck")); + expect(segments).toContain(held); + expect(segments.length).toBeLessThanOrEqual(4); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("continues after existing generations when the first listing fails", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* tempRoot("log-store-listing-"); + const directory = path.join(root, "auth", "listing"); + yield* fs.makeDirectory(directory, { recursive: true }); + yield* fs.writeFileString( + path.join(directory, segmentName(1)), + "1970-01-01T00:00:00.000Z stdout 1 | before\n", + ); + let failures = 1; + const injected: FileSystem.FileSystem = { + ...fs, + readDirectory: (target, options) => + target === directory && failures-- > 0 + ? Effect.fail( + PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "readDirectory", + pathOrDescriptor: target, + }), + ) + : fs.readDirectory(target, options), + }; + const { store } = yield* openStore(root).pipe( + Effect.provideService(FileSystem.FileSystem, injected), + ); + const instance = yield* fakeInstance("listing"); + yield* store.attach(instance); + const reader = yield* collect(store.read("listing", { from: "oldest", follow: true })); + + yield* instance.publish(instance.chunk(1, "after\n")); + yield* untilLast(reader, "after"); + + expect((yield* fs.readDirectory(directory)).toSorted()).toEqual([ + segmentName(1), + segmentName(2), + ]); + expect(texts(yield* LogStore.readStackLogs({ root }))).toEqual(["before", "after"]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("removes log directories of instances that are no longer attached", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* tempRoot("log-store-orphans-"); + const orphan = path.join(root, "rest", "ghost"); + yield* fs.makeDirectory(orphan, { recursive: true }); + yield* fs.writeFileString(path.join(orphan, segmentName(1)), ""); + const { store } = yield* openStore(root); + const instance = yield* fakeInstance("kept"); + yield* store.attach(instance); + const reader = yield* collect(store.read("kept", { from: "oldest", follow: true })); + yield* instance.publish(instance.chunk(1, "kept line\n")); + yield* untilLast(reader, "kept line"); + + yield* store.removeOrphans; + + expect(yield* fs.exists(orphan)).toBe(false); + expect(yield* fs.exists(path.join(root, "auth", "kept", segmentName(1)))).toBe(true); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("reads persisted records offline after the store closes, merged and tailed", () => + Effect.gen(function* () { + const root = yield* tempRoot("log-store-offline-"); + const opened = yield* openStore(root); + const auth = yield* fakeInstance("auth-1", { service: "auth" }); + const rest = yield* fakeInstance("rest-1", { service: "rest" }); + yield* opened.store.attach(auth); + yield* opened.store.attach(rest); + const authReader = yield* collect( + opened.store.read("auth-1", { from: "oldest", follow: true }), + ); + const restReader = yield* collect( + opened.store.read("rest-1", { from: "oldest", follow: true }), + ); + + yield* TestClock.setTime(1_000); + yield* rest.publish(rest.chunk(1, "rest first\n")); + yield* restReader.take(2); + yield* TestClock.setTime(2_000); + yield* auth.publish(auth.chunk(1, "auth second\n")); + yield* authReader.take(2); + yield* TestClock.setTime(3_000); + yield* rest.publish(rest.chunk(1, "rest third\n")); + yield* restReader.take(1); + yield* opened.close; + + const all = yield* LogStore.readStackLogs({ root }); + const tailed = yield* LogStore.readStackLogs({ root, tail: 2 }); + const recent = yield* LogStore.readStackLogs({ root, since: 2_000, instances: ["rest-1"] }); + + expect(all.map(({ service, kind }) => `${service}:${kind}`)).toEqual([ + "rest:launch", + "rest:stdout", + "auth:launch", + "auth:stdout", + "rest:stdout", + ]); + expect(texts(all)).toEqual(["rest first", "auth second", "rest third"]); + expect(texts(tailed)).toEqual(["auth second", "rest third"]); + expect(texts(recent)).toEqual(["rest third"]); + expect(recent[0]).toMatchObject({ service: "rest", instanceId: "rest-1" }); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("opens a new generation for each owner start without touching earlier ones", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* tempRoot("log-store-restart-"); + for (const text of ["before restart", "after restart"]) { + const opened = yield* openStore(root); + const instance = yield* fakeInstance("restart"); + yield* opened.store.attach(instance); + const reader = yield* collect(opened.store.read("restart", { from: "end", follow: true })); + yield* instance.publish(instance.chunk(1, `${text}\n`)); + yield* reader.take(2); + yield* opened.close; + } + + const directory = path.join(root, "auth", "restart"); + expect((yield* fs.readDirectory(directory)).toSorted()).toEqual([ + "0000000001.log", + "0000000002.log", + ]); + const records = yield* LogStore.readStackLogs({ root }); + expect(texts(records)).toEqual(["before restart", "after restart"]); + expect(records.map(({ position }) => position?.generation)).toEqual([1, 1, 2, 2]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("tails only records written after an end reader subscribes", () => + Effect.gen(function* () { + const root = yield* tempRoot("log-store-end-"); + const { store } = yield* openStore(root); + const instance = yield* fakeInstance("end"); + yield* store.attach(instance); + const history = yield* collect(store.read("end", { from: "oldest", follow: true })); + yield* instance.publish(instance.chunk(1, "old\n")); + yield* history.take(2); + + const reader = yield* collect(store.read("end", { from: "end", follow: true })); + yield* instance.publish(instance.chunk(1, "new\n")); + const [record] = yield* reader.take(1); + + expect(record).toMatchObject({ kind: "stdout", text: "new" }); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("removes an instance by ending its readers and deleting its segments", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* tempRoot("log-store-remove-"); + const { store } = yield* openStore(root); + const instance = yield* fakeInstance("removed"); + yield* store.attach(instance); + const reader = yield* collect(store.read("removed", { from: "oldest", follow: true })); + yield* instance.publish(instance.chunk(1, "doomed\n")); + yield* reader.take(2); + + yield* store.remove({ service: "auth", instanceId: "removed" }); + yield* Fiber.join(reader.fiber); + + expect(yield* fs.exists(path.join(root, "auth", "removed"))).toBe(false); + const failure = yield* store + .read("removed", { from: "oldest", follow: false }) + .pipe(Effect.flip); + expect(failure.message).toContain("has no logs"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); diff --git a/packages/stack/src/host/LogStore.ts b/packages/stack/src/host/LogStore.ts new file mode 100644 index 0000000000..1c7cbce4dc --- /dev/null +++ b/packages/stack/src/host/LogStore.ts @@ -0,0 +1,945 @@ +import { + Clock, + DateTime, + Deferred, + Duration, + Effect, + Exit, + FileSystem, + Option, + Path, + PubSub, + Ref, + Result, + Schema, + Scope, + Semaphore, + Stream, +} from "effect"; +import { failureMessage } from "../internal/failure-message.ts"; +import { errorCode, retrySharingViolation } from "../internal/sharing-violation.ts"; +import type { LaunchOutput } from "../runtime/Session.ts"; +import type { CatalogLogs } from "../services/Recipe.ts"; +import { + encodeEntry, + makeSplitter, + parseRecord, + segmentGeneration, + segmentName, + type LogEntry, + type LogPosition, + type LogRecord, + type StackLogRecord, +} from "./LogRecord.ts"; + +/** A log store operation that failed. */ +export class LogStoreError extends Schema.TaggedError()("LogStoreError", { + operation: Schema.String, + message: Schema.String, + cause: Schema.optionalKey(Schema.Defect()), +}) {} + +// Holds a whole pulled batch for a waiting subscriber while bounding a stalled one. +const liveTailCapacity = 16_384; +const storeError = (operation: string) => (cause: unknown) => + new LogStoreError({ operation, message: failureMessage(cause), cause }); + +const removeError = (directory: string) => (cause: unknown) => + new LogStoreError({ + operation: "remove", + message: `Unable to remove ${directory}: ${failureMessage(cause)}`, + cause, + }); + +export interface LogStoreOptions { + /** The stack's logs directory. */ + readonly root: string; + /** A segment reaching this size is closed and a new generation is opened. */ + readonly rotateBytes?: number; + /** Oldest closed segments are deleted while an instance's segments exceed this size. */ + readonly retainBytes?: number; + /** Oldest closed segments are deleted while an instance has more segments than this. */ + readonly retainSegments?: number; + readonly platform?: NodeJS.Platform; +} + +/** Selects the records a read returns. */ +export interface ReadOptions { + /** `end` skips every record written before the read starts. */ + readonly from: "oldest" | "end"; + /** Epoch milliseconds; older records are skipped. */ + readonly since?: number; + /** Returns only the last records of the history before following. */ + readonly tail?: number; + readonly follow: boolean; +} + +/** An instance whose output the store persists. */ +export interface AttachedInstance { + readonly service: string; + readonly instanceId: string; + readonly logs: CatalogLogs; + readonly observation: Stream.Stream<{ readonly launchId: number | undefined }>; +} + +interface Interface { + /** Subscribes to the instance's output before returning; persistence failures never fail it. */ + readonly attach: (instance: AttachedInstance) => Effect.Effect; + /** Fixes the read's start and wake subscription in the scope; the stream reads lazily. */ + readonly read: ( + instanceId: string, + options: ReadOptions, + ) => Effect.Effect, LogStoreError, Scope.Scope>; + /** Streams output lines as they are split, whether or not their append succeeds. */ + readonly tail: ( + instanceId: string, + ) => Effect.Effect< + Stream.Stream<{ readonly stream: "stdout" | "stderr"; readonly text: string }>, + LogStoreError, + Scope.Scope + >; + /** Stops the instance's writer and readers, then deletes its segments. */ + readonly remove: (instance: { + readonly service: string; + readonly instanceId: string; + }) => Effect.Effect; + /** Deletes log directories of instances that are not attached; failures are logged. */ + readonly removeOrphans: Effect.Effect; + /** Stops every writer and reader; later reads fail. */ + readonly close: Effect.Effect; +} + +const defaultRotateBytes = 5 * 1024 * 1024; +const defaultRetainBytes = 10 * 1024 * 1024; +const defaultRetainSegments = 64; +/** Bounds how much of a segment one read step holds in memory; above the longest record. */ +const readChunkBytes = 256 * 1024; +/** Concurrent read steps per instance; removal takes them all to wait out in-flight reads. */ +const readerPermits = 1024; +const retryBaseMillis = 100; +const retryMaxMillis = 30_000; +const drainBatch = 4096; +const drainTimeout = Duration.seconds(5); + +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +interface Live { + readonly lock: Semaphore.Semaphore; + readonly end: Ref.Ref; + readonly wake: PubSub.PubSub; + readonly passes: Semaphore.Semaphore; +} + +interface Step { + readonly records: ReadonlyArray; + readonly cursor: LogPosition; + /** The segment listing to reuse; `undefined` asks the next step to list again. */ + readonly listed: ReadonlyArray | undefined; + /** The cursor reached the last complete record of the newest segment. */ + readonly atEnd: boolean; + /** Bytes after the cursor did not yet form a complete record. */ + readonly partial: boolean; +} + +interface Chunk { + readonly records: ReadonlyArray; + readonly byteOffset: number; + /** The read consumed bytes; otherwise it met the end of the segment. */ + readonly progressed: boolean; + readonly partial: boolean; +} + +const before = (cursor: LogPosition, end: LogPosition) => + cursor.generation < end.generation || + (cursor.generation === end.generation && cursor.byteOffset < end.byteOffset); + +/** Parses an ISO-8601 `since` bound into epoch milliseconds. */ +export const sinceMillis = (since: string): Effect.Effect => + Option.match(DateTime.make(since), { + onNone: () => + Effect.fail( + new LogStoreError({ operation: "read", message: `Invalid since timestamp ${since}` }), + ), + onSome: (time) => Effect.succeed(DateTime.toEpochMillis(time)), + }); + +const isNotFound = (error: { readonly reason: { readonly _tag: string } }) => + error.reason._tag === "NotFound"; + +/** A missing directory, or a file where a directory was expected, holds no segments. */ +const isMissingDirectory = (error: { readonly reason: { readonly _tag: string } }) => + isNotFound(error) || errorCode(error) === "ENOTDIR"; + +interface FollowState { + readonly cursor: LogPosition; + readonly listed: ReadonlyArray | undefined; +} + +const page = ( + records: ReadonlyArray, + next: Option.Option, +): readonly [ReadonlyArray, Option.Option] => [records, next]; + +const compareText = (left: string, right: string) => (left < right ? -1 : left > right ? 1 : 0); + +const recordPosition = (record: LogRecord) => + record.position ?? record.resumeAt ?? { generation: 0, byteOffset: 0 }; + +/** Orders an instance's records by timestamp and position; a gap marker precedes its resume. */ +const compareRecords = (left: LogRecord, right: LogRecord) => { + const leftPosition = recordPosition(left); + const rightPosition = recordPosition(right); + return ( + compareText(left.timestamp, right.timestamp) || + leftPosition.generation - rightPosition.generation || + leftPosition.byteOffset - rightPosition.byteOffset || + Number(left.position !== undefined) - Number(right.position !== undefined) + ); +}; + +const nowIso = Clock.currentTimeMillis.pipe( + Effect.map((millis) => DateTime.formatIso(DateTime.makeUnsafe(millis))), +); + +/** Lists the subdirectories of `directory`; files and a missing directory list nothing. */ +const listDirectories = (fs: FileSystem.FileSystem, path: Path.Path, directory: string) => + fs.readDirectory(directory).pipe( + Effect.catchIf(isMissingDirectory, () => Effect.succeed>([])), + Effect.flatMap((names) => + Effect.filter(names, (name) => + fs.stat(path.join(directory, name)).pipe( + Effect.map((info) => info.type === "Directory"), + Effect.catchIf(isNotFound, () => Effect.succeed(false)), + ), + ), + ), + Effect.mapError(storeError("list")), + ); + +const makeReader = (fs: FileSystem.FileSystem, path: Path.Path) => { + const generations = (directory: string) => + fs.readDirectory(directory).pipe( + Effect.map((names) => + names + .map(segmentGeneration) + .filter((generation) => generation !== undefined) + .toSorted((left, right) => left - right), + ), + Effect.catchIf(isMissingDirectory, () => Effect.succeed>([])), + Effect.mapError(storeError("list")), + ); + + const segments = (directory: string) => + generations(directory).pipe( + Effect.flatMap((listed) => + Effect.forEach(listed, (generation) => + fs.stat(path.join(directory, segmentName(generation))).pipe( + Effect.map((info) => [{ generation, size: Number(info.size) }]), + Effect.catchIf(isNotFound, () => + Effect.succeed>([]), + ), + Effect.mapError(storeError("list")), + ), + ), + ), + Effect.map((listed) => listed.flat()), + ); + + const readChunk = (file: string, offset: number) => + Effect.scoped( + fs.open(file, { flag: "r" }).pipe( + Effect.tap((handle) => handle.seek(offset, "start")), + Effect.flatMap((handle) => handle.readAlloc(readChunkBytes)), + Effect.map(Option.getOrElse(() => new Uint8Array(0))), + Effect.map(Option.some), + ), + ).pipe( + Effect.catchIf(isNotFound, () => Effect.succeed(Option.none())), + Effect.mapError(storeError("read")), + ); + + const guarded = (live: Live | undefined, effect: Effect.Effect) => + live === undefined ? effect : live.passes.withPermits(1)(effect); + + /** Reads the complete records of one chunk at `position`; `none` when the segment is gone. */ + const readAt = (directory: string, position: LogPosition, live: Live | undefined) => + guarded( + live, + readChunk(path.join(directory, segmentName(position.generation)), position.byteOffset), + ).pipe( + Effect.map( + Option.map((bytes): Chunk => { + const last = bytes.lastIndexOf(0x0a); + if (last < 0) + return bytes.length === readChunkBytes + ? { + records: [], + byteOffset: position.byteOffset + bytes.length, + progressed: true, + partial: false, + } + : { + records: [], + byteOffset: position.byteOffset, + progressed: false, + partial: bytes.length > 0, + }; + const records: Array = []; + let start = 0; + while (start <= last) { + const stop = bytes.indexOf(0x0a, start); + const record = parseRecord(decoder.decode(bytes.subarray(start, stop)), { + generation: position.generation, + byteOffset: position.byteOffset + start, + }); + if (record !== undefined) records.push(record); + start = stop + 1; + } + return { + records, + byteOffset: position.byteOffset + last + 1, + progressed: true, + partial: false, + }; + }), + ), + ); + + /** + * Advances `cursor` by at most one chunk. The segment listing is reused until the cursor reaches + * the end of the listed segments or a listed segment disappears. + */ + const step = Effect.fnUntraced(function* ( + directory: string, + cursor: LogPosition, + cached: ReadonlyArray | undefined, + live: Live | undefined, + ) { + const fresh = cached === undefined; + const listed = cached ?? (yield* generations(directory)); + const relist = (partial: boolean): Step => ({ + records: [], + cursor, + listed: undefined, + atEnd: false, + partial, + }); + const atEnd = (partial: boolean): Step => ({ + records: [], + cursor, + listed, + atEnd: true, + partial, + }); + const first = listed[0]; + if (first === undefined) return fresh ? atEnd(false) : relist(false); + if (cursor.generation < first) { + const resumeAt = { generation: first, byteOffset: 0 }; + const records: ReadonlyArray = + cursor.generation === 0 ? [] : [{ kind: "lost", timestamp: yield* nowIso, resumeAt }]; + return { records, cursor: resumeAt, listed, atEnd: false, partial: false } satisfies Step; + } + const newer = listed.find((generation) => generation > cursor.generation); + const moveOn = (generation: number): Step => ({ + records: [], + cursor: { generation, byteOffset: 0 }, + listed, + atEnd: false, + partial: false, + }); + if (!listed.includes(cursor.generation)) + return newer !== undefined ? moveOn(newer) : fresh ? atEnd(false) : relist(false); + const chunk = yield* readAt(directory, cursor, live); + if (Option.isNone(chunk)) return relist(false); + if (chunk.value.progressed) + return { + records: chunk.value.records, + cursor: { ...cursor, byteOffset: chunk.value.byteOffset }, + listed, + atEnd: false, + partial: false, + } satisfies Step; + // A segment is closed once a newer one exists, so its incomplete tail never completes. + if (newer !== undefined) return moveOn(newer); + if (fresh) return atEnd(chunk.value.partial); + const writing = live === undefined ? undefined : yield* Ref.get(live.end); + return writing !== undefined && writing.generation <= cursor.generation + ? atEnd(chunk.value.partial) + : relist(chunk.value.partial); + }); + + const since = (options: Pick) => (record: LogRecord) => + options.since === undefined || Date.parse(record.timestamp) >= options.since; + + /** Reads one segment from its start; a segment deleted meanwhile yields what was read. */ + const readSegment = Effect.fnUntraced(function* ( + directory: string, + generation: number, + live: Live | undefined, + ) { + const records: Array = []; + let position: LogPosition = { generation, byteOffset: 0 }; + while (true) { + const chunk = yield* readAt(directory, position, live); + if (Option.isNone(chunk) || !chunk.value.progressed) break; + records.push(...chunk.value.records); + position = { generation, byteOffset: chunk.value.byteOffset }; + } + return { records, position }; + }); + + /** + * Reads the history after `cursor`. With `tail`, only the newest segments that hold the last + * `tail` records are read. + */ + const history = Effect.fnUntraced(function* ( + directory: string, + cursor: LogPosition, + options: Pick, + live: Live | undefined, + ) { + const keep = since(options); + const listed = yield* generations(directory); + const tail = options.tail; + if (tail !== undefined) { + if (cursor.generation !== 0) return { records: [], cursor, listed }; + if (tail === 0) + return { + records: [], + cursor: live === undefined ? cursor : yield* live.lock.withPermits(1)(Ref.get(live.end)), + listed, + }; + // A partial line is written when it ends, so a segment can hold a record older than records + // in the segment before it; one more segment is read past each stopping point. + const collected: Array = []; + let end: LogPosition = cursor; + let first = true; + let past = 0; + for (const generation of listed.toReversed()) { + const segment = yield* readSegment(directory, generation, live); + if (first) end = segment.position; + first = false; + const matching = segment.records.filter(keep); + collected.push(...matching); + const olderThanSince = + options.since !== undefined && segment.records.length > 0 && matching.length === 0; + past = collected.length >= tail || olderThanSince ? past + 1 : 0; + if (past > 1) break; + } + const records = collected.toSorted(compareRecords); + return { records: records.slice(Math.max(0, records.length - tail)), cursor: end, listed }; + } + const records: Array = []; + let current = cursor; + let cached: ReadonlyArray | undefined = listed; + while (true) { + const next: Step = yield* step(directory, current, cached, live); + for (const record of next.records) if (keep(record)) records.push(record); + current = next.cursor; + cached = next.listed; + if (next.atEnd) break; + } + return { records, cursor: current, listed: cached }; + }); + + /** Streams records from `cursor`; a follower waits on the wake signal it subscribed to first. */ + const follow = ( + directory: string, + start: { readonly cursor: LogPosition; readonly listed: ReadonlyArray | undefined }, + options: Pick, + live: Live | undefined, + wake: PubSub.Subscription | undefined, + ): Stream.Stream => + Stream.paginate(start, (current) => + Effect.gen(function* () { + const next: Step = yield* step(directory, current.cursor, current.listed, live); + const resume = Option.some({ cursor: next.cursor, listed: next.listed }); + if (!next.atEnd) return page(next.records, resume); + if (!options.follow || live === undefined || wake === undefined) + return page(next.records, Option.none()); + const end = yield* live.lock.withPermits(1)(Ref.get(live.end)); + if (!next.partial && before(next.cursor, end)) return page(next.records, resume); + yield* PubSub.take(wake); + return page(next.records, resume); + }), + ).pipe(Stream.filter(since(options))); + + /** Subscribes and fixes the start position now; the returned stream reads lazily. */ + const read = Effect.fnUntraced(function* ( + directory: string, + options: ReadOptions, + live: Live | undefined, + ) { + const wake = + options.follow && live !== undefined ? yield* PubSub.subscribe(live.wake) : undefined; + const start = + options.from === "end" && live !== undefined + ? yield* live.lock.withPermits(1)(Ref.get(live.end)) + : { generation: 0, byteOffset: 0 }; + if (options.tail === undefined) + return follow(directory, { cursor: start, listed: undefined }, options, live, wake); + return Stream.unwrap( + history(directory, start, options, live).pipe( + Effect.map((past) => + Stream.fromIterable(past.records).pipe( + Stream.concat( + options.follow + ? follow( + directory, + { cursor: past.cursor, listed: past.listed }, + options, + live, + wake, + ) + : Stream.empty, + ), + ), + ), + ), + ); + }); + + return { read, history, segments, generations }; +}; + +/** Creates the owner's log store; closing its scope flushes and closes every writer. */ +export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptions) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const storeScope = yield* Scope.Scope; + const reader = makeReader(fs, path); + const rotateBytes = options.rotateBytes ?? defaultRotateBytes; + const retainBytes = options.retainBytes ?? defaultRetainBytes; + const retainSegments = options.retainSegments ?? defaultRetainSegments; + const retryShared = retrySharingViolation(options.platform); + + interface Attached extends Live { + readonly service: string; + readonly instanceId: string; + readonly directory: string; + /** + * Records as they are split, independent of whether their append succeeds. Unbounded so the + * recipe's sliding output buffer stays the only place output is dropped. + */ + readonly live: PubSub.PubSub; + readonly closed: Deferred.Deferred; + readonly scope: Scope.Closeable; + } + const instances = yield* Ref.make>(new Map()); + const closed = yield* Ref.make(false); + const attached = (instanceId: string) => + Ref.get(instances).pipe( + Effect.flatMap((current) => { + const instance = current.get(instanceId); + return instance === undefined + ? Effect.fail( + new LogStoreError({ + operation: "read", + message: `Instance ${instanceId} has no logs`, + }), + ) + : Effect.succeed(instance); + }), + ); + + const detach = Effect.fnUntraced(function* (instance: Attached) { + yield* Deferred.succeed(instance.closed, undefined); + yield* instance.passes.take(readerPermits); + yield* Scope.close(instance.scope, Exit.void); + }); + + const attach = Effect.fn("LogStore.attach")(function* (instance: AttachedInstance) { + if (yield* Ref.get(closed)) return; + const directory = path.join(options.root, instance.service, instance.instanceId); + // An unreadable listing defers choosing the next generation to the first append. + const existing = yield* reader.segments(directory).pipe( + retryShared, + Effect.map(Option.some), + Effect.catch((error) => + Effect.logWarning(`Unable to list logs in ${directory}`, error).pipe( + Effect.as(Option.none>()), + ), + ), + ); + const latest = Option.getOrElse(existing, () => []).at(-1); + const scope = yield* Scope.fork(storeScope, "sequential"); + const handle: Attached = { + service: instance.service, + instanceId: instance.instanceId, + directory, + scope, + lock: yield* Semaphore.make(1), + passes: yield* Semaphore.make(readerPermits), + end: yield* Ref.make({ + generation: latest?.generation ?? 0, + byteOffset: latest?.size ?? 0, + }), + wake: yield* PubSub.sliding(1), + live: yield* PubSub.sliding(liveTailCapacity), + closed: yield* Deferred.make(), + }; + const splitter = makeSplitter(); + let nextGeneration: number | undefined = Option.isSome(existing) + ? (latest?.generation ?? 0) + 1 + : undefined; + let file: + | { + readonly handle: FileSystem.File; + readonly scope: Scope.Closeable; + readonly generation: number; + size: number; + } + | undefined; + /** Chunks not persisted, reported as `lost` by the next successful append. */ + const dropped = new Map>(); + let broken: { readonly attempts: number; readonly retryAt: number } | undefined; + + const closeFile = Effect.suspend(() => { + const current = file; + file = undefined; + return current === undefined ? Effect.void : Scope.close(current.scope, Exit.void); + }); + + const enforceRetention = reader.segments(directory).pipe( + Effect.flatMap((listed) => + Effect.gen(function* () { + let total = listed.reduce((sum, segment) => sum + segment.size, 0); + let count = listed.length; + // The newest closed segment holds the latest records before the open one. + const newestClosed = listed + .map(({ generation }) => generation) + .filter((generation) => generation !== file?.generation) + .at(-1); + for (const segment of listed) { + if (total <= retainBytes && count <= retainSegments) return; + if (segment.generation === file?.generation || segment.generation === newestClosed) + return; + // A reader holding the file on Windows defers its deletion to the next rotation. + const removed = yield* fs + .remove(path.join(directory, segmentName(segment.generation)), { force: true }) + .pipe( + retryShared, + Effect.as(true), + Effect.orElseSucceed(() => false), + ); + if (!removed) continue; + total -= segment.size; + count -= 1; + } + }), + ), + Effect.ignore, + ); + + const openGeneration = Effect.gen(function* () { + yield* closeFile; + yield* fs.makeDirectory(directory, { recursive: true, mode: 0o700 }); + const generation = + nextGeneration ?? + ((yield* reader.generations(directory).pipe(retryShared)).at(-1) ?? 0) + 1; + nextGeneration = generation + 1; + const fileScope = yield* Scope.make("sequential"); + const opened = yield* fs + .open(path.join(directory, segmentName(generation)), { flag: "ax", mode: 0o600 }) + .pipe( + Scope.provide(fileScope), + Effect.onError(() => + Scope.close(fileScope, Exit.void).pipe( + Effect.andThen( + Effect.sync(() => { + nextGeneration = undefined; + }), + ), + ), + ), + ); + file = { handle: opened, scope: fileScope, generation, size: 0 }; + yield* Ref.set(handle.end, { generation, byteOffset: 0 }); + yield* PubSub.publish(handle.wake, undefined); + yield* enforceRetention; + return file; + }).pipe(Effect.mapError(storeError("open"))); + + const writePieces = (pieces: ReadonlyArray) => + Effect.gen(function* () { + const current = file ?? (yield* openGeneration); + const bytes = new Uint8Array(pieces.reduce((sum, piece) => sum + piece.length, 0)); + let offset = 0; + for (const piece of pieces) { + bytes.set(piece, offset); + offset += piece.length; + } + yield* current.handle.writeAll(bytes).pipe( + Effect.mapError(storeError("write")), + Effect.tapError(() => closeFile), + ); + current.size += bytes.length; + yield* Ref.set(handle.end, { generation: current.generation, byteOffset: current.size }); + yield* PubSub.publish(handle.wake, undefined); + }); + + /** Appends records, rotating before a record that would overflow the segment. */ + const append = (entries: ReadonlyArray) => + Effect.gen(function* () { + let pieces: Array = []; + let pending = 0; + for (const entry of entries) { + const bytes = encoder.encode(encodeEntry(entry)); + const used = (file?.size ?? 0) + pending; + if (used > 0 && used + bytes.length > rotateBytes) { + if (pieces.length > 0) yield* writePieces(pieces); + yield* openGeneration; + pieces = []; + pending = 0; + } + pieces.push(bytes); + pending += bytes.length; + } + if (pieces.length > 0) yield* writePieces(pieces); + }); + + const countDropped = (chunks: ReadonlyArray) => { + for (const chunk of chunks) { + const key = `${chunk.launchId}:${chunk.stream}`; + dropped.set(key, { + kind: "lost", + timestamp: chunk.time, + launchId: chunk.launchId, + stream: chunk.stream, + count: (dropped.get(key)?.count ?? 0) + 1, + }); + } + }; + + /** + * Publishes records to live tails, then appends them. After a failure, appends are skipped + * with backoff and their chunks are reported as `lost` once writing works again. + */ + const persist = Effect.fnUntraced(function* ( + entries: ReadonlyArray, + chunks: ReadonlyArray, + ) { + if (entries.length > 0) yield* PubSub.publishAll(handle.live, entries); + const now = yield* Clock.currentTimeMillis; + if (broken !== undefined && now < broken.retryAt) return countDropped(chunks); + if (entries.length === 0 && dropped.size === 0) return; + const written = yield* append([...dropped.values(), ...entries]).pipe(Effect.exit); + if (Exit.isSuccess(written)) { + dropped.clear(); + broken = undefined; + return; + } + countDropped(chunks); + if (broken === undefined) + yield* Effect.logError( + `Unable to persist ${instance.service} logs of instance ${instance.instanceId}; retrying with a new segment`, + written.cause, + ); + const attempts = (broken?.attempts ?? 0) + 1; + broken = { + attempts, + retryAt: now + Math.min(retryBaseMillis * 2 ** (attempts - 1), retryMaxMillis), + }; + }); + + const persistBatch = (chunks: ReadonlyArray) => + Clock.currentTimeMillis.pipe( + Effect.flatMap((now) => + handle.lock.withPermits(1)( + Effect.suspend(() => + persist( + [...chunks.flatMap((chunk) => splitter.push(chunk)), ...splitter.flushEnded(now)], + chunks, + ), + ), + ), + ), + Effect.uninterruptible, + ); + + const flushLaunches = Effect.gen(function* () { + const previous = yield* Ref.make(undefined); + yield* instance.observation.pipe( + Stream.map(({ launchId }) => launchId), + Stream.changes, + Stream.runForEach((launchId) => + Ref.getAndSet(previous, launchId).pipe( + Effect.flatMap((prior) => + prior === undefined + ? Effect.void + : Clock.currentTimeMillis.pipe( + Effect.flatMap((now) => + handle.lock.withPermits(1)( + Effect.suspend(() => persist(splitter.endLaunch(prior, now), [])), + ), + ), + ), + ), + ), + ), + ); + }); + + const subscription = yield* instance.logs.pipe(Scope.provide(scope)); + /** Writes output still queued at close, within a bound; the rest is recorded as lost. */ + const drain = Effect.gen(function* () { + while (true) { + const chunks = yield* PubSub.takeUpTo(subscription, drainBatch); + if (chunks.length === 0) return; + yield* persistBatch(chunks); + } + }).pipe( + Effect.timeoutOption(drainTimeout), + Effect.flatMap( + Option.match({ + onSome: () => Effect.void, + onNone: () => + PubSub.takeUpTo(subscription, Number.MAX_SAFE_INTEGER).pipe( + Effect.flatMap((chunks) => + handle.lock.withPermits(1)( + Effect.suspend(() => { + countDropped(chunks); + broken = undefined; + return persist([], []); + }), + ), + ), + ), + }), + ), + ); + yield* Scope.addFinalizer( + scope, + drain.pipe( + Effect.andThen(Clock.currentTimeMillis), + Effect.flatMap((now) => + handle.lock.withPermits(1)(Effect.suspend(() => persist(splitter.flush(now), []))), + ), + Effect.ignore, + Effect.andThen(closeFile), + ), + ); + // A taken batch is always persisted, so interruption only lands while waiting for output. + yield* Effect.forkIn( + Effect.forever( + Effect.uninterruptibleMask((restore) => + restore(PubSub.takeAll(subscription)).pipe(Effect.flatMap(persistBatch)), + ), + ), + scope, + ); + yield* Effect.forkIn(flushLaunches, scope); + yield* Ref.update(instances, (current) => new Map(current).set(instance.instanceId, handle)); + }); + + const read = Effect.fn("LogStore.read")(function* (instanceId: string, readOptions: ReadOptions) { + const instance = yield* attached(instanceId); + const records = yield* reader.read(instance.directory, readOptions, instance); + return records.pipe(Stream.interruptWhen(Deferred.await(instance.closed))); + }); + + const tail = Effect.fn("LogStore.tail")(function* (instanceId: string) { + const instance = yield* attached(instanceId); + const subscription = yield* PubSub.subscribe(instance.live); + return Stream.fromSubscription(subscription).pipe( + Stream.filterMap((entry) => + entry.kind === "stdout" || entry.kind === "stderr" + ? Result.succeed({ stream: entry.kind, text: entry.text }) + : Result.failVoid, + ), + Stream.interruptWhen(Deferred.await(instance.closed)), + ); + }); + + const remove = Effect.fn("LogStore.remove")(function* (target: { + readonly service: string; + readonly instanceId: string; + }) { + const directory = path.join(options.root, target.service, target.instanceId); + const instance = (yield* Ref.get(instances)).get(target.instanceId); + if (instance !== undefined) { + yield* Ref.update(instances, (current) => { + const next = new Map(current); + next.delete(target.instanceId); + return next; + }); + yield* detach(instance); + } + yield* fs + .remove(directory, { recursive: true, force: true }) + .pipe(retryShared, Effect.mapError(removeError(directory))); + }); + + /** Deletes log directories whose instance is no longer attached, left by a failed removal. */ + const removeOrphans = Effect.fn("LogStore.removeOrphans")(function* () { + const current = yield* Ref.get(instances); + for (const service of yield* listDirectories(fs, path, options.root)) + for (const instanceId of yield* listDirectories(fs, path, path.join(options.root, service))) + if (!current.has(instanceId)) { + const directory = path.join(options.root, service, instanceId); + yield* fs.remove(directory, { recursive: true, force: true }).pipe( + retryShared, + Effect.catch((error) => + Effect.logWarning(`Unable to remove orphaned logs ${directory}`, error), + ), + ); + } + }); + + const close = Effect.gen(function* () { + yield* Ref.set(closed, true); + const current = yield* Ref.getAndSet(instances, new Map()); + yield* Effect.forEach(current.values(), detach, { discard: true }); + }).pipe(Effect.withSpan("LogStore.close")); + + yield* Effect.addFinalizer(() => close); + + return { attach, read, tail, remove, removeOrphans: removeOrphans(), close } satisfies Interface; +}); + +/** Reads persisted records of a stack's instances, merged by time, service, instance and position. */ +export const readStackLogs = Effect.fn("LogStore.readStackLogs")(function* (options: { + readonly root: string; + readonly instances?: ReadonlyArray; + readonly since?: number; + readonly tail?: number; +}) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const reader = makeReader(fs, path); + const list = (directory: string) => listDirectories(fs, path, directory); + const selected: Array<{ readonly service: string; readonly instanceId: string }> = []; + for (const service of yield* list(options.root)) + for (const instanceId of yield* list(path.join(options.root, service))) + if (options.instances === undefined || options.instances.includes(instanceId)) + selected.push({ service, instanceId }); + const perInstance = yield* Effect.forEach( + selected, + ({ service, instanceId }) => + reader + .history( + path.join(options.root, service, instanceId), + { generation: 0, byteOffset: 0 }, + options, + undefined, + ) + .pipe( + Effect.map(({ records }) => + records.map((record): StackLogRecord => ({ ...record, service, instanceId })), + ), + ), + { concurrency: 4 }, + ); + const merged = perInstance.flat().toSorted(compareStackRecords); + return options.tail === undefined + ? merged + : merged.slice(Math.max(0, merged.length - options.tail)); +}); + +/** Orders records by timestamp, service, instance and position. */ +const compareStackRecords = (left: StackLogRecord, right: StackLogRecord) => + compareText(left.timestamp, right.timestamp) || + compareText(left.service, right.service) || + compareText(left.instanceId, right.instanceId) || + compareRecords(left, right); diff --git a/packages/stack/src/index.ts b/packages/stack/src/index.ts index c1df2cbe77..2088cb2154 100644 --- a/packages/stack/src/index.ts +++ b/packages/stack/src/index.ts @@ -24,10 +24,14 @@ export type { DestroyResult, FindOptions, FoundStack, + LogPosition, + LogRecord, OpenOptions, PgProveOptions, PlannedInstance, + ReadStackLogsOptions, SavedStack, + StackLogRecord, StackLocations, SupabaseCompositionOptions, } from "./effect.ts"; @@ -62,3 +66,8 @@ export const discover = ( /** Reads one saved stack by id or by project identity; resolves `undefined` when none is saved. */ export const find = (options: StackEffect.FindOptions, callOptions?: CallOptions) => runOnce(StackEffect.find(options).pipe(Effect.map(Option.getOrUndefined)), callOptions); +/** Reads a stack's persisted log records without its owner. */ +export const readStackLogs = ( + options: StackEffect.ReadStackLogsOptions, + callOptions?: CallOptions, +) => runOnce(StackEffect.readStackLogs(options), callOptions); diff --git a/packages/stack/src/internal/sharing-violation.ts b/packages/stack/src/internal/sharing-violation.ts new file mode 100644 index 0000000000..4eff999c3d --- /dev/null +++ b/packages/stack/src/internal/sharing-violation.ts @@ -0,0 +1,30 @@ +import { Duration, Effect, Predicate, Schedule } from "effect"; + +/** The errno code a platform error carries on its cause, when it has one. */ +export const errorCode = (error: unknown): string | undefined => { + if (!Predicate.hasProperty(error, "cause")) return undefined; + return Predicate.hasProperty(error.cause, "code") && typeof error.cause.code === "string" + ? error.cause.code + : undefined; +}; + +/** Windows reports a file that another process holds or replaces as a transient sharing violation. */ +const isSharingViolation = + (platform: NodeJS.Platform = process.platform) => + (error: unknown): boolean => + platform === "win32" && ["EPERM", "EACCES", "EBUSY"].includes(errorCode(error) ?? ""); + +const sharingViolationSchedule = Schedule.exponential("10 millis", 2).pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.millis(100))), + ), + Schedule.upTo({ times: 12 }), +); + +/** Retries a file operation with bounded backoff while it fails with a sharing violation. */ +export const retrySharingViolation = + (platform?: NodeJS.Platform) => + (effect: Effect.Effect): Effect.Effect => + effect.pipe( + Effect.retry({ schedule: sharingViolationSchedule, while: isSharingViolation(platform) }), + ); diff --git a/packages/stack/src/runtime/Session.ts b/packages/stack/src/runtime/Session.ts index bc27d3bd44..85487fb58f 100644 --- a/packages/stack/src/runtime/Session.ts +++ b/packages/stack/src/runtime/Session.ts @@ -1,4 +1,4 @@ -import { Cause, Effect, Exit, Fiber, PubSub, Ref, Scope, Stream } from "effect"; +import { Cause, Clock, Effect, Exit, Fiber, PubSub, Ref, Scope, Stream } from "effect"; import { failureMessage } from "../internal/failure-message.ts"; import { ServiceError, type RuntimeSession } from "../Service.ts"; import type { ContainerProcess } from "./Container.ts"; @@ -62,16 +62,67 @@ export const processExit = ( ); /** - * Forks stdout/stderr drains that publish to `logs`; returns the stderr fiber so callers can await - * drain completion (e.g. before finalizing exit). Optionally captures a rolling stderr tail for - * folding into exit failure messages. + * One output chunk of a launch, stamped with its publish time in epoch milliseconds. Each process + * of the launch is a `part`; `seq` counts the launch's chunks per stream from zero across its + * parts, so chunks dropped anywhere in the launch leave a gap. + */ +export interface LaunchOutput { + readonly stream: "stdout" | "stderr"; + readonly bytes: Uint8Array; + readonly launchId: number; + readonly part: number; + readonly seq: number; + readonly time: number; +} + +/** Publishes one chunk of a process's output. */ +export type PublishOutput = (stream: "stdout" | "stderr", bytes: Uint8Array) => Effect.Effect; + +/** Tags a launch's output; each `part` call returns the publisher of the launch's next process. */ +export const launchOutputPublisher = ( + logs: PubSub.PubSub, + launchId: number, +): Effect.Effect<{ readonly part: Effect.Effect }> => + Effect.gen(function* () { + const parts = yield* Ref.make(0); + const counters = yield* Ref.make>>({ + stdout: 0, + stderr: 0, + }); + return { + part: Ref.getAndUpdate(parts, (next) => next + 1).pipe( + Effect.map( + (part): PublishOutput => + (stream, bytes) => + Ref.modify(counters, (current) => [ + current[stream], + { ...current, [stream]: current[stream] + 1 }, + ]).pipe( + Effect.flatMap((seq) => + Clock.currentTimeMillis.pipe( + Effect.flatMap((time) => + PubSub.publish(logs, { stream, bytes, launchId, part, seq, time }), + ), + ), + ), + Effect.asVoid, + ), + ), + ), + }; + }); + +/** + * Forks stdout/stderr drains that publish through `publish`; returns the stderr fiber so callers + * can await drain completion (e.g. before finalizing exit). Optionally captures a rolling stderr + * tail for folding into exit failure messages. */ export const publishProcessLogs = ( process: { readonly stdout: Stream.Stream; readonly stderr: Stream.Stream; }, - logs: PubSub.PubSub<{ readonly stream: "stdout" | "stderr"; readonly bytes: Uint8Array }>, + publish: PublishOutput, scope: Scope.Closeable, stderrTail?: Ref.Ref, ): Effect.Effect> => { @@ -85,7 +136,7 @@ export const publishProcessLogs = ( Stream.runForEach((bytes) => Effect.gen(function* () { if (decoder !== undefined) yield* appendTail(decoder.decode(bytes, { stream: true })); - yield* PubSub.publish(logs, { stream: name, bytes }); + yield* publish(name, bytes); }), ), Effect.andThen( diff --git a/packages/stack/src/runtime/Session.unit.test.ts b/packages/stack/src/runtime/Session.unit.test.ts index fcf290afef..c76a853314 100644 --- a/packages/stack/src/runtime/Session.unit.test.ts +++ b/packages/stack/src/runtime/Session.unit.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from "@effect/vitest"; -import { Cause } from "effect"; -import { mapToServiceError } from "./Session.ts"; +import { Cause, Effect, PubSub } from "effect"; +import { launchOutputPublisher, mapToServiceError, type LaunchOutput } from "./Session.ts"; describe("mapToServiceError", () => { it("names an error whose cause carries no message", () => { @@ -16,3 +16,33 @@ describe("mapToServiceError", () => { expect(failure.message).toBe("container is gone"); }); }); + +describe("launchOutputPublisher", () => { + it.effect( + "tags chunks with the launch, a part per process and a launch-wide sequence per stream", + () => + Effect.gen(function* () { + const logs = yield* PubSub.unbounded(); + const subscription = yield* PubSub.subscribe(logs); + const output = yield* launchOutputPublisher(logs, 7); + const startup = yield* output.part; + const main = yield* output.part; + const bytes = new Uint8Array([0x61]); + + yield* startup("stdout", bytes); + yield* startup("stderr", bytes); + yield* startup("stdout", bytes); + yield* main("stdout", bytes); + const published = yield* PubSub.takeAll(subscription); + + expect( + published.map(({ stream, launchId, part, seq }) => ({ stream, launchId, part, seq })), + ).toEqual([ + { stream: "stdout", launchId: 7, part: 0, seq: 0 }, + { stream: "stderr", launchId: 7, part: 0, seq: 0 }, + { stream: "stdout", launchId: 7, part: 0, seq: 1 }, + { stream: "stdout", launchId: 7, part: 1, seq: 2 }, + ]); + }).pipe(Effect.scoped), + ); +}); diff --git a/packages/stack/src/services/Catalog.ts b/packages/stack/src/services/Catalog.ts index 64e7828df5..ce843a8a1d 100644 --- a/packages/stack/src/services/Catalog.ts +++ b/packages/stack/src/services/Catalog.ts @@ -1,4 +1,4 @@ -import { Crypto, Effect, FileSystem, Path, Ref, Schema, Stream } from "effect"; +import { Crypto, Effect, FileSystem, Path, Ref, Schema } from "effect"; import { ChildProcessSpawner } from "effect/unstable/process"; import { HttpClient } from "effect/unstable/http"; import { makeContainerRuntime } from "../runtime/Container.ts"; @@ -289,12 +289,7 @@ const databaseRecipe = ( service: "database", }), ), - logs: component.logs.pipe( - Stream.mapError( - (cause) => - new CatalogError({ operation: "logs", message: cause.message, service: "database", cause }), - ), - ), + logs: component.logs, }); export const makeServiceRecipe = Effect.fn("Catalog.makeServiceRecipe")( @@ -356,73 +351,73 @@ export const makeServiceRecipe = Effect.fn("Catalog.makeServiceRecipe")( case "rest": return catalogRecipe( creation, - yield* makeProcessRecipe(creation, options, deps, Rest.makeSpec()), + yield* makeProcessRecipe(options, deps, Rest.makeSpec()), Schema.is(Rest.Creation), ); case "auth": return catalogRecipe( creation, - yield* makeProcessRecipe(creation, options, deps, Auth.makeSpec()), + yield* makeProcessRecipe(options, deps, Auth.makeSpec()), Schema.is(Auth.Creation), ); case "realtime": return catalogRecipe( creation, - yield* makeProcessRecipe(creation, options, deps, Realtime.makeSpec()), + yield* makeProcessRecipe(options, deps, Realtime.makeSpec()), Schema.is(Realtime.Creation), ); case "storage": return catalogRecipe( creation, - yield* makeProcessRecipe(creation, options, deps, Storage.makeSpec()), + yield* makeProcessRecipe(options, deps, Storage.makeSpec()), Schema.is(Storage.Creation), ); case "imgproxy": return catalogRecipe( creation, - yield* makeProcessRecipe(creation, options, deps, Imgproxy.makeSpec()), + yield* makeProcessRecipe(options, deps, Imgproxy.makeSpec()), Schema.is(Imgproxy.Creation), ); case "functions": return catalogRecipe( creation, - yield* Functions.makeRecipe(creation, options, deps), + yield* Functions.makeRecipe(options, deps), Schema.is(Functions.Creation), ); case "studio": return catalogRecipe( creation, - yield* makeProcessRecipe(creation, options, deps, Studio.makeSpec()), + yield* makeProcessRecipe(options, deps, Studio.makeSpec()), Schema.is(Studio.Creation), ); case "pgmeta": return catalogRecipe( creation, - yield* makeProcessRecipe(creation, options, deps, Pgmeta.makeSpec()), + yield* makeProcessRecipe(options, deps, Pgmeta.makeSpec()), Schema.is(Pgmeta.Creation), ); case "mail": return catalogRecipe( creation, - yield* makeProcessRecipe(creation, options, deps, Mail.makeSpec()), + yield* makeProcessRecipe(options, deps, Mail.makeSpec()), Schema.is(Mail.Creation), ); case "analytics": return catalogRecipe( creation, - yield* makeProcessRecipe(creation, options, deps, Analytics.makeSpec()), + yield* makeProcessRecipe(options, deps, Analytics.makeSpec()), Schema.is(Analytics.Creation), ); case "vector": return catalogRecipe( creation, - yield* Vector.makeRecipe(creation, options, deps), + yield* Vector.makeRecipe(options, deps), Schema.is(Vector.Creation), ); case "pooler": return catalogRecipe( creation, - yield* makeProcessRecipe(creation, options, deps, Pooler.makeSpec()), + yield* makeProcessRecipe(options, deps, Pooler.makeSpec()), Schema.is(Pooler.Creation), ); } diff --git a/packages/stack/src/services/Database.integration.test.ts b/packages/stack/src/services/Database.integration.test.ts index 08afa7397d..0e4d76937c 100644 --- a/packages/stack/src/services/Database.integration.test.ts +++ b/packages/stack/src/services/Database.integration.test.ts @@ -234,7 +234,7 @@ describe("database component", { timeout: 180_000 }, () => { }); const observed = yield* Ref.make(""); const caughtUp = yield* Deferred.make(); - yield* first.logs.pipe( + yield* Stream.fromSubscription(yield* first.logs).pipe( Stream.runForEach(({ bytes }) => Ref.updateAndGet(observed, (text) => text + new TextDecoder().decode(bytes)).pipe( Effect.flatMap((text) => diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index dab2a348f2..c1fc5eab12 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -43,6 +43,7 @@ import { processExit as sharedProcessExit, publishProcessLogs, runtimeSessionFromContainer, + launchOutputPublisher, } from "../runtime/Session.ts"; import { ServiceError, @@ -50,6 +51,7 @@ import { type RuntimeSession, type ServiceDefinition, type ServiceInstanceContext, + type ServiceLaunchContext, } from "../Service.ts"; import { defaultNativeProcessLauncher, @@ -63,7 +65,7 @@ import { resolveNativePostgresUser, type PasswdEntry, } from "../runtime/postgres-user.ts"; -import { EndpointIntent, serviceCreation } from "./Recipe.ts"; +import { EndpointIntent, serviceCreation, type CatalogLog, type CatalogLogs } from "./Recipe.ts"; import { containerInstancePath, ensureOwnedInstanceRoot, @@ -118,11 +120,6 @@ export type BackendEndpoint = | { readonly kind: "unix"; readonly path: string; readonly port: 5432 } | { readonly kind: "tcp"; readonly host: "127.0.0.1"; readonly port: number }; -interface DatabaseLog { - readonly stream: "stdout" | "stderr"; - readonly bytes: Uint8Array; -} - export class DatabaseError extends Data.TaggedError("DatabaseError")<{ readonly operation: string; readonly message: string; @@ -159,7 +156,7 @@ export interface DatabaseComponent { scope: SnapshotScope, ) => Effect.Effect; readonly endpoint: Effect.Effect; - readonly logs: Stream.Stream; + readonly logs: CatalogLogs; } const errorFor = mapToServiceError; @@ -475,7 +472,7 @@ export const makeDatabase = ( const crypto = yield* Crypto.Crypto; const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const client = yield* HttpClient.HttpClient; - const logs = yield* PubSub.sliding(256); + const logs = yield* PubSub.sliding(256); const endpoint = yield* Ref.make(undefined); const prepared = yield* Ref.make>(new Map()); if (!/^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$/u.test(String(options.stackId))) @@ -765,9 +762,10 @@ export const makeDatabase = ( const launch = Effect.fn("Database.launch")( ( - context: ServiceInstanceContext, + context: ServiceLaunchContext, ): Effect.Effect => Effect.gen(function* () { + const publish = yield* (yield* launchOutputPublisher(logs, context.launchId)).part; const postgresUser = yield* resolveNativePostgresUser(options.runtime).pipe( Effect.provideService(FileSystem.FileSystem, fs), ); @@ -858,7 +856,7 @@ export const makeDatabase = ( }; yield* Ref.set(endpoint, selectedEndpoint); const stderrTail = yield* Ref.make(""); - const stderrDrained = yield* publishLogs(process, logs, context.scope, stderrTail); + const stderrDrained = yield* publishLogs(process, publish, context.scope, stderrTail); const setup = health(selectedEndpoint, config, Effect.void, { fs, instanceRoot, @@ -888,7 +886,7 @@ export const makeDatabase = ( return yield* errorFor("launch", "Container did not publish PostgreSQL"); const selectedEndpoint: BackendEndpoint = { kind: "tcp", host: "127.0.0.1", port }; yield* Ref.set(endpoint, selectedEndpoint); - yield* publishLogs(launched, logs, context.scope); + yield* publishLogs(launched, publish, context.scope); const session = runtimeFromContainer(launched, config.stopGraceSeconds === 0); const setup = health( selectedEndpoint, @@ -989,6 +987,6 @@ export const makeDatabase = ( : Effect.succeed(value), ), ), - logs: Stream.fromPubSub(logs), + logs: PubSub.subscribe(logs), } satisfies DatabaseComponent; }); diff --git a/packages/stack/src/services/Functions.integration.test.ts b/packages/stack/src/services/Functions.integration.test.ts index 12a6ac4d60..15c1d34ac7 100644 --- a/packages/stack/src/services/Functions.integration.test.ts +++ b/packages/stack/src/services/Functions.integration.test.ts @@ -192,7 +192,7 @@ describe("service catalog", () => { }, ); const logs = yield* Ref.make(""); - yield* recipe.logs.pipe( + yield* Stream.fromSubscription(yield* recipe.logs).pipe( Stream.runForEach(({ bytes }) => Ref.update(logs, (text) => text + new TextDecoder().decode(bytes)), ), @@ -441,7 +441,7 @@ for (const runtime of ["native", "docker"] as const) { }, ); const logs = yield* Ref.make(""); - yield* recipe.logs.pipe( + yield* Stream.fromSubscription(yield* recipe.logs).pipe( Stream.runForEach(({ bytes }) => Ref.update(logs, (text) => text + new TextDecoder().decode(bytes)), ), diff --git a/packages/stack/src/services/Functions.ts b/packages/stack/src/services/Functions.ts index d3c81eb740..cbc0df1386 100644 --- a/packages/stack/src/services/Functions.ts +++ b/packages/stack/src/services/Functions.ts @@ -233,7 +233,6 @@ const makeSpec = ( export const makeRecipe = Effect.fn("Functions.makeRecipe")( ( - creation: Creation, options: CatalogOptions, deps: ProcessDependencies, ): Effect.Effect, CatalogError> => @@ -269,7 +268,6 @@ export const makeRecipe = Effect.fn("Functions.makeRecipe")( ), ); return yield* makeProcessRecipe( - creation, options, deps, makeSpec(functionsRoot, bootstrap, deps.path, deps.fs), diff --git a/packages/stack/src/services/ProcessRecipe.integration.test.ts b/packages/stack/src/services/ProcessRecipe.integration.test.ts index 9f14cc70ce..6100005b6f 100644 --- a/packages/stack/src/services/ProcessRecipe.integration.test.ts +++ b/packages/stack/src/services/ProcessRecipe.integration.test.ts @@ -189,7 +189,7 @@ describe("ProcessRecipe launch cleanup", () => { spawner: yield* ChildProcessSpawner.ChildProcessSpawner, container, } satisfies ProcessDependencies; - const recipe = yield* makeProcessRecipe(creation, options, dependencies, spec); + const recipe = yield* makeProcessRecipe(options, dependencies, spec); const service = yield* makeService(recipe.definition, { id: `rest-process-recipe-${scenario}`, config: creation, @@ -276,7 +276,7 @@ describe("ProcessRecipe launch cleanup", () => { spawner, container: undefined, } satisfies ProcessDependencies; - const recipe = yield* makeProcessRecipe(creation, nativeOptions, dependencies, nativeSpec); + const recipe = yield* makeProcessRecipe(nativeOptions, dependencies, nativeSpec); const service = yield* makeService(recipe.definition, { id: "rest-process-recipe-native", config: creation, @@ -386,7 +386,7 @@ describe("ProcessRecipe launch cleanup", () => { spawner, container: undefined, } satisfies ProcessDependencies; - const recipe = yield* makeProcessRecipe(creation, nativeOptions, dependencies, nativeSpec); + const recipe = yield* makeProcessRecipe(nativeOptions, dependencies, nativeSpec); const service = yield* makeService(recipe.definition, { id: "rest-process-recipe-port-order", config: creation, @@ -443,7 +443,6 @@ const realtimeService = Effect.fn(function* (container: ContainerRuntime) { config: { databaseUrl: "postgresql://postgres:postgres@host.docker.internal:54322/postgres" }, }; const recipe = yield* makeProcessRecipe( - creation, { stackId: "process-recipe-test", instanceId: "instance", @@ -647,7 +646,6 @@ const nativeRestRecipe = Effect.fn(function* ( const cacheRoot = path.join(root, "cache"); yield* nativeRestArtifact(cacheRoot, program); return yield* makeProcessRecipe( - creation, { ...options, root, @@ -915,7 +913,6 @@ describe("process recipe startup", () => { ), }; const recipe = yield* makeProcessRecipe( - creation, { stackId: "process-recipe-port-race", instanceId: "instance", @@ -931,6 +928,7 @@ describe("process recipe startup", () => { const scope = yield* Scope.fork(testScope, "sequential"); const runtime = yield* recipe.definition.launch({ id: "pooler", + launchId: 1, config: creation, scope, }); @@ -982,7 +980,6 @@ describe("process recipe startup", () => { }, }; const recipe = yield* makeProcessRecipe( - creation, { stackId: "process-recipe-port-exhaustion", instanceId: "instance", @@ -1116,7 +1113,6 @@ describe("process recipe startup", () => { }, }; const recipe = yield* makeProcessRecipe( - creation, { stackId: "process-recipe-deadline", instanceId: "instance", @@ -1130,7 +1126,12 @@ describe("process recipe startup", () => { ); if (recipe.definition.prepare !== undefined) yield* recipe.definition.prepare(creation); const scope = yield* Scope.fork(yield* Effect.scope, "sequential"); - const runtime = yield* recipe.definition.launch({ id: "pooler", config: creation, scope }); + const runtime = yield* recipe.definition.launch({ + id: "pooler", + launchId: 1, + config: creation, + scope, + }); const health = yield* Effect.flip(runtime.health).pipe(Effect.forkChild); yield* Deferred.await(clockAdvanced); const failure = yield* Fiber.join(health); @@ -1171,7 +1172,6 @@ describe("process recipe startup", () => { }, }; const recipe = yield* makeProcessRecipe( - creation, { stackId: "process-recipe-unrelated-failure", instanceId: "instance", @@ -1192,7 +1192,12 @@ describe("process recipe startup", () => { ); if (recipe.definition.prepare !== undefined) yield* recipe.definition.prepare(creation); const scope = yield* Scope.fork(yield* Effect.scope, "sequential"); - const runtime = yield* recipe.definition.launch({ id: "pooler", config: creation, scope }); + const runtime = yield* recipe.definition.launch({ + id: "pooler", + launchId: 1, + config: creation, + scope, + }); const health = yield* Effect.exit(runtime.health); expect(yield* Ref.get(mainLaunches)).toBe(1); expect(yield* Ref.get(startupLaunches)).toEqual(["prepare", "provision-tenant"]); @@ -1275,7 +1280,6 @@ describe("process recipe startup", () => { }, }; const recipe = yield* makeProcessRecipe( - creation, { stackId: "process-recipe-exit-failure", instanceId: "instance", @@ -1289,7 +1293,12 @@ describe("process recipe startup", () => { ); if (recipe.definition.prepare !== undefined) yield* recipe.definition.prepare(creation); const scope = yield* Scope.fork(yield* Effect.scope, "sequential"); - const runtime = yield* recipe.definition.launch({ id: "pooler", config: creation, scope }); + const runtime = yield* recipe.definition.launch({ + id: "pooler", + launchId: 1, + config: creation, + scope, + }); const health = yield* Effect.exit(runtime.health); expect(Exit.isFailure(health)).toBe(true); const exit = yield* runtime.exit; diff --git a/packages/stack/src/services/ProcessRecipe.ts b/packages/stack/src/services/ProcessRecipe.ts index 73a81799b9..8761929c6e 100644 --- a/packages/stack/src/services/ProcessRecipe.ts +++ b/packages/stack/src/services/ProcessRecipe.ts @@ -35,12 +35,19 @@ import { spawnNativeProcess, } from "../runtime/NativeProcess.ts"; import { + launchOutputPublisher, mapToServiceError, processExit as sharedProcessExit, publishProcessLogs, runtimeSessionFromContainer, + type PublishOutput, } from "../runtime/Session.ts"; -import { ServiceError, ServiceLaunchError, type RuntimeSession } from "../Service.ts"; +import { + ServiceError, + ServiceLaunchError, + type RuntimeSession, + type ServiceLaunchContext, +} from "../Service.ts"; import { containerInstancePath, ensureOwnedInstanceRoot, @@ -263,14 +270,14 @@ const awaitStartup = Effect.fn("ProcessRecipe.awaitStartup")( readonly stderr: Stream.Stream; readonly exitCode: Effect.Effect; }, - logs: PubSub.PubSub, + publish: PublishOutput, ): Effect.Effect< Readonly<{ readonly code: number; readonly output: StartupOutput }>, ServiceError > => awaitCommandOutput(process, { timeout: Duration.seconds(startupTimeoutSeconds), - onOutput: (stream, bytes) => PubSub.publish(logs, { stream, bytes }), + onOutput: publish, }).pipe( Effect.mapError((cause) => serviceError("launch", cause)), Effect.flatMap((result) => @@ -313,7 +320,7 @@ const anotherListenerHolds = (endpoints: ReadonlyMap) = const collectNativeOutput = Effect.fn("ProcessRecipe.collectNativeOutput")(function* ( process: NativeProcess, - logs: PubSub.PubSub, + publish: PublishOutput, endpoints: ReadonlyMap, readinessOutput: | ((line: string, endpoints: ReadonlyMap) => boolean) @@ -348,7 +355,7 @@ const collectNativeOutput = Effect.fn("ProcessRecipe.collectNativeOutput")(funct { discard: true }, ); yield* stream.pipe( - Stream.tap((bytes) => PubSub.publish(logs, { stream: name, bytes })), + Stream.tap((bytes) => publish(name, bytes)), Stream.decodeText, Stream.runForEach((text) => Ref.modify( @@ -408,7 +415,6 @@ const readiness = Effect.fn("ProcessRecipe.readiness")( ); export const makeProcessRecipe = >( - creation: C, options: CatalogOptions, deps: ProcessDependencies, spec: ProcessRecipeSpec, @@ -464,11 +470,8 @@ export const makeProcessRecipe = } }); - const launch = Effect.fn("ProcessRecipe.launch")(function* (context: { - readonly id: string; - readonly config: C; - readonly scope: Scope.Closeable; - }) { + const launch = Effect.fn("ProcessRecipe.launch")(function* (context: ServiceLaunchContext) { + const output = yield* launchOutputPublisher(logs, context.launchId); const portNames = Object.entries(spec.ports).filter( ([name]) => spec.enabledPort === undefined || spec.enabledPort(context.config, name), ); @@ -543,7 +546,11 @@ export const makeProcessRecipe = Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, deps.spawner), Effect.mapError((cause) => serviceError("launch", cause)), ); - const result = yield* awaitStartup(context.config.service, startupProcess, logs).pipe( + const result = yield* awaitStartup( + context.config.service, + startupProcess, + yield* output.part, + ).pipe( Effect.mapError( (failure) => new ServiceLaunchError({ failure, runtime: runtimeFromNative(startupProcess) }), @@ -588,15 +595,15 @@ export const makeProcessRecipe = Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, deps.spawner), Effect.mapError((cause) => serviceError("launch", cause)), ); - const output = yield* collectNativeOutput( + const collected = yield* collectNativeOutput( native, - logs, + yield* output.part, selected, spec.nativeReadinessOutput, scope, ); yield* Ref.set(endpoints, selected); - return { native, output, selected, scope }; + return { native, output: collected, selected, scope }; }); type NativeAttempt = Effect.Success>; @@ -777,7 +784,11 @@ export const makeProcessRecipe = ), Scope.provide(context.scope), ); - const result = yield* awaitStartup(context.config.service, startupProcess, logs).pipe( + const result = yield* awaitStartup( + context.config.service, + startupProcess, + yield* output.part, + ).pipe( Effect.mapError( (failure) => new ServiceLaunchError({ @@ -844,7 +855,7 @@ export const makeProcessRecipe = selected.set(name, { kind: "tcp", host: "127.0.0.1", port: published }); } yield* Ref.set(endpoints, selected); - yield* publishProcessLogs(launched, logs, context.scope); + yield* publishProcessLogs(launched, yield* output.part, context.scope); const runtime = runtimeFromContainer(launched); const ready = selected.get("http"); const noReadinessEndpoint = Effect.fail( @@ -879,8 +890,6 @@ export const makeProcessRecipe = ).pipe(Effect.andThen(Ref.set(endpoints, new Map()))), }, endpoints, - logs: Stream.fromPubSub(logs).pipe( - Stream.mapError((cause) => catalogError("logs", String(cause), creation.service)), - ), + logs: PubSub.subscribe(logs), }; }); diff --git a/packages/stack/src/services/Recipe.ts b/packages/stack/src/services/Recipe.ts index 940e1f5f38..4e0c0224df 100644 --- a/packages/stack/src/services/Recipe.ts +++ b/packages/stack/src/services/Recipe.ts @@ -1,7 +1,7 @@ import { Data, Schema } from "effect"; -import type { Stream } from "effect"; -import type { Effect, Ref } from "effect"; +import type { Effect, PubSub, Ref, Scope } from "effect"; import type { ServiceKind } from "../Artifacts.ts"; +import type { LaunchOutput } from "../runtime/Session.ts"; import type { ServiceDefinition } from "../Service.ts"; import type { HostGateway } from "../runtime/Container.ts"; import type { DockerHelperRegistry } from "../storage/DockerHelperRegistry.ts"; @@ -58,16 +58,16 @@ export interface CatalogOptions { readonly hostGateway?: HostGateway; } -export interface CatalogLog { - readonly stream: "stdout" | "stderr"; - readonly bytes: Uint8Array; -} +export type CatalogLog = LaunchOutput; + +/** Subscribes to a recipe's launch output; chunks published before the subscription are missed. */ +export type CatalogLogs = Effect.Effect, never, Scope.Scope>; export interface CatalogRecipe { readonly creation: C; readonly definition: ServiceDefinition; readonly endpoint: (name: string) => Effect.Effect; - readonly logs: Stream.Stream; + readonly logs: CatalogLogs; } export interface RecipeCreation { @@ -80,5 +80,5 @@ export interface RecipeCreation { export interface ProcessRecipeResult { readonly definition: ServiceDefinition; readonly endpoints: Ref.Ref>; - readonly logs: Stream.Stream; + readonly logs: CatalogLogs; } diff --git a/packages/stack/src/services/Vector.ts b/packages/stack/src/services/Vector.ts index ec830a7c99..4f0087cc7e 100644 --- a/packages/stack/src/services/Vector.ts +++ b/packages/stack/src/services/Vector.ts @@ -230,9 +230,8 @@ const makeSpec = ( }; export const makeRecipe = Effect.fn("Vector.makeRecipe")( - (creation: Creation, options: CatalogOptions, deps: ProcessDependencies) => + (options: CatalogOptions, deps: ProcessDependencies) => makeProcessRecipe( - creation, options, deps, makeSpec( From 43bc0b39e91734591c9ef002ed162f3658784e04 Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 30 Sep 2026 15:06:42 +0200 Subject: [PATCH 02/35] feat(stack): ship persisted service logs to Analytics Studio's Logs pages stayed empty because nothing forwarded the stack's service output to Analytics. The owner now ships persisted Auth, REST, Realtime, Storage, Functions and database lines to their Logflare sources, using the legacy Vector remaps ported to TypeScript. It posts to Analytics' own backend only while the composed Analytics instance is running and healthy, so shipping never wakes it or keeps it awake. Each instance keeps a cursor beside its log files: lines written while Analytics sleeps are shipped after it wakes with their original timestamps. Events carry deterministic ids and Logflare de-duplicates on them, so failed posts are retried without duplicates. Auth and target errors pause shipping and keep the cursor; malformed bodies are skipped. The CLI no longer composes Vector. Co-Authored-By: Claude Opus 5.5 --- apps/cli/src/command-internal/stack-config.ts | 20 - .../experimental/stack/logs/SIDE_EFFECTS.md | 4 +- .../stack/prepare/SIDE_EFFECTS.md | 2 +- ...ack-config-environment.integration.test.ts | 4 - .../stack/stack-config.integration.test.ts | 2 +- .../stack-forwarding.integration.test.ts | 3 - .../experimental/stack/start/SIDE_EFFECTS.md | 28 +- .../stack/start/start.integration.test.ts | 4 +- packages/stack/ARCHITECTURE.md | 11 + .../src/Owner.analytics.integration.test.ts | 228 ++++++++++ packages/stack/src/Owner.ts | 29 +- packages/stack/src/State.ts | 89 ++-- .../src/host/LogForwarder.integration.test.ts | 392 +++++++++++++++++ packages/stack/src/host/LogForwarder.ts | 404 ++++++++++++++++++ .../stack/src/host/LogForwarder.unit.test.ts | 36 ++ packages/stack/src/host/LogStore.ts | 28 +- packages/stack/src/host/LogflareEvents.ts | 153 +++++++ .../src/host/LogflareEvents.unit.test.ts | 96 +++++ 18 files changed, 1446 insertions(+), 87 deletions(-) create mode 100644 packages/stack/src/Owner.analytics.integration.test.ts create mode 100644 packages/stack/src/host/LogForwarder.integration.test.ts create mode 100644 packages/stack/src/host/LogForwarder.ts create mode 100644 packages/stack/src/host/LogForwarder.unit.test.ts create mode 100644 packages/stack/src/host/LogflareEvents.ts create mode 100644 packages/stack/src/host/LogflareEvents.unit.test.ts diff --git a/apps/cli/src/command-internal/stack-config.ts b/apps/cli/src/command-internal/stack-config.ts index c9a41e75f1..d66ff10680 100644 --- a/apps/cli/src/command-internal/stack-config.ts +++ b/apps/cli/src/command-internal/stack-config.ts @@ -1250,26 +1250,6 @@ export const loadStackConfig = Effect.fn("StackConfig.load")( } satisfies ServiceCreationType, ] : []), - ...(validatedConfig.analytics.enabled - ? [ - { - service: "vector" as const, - config: { apiKey: "api-key" }, - endpoints: { - http: endpoint( - envPortOrConfigured( - "SUPABASE_ANALYTICS_VECTOR_PORT", - document, - "analytics", - "vector_port", - validatedConfig.analytics.vector_port ?? 0, - context.projectEnvValues, - ), - ), - }, - } satisfies ServiceCreationType, - ] - : []), ...(validatedConfig.storage.image_transformation?.enabled === true ? [ { diff --git a/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md index 2a9a9e92d3..2118aca19f 100644 --- a/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md @@ -29,7 +29,9 @@ with `timestamp`, `service`, `instance_id`, `stream`, `line`, and `source: "live Lines preserve their content in machine output. UTF-8 and line fragments are assembled separately for each instance and stdout/stderr channel. Timestamps reflect receipt by the CLI. Delivery is best effort: slow subscribers can lose entries. Ordering across -stdout/stderr channels and different services is not guaranteed. +stdout/stderr channels and different services is not guaranteed. The owner's +log shipping to Analytics reads the persisted logs separately and does not +affect this command. Interrupting the command cancels its subscriptions, exits with status 130, and leaves the owner and services running. Successful stream completion exits 0; diff --git a/apps/cli/src/commands/experimental/stack/prepare/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/prepare/SIDE_EFFECTS.md index 4d51539261..d6021f9344 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/prepare/SIDE_EFFECTS.md @@ -39,7 +39,7 @@ which selects the same runtime as `stack start` (Docker, then Podman, then nativ existing stacks reuse their persisted runtime without probing, and an explicit mismatch fails. With no `--capability`, every enabled creation from the effective project configuration is prepared. Repeated `--capability` includes each capability's configured companion services (Storage/Imgproxy, -Studio/Pgmeta, Analytics/Vector); requesting a disabled service fails before instance preparation. The legacy `-o/--output` flag is rejected; use +Studio/Pgmeta); requesting a disabled service fails before instance preparation. The legacy `-o/--output` flag is rejected; use `--output-format`. Text output lists the stack ID and each prepared service and version. JSON output returns the stack diff --git a/apps/cli/src/commands/experimental/stack/stack-config-environment.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack-config-environment.integration.test.ts index 45a41ae98b..d80244649f 100644 --- a/apps/cli/src/commands/experimental/stack/stack-config-environment.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stack-config-environment.integration.test.ts @@ -79,7 +79,6 @@ site_url = "from-config" SUPABASE_API_PORT: "54321", SUPABASE_DB_PORT: "54322", SUPABASE_ANALYTICS_PORT: "55555", - SUPABASE_ANALYTICS_VECTOR_PORT: "55556", }, load(root), ); @@ -87,9 +86,6 @@ site_url = "from-config" expect(service(recipes, "database")?.endpoints).toEqual({ sql: { port: 54322 } }); expect(service(recipes, "rest")?.endpoints).toEqual({ http: { port: 54321 } }); expect(service(recipes, "analytics")?.endpoints).toEqual({ http: { port: 55555 } }); - - expect(service(recipes, "vector")?.endpoints).toEqual({ http: { port: 55556 } }); - const defaults = yield* load(root); const defaultRecipes = yield* defaults.creations("automatic-ports"); expect(service(defaultRecipes, "analytics")?.endpoints).toEqual({ diff --git a/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts index e32f41ad53..95cd2db8a2 100644 --- a/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stack-config.integration.test.ts @@ -75,7 +75,7 @@ enabled = true expect(recipes.get("analytics")?.endpoints).toEqual({ http: { port: "auto" } }); const storage = recipes.get("storage"); expect(storage?.service === "storage" && storage.config.fileSizeLimit).toBe("52428800"); - expect(recipes.has("vector")).toBe(true); + expect(recipes.has("vector")).toBe(false); expect(recipes.has("functions")).toBe(true); }).pipe(Effect.provide(BunServices.layer)), ); diff --git a/apps/cli/src/commands/experimental/stack/stack-forwarding.integration.test.ts b/apps/cli/src/commands/experimental/stack/stack-forwarding.integration.test.ts index 79ad9f79c4..d48745aeff 100644 --- a/apps/cli/src/commands/experimental/stack/stack-forwarding.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/stack-forwarding.integration.test.ts @@ -41,8 +41,6 @@ enabled = false enabled = false max_buckets = 3 max_indexes = 2 -[analytics] -vector_port = 59001 `); const loaded = yield* loadStackConfig(root); const services = yield* loaded.creations("forwarding"); @@ -60,7 +58,6 @@ vector_port = 59001 expect( realtime?.service === "realtime" ? realtime.config.secretKeyBase : undefined, ).toBeUndefined(); - expect(services.find((s) => s.service === "vector")?.endpoints?.http?.port).toBe(59001); // A composition binds each database URL before launch. const bound = (creation: C) => ({ ...creation, diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index 5251441504..f0a468dfb3 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -20,9 +20,10 @@ If the stack is otherwise in a partial lifecycle state, start fails with guidanc and start it again before applying configuration. Auth policies, OAuth providers, hooks, MFA, SMTP, email subjects and notification controls are forwarded to Auth. REST search paths, pooler limits, Realtime settings, Studio settings, Storage -S3 protocol/vector controls, and configured Vector ports are forwarded to their services. Storage -receives the local S3 access keys and region, and uses the gateway's `/storage/v1` prefix to verify -S3 signatures and to build resumable upload URLs. +S3 protocol/vector controls are forwarded to their services. Storage receives the local S3 access +keys and region, and uses the gateway's `/storage/v1` prefix to verify S3 signatures and to build +resumable upload URLs. `analytics.vector_port` and `SUPABASE_ANALYTICS_VECTOR_PORT` are accepted +but unused: the stack composes no Vector service. Encrypted JWT secrets are decrypted before shared credentials are derived. `db.health_timeout` controls database readiness; package JWT and PostgreSQL root-key defaults apply when omitted, and the effective root key is supplied through a stack-owned key file. @@ -78,10 +79,25 @@ it again to apply those changes. `--exclude` accepts repeated or comma-separated capability names: `rest`, `auth`, `realtime`, `storage`, `functions`, `studio`, `mail`, `analytics`, and `pooler`. Database cannot be excluded. -Storage includes its Imgproxy companion, Studio includes Pgmeta, and Analytics includes Vector. +Storage includes its Imgproxy companion and Studio includes Pgmeta. Studio requires REST; excluding REST while keeping Studio fails before stopping the composition. -Vector runs a stack-owned default configuration that enables its health API and forwards no service -logs; log collection into Analytics is not implemented yet. + +## Service logs in Analytics + +The owner ships the persisted Auth, REST, Realtime, Storage, Functions, and database output lines +(not launch or lost markers) to Analytics' `POST /api/logs` ingest endpoint on its direct backend, +using the Analytics API key and the legacy Logflare source names (`gotrue.logs.prod`, +`postgREST.logs.prod`, `realtime.logs.prod`, `storage.logs.prod.2`, `deno-relay-logs`, +`postgres.logs`) with the legacy per-service field remaps. This applies to the Docker, Podman, and +native runtimes. Shipping runs only while the composed Analytics service is running and healthy; +each instance keeps its position in `logs///cursor.json`, so lines written +while Analytics is stopped, starting, or unhealthy are shipped with their original timestamps once +it is healthy again, including after an owner restart. A missing or unreadable position starts from +the oldest retained line. Lines already deleted by log retention are skipped, and Analytics refusing +the API key pauses shipping until Analytics restarts or the composition changes. Each event carries an id derived from its instance and position, so a request that fails +after it may have been ingested is repeated without duplicating rows. Shipping never wakes Analytics +and does not count as idle activity, so a lazy Analytics still stops on its idle timer while other +services log. Service log streams and `supabase stack logs` are never blocked by shipping. After an explicit stop, start compares the project configuration with the saved composition through the stack package's composition plan, ignoring values the composition and stack credentials supply. diff --git a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts index c1267d0a4f..34b2620044 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts @@ -745,7 +745,7 @@ describe("experimental stack start", () => { message: "Composition start had failures", outcomes: [ { id: "database-member-1", succeeded: true }, - { id: "vector-member-1", succeeded: false, error: "Service health timed out" }, + { id: "analytics-member-1", succeeded: false, error: "Service health timed out" }, ], }), ), @@ -756,7 +756,7 @@ describe("experimental stack start", () => { expect(error).toBeInstanceOf(StackCommandStartError); expect(error).toMatchObject({ message: "Composition start had failures", - detail: "vector (vector-member-1): Service health timed out", + detail: "analytics (analytics-member-1): Service health timed out", }); }).pipe(Effect.provide(BunServices.layer)), ); diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index df434a462f..6701fc1482 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -659,6 +659,17 @@ position. Destroying an instance deletes its segments, and an owner start remove instances no longer saved, so a failed deletion is retried; destroying the stack removes `logs/`; resetting database data keeps them. +While the composed Analytics instance is running and healthy, the owner ships the persisted +stdout/stderr records of Auth, REST, Realtime, Storage, Functions and database instances to its +direct backend (never the proxy, so shipping neither wakes it nor counts as activity), in bodies of +at most 256 events and 1 MiB. Each instance reads from `cursor.json` in its logs directory, the +position of its last shipped record, written atomically after each body settles: a missing cursor +or unreadable cursor starts at the oldest retained segment, and a cursor in a deleted segment +resumes at the oldest retained one. Bodies of an instance are sequential. Event ids +derive from the instance and record position, and Logflare keeps the first row per id, so a failed +body is posted again until it settles or the target changes. A 401, 403 or 404 response pauses the +instance with its cursor until the target changes; any other 4xx except 408 and 429 skips the body. The target is re-selected when the composition or Analytics' health changes. + Registry updates use an OS-backed lock through a private `node:sqlite` connection to `/.registry-lock.sqlite`. Each `withLock` call opens its own connection, disables SQLite busy waiting, and retries `BEGIN IMMEDIATE` contention for up to five seconds through diff --git a/packages/stack/src/Owner.analytics.integration.test.ts b/packages/stack/src/Owner.analytics.integration.test.ts new file mode 100644 index 0000000000..c9dbdc6a6a --- /dev/null +++ b/packages/stack/src/Owner.analytics.integration.test.ts @@ -0,0 +1,228 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { PgClient } from "@effect/sql-pg"; +import { expect, it } from "@effect/vitest"; +import { + Context, + Crypto, + DateTime, + Effect, + Fiber, + Layer, + Option, + Path, + Redacted, + Schedule, + Stream, +} from "effect"; +import { HttpClient } from "effect/unstable/http"; +import { tmpdir } from "node:os"; +import * as State from "./State.ts"; +import type { SavedStack } from "./State.ts"; +import type { ServiceCreationInput } from "./services/Catalog.ts"; +import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; +import { ownerFor } from "../tests/owner-rpc.ts"; + +const cacheRoot = `${tmpdir()}/supabase-stack-artifacts`; + +const query = (url: string, statement: string) => + Effect.scoped( + Effect.gen(function* () { + const parsed = new URL(url); + const services = yield* Layer.build( + PgClient.layer({ + host: parsed.hostname, + port: Number(parsed.port), + database: parsed.pathname.slice(1), + username: decodeURIComponent(parsed.username), + password: Redacted.make(decodeURIComponent(parsed.password)), + }), + ); + return yield* Context.get(services, PgClient.PgClient).unsafe(statement); + }), + ); + +interface StoredEvent { + readonly message: string; + readonly severity: string | null; + readonly host: string | null; + readonly timestamp: string; +} + +it.live( + "ships persisted service logs to Analytics, catching up after it wakes from idle", + () => + Effect.scoped( + Effect.gen(function* () { + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const client = yield* HttpClient.HttpClient; + const stackId = `owner-logs-${(yield* crypto.randomUUIDv4).slice(0, 8)}`; + const root = yield* makeDockerDatabaseRoot("stack-owner-logs-", stackId); + const saved: SavedStack = { + id: stackId, + identity: { + projectRoot: "/tmp/project", + branchContext: "owner-logs", + stackName: stackId, + }, + runtime: "docker", + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + ports: [], + }; + const state = Context.get( + yield* Layer.build(State.layer({ root: path.dirname(path.dirname(root)) })), + State.Service, + ); + yield* state.save(saved); + const owner = yield* ownerFor({ saved, state, root, cacheRoot }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + + const services: ReadonlyArray = [ + { + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("owner-logs-password"), + jwtSecret: Redacted.make("owner-logs-jwt-secret-with-at-least-32-chars"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }, + { service: "rest", config: {}, endpoints: { http: { port: "auto" } } }, + { + service: "analytics", + config: { backend: "postgres", apiKey: "owner-logs-key" }, + endpoints: { http: { port: "auto" } }, + }, + ]; + const composed = yield* owner.rpc.supabaseComposition({ services }); + const idOf = (service: string) => { + const id = composed.find((entry) => entry.creation.service === service)?.id; + return id === undefined ? Effect.die(`${service} was not composed`) : Effect.succeed(id); + }; + const [databaseId, restId, analyticsId] = yield* Effect.all([ + idOf("database"), + idOf("rest"), + idOf("analytics"), + ]); + // A short idle timer lets the test observe Analytics sleeping while services keep logging. + const composition = (yield* state.read(stackId))?.composition; + if (composition === undefined) return yield* Effect.die("composition was not saved"); + yield* owner.rpc.configureComposition({ + ...composition, + members: composition.members.map((member) => + member.id === analyticsId ? { ...member, idleMillis: 3_000 } : member, + ), + }); + yield* owner.rpc.startComposition(); + + const database = yield* owner.rpc.credentials({ id: databaseId, from: "host" }); + const rest = yield* owner.rpc.credentials({ id: restId, from: "host" }); + const analytics = yield* owner.rpc.credentials({ id: analyticsId, from: "host" }); + const { databaseUrl, internalDatabaseUrl } = database; + if (databaseUrl === undefined || internalDatabaseUrl === undefined) + return yield* Effect.die("database credentials are missing"); + if (rest.url === undefined || analytics.url === undefined) + return yield* Effect.die("service URLs are missing"); + + const awaitAnalytics = ( + predicate: (observation: { + lifecycle: string; + health?: string; + wakeEnabled: boolean; + }) => boolean, + ) => + owner.rpc + .followStatus({ id: analyticsId }) + .pipe(Stream.filter(predicate), Stream.take(1), Stream.runDrain); + const wakeAnalytics = client + .get(`${analytics.url}/health`) + .pipe( + Effect.andThen( + awaitAnalytics( + ({ lifecycle, health }) => lifecycle === "running" && health === "healthy", + ), + ), + ); + // Proxy traffic is the only idle activity; this stands in for Studio while events must persist. + const keepAwake = client + .get(`${analytics.url}/health`) + .pipe(Effect.ignore, Effect.repeat(Schedule.spaced("1 second"))); + const raise = (marker: string) => + query(databaseUrl, `DO $$ BEGIN RAISE LOG '${marker}'; END $$`); + const stored = (source: string, filter: string) => + Effect.gen(function* () { + const [row] = yield* query<{ readonly token: string }>( + internalDatabaseUrl, + `SELECT replace(token::text, '-', '_') AS token FROM _analytics.sources WHERE name = '${source}'`, + ); + if (row === undefined) return []; + return yield* query( + internalDatabaseUrl, + `SELECT body->>'event_message' AS message, body->'metadata'->'parsed'->>'error_severity' AS severity, body->'metadata'->>'host' AS host, to_char(timestamp, 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') AS timestamp FROM _analytics."log_events_${row.token}" WHERE ${filter}`, + ); + }); + // Logflare ingests asynchronously and exposes no completion signal; polling is the guard. + const awaitStored = (source: string, filter: string) => + stored(source, filter).pipe( + Effect.filterOrFail((rows) => rows.length > 0), + Effect.retry(Schedule.spaced("500 millis")), + Effect.timeout("120 seconds"), + ); + + yield* wakeAnalytics; + const keeper = yield* Effect.forkScoped(keepAwake); + const awakeMarker = `${stackId}-awake`; + yield* raise(awakeMarker); + expect( + yield* awaitStored( + "postgres.logs", + `body->>'event_message' LIKE '%LOG: ${awakeMarker}'`, + ), + ).toEqual([ + { + message: expect.stringContaining(`LOG: ${awakeMarker}`), + severity: "LOG", + host: "db-default", + timestamp: expect.any(String), + }, + ]); + yield* client.get(`${rest.url}/`); + const restRows = yield* awaitStored( + "postgREST.logs.prod", + "body->'metadata'->>'host' = 'default'", + ); + expect(restRows.length).toBeGreaterThan(0); + + yield* Fiber.interrupt(keeper); + const noise = yield* Effect.forkScoped( + raise(`${stackId}-noise`).pipe(Effect.repeat(Schedule.spaced("250 millis"))), + ); + yield* awaitAnalytics( + ({ lifecycle, wakeEnabled }) => lifecycle === "stopped" && wakeEnabled, + ).pipe(Effect.timeout("90 seconds")); + yield* Fiber.interrupt(noise); + const asleepMarker = `${stackId}-asleep`; + const persistedAsleep = yield* owner.rpc.readLogs({ id: databaseId, follow: true }).pipe( + Stream.filter((record) => record.text?.includes(`LOG: ${asleepMarker}`) === true), + Stream.runHead, + Effect.forkScoped, + ); + yield* raise(asleepMarker); + const asleepRecord = Option.getOrUndefined(yield* Fiber.join(persistedAsleep)); + const wokeAt = DateTime.formatIso(yield* DateTime.now); + + yield* wakeAnalytics; + yield* Effect.forkScoped(keepAwake); + const [shippedAsleep] = yield* awaitStored( + "postgres.logs", + `body->>'event_message' LIKE '%LOG: ${asleepMarker}'`, + ); + expect(shippedAsleep?.timestamp).toBe(asleepRecord?.timestamp); + expect((shippedAsleep?.timestamp ?? "") < wokeAt).toBe(true); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 600_000 }, +); diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index 9e42fdecd8..0ee7be5178 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -67,6 +67,7 @@ import { stackError, type OwnerRpc } from "./Rpc.ts"; import * as State from "./State.ts"; import type { SavedStack, StackCredentials, StackKeysInput } from "./State.ts"; import { makeDockerHelperRegistry } from "./storage/DockerHelperRegistry.ts"; +import * as LogForwarder from "./host/LogForwarder.ts"; import * as LogStore from "./host/LogStore.ts"; export interface OwnerOptions { @@ -190,6 +191,10 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { const logStore = yield* LogStore.make({ root: options.state.logsRoot(options.saved.id) }).pipe( Effect.provideContext(services), ); + const forwarder = yield* LogForwarder.make({ + composition: orchestrator.composition, + logs: logStore, + }).pipe(Effect.provideContext(services)); const definitionGate = yield* Semaphore.make(1); const draining = yield* Ref.make(false); const { id: stackId, runtime } = options.saved; @@ -440,6 +445,13 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { logs: recipe.logs, observation: core.observation, }); + yield* forwarder.attach({ + id, + service: initial.service, + endpoint: recipe.endpoint, + creation: Ref.get(creation), + observation: core.observation, + }); }); for (const saved of options.saved.instances) @@ -448,6 +460,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { Effect.catch((cause) => Effect.logWarning("Orphaned instance logs were not removed", cause)), ); yield* orchestrator.configure(options.saved.composition); + yield* forwarder.rebind; const removeSaved = (id: string) => updateState((current) => withoutInstance(current, id)); @@ -469,14 +482,16 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { const configure = (configuration: CompositionConfig) => options.state.withLock( - orchestrator.configure( - configuration, - readSaved.pipe( - Effect.flatMap((current) => - options.state.save({ ...current, composition: configuration }), + orchestrator + .configure( + configuration, + readSaved.pipe( + Effect.flatMap((current) => + options.state.save({ ...current, composition: configuration }), + ), ), - ), - ), + ) + .pipe(Effect.andThen(forwarder.rebind)), ); // A failed cleanup must not replace the failure that triggered it. diff --git a/packages/stack/src/State.ts b/packages/stack/src/State.ts index f388ba3bef..7b71d9fdd0 100644 --- a/packages/stack/src/State.ts +++ b/packages/stack/src/State.ts @@ -231,6 +231,49 @@ const reapStaleWrites = (fs: FileSystem.FileSystem, path: Path.Path, root: strin ); }).pipe(Effect.ignore); +/** + * Replaces `target` with `content` (mode 0600) through a temporary file in a fresh directory under + * `directory`, which must share the target's file system, retrying Windows sharing violations. + */ +export const writeFileAtomically = ( + fs: FileSystem.FileSystem, + path: Path.Path, + options: { + readonly directory: string; + readonly target: string; + readonly content: string; + readonly platform?: NodeJS.Platform; + }, +): Effect.Effect => + Effect.acquireUseRelease( + fs + .makeTempDirectory({ directory: options.directory, prefix: stateWritePrefix }) + .pipe(Effect.mapError((cause) => stateError("write", cause))), + (directory) => + Effect.gen(function* () { + const temporary = path.join(directory, path.basename(options.target)); + yield* fs + .writeFileString(temporary, options.content, { mode: 0o600 }) + .pipe(Effect.mapError((cause) => stateError("write", cause))); + yield* fs.rename(temporary, options.target).pipe( + retrySharingViolation(options.platform), + Effect.mapError( + (cause) => + new StateError({ + operation: "publish", + message: `Unable to publish state to ${options.target}${errorCode(cause) ? ` (${errorCode(cause)})` : ""}: ${cause instanceof Error ? cause.message : String(cause)}`, + cause, + }), + ), + Effect.withSpan("State.publish"), + ); + }), + (directory) => + fs + .remove(directory, { recursive: true, force: true }) + .pipe(Effect.mapError((cause) => stateError("cleanup", cause))), + ); + const makeState = ( options: Options, ): Effect.Effect => @@ -261,19 +304,6 @@ const makeState = ( retryShared, Effect.mapError((cause) => stateError("read", cause)), ); - const publish = Effect.fn("State.publish")(function* (temporary: string, target: string) { - yield* fs.rename(temporary, target).pipe( - retryShared, - Effect.mapError( - (cause) => - new StateError({ - operation: "publish", - message: `Unable to publish state to ${target}${errorCode(cause) ? ` (${errorCode(cause)})` : ""}: ${cause instanceof Error ? cause.message : String(cause)}`, - cause, - }), - ), - ); - }); const removeEmptyDirectory = (directory: string) => Effect.tryPromise({ try: () => rmdir(directory), @@ -333,28 +363,17 @@ const makeState = ( ); const claims = Effect.fn("State.claims")(() => readEntries(readClaims, () => Effect.void)); const writeAtomically = (id: string, target: string, serialized: string) => - Effect.gen(function* () { - yield* fs - .makeDirectory(stackRoot(id), { recursive: true, mode: 0o700 }) - .pipe(Effect.mapError((cause) => stateError("write", cause))); - yield* Effect.acquireUseRelease( - fs - .makeTempDirectory({ directory: root, prefix: stateWritePrefix }) - .pipe(Effect.mapError((cause) => stateError("write", cause))), - (directory) => - Effect.gen(function* () { - const temporary = path.join(directory, path.basename(target)); - yield* fs - .writeFileString(temporary, serialized, { mode: 0o600 }) - .pipe(Effect.mapError((cause) => stateError("write", cause))); - yield* publish(temporary, target); - }), - (directory) => - fs - .remove(directory, { recursive: true, force: true }) - .pipe(Effect.mapError((cause) => stateError("cleanup", cause))), - ); - }); + fs.makeDirectory(stackRoot(id), { recursive: true, mode: 0o700 }).pipe( + Effect.mapError((cause) => stateError("write", cause)), + Effect.andThen( + writeFileAtomically(fs, path, { + directory: root, + target, + content: serialized, + ...(options.platform === undefined ? {} : { platform: options.platform }), + }), + ), + ); const save = Effect.fn("State.save")(function* (state: SavedStack) { yield* checkId(state.id); const serialized = yield* Schema.encodeEffect(Schema.fromJsonString(SavedStack))(state).pipe( diff --git a/packages/stack/src/host/LogForwarder.integration.test.ts b/packages/stack/src/host/LogForwarder.integration.test.ts new file mode 100644 index 0000000000..6b6ae7e775 --- /dev/null +++ b/packages/stack/src/host/LogForwarder.integration.test.ts @@ -0,0 +1,392 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { createServer } from "node:http"; // oxlint-disable-line effecttsgo/node-builtin-import -- real socket fixture. +import { + Clock, + Effect, + Exit, + FileSystem, + Layer, + Path, + PubSub, + Queue, + Schema, + Scope, + Stream, + SubscriptionRef, +} from "effect"; +import type { ServiceObservation } from "../Service.ts"; +import type { LaunchOutput } from "../runtime/Session.ts"; +import { CatalogError } from "../services/Recipe.ts"; +import type { LogRecord } from "./LogRecord.ts"; +import * as LogForwarder from "./LogForwarder.ts"; +import * as LogStore from "./LogStore.ts"; + +const Body = Schema.Struct({ + batch: Schema.Array( + Schema.Struct({ + id: Schema.String, + event_message: Schema.String, + appname: Schema.String, + timestamp: Schema.String, + metadata: Schema.Record(Schema.String, Schema.Unknown), + }), + ), +}); +type Event = Schema.Schema.Type["batch"][number]; + +interface Received { + readonly url: string; + readonly apiKey: string | undefined; + readonly events: ReadonlyArray; +} + +const decodeBody = Schema.decodeUnknownSync(Body); + +/** A Logflare stand-in that records posts and answers the next `failures` of them with `status`. */ +const makeSink = Effect.gen(function* () { + const received = yield* Queue.unbounded(); + const control = { failures: 0, status: 500 }; + const server = yield* Effect.acquireRelease( + Effect.callback<{ readonly port: number; readonly server: ReturnType }>( + (resume) => { + const server = createServer((request, response) => { + const chunks: Array = []; + request.on("data", (chunk: Uint8Array) => chunks.push(chunk)); + request.on("end", () => { + const apiKey = request.headers["x-api-key"]; + Queue.offerUnsafe(received, { + url: request.url ?? "", + apiKey: typeof apiKey === "string" ? apiKey : undefined, + events: decodeBody(JSON.parse(new TextDecoder().decode(Buffer.concat(chunks)))).batch, + }); + response.statusCode = control.failures > 0 ? control.status : 200; + control.failures = Math.max(0, control.failures - 1); + response.end(); + }); + }); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (address === null || typeof address === "string") + return resume(Effect.die("sink has no address")); + resume(Effect.succeed({ port: address.port, server })); + }); + }, + ), + ({ server }) => + Effect.callback((resume) => { + server.closeAllConnections(); + server.close(() => resume(Effect.void)); + }), + ); + return { + port: server.port, + next: Queue.take(received), + failNext: (count: number, status = 500) => + Effect.sync(() => { + control.failures = count; + control.status = status; + }), + }; +}); + +const observationOf = ( + id: string, + overrides: Partial>, +): ServiceObservation => ({ + id, + config: undefined, + lifecycle: "stopped", + health: undefined, + error: undefined, + cleanupError: undefined, + exit: undefined, + currentOperation: undefined, + launchId: undefined, + intentRevision: 0, + wakeEnabled: true, + registered: true, + ...overrides, +}); + +const serving = (id: string) => + observationOf(id, { lifecycle: "running", health: "healthy", launchId: 1 }); + +const encoder = new TextEncoder(); + +/** A database instance whose output a real log store persists. */ +const databaseSource = (store: LogStore.Interface) => + Effect.gen(function* () { + const logs = yield* PubSub.unbounded(); + const observation = yield* SubscriptionRef.make(serving("database")); + yield* store.attach({ + service: "database", + instanceId: "database", + logs: PubSub.subscribe(logs), + observation: SubscriptionRef.changes(observation).pipe( + Stream.map(({ launchId }) => ({ launchId })), + ), + }); + let seq = 0; + const written = yield* Queue.unbounded(); + yield* (yield* store.read("database", { from: "oldest", follow: true })).pipe( + Stream.runForEach((record) => Queue.offer(written, record)), + Effect.forkScoped, + ); + /** Publishes lines and waits until the store has persisted the last one. */ + const log = (...lines: ReadonlyArray) => + Effect.gen(function* () { + const time = yield* Clock.currentTimeMillis; + yield* PubSub.publish(logs, { + stream: "stdout", + bytes: encoder.encode(lines.map((line) => `${line}\n`).join("")), + launchId: 1, + part: 0, + seq: seq++, + time, + }); + const last = lines.at(-1); + while ((yield* Queue.take(written)).text !== last); + }); + return { + log, + instance: { + id: "database", + service: "database" as const, + endpoint: () => Effect.fail(new CatalogError({ operation: "endpoint", message: "unused" })), + creation: Effect.die("Only the Analytics creation is read"), + observation: SubscriptionRef.changes(observation), + } satisfies LogForwarder.ForwardedInstance, + }; + }); + +const analyticsTarget = (port: number) => + Effect.gen(function* () { + const observation = yield* SubscriptionRef.make(observationOf("analytics", {})); + return { + set: (awake: boolean) => + SubscriptionRef.set( + observation, + awake ? serving("analytics") : observationOf("analytics", { lifecycle: "stopped" }), + ), + instance: { + id: "analytics", + service: "analytics" as const, + endpoint: () => Effect.succeed({ kind: "tcp" as const, host: "127.0.0.1" as const, port }), + creation: Effect.succeed({ service: "analytics" as const, config: { apiKey: "test-key" } }), + observation: SubscriptionRef.changes(observation), + } satisfies LogForwarder.ForwardedInstance, + }; + }); + +const composition = Effect.succeed({ + members: [ + { id: "analytics", activation: "lazy" as const }, + { id: "database", activation: "eager" as const }, + ], + dependencies: [], +}); + +/** Starts a forwarder in its own scope so a test can stop it like an owner. */ +const startForwarder = ( + store: LogStore.Interface, + instances: ReadonlyArray, +) => + Effect.gen(function* () { + const scope = yield* Scope.make(); + const forwarder = yield* LogForwarder.make({ composition, logs: store }).pipe( + Scope.provide(scope), + ); + for (const instance of instances) yield* forwarder.attach(instance); + yield* forwarder.rebind; + const awaitShipping = (value: boolean) => + forwarder.shipping.pipe( + Stream.filter((current) => current === value), + Stream.take(1), + Stream.runDrain, + ); + return { awaitShipping, stop: Scope.close(scope, Exit.void) }; + }); + +const fixture = (options: Omit = {}) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "log-forwarder-" }); + const store = yield* LogStore.make({ root, ...options }); + const sink = yield* makeSink; + const database = yield* databaseSource(store); + const analytics = yield* analyticsTarget(sink.port); + return { root, store, sink, database, analytics }; + }); + +const messages = (post: Received) => post.events.map((event) => event.event_message); + +const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); + +describe("LogForwarder", () => { + it.live( + "ships records written while Analytics slept with their timestamps, without markers", + () => + Effect.gen(function* () { + const { root, store, sink, database, analytics } = yield* fixture(); + const forwarder = yield* startForwarder(store, [analytics.instance, database.instance]); + yield* database.log("while asleep 1", "while asleep 2"); + + yield* analytics.set(true); + yield* forwarder.awaitShipping(true); + const caughtUp = yield* sink.next; + yield* database.log("while awake"); + const live = yield* sink.next; + + expect(caughtUp.url).toBe("/api/logs?source_name=postgres.logs"); + expect(caughtUp.apiKey).toBe("test-key"); + expect(messages(caughtUp)).toEqual(["while asleep 1", "while asleep 2"]); + expect(messages(live)).toEqual(["while awake"]); + const persisted = (yield* LogStore.readStackLogs({ root })).filter( + (record) => record.kind === "stdout", + ); + expect([...caughtUp.events, ...live.events].map((event) => event.timestamp)).toEqual( + persisted.map((record) => record.timestamp), + ); + const ids = [...caughtUp.events, ...live.events].map((event) => event.id); + expect(new Set(ids).size).toBe(3); + expect(ids.every((id) => /^[0-9a-f]{8}-[0-9a-f]{4}-8[0-9a-f]{3}-[89ab]/u.test(id))).toBe( + true, + ); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("resumes from its persisted cursor after the forwarder restarts", () => + Effect.gen(function* () { + const { store, sink, database, analytics } = yield* fixture(); + yield* analytics.set(true); + const first = yield* startForwarder(store, [analytics.instance, database.instance]); + yield* database.log("line 1", "line 2"); + const shipped = yield* sink.next; + // Shipping line 3 implies the cursor passed lines 1 and 2 before it. + yield* database.log("line 3"); + const third = yield* sink.next; + yield* first.stop; + + yield* database.log("line 4"); + yield* startForwarder(store, [analytics.instance, database.instance]); + const resumed: Array = []; + while (resumed.at(-1)?.event_message !== "line 4") resumed.push(...(yield* sink.next).events); + + expect(messages(shipped)).toEqual(["line 1", "line 2"]); + expect(messages(third)).toEqual(["line 3"]); + expect(["line 3", "line 4"]).toEqual( + expect.arrayContaining(resumed.map((e) => e.event_message)), + ); + const repeated = resumed.find((event) => event.event_message === "line 3"); + if (repeated !== undefined) expect(repeated.id).toBe(third.events[0]?.id); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("posts an unsettled body again before any later body", () => + Effect.gen(function* () { + const { store, sink, database, analytics } = yield* fixture(); + yield* analytics.set(true); + yield* startForwarder(store, [analytics.instance, database.instance]); + yield* sink.failNext(1); + const lines = Array.from({ length: 300 }, (_, index) => `burst ${index}`); + + yield* database.log(...lines); + const failed = yield* sink.next; + const retried = yield* sink.next; + const shipped = [...retried.events]; + while (shipped.length < lines.length) shipped.push(...(yield* sink.next).events); + + expect(retried.events.map((event) => event.id)).toEqual( + failed.events.map((event) => event.id), + ); + expect(shipped.map((event) => event.event_message)).toEqual(lines); + expect(Math.max(failed.events.length, retried.events.length)).toBeLessThanOrEqual(256); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("pauses without advancing when Analytics refuses its credentials", () => + Effect.gen(function* () { + const { store, sink, database, analytics } = yield* fixture(); + yield* analytics.set(true); + const forwarder = yield* startForwarder(store, [analytics.instance, database.instance]); + yield* sink.failNext(1, 401); + + yield* database.log("refused line"); + const refused = yield* sink.next; + yield* analytics.set(false); + yield* forwarder.awaitShipping(false); + yield* analytics.set(true); + const resumed = yield* sink.next; + + expect(messages(refused)).toEqual(["refused line"]); + expect(resumed.events.map((event) => event.id)).toEqual( + refused.events.map((event) => event.id), + ); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("skips a body Analytics rejects and ships the next one", () => + Effect.gen(function* () { + const { store, sink, database, analytics } = yield* fixture(); + yield* analytics.set(true); + yield* startForwarder(store, [analytics.instance, database.instance]); + yield* sink.failNext(1, 400); + + yield* database.log("rejected line"); + const rejected = yield* sink.next; + yield* database.log("accepted line"); + const accepted = yield* sink.next; + + expect(messages(rejected)).toEqual(["rejected line"]); + expect(messages(accepted)).toEqual(["accepted line"]); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("ships from the oldest retained record when its cursor is unreadable", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { root, store, sink, database, analytics } = yield* fixture(); + yield* database.log("before a corrupt cursor"); + yield* fs.writeFileString(path.join(root, "database", "database", "cursor.json"), "{oops"); + + yield* analytics.set(true); + yield* startForwarder(store, [analytics.instance, database.instance]); + const shipped = yield* sink.next; + + expect(messages(shipped)).toEqual(["before a corrupt cursor"]); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("restarts from the oldest retained segment when its cursor's segment was deleted", () => + Effect.gen(function* () { + const { root, store, sink, database, analytics } = yield* fixture({ + rotateBytes: 200, + retainBytes: 400, + }); + yield* analytics.set(true); + const forwarder = yield* startForwarder(store, [analytics.instance, database.instance]); + yield* database.log("first"); + yield* sink.next; + yield* database.log("second"); + yield* sink.next; + yield* analytics.set(false); + yield* forwarder.awaitShipping(false); + for (let index = 0; index < 30; index++) yield* database.log(`asleep ${index}`); + const retained = (yield* LogStore.readStackLogs({ root })).filter( + (record) => record.kind === "stdout", + ); + + yield* analytics.set(true); + const shipped: Array = []; + while (shipped.at(-1)?.event_message !== "asleep 29") + shipped.push(...(yield* sink.next).events); + + expect(retained.map((record) => record.text)).not.toContain("second"); + expect(shipped.map((event) => event.event_message)).toEqual( + retained.map((record) => record.text), + ); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); +}); diff --git a/packages/stack/src/host/LogForwarder.ts b/packages/stack/src/host/LogForwarder.ts new file mode 100644 index 0000000000..cc13bc6899 --- /dev/null +++ b/packages/stack/src/host/LogForwarder.ts @@ -0,0 +1,404 @@ +import { + Crypto, + Data, + Duration, + Effect, + FileSystem, + Option, + Path, + Ref, + Schedule, + Schema, + Scope, + Semaphore, + Stream, + SubscriptionRef, +} from "effect"; +import { HttpClient, HttpClientError, HttpClientRequest } from "effect/unstable/http"; +import type { CompositionConfig } from "../Orchestrator.ts"; +import type { ServiceObservation } from "../Service.ts"; +import type { ServiceCreation } from "../services/Catalog.ts"; +import type { CatalogError, ServiceEndpoint } from "../services/Recipe.ts"; +import { writeFileAtomically } from "../State.ts"; +import { + isShippedService, + logflareEvent, + logflareSources, + type LogflareEvent, + type ShippedService, +} from "./LogflareEvents.ts"; +import { LogPosition, type LogRecord } from "./LogRecord.ts"; +import type * as LogStore from "./LogStore.ts"; + +export interface ForwardedInstance { + readonly id: string; + readonly service: ServiceCreation["service"]; + readonly endpoint: (name: string) => Effect.Effect; + /** The current saved creation; Analytics' API key is read from it per body. */ + readonly creation: Effect.Effect; + readonly observation: Stream.Stream>; +} + +interface Interface { + /** Ships a shipped service's persisted logs, or tracks an Analytics instance as the target. */ + readonly attach: (instance: ForwardedInstance) => Effect.Effect; + /** Re-selects the shipping target after the composition changes. */ + readonly rebind: Effect.Effect; + /** Emits whether records are currently shipped; the current value first. */ + readonly shipping: Stream.Stream; +} + +export interface LogForwarderOptions { + readonly composition: Effect.Effect; + readonly logs: LogStore.Interface; + readonly platform?: NodeJS.Platform; +} + +interface Candidate { + readonly instance: ForwardedInstance; + readonly serving: boolean; +} + +interface Target { + readonly instance: ForwardedInstance; + readonly epoch: number; +} + +/** A shipped event; its id makes a repeated post of the same record a no-op in Logflare. */ +interface ShippedEvent extends LogflareEvent { + readonly id: string; +} + +class StaleTarget extends Data.TaggedError("StaleTarget")<{}> {} + +const batchEvents = 256; +export const batchBytes = 1024 * 1024; +const envelopeBytes = '{"batch":[]}'.length; +const postTimeout = "5 seconds"; +const cursorFile = "cursor.json"; +const encoder = new TextEncoder(); +const decodeCursor = Schema.decodeUnknownEffect(Schema.fromJsonString(LogPosition)); +const encodeCursor = Schema.encodeEffect(Schema.fromJsonString(LogPosition)); +/** Delivery is idempotent per event id, so a body that did not settle is posted again. */ +const redeliverSchedule = Schedule.exponential("250 millis", 2).pipe( + Schedule.modifyDelay(({ duration }) => + Effect.succeed(Duration.min(duration, Duration.seconds(10))), + ), +); + +const after = (record: LogRecord, cursor: LogPosition | undefined) => + cursor === undefined || + record.position === undefined || + record.position.generation > cursor.generation || + (record.position.generation === cursor.generation && + record.position.byteOffset > cursor.byteOffset); + +/** Groups serialized events into request bodies of at most 256 events and `batchBytes` bytes. */ +export const bodies = ( + events: ReadonlyArray, +): ReadonlyArray<{ readonly body: string; readonly last: A }> => { + const result: Array<{ readonly body: string; readonly last: A }> = []; + let current: Array = []; + let size = envelopeBytes; + let last: A | undefined; + const close = () => { + if (last !== undefined) result.push({ body: `{"batch":[${current.join(",")}]}`, last }); + current = []; + size = envelopeBytes; + }; + for (const item of events) { + const serialized = JSON.stringify(item.event); + const bytes = encoder.encode(serialized).length; + if (current.length > 0 && (current.length >= batchEvents || size + 1 + bytes > batchBytes)) + close(); + size += (current.length > 0 ? 1 : 0) + bytes; + current.push(serialized); + last = item; + } + if (current.length > 0) close(); + return result; +}; + +const rejectedStatus = (error: unknown): number | undefined => + error instanceof HttpClientError.HttpClientError && error.reason._tag === "StatusCodeError" + ? error.reason.response.status + : undefined; + +/** The target refused its credentials or route, so no body can succeed until it changes. */ +const isTargetRejection = (error: unknown): boolean => + [401, 403, 404].includes(rejectedStatus(error) ?? 0); + +/** The body itself was refused and cannot succeed on a repeat. */ +const isBodyRejection = (error: unknown): boolean => { + const status = rejectedStatus(error); + return ( + status !== undefined && + status >= 400 && + status < 500 && + ![401, 403, 404, 408, 429].includes(status) + ); +}; + +/** + * Ships persisted service logs to Analytics' direct backend while the composed Analytics instance + * is running and healthy. Each instance resumes from its cursor, so records written while + * Analytics sleeps are shipped after it wakes; shipping never wakes Analytics. + */ +export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwarderOptions) { + const scope = yield* Scope.Scope; + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const client = (yield* HttpClient.HttpClient).pipe(HttpClient.filterStatusOk); + const candidates = yield* Ref.make>(new Map()); + const target = yield* SubscriptionRef.make(undefined); + const epochs = yield* Ref.make(0); + const rebinding = yield* Semaphore.make(1); + + const rebind = Effect.gen(function* () { + const members = new Set((yield* options.composition).members.map(({ id }) => id)); + const serving = [...(yield* Ref.get(candidates)).values()].find( + (candidate) => candidate.serving && members.has(candidate.instance.id), + ); + const previous = yield* SubscriptionRef.get(target); + if (serving?.instance === previous?.instance) return; + const epoch = yield* Ref.updateAndGet(epochs, (value) => value + 1); + yield* SubscriptionRef.set( + target, + serving === undefined ? undefined : { instance: serving.instance, epoch }, + ); + }).pipe(Semaphore.withPermits(rebinding, 1)); + + const trackTarget = (instance: ForwardedInstance) => + instance.observation.pipe( + Stream.takeUntil((observation) => !observation.registered), + Stream.runForEach((observation) => + Ref.update(candidates, (current) => + new Map(current).set(instance.id, { + instance, + serving: observation.lifecycle === "running" && observation.health === "healthy", + }), + ).pipe(Effect.andThen(rebind)), + ), + Effect.ensuring( + Ref.update(candidates, (current) => { + const next = new Map(current); + next.delete(instance.id); + return next; + }).pipe(Effect.andThen(rebind)), + ), + ); + + const serving = SubscriptionRef.changes(target).pipe( + Stream.filter((current) => current !== undefined), + Stream.take(1), + Stream.runHead, + Effect.flatMap(Effect.fromOption), + Effect.orDie, + ); + const retargeted = (current: Target) => + SubscriptionRef.changes(target).pipe( + Stream.filter((latest) => latest?.epoch !== current.epoch), + Stream.take(1), + Stream.runDrain, + ); + const requireCurrent = (current: Target) => + SubscriptionRef.get(target).pipe( + Effect.flatMap((latest) => + latest?.epoch === current.epoch ? Effect.void : Effect.fail(new StaleTarget()), + ), + ); + + /** A missing or unreadable cursor ships from the oldest retained segment. */ + const readCursor = (directory: string) => + fs.readFileString(path.join(directory, cursorFile)).pipe( + Effect.map(Option.some), + Effect.catchIf( + (error) => error.reason._tag === "NotFound", + () => Effect.succeed(Option.none()), + ), + Effect.flatMap( + Option.match({ + onNone: () => Effect.succeed("oldest"), + onSome: (text) => decodeCursor(text), + }), + ), + Effect.catch(() => + Effect.logWarning( + `Unreadable log cursor in ${directory}; shipping from the oldest retained record`, + ).pipe(Effect.as("oldest")), + ), + ); + const writeCursor = (directory: string, position: LogPosition) => + encodeCursor(position).pipe( + Effect.flatMap((content) => + writeFileAtomically(fs, path, { + directory, + target: path.join(directory, cursorFile), + content, + ...(options.platform === undefined ? {} : { platform: options.platform }), + }), + ), + ); + + const eventId = Effect.fnUntraced(function* (instanceId: string, position: LogPosition) { + const digest = yield* crypto.digest( + "SHA-256", + encoder.encode(`${instanceId}\0${position.generation}\0${position.byteOffset}`), + ); + const hex = Array.from(digest.slice(0, 16), (byte) => byte.toString(16).padStart(2, "0")); + hex[6] = `8${(hex[6] ?? "00").slice(1)}`; + hex[8] = ((Number.parseInt(hex[8] ?? "00", 16) & 0x3f) | 0x80).toString(16).padStart(2, "0"); + const text = hex.join(""); + return `${text.slice(0, 8)}-${text.slice(8, 12)}-${text.slice(12, 16)}-${text.slice(16, 20)}-${text.slice(20)}`; + }); + + const post = Effect.fn("LogForwarder.post")(function* ( + current: Target, + source: string, + body: string, + ) { + yield* requireCurrent(current); + const creation = yield* current.instance.creation; + const apiKey = creation.service === "analytics" ? creation.config.apiKey : undefined; + if (apiKey === undefined) return yield* new StaleTarget(); + const endpoint = yield* current.instance.endpoint("http"); + yield* client + .execute( + HttpClientRequest.post( + `http://${endpoint.host ?? "127.0.0.1"}:${endpoint.port}/api/logs`, + ).pipe( + HttpClientRequest.setUrlParam("source_name", source), + HttpClientRequest.setHeader("x-api-key", apiKey), + HttpClientRequest.bodyText(body, "application/json"), + ), + ) + .pipe( + Effect.flatMap((response) => response.text), + Effect.timeout(postTimeout), + ); + }); + + /** + * Posts one body until it settles; the cursor passes it only afterwards. A rejected target pauses + * shipping until the next retarget, and a rejected body is skipped. + */ + const deliver = Effect.fnUntraced(function* (current: Target, source: string, body: string) { + const warned = yield* Ref.make(false); + yield* post(current, source, body).pipe( + Effect.catchIf(isTargetRejection, (error) => + Effect.logWarning(`Analytics refused ${source} logs; pausing until it changes`, error).pipe( + Effect.andThen(Effect.fail(new StaleTarget())), + ), + ), + Effect.tapError((error) => + error._tag === "StaleTarget" || isBodyRejection(error) + ? Effect.void + : Ref.getAndSet(warned, true).pipe( + Effect.flatMap((already) => + already + ? Effect.void + : Effect.logWarning(`Posting ${source} logs failed; retrying`, error), + ), + ), + ), + Effect.retry({ + schedule: redeliverSchedule, + while: (error) => error._tag !== "StaleTarget" && !isBodyRejection(error), + }), + Effect.catch((error) => + error._tag === "StaleTarget" + ? Effect.fail(error) + : Effect.logWarning(`Analytics rejected a ${source} log body; skipping it`, error), + ), + ); + }); + + /** Ships one instance's records from its cursor until the target changes. */ + const session = (instanceId: string, service: ShippedService, current: Target) => + Effect.scoped( + Effect.gen(function* () { + const directory = yield* options.logs.directory(instanceId); + const from = yield* readCursor(directory); + const cursor = yield* Ref.make( + typeof from === "string" ? undefined : from, + ); + const records = yield* options.logs.read(instanceId, { from, follow: true }); + const source = logflareSources[service]; + yield* records.pipe( + Stream.runForEachArray((page) => + Effect.gen(function* () { + const shipped = yield* Ref.get(cursor); + const pending: Array<{ + readonly event: ShippedEvent; + readonly position: LogPosition; + }> = []; + for (const record of page) { + if (record.position === undefined) { + yield* Effect.logWarning( + `Logs of ${service} instance ${instanceId} were deleted before shipping; resuming from the oldest retained record`, + ); + continue; + } + if (!after(record, shipped)) continue; + if (record.kind !== "stdout" && record.kind !== "stderr") continue; + pending.push({ + position: record.position, + event: { + id: yield* eventId(instanceId, record.position), + ...logflareEvent(service, record.timestamp, record.text ?? ""), + }, + }); + } + for (const { body, last } of bodies(pending)) { + yield* deliver(current, source, body); + yield* Ref.set(cursor, last.position); + yield* writeCursor(directory, last.position).pipe( + Effect.catch((error) => + Effect.logDebug(`Log cursor of ${instanceId} was not saved`, error), + ), + ); + } + }), + ), + ); + }), + ); + + const unregistered = (instance: ForwardedInstance) => + instance.observation.pipe( + Stream.filter((observation) => !observation.registered), + Stream.take(1), + Stream.runDrain, + ); + + const forward = (instance: ForwardedInstance, service: ShippedService) => + Effect.gen(function* () { + const current = yield* serving; + // A stopped target pauses shipping; the cursor keeps the position to resume from. + yield* session(instance.id, service, current).pipe( + Effect.catch((error) => + (error._tag === "StaleTarget" + ? Effect.void + : Effect.logWarning(`Log shipping of ${instance.id} paused`, error) + ).pipe(Effect.andThen(retargeted(current))), + ), + Effect.raceFirst(retargeted(current)), + ); + }).pipe(Effect.forever, Effect.raceFirst(unregistered(instance))); + + const attach = Effect.fn("LogForwarder.attach")(function* (instance: ForwardedInstance) { + if (instance.service === "analytics") yield* Effect.forkIn(trackTarget(instance), scope); + else if (isShippedService(instance.service)) + yield* Effect.forkIn(forward(instance, instance.service), scope); + }); + + return { + attach, + rebind, + shipping: SubscriptionRef.changes(target).pipe( + Stream.map((current) => current !== undefined), + Stream.changes, + ), + } satisfies Interface; +}); diff --git a/packages/stack/src/host/LogForwarder.unit.test.ts b/packages/stack/src/host/LogForwarder.unit.test.ts new file mode 100644 index 0000000000..c81292313a --- /dev/null +++ b/packages/stack/src/host/LogForwarder.unit.test.ts @@ -0,0 +1,36 @@ +import { expect, it } from "@effect/vitest"; +import { batchBytes, bodies } from "./LogForwarder.ts"; + +const encoder = new TextEncoder(); + +const eventWith = (message: string) => ({ + event: { + id: "00000000-0000-8000-8000-000000000000", + appname: "database" as const, + event_message: message, + timestamp: "2026-09-30T10:00:00.000Z", + metadata: {}, + }, +}); + +// Ten events whose single `{"batch":[...]}` body would be exactly `total` bytes. +const eventsForBody = (total: number) => { + const empty = JSON.stringify(eventWith("").event).length; + const available = total - '{"batch":[]}'.length - 9 - 10 * empty; + return Array.from({ length: 10 }, (_, index) => + eventWith("x".repeat(Math.floor(available / 10) + (index === 9 ? available % 10 : 0))), + ); +}; + +const sizes = (events: ReturnType) => + bodies(events).map(({ body }) => encoder.encode(body).length); + +it("keeps a body whose full JSON envelope is exactly the byte limit in one request", () => { + expect(sizes(eventsForBody(batchBytes))).toEqual([batchBytes]); +}); + +it("splits a body that the JSON envelope would push past the byte limit", () => { + const split = sizes(eventsForBody(batchBytes + 1)); + expect(split).toHaveLength(2); + expect(split.every((size) => size <= batchBytes)).toBe(true); +}); diff --git a/packages/stack/src/host/LogStore.ts b/packages/stack/src/host/LogStore.ts index 1c7cbce4dc..8e81a28238 100644 --- a/packages/stack/src/host/LogStore.ts +++ b/packages/stack/src/host/LogStore.ts @@ -65,8 +65,8 @@ export interface LogStoreOptions { /** Selects the records a read returns. */ export interface ReadOptions { - /** `end` skips every record written before the read starts. */ - readonly from: "oldest" | "end"; + /** `end` skips every record written before the read starts; a position resumes there. */ + readonly from: "oldest" | "end" | LogPosition; /** Epoch milliseconds; older records are skipped. */ readonly since?: number; /** Returns only the last records of the history before following. */ @@ -82,7 +82,7 @@ export interface AttachedInstance { readonly observation: Stream.Stream<{ readonly launchId: number | undefined }>; } -interface Interface { +export interface Interface { /** Subscribes to the instance's output before returning; persistence failures never fail it. */ readonly attach: (instance: AttachedInstance) => Effect.Effect; /** Fixes the read's start and wake subscription in the scope; the stream reads lazily. */ @@ -90,6 +90,8 @@ interface Interface { instanceId: string, options: ReadOptions, ) => Effect.Effect, LogStoreError, Scope.Scope>; + /** The directory holding an attached instance's segments and its forwarding cursor. */ + readonly directory: (instanceId: string) => Effect.Effect; /** Streams output lines as they are split, whether or not their append succeeds. */ readonly tail: ( instanceId: string, @@ -474,9 +476,13 @@ const makeReader = (fs: FileSystem.FileSystem, path: Path.Path) => { const wake = options.follow && live !== undefined ? yield* PubSub.subscribe(live.wake) : undefined; const start = - options.from === "end" && live !== undefined - ? yield* live.lock.withPermits(1)(Ref.get(live.end)) - : { generation: 0, byteOffset: 0 }; + options.from === "oldest" + ? { generation: 0, byteOffset: 0 } + : options.from === "end" + ? live === undefined + ? { generation: 0, byteOffset: 0 } + : yield* live.lock.withPermits(1)(Ref.get(live.end)) + : options.from; if (options.tail === undefined) return follow(directory, { cursor: start, listed: undefined }, options, live, wake); return Stream.unwrap( @@ -895,7 +901,15 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio yield* Effect.addFinalizer(() => close); - return { attach, read, tail, remove, removeOrphans: removeOrphans(), close } satisfies Interface; + return { + attach, + read, + directory: (instanceId) => Effect.map(attached(instanceId), (instance) => instance.directory), + tail, + remove, + removeOrphans: removeOrphans(), + close, + } satisfies Interface; }); /** Reads persisted records of a stack's instances, merged by time, service, instance and position. */ diff --git a/packages/stack/src/host/LogflareEvents.ts b/packages/stack/src/host/LogflareEvents.ts new file mode 100644 index 0000000000..4e144b1460 --- /dev/null +++ b/packages/stack/src/host/LogflareEvents.ts @@ -0,0 +1,153 @@ +import { DateTime, Option } from "effect"; +import type { ServiceCreation } from "../services/Catalog.ts"; + +/** Logflare source per shipped service kind; Studio's Logs pages query these names. */ +export const logflareSources = { + auth: "gotrue.logs.prod", + rest: "postgREST.logs.prod", + realtime: "realtime.logs.prod", + storage: "storage.logs.prod.2", + functions: "deno-relay-logs", + database: "postgres.logs", +} as const satisfies Partial>; + +export type ShippedService = keyof typeof logflareSources; + +export const isShippedService = (service: ServiceCreation["service"]): service is ShippedService => + Object.hasOwn(logflareSources, service); + +/** One ingest event in the shape Studio's local log queries expect. */ +export interface LogflareEvent { + readonly project?: string; + readonly event_message: string; + readonly appname: ShippedService; + readonly timestamp: string; + readonly metadata: Record; +} + +const isRecord = (value: unknown): value is Record => + typeof value === "object" && value !== null && !Array.isArray(value); + +const parseJsonObject = (text: string): Record | undefined => { + if (!text.startsWith("{")) return undefined; + try { + const value: unknown = JSON.parse(text); + return isRecord(value) ? value : undefined; + } catch { + return undefined; + } +}; + +const months: Record = { + jan: 0, + feb: 1, + mar: 2, + apr: 3, + may: 4, + jun: 5, + jul: 6, + aug: 7, + sep: 8, + oct: 9, + nov: 10, + dec: 11, +}; + +/** Parses PostgREST's `%d/%b/%Y:%H:%M:%S %z` prefix into an ISO-8601 timestamp. */ +const parsePostgrestTime = (text: string): string | undefined => { + const match = + /^(\d{2})\/([A-Za-z]{3})\/(\d{4}):(\d{2}):(\d{2}):(\d{2}) ([+-])(\d{2})(\d{2})$/u.exec(text); + if (match === null) return undefined; + const [, day, month, year, hour, minute, second, sign, zoneHours, zoneMinutes] = match; + const monthIndex = months[month?.toLowerCase() ?? ""]; + if (monthIndex === undefined) return undefined; + const utc = Date.UTC( + Number(year), + monthIndex, + Number(day), + Number(hour), + Number(minute), + Number(second), + ); + const offsetMinutes = (sign === "-" ? -1 : 1) * (Number(zoneHours) * 60 + Number(zoneMinutes)); + return Option.getOrUndefined( + Option.map(DateTime.make(utc - offsetMinutes * 60_000), DateTime.formatIso), + ); +}; + +const withoutProject = ({ project: _project, ...event }: LogflareEvent): LogflareEvent => event; + +const remaps: Record LogflareEvent> = { + auth: (event) => { + const parsed = parseJsonObject(event.event_message); + return parsed === undefined + ? event + : { ...event, metadata: { ...event.metadata, timestamp: parsed.time, ...parsed } }; + }, + rest: (event) => { + const match = /^(.*): (.*)$/u.exec(event.event_message); + const timestamp = match === null ? undefined : parsePostgrestTime(match[1] ?? ""); + return match === null || timestamp === undefined + ? event + : { + ...event, + event_message: match[2] ?? "", + timestamp, + metadata: { ...event.metadata, host: event.project }, + }; + }, + realtime: (event) => { + const match = /^(\d+:\d+:\d+\.\d+) \[(\w+)\] (.*)$/u.exec(event.event_message); + const metadata = { ...event.metadata, project: event.project, external_id: event.project }; + return withoutProject( + match === null + ? { ...event, metadata } + : { ...event, event_message: match[3] ?? "", metadata: { ...metadata, level: match[2] } }, + ); + }, + storage: (event) => { + const parsed = parseJsonObject(event.event_message); + const metadata = { ...event.metadata, project: event.project, tenantId: event.project }; + return withoutProject( + parsed === undefined + ? { ...event, metadata } + : { + ...event, + event_message: typeof parsed.msg === "string" ? parsed.msg : event.event_message, + metadata: { + ...metadata, + level: parsed.level, + timestamp: parsed.time, + context: [{ host: parsed.hostname, pid: parsed.pid }], + }, + }, + ); + }, + functions: (event) => + withoutProject({ ...event, metadata: { ...event.metadata, project_ref: event.project } }), + database: (event) => { + const match = /.*(INFO|NOTICE|WARNING|ERROR|LOG|FATAL|PANIC):/u.exec(event.event_message); + return { + ...event, + metadata: { + ...event.metadata, + host: "db-default", + parsed: { timestamp: event.timestamp, error_severity: match?.[1] ?? "LOG" }, + }, + }; + }, +}; + +/** Builds the Logflare event for one service log line received at `timestamp`. */ +export const logflareEvent = ( + service: ShippedService, + timestamp: string, + message: string, +): LogflareEvent => + remaps[service]({ + project: "default", + event_message: message, + appname: service, + timestamp, + metadata: {}, + }); diff --git a/packages/stack/src/host/LogflareEvents.unit.test.ts b/packages/stack/src/host/LogflareEvents.unit.test.ts new file mode 100644 index 0000000000..32b4fcecea --- /dev/null +++ b/packages/stack/src/host/LogflareEvents.unit.test.ts @@ -0,0 +1,96 @@ +import { expect, it } from "@effect/vitest"; +import { logflareEvent } from "./LogflareEvents.ts"; + +const received = "2026-09-28T10:00:00.000Z"; + +it("merges structured Auth lines into metadata and keeps the raw message", () => { + const line = + '{"level":"info","msg":"request completed","time":"2026-09-28T09:59:59Z","path":"/health"}'; + expect(logflareEvent("auth", received, line)).toEqual({ + project: "default", + appname: "auth", + event_message: line, + timestamp: received, + metadata: { + timestamp: "2026-09-28T09:59:59Z", + level: "info", + msg: "request completed", + time: "2026-09-28T09:59:59Z", + path: "/health", + }, + }); +}); + +it("splits PostgREST's timestamp prefix from the message and converts its zone offset", () => { + expect( + logflareEvent("rest", received, "28/Sep/2026:12:30:15 +0200: Starting PostgREST 13.0.7..."), + ).toEqual({ + project: "default", + appname: "rest", + event_message: "Starting PostgREST 13.0.7...", + timestamp: "2026-09-28T10:30:15.000Z", + metadata: { host: "default" }, + }); +}); + +it("keeps a PostgREST line whose prefix is not a timestamp", () => { + const event = logflareEvent("rest", received, "Config: schema cache loaded"); + expect(event.event_message).toBe("Config: schema cache loaded"); + expect(event.timestamp).toBe(received); +}); + +it("extracts the Realtime level and moves the project into metadata", () => { + expect( + logflareEvent("realtime", received, "10:00:01.123 [info] Running RealtimeWeb.Endpoint"), + ).toEqual({ + appname: "realtime", + event_message: "Running RealtimeWeb.Endpoint", + timestamp: received, + metadata: { project: "default", external_id: "default", level: "info" }, + }); +}); + +it("parses Storage JSON lines into the tenant metadata shape", () => { + expect( + logflareEvent( + "storage", + received, + '{"level":"info","time":"2026-09-28T09:59:58Z","pid":7,"hostname":"storage","msg":"listening"}', + ), + ).toEqual({ + appname: "storage", + event_message: "listening", + timestamp: received, + metadata: { + project: "default", + tenantId: "default", + level: "info", + timestamp: "2026-09-28T09:59:58Z", + context: [{ host: "storage", pid: 7 }], + }, + }); +}); + +it("tags Functions lines with the project ref", () => { + expect(logflareEvent("functions", received, "booted worker")).toEqual({ + appname: "functions", + event_message: "booted worker", + timestamp: received, + metadata: { project_ref: "default" }, + }); +}); + +it("derives the Postgres severity from the last level marker and defaults to LOG", () => { + const line = "2026-09-28 10:00:00.000 UTC [77] ERROR: relation does not exist"; + expect(logflareEvent("database", received, line)).toEqual({ + project: "default", + appname: "database", + event_message: line, + timestamp: received, + metadata: { host: "db-default", parsed: { timestamp: received, error_severity: "ERROR" } }, + }); + expect(logflareEvent("database", received, "\tat character 15").metadata.parsed).toEqual({ + timestamp: received, + error_severity: "LOG", + }); +}); From 9da34285e05988790dab22562f06e0312056f803 Mon Sep 17 00:00:00 2001 From: avallete Date: Wed, 30 Sep 2026 16:15:54 +0200 Subject: [PATCH 03/35] feat(stack): read log history, drop Vector, and log PostgREST requests `supabase stack logs` now prints retained history and exits by default. `-f/--follow` hands over from history to live output per instance without gaps, `--tail` (default 200), `--since ` and a repeatable `--service` select what is shown, and history is read from the log files when the stack is down. Output keeps the `log-entry` contract with `source: "history" | "live"` and adds `log-marker` events. The legacy live-only `logs` RPC is replaced by `readLogs`, which the Promise client exposes as an async iterable. The experimental stack no longer has a Vector service: saved stacks that contain one are migrated on owner start, and leftover Vector files and containers are removed. `analytics.vector_port` stays in the config schema because the legacy `supabase start` still reads it. PostgREST now logs every request, and request lines reach Analytics with their method, path, protocol and status. Co-Authored-By: Claude Opus 5.5 --- apps/cli/docs/stack-commands.md | 35 +- apps/cli/src/command-internal/colors.ts | 15 + .../db-config.integration.test.ts | 2 +- apps/cli/src/command-internal/stack-config.ts | 11 - .../commands/db/dump/dump.integration.test.ts | 2 +- .../db/reset/reset.integration.test.ts | 2 +- .../generate/generate.integration.test.ts | 2 +- .../declarative/sync/sync.integration.test.ts | 2 +- .../db/start/start.integration.test.ts | 2 +- .../experimental/stack/logs/SIDE_EFFECTS.md | 95 ++- .../experimental/stack/logs/logs.command.ts | 48 +- .../experimental/stack/logs/logs.format.ts | 238 +++++++ .../stack/logs/logs.format.unit.test.ts | 122 ++++ .../experimental/stack/logs/logs.handler.ts | 197 ++++-- .../stack/logs/logs.integration.test.ts | 597 ++++++++++++------ .../stack/prepare/SIDE_EFFECTS.md | 2 + .../stack/prepare/prepare.integration.test.ts | 2 +- .../stack/stack-config.integration.test.ts | 1 - .../experimental/stack/stack.shared.ts | 2 - .../experimental/stack/start/SIDE_EFFECTS.md | 7 +- .../stack/start/start.e2e.test.ts | 68 +- .../stack/start/start.integration.test.ts | 4 +- .../experimental/stack/status/SIDE_EFFECTS.md | 3 +- .../stack/status/status.integration.test.ts | 2 +- .../commands/functions/serve/SIDE_EFFECTS.md | 3 +- .../functions/serve/serve.stack.handler.ts | 47 +- .../serve/serve.stack.integration.test.ts | 48 +- .../services/services.integration.test.ts | 9 +- apps/cli/src/shared/output/types.ts | 10 + apps/cli/tests/helpers/storage.ts | 2 +- docs/output.md | 12 +- packages/stack/ARCHITECTURE.md | 29 +- packages/stack/README.md | 2 +- packages/stack/src/Artifacts.ts | 29 +- .../src/Owner.analytics.integration.test.ts | 18 +- .../stack/src/Owner.logs.integration.test.ts | 43 +- packages/stack/src/Owner.ts | 11 +- packages/stack/src/PromiseClient.ts | 14 +- packages/stack/src/Rpc.ts | 8 +- .../stack/src/StackHost.integration.test.ts | 4 +- packages/stack/src/StackHost.ts | 2 + packages/stack/src/State.ts | 111 +++- ...State.vector-migration.integration.test.ts | 203 ++++++ packages/stack/src/Sweep.ts | 3 +- packages/stack/src/composition/Supabase.ts | 7 - packages/stack/src/effect.ts | 55 +- packages/stack/src/host/Credentials.ts | 1 - .../src/host/LogStore.integration.test.ts | 63 +- packages/stack/src/host/LogStore.ts | 100 +-- packages/stack/src/host/LogflareEvents.ts | 18 + .../src/host/LogflareEvents.unit.test.ts | 18 + packages/stack/src/index.ts | 1 + .../stack/src/promise-api.integration.test.ts | 9 +- packages/stack/src/promise-api.unit.test.ts | 3 + ... => AnalyticsImgproxy.integration.test.ts} | 34 +- packages/stack/src/services/Catalog.ts | 12 - .../services/Functions.integration.test.ts | 7 +- packages/stack/src/services/Rest.ts | 2 + .../src/services/Vector.integration.test.ts | 122 ---- packages/stack/src/services/Vector.ts | 244 ------- packages/stack/tests/public-client-fixture.ts | 4 +- packages/stack/tests/whole-stack/analytics.ts | 58 +- packages/stack/tests/whole-stack/fixture.ts | 18 +- packages/stack/tests/whole-stack/scenarios.ts | 24 +- 64 files changed, 1797 insertions(+), 1072 deletions(-) create mode 100644 apps/cli/src/commands/experimental/stack/logs/logs.format.ts create mode 100644 apps/cli/src/commands/experimental/stack/logs/logs.format.unit.test.ts create mode 100644 packages/stack/src/State.vector-migration.integration.test.ts rename packages/stack/src/services/{AnalyticsVectorImgproxy.integration.test.ts => AnalyticsImgproxy.integration.test.ts} (72%) delete mode 100644 packages/stack/src/services/Vector.integration.test.ts delete mode 100644 packages/stack/src/services/Vector.ts diff --git a/apps/cli/docs/stack-commands.md b/apps/cli/docs/stack-commands.md index dd8cbdd896..f6d4e42511 100644 --- a/apps/cli/docs/stack-commands.md +++ b/apps/cli/docs/stack-commands.md @@ -16,7 +16,7 @@ and password-reconciliation connection uses that administrative role, so a nativ | `supabase stack prepare` | Download artifacts without starting services. | | `supabase stack start` | Create or resume the project's stack. | | `supabase stack status` | Show identity, readiness, and drift, or export connection variables with `--env`. | -| `supabase stack logs` | Stream live stack logs. | +| `supabase stack logs` | Print retained stack logs; `-f` streams new lines. | | `supabase stack restart` | Restart an existing stack using its saved effective configuration. | | `supabase stack stop` | Stop a stack while retaining its data. | @@ -160,7 +160,7 @@ Other values are rejected. The override is applied before reading the project co `supabase services` follows the same backend selection. In stack mode it lists image versions and canonical `ghcr.io/supabase/cli/...` names from the installed CLI's artifact catalog, including -Mailpit and Vector. PostgreSQL uses the selected major version (15 or 17); invalid configuration +Mailpit. PostgreSQL uses the selected major version (15 or 17); invalid configuration or an unsupported PostgreSQL major warns with the cause and uses catalog defaults. This inventory describes the CLI catalog, not running containers, downloaded images, or service health. The Docker and native stack runtimes use the same catalog versions, though a running stack launched by another CLI @@ -237,18 +237,25 @@ lint transaction (always rolled back). It does not launch a client binary. ## Reading stack logs -`supabase stack logs` streams live stdout/stderr from composition members without -starting an owner or service. Select `--stack ` or `--stack-id `; -`--service ` can include standalone services too. The command -requires a reachable owner and streams until interrupted. Ctrl-C leaves services -running. There is no retained history, `--tail`, or `--follow` flag. - -Text uses ` //: ` and strips terminal -control sequences. For automation use `--output-format stream-json`: each -`log-entry` contains `timestamp`, `service`, `instance_id`, `stream`, `line`, and -`source: "live"`. Finite JSON output is not supported. Delivery is best effort; -stdout/stderr and different services may interleave. Missing stacks, unavailable -owners, and unmatched services fail with status 1; interruption exits 130. +`supabase stack logs` prints the retained stdout/stderr of composition members and +exits; `-f/--follow` then streams new lines until interrupted. Select `--stack ` +or `--stack-id `; the repeatable `--service ` can include +standalone services too. History is read from the persisted log files, so it works +while the stack is stopped; `--follow` requires a running owner and fails before +printing anything without one. Neither mode starts an owner or service, and Ctrl-C +leaves services running. + +`--tail N` (default 200) keeps the newest lines across the selected services and +`--since` takes a duration (`10m`, `1h30m`), an ISO-8601 time, or `start` for each +service's latest launch. Text uses ` | ` with aligned +labels, shows launches and lost output as dim separators, strips terminal control +sequences, and notes on stderr when the tail hides older lines. For automation use +`--output-format stream-json`: each `log-entry` contains `timestamp`, `service`, +`instance_id`, `stream`, `line`, and `source` (`history` or `live`), and markers are +`log-marker` events with `kind` and, for lost output, `count`. `--output-format json` +prints the history as one array and does not accept `--follow`. Missing stacks, +unmatched services, and `--follow` without an owner fail with status 1; interruption +exits 130. ## Data and configuration diff --git a/apps/cli/src/command-internal/colors.ts b/apps/cli/src/command-internal/colors.ts index 50addc7657..1cce53c10f 100644 --- a/apps/cli/src/command-internal/colors.ts +++ b/apps/cli/src/command-internal/colors.ts @@ -77,3 +77,18 @@ export function green(text: string, stream: ColorStream = process.stderr): strin export function gray(text: string, stream: ColorStream = process.stderr): string { return supportsColor(stream) ? styleText("gray", text, { validateStream: false }) : text; } + +/** Renders in bright magenta. */ +export function magenta(text: string, stream: ColorStream = process.stderr): string { + return supportsColor(stream) ? styleText("magenta", text, { validateStream: false }) : text; +} + +/** Renders in bright blue. */ +export function blue(text: string, stream: ColorStream = process.stderr): string { + return supportsColor(stream) ? styleText("blue", text, { validateStream: false }) : text; +} + +/** Renders dimmed. */ +export function dim(text: string, stream: ColorStream = process.stderr): string { + return supportsColor(stream) ? styleText("dim", text, { validateStream: false }) : text; +} diff --git a/apps/cli/src/command-internal/db-config.integration.test.ts b/apps/cli/src/command-internal/db-config.integration.test.ts index 9e8a09e3ae..1002a2b5d7 100644 --- a/apps/cli/src/command-internal/db-config.integration.test.ts +++ b/apps/cli/src/command-internal/db-config.integration.test.ts @@ -465,7 +465,7 @@ describe("dbConfigResolver (db-url under the stack backend)", () => { registered: true, }), followStatus: Stream.empty, - logs: Stream.empty, + readLogs: () => Stream.empty, credentials: () => unused, saveSnapshot: () => unused, restoreSnapshot: () => unused, diff --git a/apps/cli/src/command-internal/stack-config.ts b/apps/cli/src/command-internal/stack-config.ts index d66ff10680..ba89f6bc74 100644 --- a/apps/cli/src/command-internal/stack-config.ts +++ b/apps/cli/src/command-internal/stack-config.ts @@ -522,17 +522,6 @@ const resolveEffectiveCliConfig = ( env, ), port: resolvedPort("SUPABASE_ANALYTICS_PORT", analytics.port, "analytics.port", env), - ...(analytics.vector_port === undefined && - envOverride("SUPABASE_ANALYTICS_VECTOR_PORT", undefined, env) === undefined - ? {} - : { - vector_port: resolvedPort( - "SUPABASE_ANALYTICS_VECTOR_PORT", - analytics.vector_port ?? 0, - "analytics.vector_port", - env, - ), - }), backend: envOverrideAnalyticsBackend(analytics.backend, env), gcp_project_id: envOverride("SUPABASE_ANALYTICS_GCP_PROJECT_ID", analytics.gcp_project_id, env), gcp_project_number: envOverride( diff --git a/apps/cli/src/commands/db/dump/dump.integration.test.ts b/apps/cli/src/commands/db/dump/dump.integration.test.ts index af9f38fc3b..e020809e10 100644 --- a/apps/cli/src/commands/db/dump/dump.integration.test.ts +++ b/apps/cli/src/commands/db/dump/dump.integration.test.ts @@ -124,7 +124,7 @@ const managedDumpStackApi = (runtime: "native" | "docker") => { registered: true, }), followStatus: Stream.empty, - logs: Stream.empty, + readLogs: () => Stream.empty, credentials: () => Effect.succeed({ databaseUrl: "postgresql://postgres:secret@127.0.0.1:54322/postgres" }), saveSnapshot: () => Effect.die("unused"), diff --git a/apps/cli/src/commands/db/reset/reset.integration.test.ts b/apps/cli/src/commands/db/reset/reset.integration.test.ts index 4e960d99b1..1f73deb1b8 100644 --- a/apps/cli/src/commands/db/reset/reset.integration.test.ts +++ b/apps/cli/src/commands/db/reset/reset.integration.test.ts @@ -565,7 +565,7 @@ const stackService = ( prepare: Effect.void, status: observation, followStatus: Stream.empty, - logs: Stream.empty, + readLogs: () => Stream.empty, credentials: () => Effect.succeed({}), } satisfies Omit; switch (creation.service) { diff --git a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts index 7344c9711b..f6838085f6 100644 --- a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts @@ -112,7 +112,7 @@ function generateStackApi(workdir: string) { saveSnapshot: unusedStackFn, restoreSnapshot: unusedStackFn, resetData: unusedStack, - logs: Stream.empty, + readLogs: () => Stream.empty, followStatus: Stream.empty, status: Effect.succeed({ id: "primary", diff --git a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts index 85ebc7eaaa..c7035c2c4f 100644 --- a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts @@ -114,7 +114,7 @@ function syncStackApi(workdir: string, port: number) { saveSnapshot: unusedSyncFn, restoreSnapshot: unusedSyncFn, resetData: unusedSync, - logs: Stream.empty, + readLogs: () => Stream.empty, followStatus: Stream.empty, status: Effect.succeed({ id: "primary", diff --git a/apps/cli/src/commands/db/start/start.integration.test.ts b/apps/cli/src/commands/db/start/start.integration.test.ts index debb4eae72..48f591f4dc 100644 --- a/apps/cli/src/commands/db/start/start.integration.test.ts +++ b/apps/cli/src/commands/db/start/start.integration.test.ts @@ -1633,7 +1633,7 @@ describe("db start stack backend", () => { registered: true, })), followStatus: Stream.empty, - logs: Stream.empty, + readLogs: () => Stream.empty, credentials: () => Effect.sync(() => { state.credentialsCalled = true; diff --git a/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md index 2118aca19f..c9b52d568b 100644 --- a/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md @@ -1,44 +1,73 @@ # `supabase stack logs` -Streams live stdout/stderr from the selected saved stack. The experimental -feature flag controls command registration. It connects to an existing owner; -it never launches an owner or starts/stops a service. +Prints the retained service logs of the selected saved stack and exits; with +`-f/--follow` it then streams new lines until interrupted. The experimental +feature flag controls command registration. It never launches an owner or +starts/stops a service. ## Selection and files Select the current project/branch/name, `--stack `, or `--stack-id `. The selectors are mutually exclusive. By default, only composition members are -included. `--service ` can also select standalone instances. -An unavailable owner, missing stack, or unmatched service fails with status 1. - -Reads saved definitions under `/stacks//`. -Discovery ensures the registry directory exists with mode 0700 and probes local -owners. Shared routing/settings may read project config and profiles. No project -files, service configuration, artifacts, or data are changed. - -## Output and cancellation - -This is a live-only stream with no retained history, cursor, or `--tail` option. -It continues until interrupted or the selected streams close. `--follow` is -unnecessary and is not accepted. Legacy `-o/--output` and finite JSON output are -rejected; use text or `--output-format stream-json`. - -Text writes ` //: `. Terminal -control sequences are stripped from text. Stream JSON emits `log-entry` events -with `timestamp`, `service`, `instance_id`, `stream`, `line`, and `source: "live"`. -Lines preserve their content in machine output. UTF-8 and line fragments are -assembled separately for each instance and stdout/stderr channel. Timestamps -reflect receipt by the CLI. Delivery is best effort: slow subscribers can lose entries. Ordering across -stdout/stderr channels and different services is not guaranteed. The owner's -log shipping to Analytics reads the persisted logs separately and does not -affect this command. - -Interrupting the command cancels its subscriptions, exits with status 130, and -leaves the owner and services running. Successful stream completion exits 0; -selection, connection, or stream failures exit 1. +included. `--service ` is repeatable and can also select +standalone instances; a value that matches no saved instance fails with status 1. +A missing stack fails with status 1. + +Reads saved definitions under `/stacks//` and +the persisted segments under +`/stacks//logs///.log` +directly, so history is readable while the owner is down. Discovery ensures the +registry directory exists with mode 0700 and probes local owners. Shared +routing/settings may read project config and profiles. No project files, +service configuration, artifacts, logs, or data are changed. + +## History, follow, and flags + +`--tail N` (default 200, 0 prints none) keeps the newest N output lines across +the selected services, with the `launch` and `lost` markers between them. +`--since` accepts a duration before now (`30s`, `10m`, `1h30m`, `2d`), an +ISO-8601 time, or `start`, which keeps each instance's records from its latest +launch (all of them when retention removed that launch record). Records are +ordered by timestamp, service, instance, and file position. Timestamps are the +owner's clock at each line's first byte. History is read as a stream holding at +most N lines per instance, while every matching line is counted for the footer. + +`-f/--follow` requires a reachable owner: without one it fails with status 1 +before printing anything and suggests running without `--follow`. It prints the +history, then streams each instance's new records through the owner from the +last record read, so no record is repeated or skipped between history and live +output. With `--tail 0` no history is read and the owner starts each follow at +its current end. The owner's log shipping to Analytics reads the persisted logs +separately and does not affect this command. + +## Output + +Text writes ` | ` with the service labels padded +to one width; a service kind selected more than once is labelled +`:`. Times are local. `launch` +and `lost` records are dim separator lines (`--- launch 2 ---`, +`--- 3 stdout chunks lost ---`, `--- older records were removed by retention ---`). +Labels are coloured and markers dimmed only when stdout is a colour-capable +terminal. Terminal control sequences are stripped from text. When a non-zero +tail hides older lines, stderr gets +`showing last 200 of 5,234 lines, use --tail/--since`; an empty history without +`--follow` prints `No retained log lines for the selected services.` on stderr. + +Stream JSON emits a `log-entry` event per output line with `timestamp`, +`service`, `instance_id`, `stream` (`stdout` or `stderr`), `line`, and `source` +(`history` or `live`), and a `log-marker` event per marker with `timestamp`, +`service`, `instance_id`, `kind` (`launch` or `lost`), `source`, and, for chunks +lost before they were written, `stream` and `count`. A `lost` marker without +`count` reports segments removed by retention. `--output-format json` prints +one array of the same objects; it cannot be combined with `--follow`. Lines +preserve their content in machine output. Legacy `-o/--output` is rejected. + +Interrupting a follow cancels its subscriptions, exits with status 130, and +leaves the owner and services running. History reads and completed follows exit +0; selection, owner, or read failures exit 1. ## Telemetry -Standard command telemetry is retained; log contents are not custom telemetry -properties. Telemetry flushes on every exit to +Standard command telemetry is retained, with `-f` reported as `follow`; log +contents are not custom telemetry properties. Telemetry flushes on every exit to `/telemetry.json`. diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.command.ts b/apps/cli/src/commands/experimental/stack/logs/logs.command.ts index f895e7151d..0e3d972b62 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.command.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.command.ts @@ -17,7 +17,30 @@ const config = { ), service: Flag.string("service").pipe( Flag.withDescription( - "Stream one service kind or instance ID; defaults to composition members.", + "Read one service kind or instance ID; repeat to select several. Defaults to composition members.", + ), + Flag.atLeast(0), + ), + follow: Flag.boolean("follow").pipe( + Flag.withAlias("f"), + Flag.withDescription( + "After the history, keep streaming new lines until interrupted; requires a running stack.", + ), + Flag.withDefault(false), + ), + tail: Flag.integer("tail").pipe( + Flag.filter( + (tail) => tail >= 0, + (tail) => `Expected --tail to be 0 or more, got ${tail}`, + ), + Flag.withDescription( + "Number of retained lines to print across the selected services (default 200); 0 prints none.", + ), + Flag.withDefault(200), + ), + since: Flag.string("since").pipe( + Flag.withDescription( + "Only print lines since a duration ago (10m, 1h30m), an ISO-8601 time, or start for each service's latest launch.", ), Flag.optional, ), @@ -26,19 +49,28 @@ const config = { export type StackLogsFlags = CliCommand.Command.Config.Infer; export const stackLogsCommand = Command.make("logs", config).pipe( - Command.withDescription("Stream live logs until interrupted; retained history is unavailable."), - Command.withShortDescription("Stream managed local stack logs"), + Command.withDescription( + "Print retained service logs and exit; with --follow, keep streaming new lines. History is readable while the stack is stopped.", + ), + Command.withShortDescription("Show managed local stack logs"), Command.withExamples([ { - command: "supabase stack logs --service database", - description: "Stream new database logs", + command: "supabase stack logs --service database --since 10m", + description: "Print database lines from the last ten minutes", }, { - command: "supabase stack logs --output-format stream-json", - description: "Stream new stack logs as structured events", + command: "supabase stack logs -f --tail 0", + description: "Stream only new lines until interrupted", + }, + { + command: "supabase stack logs --follow --output-format stream-json", + description: "Stream history and new lines as structured events", }, ]), Command.withHandler((flags) => - stackLogs(flags).pipe(withCommandTelemetry({ flags, config }), withJsonErrorHandling), + stackLogs(flags).pipe( + withCommandTelemetry({ flags, config, aliases: { f: "follow" } }), + withJsonErrorHandling, + ), ), ); diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.format.ts b/apps/cli/src/commands/experimental/stack/logs/logs.format.ts new file mode 100644 index 0000000000..3df0d15ab3 --- /dev/null +++ b/apps/cli/src/commands/experimental/stack/logs/logs.format.ts @@ -0,0 +1,238 @@ +import { DateTime, Option } from "effect"; +import type { LogPosition, LogRecord, StackLogRecord } from "@supabase/stack/effect"; +import { + aqua, + blue, + type ColorStream, + dim, + green, + magenta, + yellow, +} from "../../../../command-internal/colors.ts"; +import { stripControlSequences } from "../../../../shared/output/strip-control-sequences.ts"; +import type { StreamEvent } from "../../../../shared/output/types.ts"; + +export type LogSource = "history" | "live"; + +/** A `--since` bound: an absolute time, or each instance's latest launch. */ +export type SinceBound = + | { readonly kind: "time"; readonly iso: string } + | { readonly kind: "start" }; + +const durationShape = /^(?:\d+(?:\.\d+)?(?:ms|s|m|h|d))+$/u; +const durationPart = /(\d+(?:\.\d+)?)(ms|s|m|h|d)/gu; +const unitMillis: Readonly> = { + ms: 1, + s: 1_000, + m: 60_000, + h: 3_600_000, + d: 86_400_000, +}; +const isoDate = /^\d{4}-\d{2}-\d{2}/u; + +/** Parses `start`, a duration before `nowMillis` such as `1h30m`, or an ISO-8601 time. */ +export const parseSince = (value: string, nowMillis: number): SinceBound | undefined => { + if (value === "start") return { kind: "start" }; + if (durationShape.test(value)) { + let millis = 0; + for (const [, amount, unit] of value.matchAll(durationPart)) + millis += Number(amount) * (unitMillis[unit ?? ""] ?? 0); + return Option.match(DateTime.make(nowMillis - millis), { + onNone: () => undefined, + onSome: (time) => ({ kind: "time", iso: DateTime.formatIso(time) }), + }); + } + if (!isoDate.test(value)) return undefined; + return Option.match(DateTime.make(value), { + onNone: () => undefined, + onSome: (time) => ({ kind: "time", iso: DateTime.formatIso(time) }), + }); +}; + +const isLine = (record: LogRecord) => record.kind === "stdout" || record.kind === "stderr"; + +/** The newest `tail` output lines, with the markers between them, and how many lines exist. */ +export interface LogWindow { + readonly records: ReadonlyArray; + readonly shown: number; + readonly total: number; +} + +export const tailWindow = (records: ReadonlyArray, tail: number): LogWindow => { + const total = records.filter(isLine).length; + if (total <= tail) return { records, shown: total, total }; + let seen = 0; + let start = records.length; + while (seen < tail && start > 0) { + start -= 1; + const record = records[start]; + if (record !== undefined && isLine(record)) seen += 1; + } + return { records: seen === 0 ? [] : records.slice(start), shown: seen, total }; +}; + +const comparePositions = (left: LogPosition, right: LogPosition) => + left.generation - right.generation || left.byteOffset - right.byteOffset; + +const origin: LogPosition = { generation: 0, byteOffset: 0 }; +const compareText = (left: string, right: string) => (left < right ? -1 : left > right ? 1 : 0); +const compareRecords = (left: StackLogRecord, right: StackLogRecord) => + compareText(left.timestamp, right.timestamp) || + compareText(left.service, right.service) || + compareText(left.instanceId, right.instanceId) || + comparePositions( + left.position ?? left.resumeAt ?? origin, + right.position ?? right.resumeAt ?? origin, + ); + +interface InstanceTail { + records: Array; + head: number; + lines: number; + total: number; + launched: string | undefined; +} + +/** + * Collects the history window from records streamed per instance in file order, holding each + * instance's newest `tail` lines in display order; `positions` holds each instance's last record + * in file order, where a follow resumes. With `fromLatestLaunch`, an instance's records before its latest + * `launch` record are dropped; an instance whose launch record was not retained keeps all of them. + */ +export const makeHistoryCollector = (tail: number, fromLatestLaunch: boolean) => { + const instances = new Map(); + const positions = new Map(); + const push = (record: StackLogRecord) => { + const position = record.position; + const last = positions.get(record.instanceId); + if (position !== undefined && (last === undefined || comparePositions(position, last) > 0)) + positions.set(record.instanceId, position); + let state = instances.get(record.instanceId); + if (state === undefined) { + state = { records: [], head: 0, lines: 0, total: 0, launched: undefined }; + instances.set(record.instanceId, state); + } + if (fromLatestLaunch) { + if (record.kind === "launch") { + const launched = record.timestamp; + state.records = state.records + .slice(state.head) + .filter((kept) => kept.timestamp >= launched); + state.head = 0; + state.lines = state.records.filter(isLine).length; + state.total = state.lines; + state.launched = launched; + } else if (state.launched !== undefined && record.timestamp < state.launched) return; + } + const line = isLine(record); + if (line) state.total += 1; + // Kept records stay in display order; a record older than every kept line of a full window + // would be trimmed at once. + const oldest = state.records[state.head]; + if (state.lines >= tail && oldest !== undefined && compareRecords(record, oldest) < 0) return; + let low = state.head; + let high = state.records.length; + while (low < high) { + const middle = (low + high) >>> 1; + const kept = state.records[middle]; + if (kept !== undefined && compareRecords(kept, record) <= 0) low = middle + 1; + else high = middle; + } + if (low === state.records.length) state.records.push(record); + else state.records.splice(low, 0, record); + if (line) state.lines += 1; + while (state.lines > tail && state.head < state.records.length) { + const dropped = state.records[state.head]; + state.head += 1; + if (dropped !== undefined && isLine(dropped)) state.lines -= 1; + } + if (state.head > 1024 && state.head * 2 > state.records.length) { + state.records = state.records.slice(state.head); + state.head = 0; + } + }; + const finish = (): { readonly window: LogWindow; readonly positions: typeof positions } => { + const kept = [...instances.values()].flatMap((state) => state.records.slice(state.head)); + const total = [...instances.values()].reduce((sum, state) => sum + state.total, 0); + const window = tailWindow(kept.toSorted(compareRecords), tail); + return { window: { ...window, total }, positions }; + }; + return { push, finish }; +}; + +/** Whether a followed record comes after the history already printed. */ +export const isAfter = (record: LogRecord, printed: LogPosition | undefined) => + printed === undefined || + record.position === undefined || + comparePositions(record.position, printed) > 0; + +/** The `log-entry` or `log-marker` event of a record. */ +export const logEvent = (record: StackLogRecord, source: LogSource): StreamEvent => { + const subject = { service: record.service, instance_id: record.instanceId }; + if (record.kind === "stdout" || record.kind === "stderr") + return { + type: "log-entry", + timestamp: record.timestamp, + source, + ...subject, + stream: record.kind, + line: record.text ?? "", + }; + return { + type: "log-marker", + timestamp: record.timestamp, + source, + ...subject, + kind: record.kind, + ...(record.stream === undefined ? {} : { stream: record.stream }), + ...(record.count === undefined ? {} : { count: record.count }), + }; +}; + +const palette = [aqua, yellow, green, magenta, blue]; +const pad = (value: number, width: number) => String(value).padStart(width, "0"); + +const localTime = (iso: string) => + Option.match(DateTime.make(iso), { + onNone: () => iso, + onSome: (time) => { + const parts = DateTime.toParts(DateTime.setZone(time, DateTime.zoneMakeLocal())); + return `${pad(parts.hour, 2)}:${pad(parts.minute, 2)}:${pad(parts.second, 2)}.${pad(parts.millisecond, 3)}`; + }, + }); + +const markerText = (record: LogRecord) => { + if (record.kind === "launch") + return record.launchId === undefined ? "--- launch ---" : `--- launch ${record.launchId} ---`; + if (record.count === undefined) return "--- older records were removed by retention ---"; + const unit = record.count === 1 ? "chunk" : "chunks"; + return `--- ${record.count} ${record.stream ?? "output"} ${unit} lost ---`; +}; + +/** Formats records as `

) => AsyncIterable + : T : T extends object ? { readonly [K in keyof T]: Promised } : T; @@ -264,7 +266,11 @@ const isCallOptions = (value: unknown): value is CallOptions => /** Adapts the operations of an Effect handle; their results pass through as data. */ const makeAdapter = (client: Client) => { const call = (result: unknown, options: CallOptions | undefined) => - isOperation(result) ? client.run(result, options) : result; + isOperation(result) + ? client.run(result, options) + : isStream(result) + ? client.iterable(result) + : result; function promised(value: T): Promised; function promised(value: unknown): unknown { if (isStream(value)) return () => client.iterable(value); diff --git a/packages/stack/src/Rpc.ts b/packages/stack/src/Rpc.ts index 6c8b7cf6d1..9b9203cc65 100644 --- a/packages/stack/src/Rpc.ts +++ b/packages/stack/src/Rpc.ts @@ -6,7 +6,7 @@ import { causeMessage, CompositionConfig, OrchestratorError } from "./Orchestrat import { CommandInvocation } from "./Commands.ts"; import { StackKeysInput } from "./State.ts"; import { failureMessage } from "./internal/failure-message.ts"; -import { LogRecord } from "./host/LogRecord.ts"; +import { LogPosition, LogRecord } from "./host/LogRecord.ts"; const Outcome = Schema.Struct({ id: Schema.String, @@ -81,10 +81,6 @@ export interface Definition extends Schema.Schema.Type {} const Instance = { id: Schema.String }; const SnapshotScope = Schema.Literals(snapshotScopes); -const Log = Schema.Struct({ - stream: Schema.Literals(["stdout", "stderr"]), - bytes: Schema.Uint8ArrayFromBase64, -}); export const CommandEvent = Schema.TaggedUnion({ Attached: { attachmentId: Schema.String }, @@ -121,10 +117,10 @@ export const OwnerRpc = RpcGroup.make( error: StackError, stream: true, }), - Rpc.make("logs", { payload: Instance, success: Log, error: StackError, stream: true }), Rpc.make("readLogs", { payload: { ...Instance, + from: Schema.optionalKey(LogPosition), since: Schema.optionalKey(Schema.String), tail: Schema.optionalKey(Schema.Int.check(Schema.isGreaterThanOrEqualTo(0))), follow: Schema.Boolean, diff --git a/packages/stack/src/StackHost.integration.test.ts b/packages/stack/src/StackHost.integration.test.ts index 914f877ed2..0bb65d4a65 100644 --- a/packages/stack/src/StackHost.integration.test.ts +++ b/packages/stack/src/StackHost.integration.test.ts @@ -626,7 +626,9 @@ it.live( const idleSocket = yield* Effect.acquireRelease(openIdleSocket(endpoint.port), (socket) => Effect.sync(() => socket.destroy()), ); - const logs = yield* Effect.forkScoped(client.logs({ id: mail.id }).pipe(Stream.runDrain)); + const logs = yield* Effect.forkScoped( + client.readLogs({ id: mail.id, follow: true }).pipe(Stream.runDrain), + ); const ready = yield* Deferred.make(); const drainingTool = yield* Effect.forkScoped( client diff --git a/packages/stack/src/StackHost.ts b/packages/stack/src/StackHost.ts index e2571409c3..2e48da56b0 100644 --- a/packages/stack/src/StackHost.ts +++ b/packages/stack/src/StackHost.ts @@ -437,6 +437,8 @@ export const runStackHost = Effect.fn("StackHost.run")( ), ), ); + // After the sweep, so no leftover Vector container still mounts the files it removes. + yield* state.migrate(id); const hostGateway = yield* makeHostGateway; const services = yield* Layer.build( Layer.merge( diff --git a/packages/stack/src/State.ts b/packages/stack/src/State.ts index 7b71d9fdd0..e035d5519d 100644 --- a/packages/stack/src/State.ts +++ b/packages/stack/src/State.ts @@ -128,7 +128,13 @@ export class StateError extends Data.TaggedError("StateError")<{ }> {} export interface Interface { + /** Omits saved Vector instances and their composition entries without rewriting the file. */ readonly read: (id: string) => Effect.Effect; + /** + * Persists the removal of saved Vector instances and deletes their stack-owned files. Call while + * holding the stack's lease, before any of its services run. + */ + readonly migrate: (id: string) => Effect.Effect; /** Skips each stack entry that stays unreadable after transient retries, reporting it to `onInvalidState`. */ readonly list: Effect.Effect, StateError>; /** Decodes only each stack's port claims and silently skips entries that cannot provide them. */ @@ -171,12 +177,68 @@ const checkId = (id: string): Effect.Effect => ? Effect.void : Effect.fail(stateError("identity", `Invalid state id: ${id}`)); +const isRecord = (value: unknown): value is Readonly> => + typeof value === "object" && value !== null && !Array.isArray(value); + +const retainedEntries = ( + entries: unknown, + retired: (entry: Readonly>) => boolean, +): unknown => + Array.isArray(entries) + ? entries.filter((entry: unknown) => !(isRecord(entry) && retired(entry))) + : entries; + +/** + * Drops saved Vector instances, which the stack no longer runs, with every composition member, + * dependency and port claim that references them. + */ +const withoutVectorInstances = ( + document: unknown, +): { readonly document: unknown; readonly removed: ReadonlyArray } => { + if (!isRecord(document) || !Array.isArray(document.instances)) return { document, removed: [] }; + const removed = document.instances.flatMap((instance: unknown) => + isRecord(instance) && + typeof instance.id === "string" && + isRecord(instance.creation) && + instance.creation.service === "vector" + ? [instance.id] + : [], + ); + if (removed.length === 0) return { document, removed }; + const ids: ReadonlySet = new Set(removed); + const { composition } = document; + return { + removed, + document: { + ...document, + instances: retainedEntries(document.instances, (instance) => ids.has(instance.id)), + composition: isRecord(composition) + ? { + ...composition, + members: retainedEntries(composition.members, (member) => ids.has(member.id)), + dependencies: retainedEntries( + composition.dependencies, + (dependency) => ids.has(dependency.from) || ids.has(dependency.to), + ), + } + : composition, + ports: retainedEntries( + document.ports, + ({ key }) => typeof key === "string" && removed.some((id) => key.startsWith(`${id}:`)), + ), + }, + }; +}; + const decodeState = ( text: string, id: string, target: string, ): Effect.Effect => - Schema.decodeEffect(Schema.fromJsonString(SavedStack))(text).pipe( + Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))(text).pipe( + Effect.flatMap((document) => + Schema.decodeUnknownEffect(SavedStack)(withoutVectorInstances(document).document), + ), Effect.mapError( (cause) => new StateError({ @@ -598,8 +660,55 @@ const makeState = ( .remove(ownerPath(id), { force: true }) .pipe(Effect.mapError((cause) => stateError("remove", cause))); }); + // A caller's Vector configPath may live under the instance root, so only recipe files and + // empty directories go. + const removeVectorData = (id: string, instanceId: string) => + Effect.gen(function* () { + const instanceRoot = path.join(stackRoot(id), "data", instanceId); + const configRoot = path.join(instanceRoot, "runtime", "vector"); + if (!(yield* fs.exists(configRoot))) return; + for (const entry of yield* fs.readDirectory(configRoot)) + if ( + entry === "vector.yaml" || + entry === "vector-api.yaml" || + entry.startsWith(".vector-write-") + ) + yield* fs.remove(path.join(configRoot, entry), { recursive: true, force: true }); + for (const directory of [configRoot, path.dirname(configRoot), instanceRoot]) + yield* removeEmptyDirectory(directory); + }).pipe( + Effect.catchCause((cause) => + Effect.logWarning(`Unable to remove the files of Vector instance ${instanceId}`, cause), + ), + ); + const migrate = Effect.fn("State.migrate")(function* (id: string) { + yield* checkId(id); + const removed = yield* withLock( + Effect.gen(function* () { + const target = statePath(id); + if (!(yield* fs.exists(target).pipe(retryTransientRead))) return []; + const text = yield* fs.readFileString(target).pipe(retryTransientRead); + const { removed } = yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))( + text, + ).pipe( + Effect.map(withoutVectorInstances), + Effect.orElseSucceed(() => ({ removed: [] })), + ); + if (removed.length === 0) return removed; + const state = yield* decodeState(text, id, target); + if (state.id !== id) + return yield* stateError("identity", "State document identity does not match its path"); + yield* save(state); + return removed; + }), + ); + yield* Effect.forEach(removed, (instanceId) => removeVectorData(id, instanceId), { + discard: true, + }); + }); return { read, + migrate, list: list(), claims: claims(), save, diff --git a/packages/stack/src/State.vector-migration.integration.test.ts b/packages/stack/src/State.vector-migration.integration.test.ts new file mode 100644 index 0000000000..6c6b4fc9d3 --- /dev/null +++ b/packages/stack/src/State.vector-migration.integration.test.ts @@ -0,0 +1,203 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { describe, expect, it } from "@effect/vitest"; +import { Context, Effect, FileSystem, Layer, Path, Schema } from "effect"; +import { create, open } from "./effect.ts"; +import * as State from "./State.ts"; +import { destroyTestStack } from "../tests/stack-cleanup.ts"; + +const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); +const Json = Schema.fromJsonString(Schema.Unknown); +const savedServices = (text: string) => + Schema.decodeEffect( + Schema.fromJsonString( + Schema.Struct({ + instances: Schema.Array( + Schema.Struct({ creation: Schema.Struct({ service: Schema.String }) }), + ), + }), + ), + )(text).pipe(Effect.map(({ instances }) => instances.map(({ creation }) => creation.service))); + +const stateFor = (root: string) => + Layer.build(State.layer({ root })).pipe( + Effect.map((context) => Context.get(context, State.Service)), + ); + +/** A state document saved before the stack dropped its Vector service kind. */ +const legacyDocument = (id: string, projectRoot: string) => ({ + id, + lifetime: "detached", + identity: { projectRoot, branchContext: "main", stackName: "local" }, + runtime: "native", + instances: [ + { id: "analytics", creation: { service: "analytics", config: { backend: "postgres" } } }, + { id: "mail", creation: { service: "mail", config: {} } }, + { + id: "vector", + creation: { + service: "vector", + config: { analyticsUrl: "http://127.0.0.1:1", configPath: `${projectRoot}/vector.yaml` }, + }, + }, + { id: "vector-owned", creation: { service: "vector", config: {} } }, + ], + composition: { + members: [ + { id: "analytics", activation: "eager" }, + { id: "mail", activation: "eager" }, + { id: "vector", activation: "eager" }, + { id: "vector-owned", activation: "eager" }, + ], + dependencies: [ + { + from: "analytics", + to: "vector", + bindings: [{ output: "url", input: "analyticsUrl" }], + }, + { from: "analytics", to: "vector-owned" }, + { from: "mail", to: "analytics" }, + ], + }, + ports: [ + { key: "vector:http", host: "127.0.0.1", port: 24_501 }, + { key: "mail:http", host: "127.0.0.1", port: 24_502 }, + ], +}); + +const writeLegacyState = Effect.fn("test.writeLegacyState")(function* ( + stateRoot: string, + id: string, + projectRoot: string, +) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const data = path.join(stateRoot, id, "data"); + yield* fs.makeDirectory(path.join(data, "vector", "runtime", "vector"), { recursive: true }); + yield* fs.writeFileString(path.join(data, "vector", "runtime", "vector", "vector.yaml"), ""); + yield* fs.writeFileString(path.join(data, "vector", "runtime", "vector", "vector-api.yaml"), ""); + yield* fs.writeFileString(path.join(data, "vector", "pipeline.yaml"), "caller"); + const owned = path.join(data, "vector-owned", "runtime", "vector"); + yield* fs.makeDirectory(path.join(owned, ".vector-write-1"), { recursive: true }); + yield* fs.writeFileString(path.join(owned, "vector-api.yaml"), ""); + const file = path.join(stateRoot, id, "state.json"); + yield* fs.writeFileString( + file, + yield* Schema.encodeEffect(Json)(legacyDocument(id, projectRoot)), + ); + return file; +}); + +describe("saved Vector instance migration", () => { + it.live("reads a legacy state without Vector and leaves the file for the lease holder", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-vector-read-" }); + const state = yield* stateFor(root); + const file = yield* writeLegacyState(root, "legacy", root); + const before = yield* fs.readFileString(file); + + const saved = yield* state.read("legacy"); + + expect(saved?.instances.map(({ id }) => id)).toEqual(["analytics", "mail"]); + expect(saved?.composition).toEqual({ + members: [ + { id: "analytics", activation: "eager" }, + { id: "mail", activation: "eager" }, + ], + dependencies: [{ from: "mail", to: "analytics" }], + }); + expect(saved?.ports.map(({ key }) => key)).toEqual(["mail:http"]); + expect((yield* state.list).map(({ id }) => id)).toEqual(["legacy"]); + expect(yield* fs.readFileString(file)).toBe(before); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("persists the migration once and keeps files a caller owns", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-vector-migrate-" }); + const state = yield* stateFor(root); + const file = yield* writeLegacyState(root, "legacy", root); + const data = path.join(root, "legacy", "data"); + + yield* state.migrate("legacy"); + + const migrated = yield* fs.readFileString(file); + expect(yield* savedServices(migrated)).toEqual(["analytics", "mail"]); + expect(yield* Schema.decodeEffect(Schema.fromJsonString(State.SavedStack))(migrated)).toEqual( + yield* state.read("legacy"), + ); + expect(yield* fs.exists(path.join(data, "vector", "runtime"))).toBe(false); + expect(yield* fs.readFileString(path.join(data, "vector", "pipeline.yaml"))).toBe("caller"); + expect(yield* fs.exists(path.join(data, "vector-owned"))).toBe(false); + + yield* state.migrate("legacy"); + expect(yield* fs.readFileString(file)).toBe(migrated); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("refuses to migrate a state document copied from another stack", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-vector-copied-" }); + const state = yield* stateFor(root); + const file = yield* writeLegacyState(root, "copied", root); + yield* fs.writeFileString( + file, + yield* Schema.encodeEffect(Json)(legacyDocument("original", root)), + ); + const before = yield* fs.readFileString(file); + + const failure = yield* state.migrate("copied").pipe(Effect.flip); + + expect(failure.operation).toBe("identity"); + expect(yield* fs.readFileString(file)).toBe(before); + expect(yield* fs.exists(path.join(root, "original"))).toBe(false); + expect( + yield* fs.exists(path.join(root, "copied", "data", "vector", "runtime", "vector")), + ).toBe(true); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("starts the owner of a legacy stack and drops Vector from its saved state", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-vector-owner-" }); + const options = { + projectRoot: root, + stateRoot: `${root}/state`, + cacheRoot: `${root}/cache`, + runtime: "native", + } satisfies Parameters[0]; + const created = yield* create(options); + const file = yield* writeLegacyState(options.stateRoot, created.id, root); + + const stack = yield* open({ ...options, id: created.id, startOwner: true }); + yield* Effect.ensuring( + Effect.gen(function* () { + expect((yield* stack.services.list).map(({ id }) => id)).toEqual(["analytics", "mail"]); + expect(yield* stack.composition.describe).toEqual({ + members: [ + { id: "analytics", activation: "eager" }, + { id: "mail", activation: "eager" }, + ], + dependencies: [{ from: "mail", to: "analytics" }], + }); + const migrated = yield* fs.readFileString(file); + expect(yield* savedServices(migrated)).toEqual(["analytics", "mail"]); + + yield* stack.stop; + const reopened = yield* open({ ...options, id: created.id, startOwner: true }); + expect((yield* reopened.services.list).map(({ id }) => id)).toEqual([ + "analytics", + "mail", + ]); + expect(yield* fs.readFileString(file)).toBe(migrated); + }), + destroyTestStack(stack), + ); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); +}); diff --git a/packages/stack/src/Sweep.ts b/packages/stack/src/Sweep.ts index 6817993f21..413b422d0c 100644 --- a/packages/stack/src/Sweep.ts +++ b/packages/stack/src/Sweep.ts @@ -47,9 +47,10 @@ export const reclaimStack = Effect.fn("Sweep.reclaimStack")(function* (options: const saved = yield* state.read(id); if (saved === undefined) return true; const dataRoot = path.join(yield* fs.realPath(options.stateRoot), id, "data"); + yield* Owner.sweepContainers(saved, dataRoot); + yield* state.migrate(id); if (saved.lifetime === "session") yield* destroyStack(state, saved, dataRoot, options.cacheRoot); - else yield* Owner.sweepContainers(saved, dataRoot); return true; }), ).pipe(Effect.timeout(sweepTimeout)); diff --git a/packages/stack/src/composition/Supabase.ts b/packages/stack/src/composition/Supabase.ts index 19fb3eeaf9..7cdf0538c6 100644 --- a/packages/stack/src/composition/Supabase.ts +++ b/packages/stack/src/composition/Supabase.ts @@ -104,13 +104,6 @@ const managedBindings: ReadonlyArray<{ targetKind: "studio", input: "analyticsUrl", }, - { - sourceKind: "analytics", - sourceEndpoint: "http", - output: "url", - targetKind: "vector", - input: "analyticsUrl", - }, { sourceKind: "functions", sourceEndpoint: "http", diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index c8678f5287..559caf0c98 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -45,7 +45,12 @@ import { failureMessage } from "./internal/failure-message.ts"; import * as State from "./State.ts"; import type { SavedStack, StackCredentials, StackKeysInput } from "./State.ts"; import { StackError, type Definition, type Observation } from "./Rpc.ts"; -import { readStackLogs as readPersistedLogs, sinceMillis } from "./host/LogStore.ts"; +import { + readStackLogs as readPersistedLogs, + sinceMillis, + streamStackLogs as streamPersistedLogs, +} from "./host/LogStore.ts"; +import type { LogPosition, LogRecord, StackLogRecord } from "./host/LogRecord.ts"; import { reclaimStack } from "./Sweep.ts"; import { ServiceCreationInput as ServiceCreationInputSchema, @@ -78,7 +83,7 @@ export { StackIdSchema as StackId } from "./identity/StackId.ts"; export type { SavedStack } from "./State.ts"; export type { StackCredentials, StackKeysInput }; export type { Observation } from "./Rpc.ts"; -export type { LogPosition, LogRecord, StackLogRecord } from "./host/LogRecord.ts"; +export type { LogPosition, LogRecord, StackLogRecord }; export type { Command, InitializationCommand, @@ -154,14 +159,23 @@ export interface ServiceInstance { readonly prepare: Effect.Effect; readonly status: Effect.Effect; readonly followStatus: Stream.Stream; - readonly logs: Stream.Stream< - { readonly stream: "stdout" | "stderr"; readonly bytes: Uint8Array }, - StackError - >; + /** Streams the instance's persisted records through its live owner, which it never launches. */ + readonly readLogs: (options?: ReadLogsOptions) => Stream.Stream; readonly credentials: (options?: { readonly from?: "host" | "runtime"; }) => Effect.Effect>, StackError>; } +/** Selects the records {@link ServiceInstance.readLogs} streams. */ +export interface ReadLogsOptions { + /** Starts at this record instead of the oldest retained one; the read fails with `tail`. */ + readonly from?: LogPosition; + /** An ISO-8601 timestamp; older records are skipped. */ + readonly since?: string; + /** Starts with only this many of the latest records; the read fails with `from`. */ + readonly tail?: number; + /** Keeps streaming new records until interrupted; `false` by default. */ + readonly follow?: boolean; +} /** Snapshot placement; `cache` is the default. */ export interface DatabaseSnapshotOptions { /** @@ -647,7 +661,16 @@ const makeHandle = Effect.fn("Stack.makeHandle")(function* ( prepare: call("prepare", (rpc) => rpc.prepareService({ id })), status: call("status", (rpc) => rpc.status({ id }), "attach"), followStatus: stream("followStatus", (rpc) => rpc.followStatus({ id })), - logs: stream("logs", (rpc) => rpc.logs({ id })), + readLogs: (options) => + stream("readLogs", (rpc) => + rpc.readLogs({ + id, + follow: options?.follow ?? false, + ...(options?.from === undefined ? {} : { from: options.from }), + ...(options?.since === undefined ? {} : { since: options.since }), + ...(options?.tail === undefined ? {} : { tail: options.tail }), + }), + ), credentials: (options) => call("credentials", (rpc) => rpc.credentials({ id, from: options?.from ?? "host" })), }); @@ -689,8 +712,6 @@ const makeHandle = Effect.fn("Stack.makeHandle")(function* ( return common(id, "mail"); case "analytics": return common(id, "analytics"); - case "vector": - return common(id, "vector"); case "pooler": return common(id, "pooler"); } @@ -1042,3 +1063,19 @@ export const readStackLogs = Effect.fn("Stack.readStackLogs")( }, Effect.mapError((cause) => failure("readStackLogs", cause)), ); + +/** Streams a stack's persisted records one instance after another, each in file order. */ +export const streamStackLogs = ( + options: Omit, +): Stream.Stream => + Stream.unwrap( + Effect.gen(function* () { + const path = yield* Path.Path; + const root = yield* State.stackLogsRoot(path, options.stateRoot, options.stackId); + return streamPersistedLogs({ + root, + ...(options.instances === undefined ? {} : { instances: options.instances }), + ...(options.since === undefined ? {} : { since: yield* sinceMillis(options.since) }), + }); + }), + ).pipe(Stream.mapError((cause) => failure("streamStackLogs", cause))); diff --git a/packages/stack/src/host/Credentials.ts b/packages/stack/src/host/Credentials.ts index 64705e0452..55d95eed28 100644 --- a/packages/stack/src/host/Credentials.ts +++ b/packages/stack/src/host/Credentials.ts @@ -184,7 +184,6 @@ const credentialInputs: { pgmeta: {}, mail: {}, analytics: {}, - vector: {}, }; /** Names the config inputs the owner fills from the stack credential record. */ diff --git a/packages/stack/src/host/LogStore.integration.test.ts b/packages/stack/src/host/LogStore.integration.test.ts index 1abf095396..e3ca8424c5 100644 --- a/packages/stack/src/host/LogStore.integration.test.ts +++ b/packages/stack/src/host/LogStore.integration.test.ts @@ -415,30 +415,6 @@ describe("LogStore", () => { }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); - it.effect("keeps the live tail flowing while appends fail", () => - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const base = yield* tempRoot("log-store-unwritable-"); - const root = path.join(base, "not-a-directory"); - yield* fs.writeFileString(root, ""); - const { store } = yield* openStore(root); - const instance = yield* fakeInstance("unwritable"); - yield* store.attach(instance); - const lines = yield* Queue.unbounded(); - yield* (yield* store.tail("unwritable")).pipe( - Stream.runForEach(({ text }) => Queue.offer(lines, text)), - Effect.forkScoped, - ); - - yield* instance.publish(instance.chunk(1, "first live line\n")); - yield* instance.publish(instance.chunk(1, "second live line\n")); - const received = yield* takeExactly(lines, 2); - - expect(Array.from(received)).toEqual(["first live line", "second live line"]); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.effect("tails with since past a newer segment that holds an older flushed line", () => Effect.gen(function* () { const root = yield* tempRoot("log-store-late-flush-"); @@ -470,26 +446,6 @@ describe("LogStore", () => { }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); - it.effect("delivers one batch larger than any buffer to a waiting live tail", () => - Effect.gen(function* () { - const root = yield* tempRoot("log-store-live-batch-"); - const { store } = yield* openStore(root); - const instance = yield* fakeInstance("live-batch"); - yield* store.attach(instance); - const lines = yield* Queue.unbounded(); - yield* (yield* store.tail("live-batch")).pipe( - Stream.runForEach(({ text }) => Queue.offer(lines, text)), - Effect.forkScoped, - ); - const expected = Array.from({ length: 2_000 }, (_, index) => `burst line ${index}`); - - yield* instance.publish(instance.chunk(1, `${expected.join("\n")}\n`)); - const received = yield* takeExactly(lines, expected.length); - - expect(received).toEqual(expected); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.effect("keeps sweeping older segments past one it cannot delete", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -625,7 +581,11 @@ describe("LogStore", () => { const all = yield* LogStore.readStackLogs({ root }); const tailed = yield* LogStore.readStackLogs({ root, tail: 2 }); const recent = yield* LogStore.readStackLogs({ root, since: 2_000, instances: ["rest-1"] }); + const streamed = yield* LogStore.streamStackLogs({ root, instances: ["rest-1"] }).pipe( + Stream.runCollect, + ); + expect(Array.from(streamed)).toEqual(all.filter(({ service }) => service === "rest")); expect(all.map(({ service, kind }) => `${service}:${kind}`)).toEqual([ "rest:launch", "rest:stdout", @@ -684,6 +644,21 @@ describe("LogStore", () => { }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); + it.effect("rejects a tailed read that also sets a start position", () => + Effect.gen(function* () { + const root = yield* tempRoot("log-store-tail-from-"); + const { store } = yield* openStore(root); + yield* store.attach(yield* fakeInstance("tail-from")); + + const error = yield* store + .read("tail-from", { from: { generation: 1, byteOffset: 0 }, tail: 5, follow: false }) + .pipe(Effect.flip); + + expect(error).toBeInstanceOf(LogStore.LogStoreError); + expect(error.message).toContain("cannot also set a start position"); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + it.effect("removes an instance by ending its readers and deleting its segments", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/packages/stack/src/host/LogStore.ts b/packages/stack/src/host/LogStore.ts index 8e81a28238..e716b7e069 100644 --- a/packages/stack/src/host/LogStore.ts +++ b/packages/stack/src/host/LogStore.ts @@ -10,7 +10,6 @@ import { Path, PubSub, Ref, - Result, Schema, Scope, Semaphore, @@ -39,8 +38,6 @@ export class LogStoreError extends Schema.TaggedError()("LogStore cause: Schema.optionalKey(Schema.Defect()), }) {} -// Holds a whole pulled batch for a waiting subscriber while bounding a stalled one. -const liveTailCapacity = 16_384; const storeError = (operation: string) => (cause: unknown) => new LogStoreError({ operation, message: failureMessage(cause), cause }); @@ -69,7 +66,7 @@ export interface ReadOptions { readonly from: "oldest" | "end" | LogPosition; /** Epoch milliseconds; older records are skipped. */ readonly since?: number; - /** Returns only the last records of the history before following. */ + /** Returns only the last records of the history before following; requires `from: "oldest"`. */ readonly tail?: number; readonly follow: boolean; } @@ -92,14 +89,6 @@ export interface Interface { ) => Effect.Effect, LogStoreError, Scope.Scope>; /** The directory holding an attached instance's segments and its forwarding cursor. */ readonly directory: (instanceId: string) => Effect.Effect; - /** Streams output lines as they are split, whether or not their append succeeds. */ - readonly tail: ( - instanceId: string, - ) => Effect.Effect< - Stream.Stream<{ readonly stream: "stdout" | "stderr"; readonly text: string }>, - LogStoreError, - Scope.Scope - >; /** Stops the instance's writer and readers, then deletes its segments. */ readonly remove: (instance: { readonly service: string; @@ -405,7 +394,6 @@ const makeReader = (fs: FileSystem.FileSystem, path: Path.Path) => { const listed = yield* generations(directory); const tail = options.tail; if (tail !== undefined) { - if (cursor.generation !== 0) return { records: [], cursor, listed }; if (tail === 0) return { records: [], @@ -473,6 +461,11 @@ const makeReader = (fs: FileSystem.FileSystem, path: Path.Path) => { options: ReadOptions, live: Live | undefined, ) { + if (options.tail !== undefined && options.from !== "oldest") + return yield* new LogStoreError({ + operation: "read", + message: "A tailed read starts at the oldest record and cannot also set a start position", + }); const wake = options.follow && live !== undefined ? yield* PubSub.subscribe(live.wake) : undefined; const start = @@ -506,7 +499,17 @@ const makeReader = (fs: FileSystem.FileSystem, path: Path.Path) => { ); }); - return { read, history, segments, generations }; + /** Streams a directory's records in file order without an owner. */ + const records = (directory: string, options: Pick) => + follow( + directory, + { cursor: { generation: 0, byteOffset: 0 }, listed: undefined }, + { ...options, follow: false }, + undefined, + undefined, + ); + + return { read, history, records, segments, generations }; }; /** Creates the owner's log store; closing its scope flushes and closes every writer. */ @@ -524,11 +527,6 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio readonly service: string; readonly instanceId: string; readonly directory: string; - /** - * Records as they are split, independent of whether their append succeeds. Unbounded so the - * recipe's sliding output buffer stays the only place output is dropped. - */ - readonly live: PubSub.PubSub; readonly closed: Deferred.Deferred; readonly scope: Scope.Closeable; } @@ -582,7 +580,6 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio byteOffset: latest?.size ?? 0, }), wake: yield* PubSub.sliding(1), - live: yield* PubSub.sliding(liveTailCapacity), closed: yield* Deferred.make(), }; const splitter = makeSplitter(); @@ -719,14 +716,13 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio }; /** - * Publishes records to live tails, then appends them. After a failure, appends are skipped - * with backoff and their chunks are reported as `lost` once writing works again. + * Appends records. After a failure, appends are skipped with backoff and their chunks are + * reported as `lost` once writing works again. */ const persist = Effect.fnUntraced(function* ( entries: ReadonlyArray, chunks: ReadonlyArray, ) { - if (entries.length > 0) yield* PubSub.publishAll(handle.live, entries); const now = yield* Clock.currentTimeMillis; if (broken !== undefined && now < broken.retryAt) return countDropped(chunks); if (entries.length === 0 && dropped.size === 0) return; @@ -845,19 +841,6 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio return records.pipe(Stream.interruptWhen(Deferred.await(instance.closed))); }); - const tail = Effect.fn("LogStore.tail")(function* (instanceId: string) { - const instance = yield* attached(instanceId); - const subscription = yield* PubSub.subscribe(instance.live); - return Stream.fromSubscription(subscription).pipe( - Stream.filterMap((entry) => - entry.kind === "stdout" || entry.kind === "stderr" - ? Result.succeed({ stream: entry.kind, text: entry.text }) - : Result.failVoid, - ), - Stream.interruptWhen(Deferred.await(instance.closed)), - ); - }); - const remove = Effect.fn("LogStore.remove")(function* (target: { readonly service: string; readonly instanceId: string; @@ -905,13 +888,49 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio attach, read, directory: (instanceId) => Effect.map(attached(instanceId), (instance) => instance.directory), - tail, remove, removeOrphans: removeOrphans(), close, } satisfies Interface; }); +/** Lists the instance directories of a stack's logs, limited to `instances` when given. */ +const persistedInstances = Effect.fnUntraced(function* ( + fs: FileSystem.FileSystem, + path: Path.Path, + root: string, + instances: ReadonlyArray | undefined, +) { + const selected: Array<{ readonly service: string; readonly instanceId: string }> = []; + for (const service of yield* listDirectories(fs, path, root)) + for (const instanceId of yield* listDirectories(fs, path, path.join(root, service))) + if (instances === undefined || instances.includes(instanceId)) + selected.push({ service, instanceId }); + return selected; +}); + +/** Streams persisted records of a stack's instances, one instance after another in file order. */ +export const streamStackLogs = (options: { + readonly root: string; + readonly instances?: ReadonlyArray; + readonly since?: number; +}): Stream.Stream => + Stream.unwrap( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const reader = makeReader(fs, path); + const selected = yield* persistedInstances(fs, path, options.root, options.instances); + return Stream.fromIterable(selected).pipe( + Stream.flatMap(({ service, instanceId }) => + reader + .records(path.join(options.root, service, instanceId), options) + .pipe(Stream.map((record): StackLogRecord => ({ ...record, service, instanceId }))), + ), + ); + }), + ); + /** Reads persisted records of a stack's instances, merged by time, service, instance and position. */ export const readStackLogs = Effect.fn("LogStore.readStackLogs")(function* (options: { readonly root: string; @@ -922,12 +941,7 @@ export const readStackLogs = Effect.fn("LogStore.readStackLogs")(function* (opti const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const reader = makeReader(fs, path); - const list = (directory: string) => listDirectories(fs, path, directory); - const selected: Array<{ readonly service: string; readonly instanceId: string }> = []; - for (const service of yield* list(options.root)) - for (const instanceId of yield* list(path.join(options.root, service))) - if (options.instances === undefined || options.instances.includes(instanceId)) - selected.push({ service, instanceId }); + const selected = yield* persistedInstances(fs, path, options.root, options.instances); const perInstance = yield* Effect.forEach( selected, ({ service, instanceId }) => diff --git a/packages/stack/src/host/LogflareEvents.ts b/packages/stack/src/host/LogflareEvents.ts index 4e144b1460..432200106e 100644 --- a/packages/stack/src/host/LogflareEvents.ts +++ b/packages/stack/src/host/LogflareEvents.ts @@ -75,6 +75,9 @@ const parsePostgrestTime = (text: string): string | undefined => { ); }; +/** The time and request of PostgREST's Apache combined request line. */ +const postgrestRequest = /^\S+ \S+ \S+ \[([^\]]+)\] "([A-Z]+) (\S+) ([^"\s]+)" (\d{3}) /u; + const withoutProject = ({ project: _project, ...event }: LogflareEvent): LogflareEvent => event; const remaps: Record LogflareEvent> = { @@ -85,6 +88,21 @@ const remaps: Record LogflareEvent> = : { ...event, metadata: { ...event.metadata, timestamp: parsed.time, ...parsed } }; }, rest: (event) => { + const request = postgrestRequest.exec(event.event_message); + const requestTime = request === null ? undefined : parsePostgrestTime(request[1] ?? ""); + if (request !== null && requestTime !== undefined) + return { + ...event, + timestamp: requestTime, + metadata: { + ...event.metadata, + host: event.project, + method: request[2], + path: request[3], + protocol: request[4], + status: Number(request[5]), + }, + }; const match = /^(.*): (.*)$/u.exec(event.event_message); const timestamp = match === null ? undefined : parsePostgrestTime(match[1] ?? ""); return match === null || timestamp === undefined diff --git a/packages/stack/src/host/LogflareEvents.unit.test.ts b/packages/stack/src/host/LogflareEvents.unit.test.ts index 32b4fcecea..ebbf816ad6 100644 --- a/packages/stack/src/host/LogflareEvents.unit.test.ts +++ b/packages/stack/src/host/LogflareEvents.unit.test.ts @@ -33,6 +33,24 @@ it("splits PostgREST's timestamp prefix from the message and converts its zone o }); }); +it("parses a PostgREST request line into request metadata stamped with the request time", () => { + const line = + '172.18.0.1 - anon [28/Sep/2026:12:30:15 +0200] "GET /items?select=id HTTP/1.1" 200 2 "" "curl/8.7.1"'; + expect(logflareEvent("rest", received, line)).toEqual({ + project: "default", + appname: "rest", + event_message: line, + timestamp: "2026-09-28T10:30:15.000Z", + metadata: { + host: "default", + method: "GET", + path: "/items?select=id", + protocol: "HTTP/1.1", + status: 200, + }, + }); +}); + it("keeps a PostgREST line whose prefix is not a timestamp", () => { const event = logflareEvent("rest", received, "Config: schema cache loaded"); expect(event.event_message).toBe("Config: schema cache loaded"); diff --git a/packages/stack/src/index.ts b/packages/stack/src/index.ts index 2088cb2154..cf415fa053 100644 --- a/packages/stack/src/index.ts +++ b/packages/stack/src/index.ts @@ -29,6 +29,7 @@ export type { OpenOptions, PgProveOptions, PlannedInstance, + ReadLogsOptions, ReadStackLogsOptions, SavedStack, StackLogRecord, diff --git a/packages/stack/src/promise-api.integration.test.ts b/packages/stack/src/promise-api.integration.test.ts index d17f2f7792..394f99e4ea 100644 --- a/packages/stack/src/promise-api.integration.test.ts +++ b/packages/stack/src/promise-api.integration.test.ts @@ -1,7 +1,7 @@ import { NodeServices, NodeSocketServer } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; import { Effect, Exit, FileSystem, Redacted, Schema } from "effect"; -import { discover, StackError, type Observation } from "./index.ts"; +import { discover, StackError, type LogRecord, type Observation } from "./index.ts"; import { createTestStack } from "./testing.ts"; const databaseSecret = (observation: Observation) => @@ -25,6 +25,13 @@ it.live( const stopped = yield* Effect.promise(() => database.status()); expect(Exit.isExit(stopped.exit)).toBe(true); expect(Redacted.isRedacted(databaseSecret(stopped))).toBe(true); + // oxlint-disable-next-line effecttsgo/async-function -- This test exercises the Promise client API. + const tailed = yield* Effect.promise(async () => { + const records: Array = []; + for await (const record of database.readLogs({ tail: 1 })) records.push(record); + return records; + }); + expect(tailed).toHaveLength(1); yield* Effect.promise(() => test.stack.composition.start()); const members = yield* Effect.promise(() => test.stack.composition.stop()); diff --git a/packages/stack/src/promise-api.unit.test.ts b/packages/stack/src/promise-api.unit.test.ts index 499801b95d..3f0cf7c683 100644 --- a/packages/stack/src/promise-api.unit.test.ts +++ b/packages/stack/src/promise-api.unit.test.ts @@ -77,6 +77,9 @@ describe("Promise API derived from the Effect API", () => { expectTypeOf().toEqualTypeOf< () => AsyncIterable >(); + expectTypeOf().toEqualTypeOf< + (options?: PromiseApi.ReadLogsOptions) => AsyncIterable + >(); }); it("types created and returned handles by service kind", () => { diff --git a/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts b/packages/stack/src/services/AnalyticsImgproxy.integration.test.ts similarity index 72% rename from packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts rename to packages/stack/src/services/AnalyticsImgproxy.integration.test.ts index de79a2f854..134c3f0ea1 100644 --- a/packages/stack/src/services/AnalyticsVectorImgproxy.integration.test.ts +++ b/packages/stack/src/services/AnalyticsImgproxy.integration.test.ts @@ -4,7 +4,7 @@ import { Effect, FileSystem, Layer, Redacted } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { makeService } from "../Service.ts"; import { makeServiceRecipe } from "./Catalog.ts"; -import { makeDockerHttpRelay, makeDockerTcpRelay } from "../../tests/docker-relay.ts"; +import { makeDockerTcpRelay } from "../../tests/docker-relay.ts"; import { makeDockerDatabaseRoot } from "../../tests/docker-fixture.ts"; const options = (root: string) => ({ @@ -22,7 +22,7 @@ const dockerOptions = (root: string) => ({ describe("service catalog", () => { it.live( - "serves Analytics, Vector, and Imgproxy with their real endpoints", + "serves Analytics and Imgproxy with their real endpoints", () => Effect.scoped( Effect.gen(function* () { @@ -65,7 +65,6 @@ describe("service catalog", () => { yield* analytics.start; yield* analytics.ready; const analyticsEndpoint = yield* analyticsRecipe.endpoint("http"); - const analyticsRelay = yield* makeDockerHttpRelay(analyticsRecipe.endpoint("http")); const analyticsResponse = yield* client.execute( HttpClientRequest.get( `http://${analyticsEndpoint.host}:${analyticsEndpoint.port}/health`, @@ -73,34 +72,6 @@ describe("service catalog", () => { ); expect(analyticsResponse.status).toBe(200); - const vectorRecipe = yield* makeServiceRecipe( - { - service: "vector", - config: { - analyticsUrl: `http://${analyticsRelay.host}:${analyticsRelay.port}`, - apiKey: "catalog-analytics", - configPath: `${root}/vector.yaml`, - }, - }, - dockerOptions(root), - ); - yield* fs.writeFileString( - `${root}/vector.yaml`, - "sources:\n dummy:\n type: demo_logs\n format: syslog\n interval: 60\n" + - "sinks:\n print:\n type: console\n inputs: [dummy]\n encoding:\n codec: json\n", - ); - const vector = yield* makeService(vectorRecipe.definition, { - id: "vector", - config: vectorRecipe.creation, - }); - yield* vector.start; - yield* vector.ready; - const vectorEndpoint = yield* vectorRecipe.endpoint("http"); - const vectorResponse = yield* client.execute( - HttpClientRequest.get(`http://${vectorEndpoint.host}:${vectorEndpoint.port}/health`), - ); - expect(vectorResponse.status).toBe(200); - const imageRoot = `${root}/images`; yield* fs.makeDirectory(imageRoot, { recursive: true }); const imgproxyRecipe = yield* makeServiceRecipe( @@ -122,7 +93,6 @@ describe("service catalog", () => { expect(imgproxyResponse.status).toBe(200); yield* imgproxy.stop; - yield* vector.stop; yield* analytics.stop; yield* database.stop; }), diff --git a/packages/stack/src/services/Catalog.ts b/packages/stack/src/services/Catalog.ts index ce843a8a1d..be28c3d37b 100644 --- a/packages/stack/src/services/Catalog.ts +++ b/packages/stack/src/services/Catalog.ts @@ -21,7 +21,6 @@ import * as Realtime from "./Realtime.ts"; import * as Rest from "./Rest.ts"; import * as Storage from "./Storage.ts"; import * as Studio from "./Studio.ts"; -import * as Vector from "./Vector.ts"; import { CatalogError, serviceCreation, @@ -47,7 +46,6 @@ const endpointSchemas = [ ["pgmeta", Pgmeta.Endpoints], ["mail", Mail.Endpoints], ["analytics", Analytics.Endpoints], - ["vector", Vector.Endpoints], ["pooler", Pooler.Endpoints], ] as const; @@ -87,7 +85,6 @@ export const ServiceCreation = Schema.Union([ Pgmeta.Creation, Mail.Creation, Analytics.Creation, - Vector.Creation, Pooler.Creation, ]); export type ServiceCreation = Schema.Schema.Type; @@ -101,7 +98,6 @@ const requiredInputs: { readonly [K in ServiceKind]?: ReadonlyArray } = pgmeta: ["databaseUrl"], analytics: ["databaseUrl"], pooler: ["databaseUrl"], - vector: ["analyticsUrl"], }; /** Rejects a creation that lacks a required input before any lifecycle change. */ @@ -137,7 +133,6 @@ export const ServiceCreationInput = Schema.Union([ Pgmeta.Creation, Mail.Creation, Analytics.Creation, - Vector.Creation, Pooler.Creation, ]); export type ServiceCreationInput = Schema.Schema.Type; @@ -153,7 +148,6 @@ export const serviceSchemas = { pgmeta: Pgmeta.Config, mail: Mail.Config, analytics: Analytics.Config, - vector: Vector.Config, pooler: Pooler.Config, } as const; @@ -408,12 +402,6 @@ export const makeServiceRecipe = Effect.fn("Catalog.makeServiceRecipe")( yield* makeProcessRecipe(options, deps, Analytics.makeSpec()), Schema.is(Analytics.Creation), ); - case "vector": - return catalogRecipe( - creation, - yield* Vector.makeRecipe(options, deps), - Schema.is(Vector.Creation), - ); case "pooler": return catalogRecipe( creation, diff --git a/packages/stack/src/services/Functions.integration.test.ts b/packages/stack/src/services/Functions.integration.test.ts index 15c1d34ac7..c867b6ff96 100644 --- a/packages/stack/src/services/Functions.integration.test.ts +++ b/packages/stack/src/services/Functions.integration.test.ts @@ -260,7 +260,7 @@ describe("service catalog", () => { }, ); const logs = yield* Ref.make(""); - yield* recipe.logs.pipe( + yield* Stream.fromSubscription(yield* recipe.logs).pipe( Stream.runForEach(({ bytes }) => Ref.update(logs, (text) => text + new TextDecoder().decode(bytes)), ), @@ -332,7 +332,7 @@ describe("service catalog", () => { ); const logs = yield* Ref.make(""); const warned = yield* Deferred.make(); - yield* recipe.logs.pipe( + yield* Stream.fromSubscription(yield* recipe.logs).pipe( Stream.runForEach(({ bytes }) => Ref.updateAndGet(logs, (text) => text + new TextDecoder().decode(bytes)).pipe( Effect.flatMap((text) => @@ -534,7 +534,6 @@ it.effect("passes POSIX project paths to a docker Functions container from a Win }, }; const recipe = yield* Functions.makeRecipe( - creation, { stackId: "e".repeat(64), instanceId: "windows", @@ -553,7 +552,7 @@ it.effect("passes POSIX project paths to a docker Functions container from a Win ); const scope = yield* Scope.make(); yield* recipe.definition - .launch({ id: "windows", config: creation, scope }) + .launch({ id: "windows", config: creation, scope, launchId: 1 }) .pipe(Effect.flip, Effect.ensuring(Scope.close(scope, Exit.void))); const spec = yield* Ref.get(launched); diff --git a/packages/stack/src/services/Rest.ts b/packages/stack/src/services/Rest.ts index bbd80ef641..0d913c6158 100644 --- a/packages/stack/src/services/Rest.ts +++ b/packages/stack/src/services/Rest.ts @@ -42,6 +42,8 @@ export const makeSpec = (): ProcessRecipeSpec => ({ : { PGRST_DB_EXTRA_SEARCH_PATH: creation.config.extraSearchPath }), PGRST_DB_ANON_ROLE: creation.config.anonRole ?? "anon", PGRST_DB_MAX_ROWS: String(creation.config.maxRows ?? 1000), + // `info` logs every request, which Studio's PostgREST logs page shows. + PGRST_LOG_LEVEL: "info", ...(jwtSecret === undefined ? {} : { PGRST_JWT_SECRET: jwtSecret }), ...(creation.config.externalApiUrl === undefined ? {} diff --git a/packages/stack/src/services/Vector.integration.test.ts b/packages/stack/src/services/Vector.integration.test.ts deleted file mode 100644 index 2eba9f07ec..0000000000 --- a/packages/stack/src/services/Vector.integration.test.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { NodeHttpClient, NodeServices } from "@effect/platform-node"; -import { describe, expect, it } from "@effect/vitest"; -import { tmpdir } from "node:os"; -import { Effect, FileSystem, Layer } from "effect"; -import { HttpClient, HttpClientRequest } from "effect/unstable/http"; -import { makeService } from "../Service.ts"; -import { makeServiceRecipe } from "./Catalog.ts"; - -const options = (root: string, runtime: "docker" | "native") => ({ - stackId: "catalog-test", - instanceId: "vector", - root, - cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, - runtime, -}); - -describe("vector recipe", () => { - for (const runtime of ["docker", "native"] as const) { - it.live( - `serves health without a custom config and cleans up on destroy, including interrupted writes (${runtime})`, - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const client = yield* HttpClient.HttpClient; - const root = yield* fs.makeTempDirectoryScoped({ - prefix: `catalog-vector-${runtime}-`, - }); - const recipe = yield* makeServiceRecipe( - { - service: "vector", - config: { analyticsUrl: "http://analytics", apiKey: "api-key" }, - endpoints: { http: { port: "auto" } }, - }, - options(root, runtime), - ); - const vector = yield* makeService(recipe.definition, { - id: "vector", - config: recipe.creation, - }); - yield* vector.start; - yield* vector.ready; - const endpoint = yield* recipe.endpoint("http"); - const response = yield* client.execute( - HttpClientRequest.get(`http://${endpoint.host}:${endpoint.port}/health`), - ); - expect(response.status).toBe(200); - yield* fs.makeDirectory(`${root}/vector/runtime/vector/.vector-write-interrupted`); - yield* vector.destroy; - expect(yield* fs.exists(`${root}/vector`)).toBe(false); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), - { timeout: 120_000 }, - ); - } - - it.live( - "keeps a caller pipeline stored beside the recipe config on destroy", - () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-vector-caller-" }); - const pipeline = `${root}/vector/runtime/vector/pipeline.yaml`; - yield* fs.makeDirectory(`${root}/vector/runtime/vector`, { recursive: true }); - yield* fs.writeFileString( - pipeline, - "sources:\n s:\n type: internal_logs\nsinks:\n d:\n type: blackhole\n inputs: [s]\n", - ); - const recipe = yield* makeServiceRecipe( - { - service: "vector", - config: { analyticsUrl: "http://analytics", configPath: pipeline }, - endpoints: { http: { port: "auto" } }, - }, - options(root, "docker"), - ); - const vector = yield* makeService(recipe.definition, { - id: "vector", - config: recipe.creation, - }); - yield* vector.start; - yield* vector.ready; - yield* vector.destroy; - expect(yield* fs.exists(pipeline)).toBe(true); - expect(yield* fs.exists(`${root}/vector/runtime/vector/vector-api.yaml`)).toBe(false); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), - { timeout: 120_000 }, - ); - - it.live("rejects a caller pipeline that resolves to a stack-owned config file", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const root = yield* fs.makeTempDirectoryScoped({ prefix: "catalog-vector-reject-" }); - const owned = `${root}/vector/runtime/vector/vector-api.yaml`; - yield* fs.makeDirectory(`${root}/vector/runtime/vector`, { recursive: true }); - yield* fs.writeFileString(owned, "api:\n enabled: true\n"); - yield* fs.symlink(owned, `${root}/alias.yaml`); - const start = (configPath: string) => - Effect.gen(function* () { - const recipe = yield* makeServiceRecipe( - { - service: "vector", - config: { analyticsUrl: "http://analytics", configPath }, - endpoints: { http: { port: "auto" } }, - }, - options(root, "docker"), - ); - const vector = yield* makeService(recipe.definition, { - id: "vector", - config: recipe.creation, - }); - return yield* vector.start.pipe(Effect.flip); - }); - for (const configPath of [owned, `${root}/alias.yaml`]) - expect((yield* start(configPath)).message).toContain("stack-owned Vector config file"); - }), - ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), - ); -}); diff --git a/packages/stack/src/services/Vector.ts b/packages/stack/src/services/Vector.ts deleted file mode 100644 index 4f0087cc7e..0000000000 --- a/packages/stack/src/services/Vector.ts +++ /dev/null @@ -1,244 +0,0 @@ -import { Effect, type FileSystem, type Path, Schema } from "effect"; -import { ServiceError } from "../Service.ts"; -import { type CatalogOptions, EndpointIntent, serviceCreation } from "./Recipe.ts"; -import { requiredInput } from "./ServiceConfig.ts"; -import { - makeProcessRecipe, - type ProcessDependencies, - type ProcessRecipeSpec, -} from "./ProcessRecipe.ts"; - -export const Config = Schema.Struct({ - analyticsUrl: Schema.optionalKey(Schema.String), - apiKey: Schema.optionalKey(Schema.String), - /** Pipeline config without an `api` block; the recipe adds its own. */ - configPath: Schema.optionalKey(Schema.String), -}); -export interface Config extends Schema.Schema.Type {} -export const Endpoints = Schema.Struct({ http: Schema.optionalKey(EndpointIntent) }); -export interface Endpoints extends Schema.Schema.Type {} -export const Creation = serviceCreation("vector", Config, Endpoints); -export interface Creation extends Schema.Schema.Type {} - -/** - * Vector has no API flag or env var, so the recipe loads this alongside the pipeline config. - * `address` is written into the file because Vector 0.58 disables `${VAR}` interpolation. - */ -const apiConfigFor = (address: string | undefined): string => - address === undefined - ? "api:\n enabled: false\n" - : `api:\n enabled: true\n address: "${address}"\n`; - -/** - * Substitutes the recipe-owned `${VAR}` names a caller pipeline (`Config.configPath`) may use, so - * it runs without `--dangerously-allow-env-var-interpolation`, which exposes every env var. - */ -const renderKnownPlaceholders = ( - content: string, - values: Readonly>, -): string => - Object.entries(values).reduce( - (rendered, [name, value]) => - value === undefined ? rendered : rendered.replaceAll(`\${${name}}`, value), - content, - ); - -// Vector requires at least one source and sink; the default forwards nothing. -const defaultPipelineConfig = [ - "sources:", - " vector_logs:", - " type: internal_logs", - "sinks:", - " discard:", - " type: blackhole", - " inputs: [vector_logs]", - " print_interval_secs: 0", - "", -].join("\n"); - -const containerPipelinePath = "/etc/vector/vector.yaml"; -const containerApiPath = "/etc/supabase/vector-api.yaml"; -const temporaryPrefix = ".vector-write-"; - -const writeAtomically = Effect.fn("Vector.writeAtomically")( - function* (fs: FileSystem.FileSystem, path: Path.Path, target: string, content: string) { - const directory = path.dirname(target); - yield* fs.makeDirectory(directory, { recursive: true, mode: 0o700 }); - yield* Effect.acquireUseRelease( - fs.makeTempDirectory({ directory, prefix: temporaryPrefix }), - (temporaryDirectory) => - Effect.gen(function* () { - const temporary = path.join(temporaryDirectory, path.basename(target)); - yield* fs.writeFileString(temporary, content, { mode: 0o644 }); - yield* fs.rename(temporary, target); - }), - (temporaryDirectory) => - fs - .remove(temporaryDirectory, { recursive: true, force: true }) - .pipe(Effect.catchTag("PlatformError", () => Effect.void)), - ); - }, - Effect.mapError( - (cause) => - new ServiceError({ operation: "prepare", message: "Unable to write Vector config", cause }), - ), -); - -const makeSpec = ( - instanceId: string, - instanceRoot: string, - fs: FileSystem.FileSystem, - path: Path.Path, -): ProcessRecipeSpec => { - const configRoot = path.join(instanceRoot, "runtime", "vector"); - const apiConfigPath = path.join(configRoot, "vector-api.yaml"); - const defaultPipelinePath = path.join(configRoot, "vector.yaml"); - const renderedPipelinePath = path.join(configRoot, "vector.rendered.yaml"); - /** What Vector actually loads: a caller's file is rendered to a recipe-owned copy first. */ - const resolvedPipelinePath = (creation: Creation) => - creation.config.configPath === undefined ? defaultPipelinePath : renderedPipelinePath; - const ownedInstance = (operation: string) => - /^[a-zA-Z0-9_-]+$/u.test(instanceId) - ? Effect.void - : Effect.fail(new ServiceError({ operation, message: "Invalid Vector instance identity" })); - return { - service: "vector", - executable: "bin/vector", - ports: { http: 9001 }, - healthPath: "/health", - env: (creation, _endpoints, _container) => - requiredInput("vector", "analyticsUrl", creation.config.analyticsUrl).pipe( - Effect.map((analyticsUrl) => ({ - LOGFLARE_URL: analyticsUrl, - ...(creation.config.apiKey === undefined - ? {} - : { LOGFLARE_PRIVATE_ACCESS_TOKEN: creation.config.apiKey }), - })), - ), - args: (creation, endpoints, context) => - Effect.gen(function* () { - const http = endpoints.get("http"); - const address = - http === undefined - ? undefined - : `${context.container ? "0.0.0.0" : "127.0.0.1"}:${http.port}`; - yield* writeAtomically(fs, path, apiConfigPath, apiConfigFor(address)); - if (creation.config.configPath !== undefined) { - const source = yield* fs.readFileString(creation.config.configPath).pipe( - Effect.mapError( - (cause) => - new ServiceError({ - operation: "prepare", - message: "Unable to read Vector configPath", - cause, - }), - ), - ); - const rendered = renderKnownPlaceholders(source, { - LOGFLARE_URL: creation.config.analyticsUrl, - LOGFLARE_PRIVATE_ACCESS_TOKEN: creation.config.apiKey, - }); - yield* writeAtomically(fs, path, renderedPipelinePath, rendered); - } - return context.container - ? ["--config", containerPipelinePath, "--config", containerApiPath] - : ["--config", resolvedPipelinePath(creation), "--config", apiConfigPath]; - }), - mounts: (creation) => - Effect.succeed([ - { source: resolvedPipelinePath(creation), target: containerPipelinePath, readOnly: true }, - { source: apiConfigPath, target: containerApiPath, readOnly: true }, - ]), - startupCommands: [], - prepare: (creation) => - Effect.gen(function* () { - yield* ownedInstance("prepare"); - const callerPath = creation.config.configPath; - if (callerPath !== undefined) { - const canonical = (file: string) => - fs - .exists(file) - .pipe( - Effect.flatMap((exists) => - exists ? fs.realPath(file) : Effect.succeed(path.resolve(file)), - ), - ); - const caller = yield* fs.realPath(callerPath).pipe( - Effect.mapError( - (cause) => - new ServiceError({ - operation: "prepare", - message: "Unable to read Vector configPath", - cause, - }), - ), - ); - const owned = yield* Effect.all([ - canonical(apiConfigPath), - canonical(defaultPipelinePath), - canonical(renderedPipelinePath), - ]).pipe( - Effect.mapError( - (cause) => - new ServiceError({ - operation: "prepare", - message: "Unable to resolve Vector config paths", - cause, - }), - ), - ); - if (owned.includes(caller)) - return yield* new ServiceError({ - operation: "prepare", - message: "Vector configPath must not point at a stack-owned Vector config file", - }); - } - // A safe placeholder until `args` writes the real address — the listening port isn't - // known until endpoints are reserved for an actual launch, which happens after `prepare`. - yield* writeAtomically(fs, path, apiConfigPath, apiConfigFor(undefined)); - if (callerPath === undefined) - yield* writeAtomically(fs, path, defaultPipelinePath, defaultPipelineConfig); - }), - // A caller configPath may live anywhere under the instance root, so only recipe files and empty directories go. - removeData: () => - Effect.gen(function* () { - yield* ownedInstance("destroy"); - yield* fs.remove(apiConfigPath, { force: true }); - yield* fs.remove(defaultPipelinePath, { force: true }); - yield* fs.remove(renderedPipelinePath, { force: true }); - if (yield* fs.exists(configRoot)) - for (const entry of yield* fs.readDirectory(configRoot)) - if (entry.startsWith(temporaryPrefix)) - yield* fs.remove(path.join(configRoot, entry), { recursive: true, force: true }); - for (const directory of [configRoot, path.dirname(configRoot), instanceRoot]) { - if (!(yield* fs.exists(directory))) continue; - if ((yield* fs.readDirectory(directory)).length > 0) return; - yield* fs.remove(directory, { recursive: true }); - } - }).pipe( - Effect.mapError((cause) => - cause instanceof ServiceError - ? cause - : new ServiceError({ - operation: "destroy", - message: "Unable to remove Vector config", - cause, - }), - ), - ), - }; -}; - -export const makeRecipe = Effect.fn("Vector.makeRecipe")( - (options: CatalogOptions, deps: ProcessDependencies) => - makeProcessRecipe( - options, - deps, - makeSpec( - options.instanceId, - deps.path.join(options.root, options.instanceId), - deps.fs, - deps.path, - ), - ), -); diff --git a/packages/stack/tests/public-client-fixture.ts b/packages/stack/tests/public-client-fixture.ts index 5e905361ef..4f9bdaa20e 100644 --- a/packages/stack/tests/public-client-fixture.ts +++ b/packages/stack/tests/public-client-fixture.ts @@ -20,8 +20,8 @@ try { }); // oxlint-disable-next-line effecttsgo/async-function -- This fixture exercises the Promise client API. readLogs = (async () => { - for await (const entry of mail.logs()) { - logs += `[${entry.stream}] ${new TextDecoder().decode(entry.bytes)}`; + for await (const record of mail.readLogs({ follow: true })) { + if (record.text !== undefined) logs += `[${record.kind}] ${record.text}\n`; } })().catch((cause) => { logReaderFailure = cause; diff --git a/packages/stack/tests/whole-stack/analytics.ts b/packages/stack/tests/whole-stack/analytics.ts index 810025b9fe..898d28acae 100644 --- a/packages/stack/tests/whole-stack/analytics.ts +++ b/packages/stack/tests/whole-stack/analytics.ts @@ -21,37 +21,6 @@ class AnalyticsFlowError extends Schema.TaggedError()( }, ) {} -/** Builds the Vector config used by the whole-stack Analytics flow. */ -export const vectorAnalyticsConfig = (marker: string): string => - `sources: - demo_logs: - type: demo_logs - format: json - interval: 1 -transforms: - fixture_marker: - type: remap - inputs: [demo_logs] - source: | - .event_message = ${JSON.stringify(marker)} - .message = ${JSON.stringify(marker)} -sinks: - analytics: - type: http - inputs: [fixture_marker] - uri: "${"${LOGFLARE_URL}"}/api/logs?source_name=postgres.logs" - method: post - encoding: - codec: json - request: - headers: - x-api-key: "${"${LOGFLARE_PRIVATE_ACCESS_TOKEN}"}" - batch: - max_events: 1 - timeout_secs: 1 - healthcheck: false -`; - const request = Effect.fn("WholeStack.analyticsRequest")( (request: HttpClientRequest.HttpClientRequest) => Effect.gen(function* () { @@ -60,7 +29,7 @@ const request = Effect.fn("WholeStack.analyticsRequest")( }).pipe(Effect.mapError((cause) => new AnalyticsFlowError({ operation: "request", cause }))), ); -/** Sends one marker directly to Logflare, for separating API ingest from Vector ingest. */ +/** Sends one marker directly to Logflare. */ const postAnalyticsMarker = Effect.fn("WholeStack.postAnalyticsMarker")( (analyticsUrl: string, apiKey: string, marker: string, timestampMillis: number) => Effect.gen(function* () { @@ -144,25 +113,22 @@ const queryMarkerUntilVisible = Effect.fn("WholeStack.queryMarkerUntilVisible")( ), ); -/** Exercises direct and Vector Analytics ingestion for one whole-stack phase. */ +/** Names the marker every phase of a stack ingests, for cross-stack isolation checks. */ +export const stackAnalyticsMarker = (stackId: string): string => `stack-${stackId}`; + +/** Exercises Analytics ingestion for one whole-stack phase. */ export const exerciseAnalytics = Effect.fn("WholeStack.exerciseAnalytics")( - (analyticsUrl: string, vectorUrl: string, apiKey: string, stackId: string, phase: string) => + (analyticsUrl: string, apiKey: string, stackId: string, phase: string) => Effect.gen(function* () { const phaseStartedAtMillis = yield* Clock.currentTimeMillis; - const directMarker = `direct-${stackId}-${phase}-${phaseStartedAtMillis}`; - const vectorMarker = `vector-${stackId}`; - yield* postAnalyticsMarker(analyticsUrl, apiKey, directMarker, phaseStartedAtMillis); - const vectorHealth = yield* request(HttpClientRequest.get(`${vectorUrl}/health`)); - yield* vectorHealth.text; - if (vectorHealth.status >= 400) - return yield* new AnalyticsFlowError({ - operation: "vector-health", - cause: vectorHealth.status, - }); + const phaseMarker = `direct-${stackId}-${phase}-${phaseStartedAtMillis}`; + const stackMarker = stackAnalyticsMarker(stackId); + yield* postAnalyticsMarker(analyticsUrl, apiKey, phaseMarker, phaseStartedAtMillis); + yield* postAnalyticsMarker(analyticsUrl, apiKey, stackMarker, phaseStartedAtMillis); yield* Effect.all( [ - queryMarkerUntilVisible(analyticsUrl, apiKey, directMarker, phaseStartedAtMillis), - queryMarkerUntilVisible(analyticsUrl, apiKey, vectorMarker, phaseStartedAtMillis), + queryMarkerUntilVisible(analyticsUrl, apiKey, phaseMarker, phaseStartedAtMillis), + queryMarkerUntilVisible(analyticsUrl, apiKey, stackMarker, phaseStartedAtMillis), ], { concurrency: 2, discard: true }, ); diff --git a/packages/stack/tests/whole-stack/fixture.ts b/packages/stack/tests/whole-stack/fixture.ts index 98e048cf87..c34c578377 100644 --- a/packages/stack/tests/whole-stack/fixture.ts +++ b/packages/stack/tests/whole-stack/fixture.ts @@ -17,7 +17,6 @@ import { homedir, tmpdir } from "node:os"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { create, type Stack } from "../../src/effect.ts"; import type { Observation } from "../../src/Rpc.ts"; -import { vectorAnalyticsConfig } from "./analytics.ts"; import { cleanupDockerRoot } from "../docker-cleanup.ts"; import { destroyTestStack } from "../stack-cleanup.ts"; @@ -35,7 +34,6 @@ export const serviceNames = [ "pgmeta", "mail", "analytics", - "vector", "pooler", ] as const; @@ -62,10 +60,10 @@ const watchServiceLogs = Effect.fn("WholeStack.watchServiceLogs")( services, (instance) => Effect.forkScoped( - instance.logs.pipe( - Stream.runForEach(({ bytes }) => + instance.readLogs({ follow: true }).pipe( + Stream.runForEach((record) => Ref.update(logTails, (tails) => { - const text = new TextDecoder().decode(bytes); + const text = record.text === undefined ? "" : `${record.text}\n`; const existing = tails.find(([name]) => name === instance.service)?.[1] ?? ""; const updated = `${existing}${text}`.slice(-8192); const without = tails.filter(([name]) => name !== instance.service); @@ -119,7 +117,6 @@ export const wholeStack = Effect.fn("WholeStack.fixture")((runtime: Runtime) => }); const functionsRoot = `${root}/functions`; const storageRoot = `${root}/storage`; - const vectorConfigPath = `${root}/vector.yaml`; yield* fs.makeDirectory(`${functionsRoot}/hello`, { recursive: true }); yield* fs.makeDirectory(storageRoot, { recursive: true }); yield* fs.writeFileString( @@ -138,7 +135,6 @@ export const wholeStack = Effect.fn("WholeStack.fixture")((runtime: Runtime) => runtime, name: `whole-${runtime}`, }); - yield* fs.writeFileString(vectorConfigPath, vectorAnalyticsConfig(`vector-${stack.id}`)); const owner = yield* Ref.make>(Option.some(stack)); yield* Effect.addFinalizer(() => Effect.gen(function* () { @@ -210,14 +206,6 @@ export const wholeStack = Effect.fn("WholeStack.fixture")((runtime: Runtime) => config: { backend: "postgres", apiKey: secret }, endpoints: { http: endpoint("auto") }, }, - { - service: "vector", - config: { - apiKey: secret, - configPath: vectorConfigPath, - }, - endpoints: { http: endpoint("auto") }, - }, { service: "pooler", config: { diff --git a/packages/stack/tests/whole-stack/scenarios.ts b/packages/stack/tests/whole-stack/scenarios.ts index 5edcc136fa..0c1e42633b 100644 --- a/packages/stack/tests/whole-stack/scenarios.ts +++ b/packages/stack/tests/whole-stack/scenarios.ts @@ -23,7 +23,7 @@ import { exerciseMetadataAndPooler, exerciseStorageAndFunctions, } from "./service-flows.ts"; -import { exerciseAnalytics, queryAnalyticsMarker } from "./analytics.ts"; +import { exerciseAnalytics, queryAnalyticsMarker, stackAnalyticsMarker } from "./analytics.ts"; import { subscribeRealtime } from "./websocket.ts"; import { watchRecoveryMail } from "./realtime-mail.ts"; import { assertWorkloadsGone, captureWorkloads } from "./workloads.ts"; @@ -289,10 +289,8 @@ const exerciseStack = Effect.fn("WholeStack.exerciseStack")( }), ); const analyticsUrl = (yield* service(fixture, "analytics").credentials()).url; - const vectorUrl = (yield* service(fixture, "vector").credentials()).url; - if (analyticsUrl === undefined || vectorUrl === undefined) - return yield* Effect.die("Analytics or Vector URL missing"); - yield* exerciseAnalytics(analyticsUrl, vectorUrl, fixture.secret, fixture.stack.id, phase); + if (analyticsUrl === undefined) return yield* Effect.die("Analytics URL missing"); + yield* exerciseAnalytics(analyticsUrl, fixture.secret, fixture.stack.id, phase); yield* assertLifecycle(fixture, "running"); return { accessToken: loginBody.access_token, @@ -444,12 +442,20 @@ export const parallel = (runtime: Runtime) => if (leftAnalyticsUrl === undefined || rightAnalyticsUrl === undefined) return yield* Effect.die("Parallel Analytics URL missing"); expect( - (yield* queryAnalyticsMarker(leftAnalyticsUrl, left.secret, `vector-${right.stack.id}`, 0)) - .length, + (yield* queryAnalyticsMarker( + leftAnalyticsUrl, + left.secret, + stackAnalyticsMarker(right.stack.id), + 0, + )).length, ).toBe(0); expect( - (yield* queryAnalyticsMarker(rightAnalyticsUrl, right.secret, `vector-${left.stack.id}`, 0)) - .length, + (yield* queryAnalyticsMarker( + rightAnalyticsUrl, + right.secret, + stackAnalyticsMarker(left.stack.id), + 0, + )).length, ).toBe(0); const rightRest = service(right, "rest"); const rightRestUrl = (yield* rightRest.credentials()).url; From 2ba2d898f3bdbaa525983e83579328b7a1ea5620 Mon Sep 17 00:00:00 2001 From: avallete Date: Thu, 1 Oct 2026 10:49:05 +0200 Subject: [PATCH 04/35] fix(stack): keep the Vector artifact pin for the legacy start The artifacts catalog is also the version table the legacy `supabase start` reads in slim mode, so dropping Vector's entry would silently fall back to the upstream Vector image there. Vector stays in the catalog as an artifact kind that only the legacy start runs; the experimental stack's service kinds, `supabase services` and stack prepare still exclude it. Docker-backed log tests get the same timeout guard as the package's other Docker tests. Co-Authored-By: Claude Opus 5.5 --- .../start-export-pointer.integration.test.ts | 2 +- .../start/start.slim-images.e2e.test.ts | 4 +- packages/stack/src/Artifacts.ts | 54 +++++++++++++++---- .../stack/src/Owner.logs.integration.test.ts | 2 + packages/stack/src/internal/artifacts.ts | 1 + 5 files changed, 49 insertions(+), 14 deletions(-) diff --git a/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts index a15901b14b..b0b01e52d0 100644 --- a/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts @@ -84,7 +84,7 @@ function makeDatabaseStack(sqlPort: number, credentials: StackCredentials): Stac prepare: Effect.die("unused"), status: Effect.succeed(observation), followStatus: Stream.empty, - logs: Stream.empty, + readLogs: () => Stream.empty, credentials: () => Effect.succeed({}), saveSnapshot: () => Effect.die("unused"), restoreSnapshot: () => Effect.die("unused"), diff --git a/apps/cli/src/commands/start/start.slim-images.e2e.test.ts b/apps/cli/src/commands/start/start.slim-images.e2e.test.ts index 2a44aacfb3..8ee679bc6d 100644 --- a/apps/cli/src/commands/start/start.slim-images.e2e.test.ts +++ b/apps/cli/src/commands/start/start.slim-images.e2e.test.ts @@ -2,7 +2,7 @@ import { FetchHttpClient, HttpClient, HttpClientRequest } from "effect/unstable/ import { BunServices } from "@effect/platform-bun"; import { Clock, Data, Effect, FileSystem, Layer, Path, Schema } from "effect"; import { beforeAll, describe, expect, it } from "@effect/vitest"; -import { catalogPins, type ServiceKind } from "@supabase/stack/internal/artifacts"; +import { catalogPins, type ArtifactKind } from "@supabase/stack/internal/artifacts"; import { dockerfileServiceImageRaw } from "../../shared/services/dockerfile-images.ts"; import { isSlimImageRef, toSlimImage } from "../../shared/services/slim-images.ts"; @@ -73,7 +73,7 @@ const PULL_ALIASES = [ * Dockerfile-derived image: whether the Dockerfile's tag currently matches that catalog pin is * unrelated to whether the pinned slim image itself accepts the BusyBox argv). */ -const WGET_PROBE_SERVICES: ReadonlyArray = [ +const WGET_PROBE_SERVICES: ReadonlyArray = [ "auth", "realtime", "storage", diff --git a/packages/stack/src/Artifacts.ts b/packages/stack/src/Artifacts.ts index 31d5f88382..2651be440d 100644 --- a/packages/stack/src/Artifacts.ts +++ b/packages/stack/src/Artifacts.ts @@ -21,6 +21,11 @@ export type ServiceKind = | "analytics" | "pooler"; +/** A catalog artifact: a stack service kind, or an artifact only the legacy `supabase start` runs. */ +export type ArtifactKind = ServiceKind | "vector"; + +const isServiceKind = (kind: ArtifactKind): kind is ServiceKind => kind !== "vector"; + export class ArtifactError extends Data.TaggedError("ArtifactError")<{ readonly message: string; readonly service?: string; @@ -47,7 +52,7 @@ export interface ArtifactPin { /** * The exact upstream image this release was built or mirrored from, as slim-services recorded * it: the release manifest's `upstream_image` for a derived service, or the release's - * `oci-provenance.json` `source` for a mirrored one (mailpit, imgproxy). Normalized to + * `oci-provenance.json` `source` for a mirrored one (vector, mailpit, imgproxy). Normalized to * the Dockerfile's `FROM` form — no leading `docker.io/`, no digest — so legacy non-slim mode * can use it as the upstream tag directly. */ @@ -59,7 +64,7 @@ export interface ArtifactPin { const releaseVersion = (pin: ArtifactPin): string => `${pin.upstreamVersion}-r${pin.revision}`; interface ArtifactResolution { - readonly service: ServiceKind; + readonly service: ArtifactKind; /** Upstream version. */ readonly version: string; readonly releaseVersion: string; @@ -70,7 +75,7 @@ interface ArtifactResolution { } export interface PreparedNativeArtifact { - readonly service: ServiceKind; + readonly service: ArtifactKind; readonly version: string; readonly root: string; readonly executable: string; @@ -101,7 +106,7 @@ const definition = ( const SLIM_IMAGE_GHCR_REGISTRY = "ghcr.io/supabase/cli/"; -const definitions: Readonly> = { +const definitions: Readonly> = { database: definition( "postgres", { @@ -404,6 +409,32 @@ const definitions: Readonly> = { "bin/logflare", ["bin/logflare", "bin/prepare"], ), + vector: definition( + "vector", + { + upstreamVersion: "0.58.0", + revision: 0, + image: + "ghcr.io/supabase/cli/vector:0.58.0-r0@sha256:5dcf67db0ee378caa87f3395cb9484ebe3e97bb0334d119f2ac33116e00c5773", + upstreamImage: "timberio/vector:0.58.0-alpine", + natives: { + "darwin-arm64": { + archive: "567245cf9a7d54eee45ecf74e1c9a61ca6d02cdca103edc7b32b005e54f4e632", + manifest: "a973a763b00599858ceae8f304714fb99f785860112e9e0812ef0df8f81dd2ee", + }, + "linux-amd64": { + archive: "697f4fae35be3026474695bef16336f6fcfd429ce8cfba884c595896e96c30ec", + manifest: "8989b8b061f08bd7653e9ae71c6ee0e5cf01a0b10f2d358fe3dbc671ec5b63e1", + }, + "linux-arm64": { + archive: "c66b8ad0a0dd0fdcb3e4ee36bae8040b7b23b44ec2b4023565ca335884268c1d", + manifest: "992467ec68a99a6413468b9311294abb7a1f86b7857ad37f1f3bd9e6aecc9dbb", + }, + }, + }, + "bin/vector", + ["bin/vector", "share/doc/vector/config/vector.yaml"], + ), pooler: definition( "pooler", { @@ -474,7 +505,7 @@ export const slimImageMirrors = (image: string): ReadonlyArray => : []; const artifactFor = ( - service: ServiceKind, + service: ArtifactKind, resolved: ArtifactResolution, target: NativeTarget, ): SlimServicesArtifact => { @@ -510,7 +541,7 @@ const artifactFor = ( }; export const resolveArtifact = Effect.fn("Artifacts.resolveArtifact")(function* (request: { - readonly service: ServiceKind; + readonly service: ArtifactKind; readonly version?: string; }) { if (!Object.hasOwn(definitions, request.service)) @@ -540,8 +571,9 @@ export const postgresVersion = (version: string): string => Object.keys(definitions.database.pins).find((candidate) => candidate.split(".")[0] === version) ?? version; -/** Service kinds in artifact catalog order. */ -export const artifactServiceKinds = (): ReadonlyArray => Record.keys(definitions); +/** Service kinds the stack runs, in artifact catalog order; legacy-only artifacts are omitted. */ +export const artifactServiceKinds = (): ReadonlyArray => + Record.keys(definitions).filter(isServiceKind); /** * Every catalog pin in catalog order, including additional upstream lines. `isDefault` marks the @@ -549,12 +581,12 @@ export const artifactServiceKinds = (): ReadonlyArray => Record.key * other pin (postgres's 15.x additional line) carries `isDefault: false`. */ export const catalogPins = (): ReadonlyArray<{ - readonly service: ServiceKind; + readonly service: ArtifactKind; readonly sourceService: string; readonly pin: ArtifactPin; readonly isDefault: boolean; }> => - artifactServiceKinds().flatMap((service) => { + Record.keys(definitions).flatMap((service) => { const { sourceService, defaultVersion, pins } = definitions[service]; return Object.values(pins).map((pin) => ({ service, @@ -568,7 +600,7 @@ const artifactKey = (artifact: SlimServicesArtifact): string => `slim-services/${artifact.service}/${artifact.version}/${artifact.target}`; export const prepareNativeArtifact = Effect.fn("Artifacts.prepareNativeArtifact")(function* ( - request: { readonly service: ServiceKind; readonly version?: string }, + request: { readonly service: ArtifactKind; readonly version?: string }, cacheRoot: string, platform: { readonly os: string; readonly arch: string } = { os: process.platform, diff --git a/packages/stack/src/Owner.logs.integration.test.ts b/packages/stack/src/Owner.logs.integration.test.ts index aca990132c..b1793e1441 100644 --- a/packages/stack/src/Owner.logs.integration.test.ts +++ b/packages/stack/src/Owner.logs.integration.test.ts @@ -212,6 +212,7 @@ describe("owner persisted logs", () => { expect(yield* fs.exists(owner.logsRoot)).toBe(false); }), ).pipe(Effect.provide(services)), + { timeout: 120_000 }, ); it.live.skipIf(process.platform === "win32")("persists container output", () => @@ -236,5 +237,6 @@ describe("owner persisted logs", () => { ]); }), ).pipe(Effect.provide(services)), + { timeout: 120_000 }, ); }); diff --git a/packages/stack/src/internal/artifacts.ts b/packages/stack/src/internal/artifacts.ts index 0ee05f1fac..5d9e1fb81b 100644 --- a/packages/stack/src/internal/artifacts.ts +++ b/packages/stack/src/internal/artifacts.ts @@ -7,6 +7,7 @@ export { postgresVersion, prepareNativeArtifact, resolveArtifact, + type ArtifactKind, type ArtifactPin, type NativePin, type ServiceKind, From 656c7b580961ef25c86a744181c23350c7eb7d9b Mon Sep 17 00:00:00 2001 From: avallete Date: Thu, 1 Oct 2026 12:58:25 +0200 Subject: [PATCH 05/35] fix(stack): bound log shutdown and harden log store accounting - Abort log persistence 5 seconds into close, detaching instances under one deadline; aborted segments are retired and still release their handles. - Flush quiet late partial lines when their grace expires. - Count unwritten records as lost once, and clear lost markers only after they are written. - Read every segment for tails and report interior segment gaps with resumeAt. - Keep the first byte's time for UTF-8 characters split across chunks. - Retry wrapped sharing violations, claim removed instances atomically, and validate the readStackLogs tail. Co-Authored-By: Claude Opus 5.5 --- .../experimental/stack/logs/SIDE_EFFECTS.md | 6 +- packages/stack/ARCHITECTURE.md | 12 +- .../stack/src/Owner.logs.integration.test.ts | 133 ++++----- packages/stack/src/Rpc.ts | 4 +- packages/stack/src/effect.ts | 6 +- packages/stack/src/host/LogForwarder.ts | 2 +- packages/stack/src/host/LogRecord.ts | 47 +++- .../stack/src/host/LogRecord.unit.test.ts | 17 +- .../src/host/LogStore.integration.test.ts | 218 ++++++++++++++- packages/stack/src/host/LogStore.ts | 255 ++++++++++++------ .../stack/src/internal/sharing-violation.ts | 13 +- .../internal/sharing-violation.unit.test.ts | 53 ++++ 12 files changed, 592 insertions(+), 174 deletions(-) create mode 100644 packages/stack/src/internal/sharing-violation.unit.test.ts diff --git a/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md index c9b52d568b..8cd7e8e4cf 100644 --- a/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md @@ -29,8 +29,10 @@ the selected services, with the `launch` and `lost` markers between them. ISO-8601 time, or `start`, which keeps each instance's records from its latest launch (all of them when retention removed that launch record). Records are ordered by timestamp, service, instance, and file position. Timestamps are the -owner's clock at each line's first byte. History is read as a stream holding at -most N lines per instance, while every matching line is counted for the footer. +owner's clock at each line's first byte. Lines end at `\n`, `\r\n`, or a lone +`\r`, so carriage-return progress updates print as separate lines. History is +read as a stream holding at most N lines per instance, while every matching line +is counted for the footer. `-f/--follow` requires a reachable owner: without one it fails with status 1 before printing anything and suggests running without `--follow`. It prints the diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index eb6e9af316..0d558590ce 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -642,7 +642,8 @@ The state root is the stack registry root. Each stack keeps one state document a //logs///.log persisted service output ``` -The owner is the only subscriber of each instance's output. It writes one record per line, ` [ truncated] | `, into immutable generation-named segments: each owner start opens a new generation on the instance's first output, and a record that would take a segment past 5 MiB starts the next one. The oldest closed segments except the newest @@ -650,8 +651,11 @@ are deleted while an instance holds more than 10 MiB or 64 segments. Records car of their first byte. Line state is kept per launch, process and stream; chunk sequence numbers span a launch's processes, so a process whose output was all dropped still shows as `lost`. A late partial line of an ended launch waits for its newline for two seconds of quiet, or until the store -closes. Lines are cut at 32 KiB; chunks the in-memory output buffer dropped, that failed to write, -or that were still queued when a bounded drain at close ran out are recorded as `lost`. Every +closes. Lines are cut at 32 KiB; chunks the in-memory output buffer dropped or that were still +queued when a bounded drain at close ran out, and records that failed to write, are recorded as +`lost`; a write that fails part-way can count records that did land, so the count is an upper bound. +Closing aborts filesystem work still pending after 5 seconds; output not yet written by then is not +recorded. Every reader, live or offline, reads segments by position, so history and following share one path; a follow can resume at a record position, and a reader that finds its segment deleted reports a `lost` gap marker, which carries `resumeAt` instead of a record position. Destroying an instance deletes its segments, and an owner start removes directories of @@ -664,7 +668,7 @@ direct backend (never the proxy, so shipping neither wakes it nor counts as acti at most 256 events and 1 MiB. Each instance reads from `cursor.json` in its logs directory, the position of its last shipped record, written atomically after each body settles: a missing cursor or unreadable cursor starts at the oldest retained segment, and a cursor in a deleted segment -resumes at the oldest retained one. Bodies of an instance are sequential. Event ids +resumes at the next retained one. Bodies of an instance are sequential. Event ids derive from the instance and record position, and Logflare keeps the first row per id, so a failed body is posted again until it settles or the target changes. A 401, 403 or 404 response pauses the instance with its cursor until the target changes; any other 4xx except 408 and 429 skips the body. The target is re-selected when the composition or Analytics' health changes. diff --git a/packages/stack/src/Owner.logs.integration.test.ts b/packages/stack/src/Owner.logs.integration.test.ts index b1793e1441..684eebfbc6 100644 --- a/packages/stack/src/Owner.logs.integration.test.ts +++ b/packages/stack/src/Owner.logs.integration.test.ts @@ -128,20 +128,21 @@ describe("owner persisted logs", () => { const records = yield* Effect.promise(() => readStackLogs({ stateRoot: owner.stateRoot, stackId: owner.stack.id }), ); - const invalid = yield* Effect.promise(() => - readStackLogs({ - stateRoot: owner.stateRoot, - stackId: owner.stack.id, - since: "soon", - }).then( - () => undefined, - (error: unknown) => error, - ), + const rejected = (options: { readonly since?: string; readonly tail?: number }) => + Effect.promise(() => + readStackLogs({ stateRoot: owner.stateRoot, stackId: owner.stack.id, ...options }).then( + () => undefined, + (error: unknown) => error, + ), + ); + const invalid = yield* Effect.forEach( + [{ since: "soon" }, { tail: -1 }, { tail: 1.5 }, { tail: Number.POSITIVE_INFINITY }], + rejected, ); expect(records[0]).toMatchObject({ kind: "launch", service: "mail", instanceId: mail.id }); expect(records.map(({ text }) => text)).toContain(followed?.text); - expect(invalid).toBeInstanceOf(StackError); + for (const error of invalid) expect(error).toBeInstanceOf(StackError); }), ).pipe(Effect.provide(services)), ); @@ -180,63 +181,67 @@ describe("owner persisted logs", () => { ).pipe(Effect.provide(services)), ); - it.live("keeps database logs across a data reset and removes all logs with the stack", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const owner = yield* openOwner("owner-logs-reset-", "native"); - const database = yield* owner.rpc.createService({ - service: "database", - config: { - version: "17", - databasePassword: Redacted.make("owner-logs-reset-password"), - jwtExpiry: 3600, - }, - endpoints: { sql: { port: "auto" } }, - }); - yield* owner.rpc.startService({ id: database.id }); - yield* owner.rpc.readyService({ id: database.id }); - yield* owner.rpc.stopService({ id: database.id }); - const before = Array.from( - yield* owner.rpc.readLogs({ id: database.id, follow: false }).pipe(Stream.runCollect), - ); - - yield* owner.rpc.resetData({ id: database.id }); - const after = Array.from( - yield* owner.rpc.readLogs({ id: database.id, follow: false }).pipe(Stream.runCollect), - ); - - expect(before.some(isOutput)).toBe(true); - expect(after.slice(0, before.length)).toEqual(before); - yield* owner.namespace.destroy; - expect(yield* fs.exists(owner.logsRoot)).toBe(false); - }), - ).pipe(Effect.provide(services)), + it.live( + "keeps database logs across a data reset and removes all logs with the stack", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const owner = yield* openOwner("owner-logs-reset-", "native"); + const database = yield* owner.rpc.createService({ + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("owner-logs-reset-password"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, + }); + yield* owner.rpc.startService({ id: database.id }); + yield* owner.rpc.readyService({ id: database.id }); + yield* owner.rpc.stopService({ id: database.id }); + const before = Array.from( + yield* owner.rpc.readLogs({ id: database.id, follow: false }).pipe(Stream.runCollect), + ); + + yield* owner.rpc.resetData({ id: database.id }); + const after = Array.from( + yield* owner.rpc.readLogs({ id: database.id, follow: false }).pipe(Stream.runCollect), + ); + + expect(before.some(isOutput)).toBe(true); + expect(after.slice(0, before.length)).toEqual(before); + yield* owner.namespace.destroy; + expect(yield* fs.exists(owner.logsRoot)).toBe(false); + }), + ).pipe(Effect.provide(services)), { timeout: 120_000 }, ); - it.live.skipIf(process.platform === "win32")("persists container output", () => - Effect.scoped( - Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const owner = yield* openOwner("owner-logs-docker-", "docker"); - const mail = yield* owner.rpc.createService({ - service: "mail", - config: {}, - endpoints: { http: { port: "auto" } }, - }); - yield* owner.rpc.startService({ id: mail.id }); - yield* owner.rpc.readyService({ id: mail.id }); - - const [followed] = yield* firstOutput(owner.rpc.readLogs({ id: mail.id, follow: true })); - - expect(followed).toMatchObject({ launchId: 1 }); - expect(yield* fs.readDirectory(path.join(owner.logsRoot, "mail", mail.id))).toEqual([ - "0000000001.log", - ]); - }), - ).pipe(Effect.provide(services)), + it.live.skipIf(process.platform === "win32")( + "persists container output", + () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const owner = yield* openOwner("owner-logs-docker-", "docker"); + const mail = yield* owner.rpc.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + yield* owner.rpc.startService({ id: mail.id }); + yield* owner.rpc.readyService({ id: mail.id }); + + const [followed] = yield* firstOutput(owner.rpc.readLogs({ id: mail.id, follow: true })); + + expect(followed).toMatchObject({ launchId: 1 }); + expect(yield* fs.readDirectory(path.join(owner.logsRoot, "mail", mail.id))).toEqual([ + "0000000001.log", + ]); + }), + ).pipe(Effect.provide(services)), { timeout: 120_000 }, ); }); diff --git a/packages/stack/src/Rpc.ts b/packages/stack/src/Rpc.ts index 9b9203cc65..b9e7af8a94 100644 --- a/packages/stack/src/Rpc.ts +++ b/packages/stack/src/Rpc.ts @@ -80,6 +80,8 @@ export const Definition = Schema.Struct({ id: Schema.String, creation: ServiceCr export interface Definition extends Schema.Schema.Type {} const Instance = { id: Schema.String }; +/** How many of the latest log records a read starts with. */ +export const LogTail = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)); const SnapshotScope = Schema.Literals(snapshotScopes); export const CommandEvent = Schema.TaggedUnion({ @@ -122,7 +124,7 @@ export const OwnerRpc = RpcGroup.make( ...Instance, from: Schema.optionalKey(LogPosition), since: Schema.optionalKey(Schema.String), - tail: Schema.optionalKey(Schema.Int.check(Schema.isGreaterThanOrEqualTo(0))), + tail: Schema.optionalKey(LogTail), follow: Schema.Boolean, }, success: LogRecord, diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index 559caf0c98..b8ab3d857d 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -44,7 +44,7 @@ import { deriveStackId, resolveStackIdentity } from "./identity/Identity.ts"; import { failureMessage } from "./internal/failure-message.ts"; import * as State from "./State.ts"; import type { SavedStack, StackCredentials, StackKeysInput } from "./State.ts"; -import { StackError, type Definition, type Observation } from "./Rpc.ts"; +import { LogTail, StackError, type Definition, type Observation } from "./Rpc.ts"; import { readStackLogs as readPersistedLogs, sinceMillis, @@ -1058,7 +1058,9 @@ export const readStackLogs = Effect.fn("Stack.readStackLogs")( root, ...(options.instances === undefined ? {} : { instances: options.instances }), ...(options.since === undefined ? {} : { since: yield* sinceMillis(options.since) }), - ...(options.tail === undefined ? {} : { tail: options.tail }), + ...(options.tail === undefined + ? {} + : { tail: yield* Schema.decodeEffect(LogTail)(options.tail) }), }); }, Effect.mapError((cause) => failure("readStackLogs", cause)), diff --git a/packages/stack/src/host/LogForwarder.ts b/packages/stack/src/host/LogForwarder.ts index cc13bc6899..9f81111cca 100644 --- a/packages/stack/src/host/LogForwarder.ts +++ b/packages/stack/src/host/LogForwarder.ts @@ -336,7 +336,7 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar for (const record of page) { if (record.position === undefined) { yield* Effect.logWarning( - `Logs of ${service} instance ${instanceId} were deleted before shipping; resuming from the oldest retained record`, + `Logs of ${service} instance ${instanceId} were deleted before shipping; resuming from the next retained record`, ); continue; } diff --git a/packages/stack/src/host/LogRecord.ts b/packages/stack/src/host/LogRecord.ts index bb7597f0ec..1481c932e2 100644 --- a/packages/stack/src/host/LogRecord.ts +++ b/packages/stack/src/host/LogRecord.ts @@ -121,7 +121,9 @@ interface LineState { text: string; bytes: number; since: number | undefined; - /** Publish time of bytes the decoder holds until their character completes. */ + /** Bytes the decoder holds until their character completes. */ + held: Uint8Array; + /** Publish time of the first held byte. */ heldSince: number | undefined; /** Publish time of the latest chunk. */ lastChunk: number; @@ -137,6 +139,8 @@ export interface Splitter { readonly endLaunch: (launchId: number, now: number) => ReadonlyArray; /** Flushes partial lines late chunks of ended launches left once they are idle for a grace. */ readonly flushEnded: (now: number) => ReadonlyArray; + /** The earliest time `flushEnded` has a partial line to flush. */ + readonly endedDue: () => number | undefined; readonly flush: (now: number) => ReadonlyArray; } @@ -147,15 +151,26 @@ const seenLaunches = 64; /** A late partial line of an ended launch waits this long for its newline. */ export const endedLineGraceMillis = 2_000; -/** Reports whether `bytes` ends inside a UTF-8 sequence the next chunk completes. */ -const endsMidCharacter = (bytes: Uint8Array) => { +const noBytes = new Uint8Array(0); + +/** The length of the UTF-8 sequence that `bytes` ends inside and a later chunk completes. */ +const incompleteLength = (bytes: Uint8Array) => { for (let index = bytes.length - 1; index >= Math.max(0, bytes.length - 4); index--) { const byte = bytes[index] ?? 0; if ((byte & 0xc0) === 0x80) continue; const width = byte >= 0xf0 ? 4 : byte >= 0xe0 ? 3 : byte >= 0xc0 ? 2 : 1; - return bytes.length - index < width; + return bytes.length - index < width ? bytes.length - index : 0; } - return false; + return 0; +}; + +/** The last four bytes of the held bytes followed by `bytes`. */ +const lastBytes = (held: Uint8Array, bytes: Uint8Array) => { + if (bytes.length >= 4) return bytes.subarray(bytes.length - 4); + const joined = new Uint8Array(held.length + bytes.length); + joined.set(held); + joined.set(bytes, held.length); + return joined.subarray(Math.max(0, joined.length - 4)); }; /** Splits lines like `Stream.decodeText` plus `Stream.splitLines`, per launch, part and stream. */ @@ -211,6 +226,7 @@ export const makeSplitter = (limit = maxLineBytes): Splitter => { const flushLine = (state: LineState, now: number, out: Array) => { append(state, state.decoder.decode(), state.heldSince ?? now, out); state.decoder = new TextDecoder(); + state.held = noBytes; state.heldSince = undefined; state.midCarriageReturn = false; if (state.since !== undefined && !state.discarding) terminate(state, now, out); @@ -255,6 +271,7 @@ export const makeSplitter = (limit = maxLineBytes): Splitter => { text: "", bytes: 0, since: undefined, + held: noBytes, heldSince: undefined, lastChunk: now, midCarriageReturn: false, @@ -275,17 +292,17 @@ export const makeSplitter = (limit = maxLineBytes): Splitter => { }); resetLine(state); state.decoder = new TextDecoder(); + state.held = noBytes; state.heldSince = undefined; state.midCarriageReturn = false; } expected.set(sequence, Math.max(next, chunk.seq + 1)); const carried = state.heldSince; const text = state.decoder.decode(chunk.bytes, { stream: true }); - state.heldSince = endsMidCharacter(chunk.bytes) - ? text.length === 0 - ? (carried ?? now) - : now - : undefined; + const tail = lastBytes(state.held, chunk.bytes); + const held = incompleteLength(tail); + state.held = tail.slice(tail.length - held); + state.heldSince = held === 0 ? undefined : held > chunk.bytes.length ? (carried ?? now) : now; if (text.length > 0) { let from = state.midCarriageReturn && text.startsWith("\n") ? 1 : 0; state.midCarriageReturn = false; @@ -324,11 +341,19 @@ export const makeSplitter = (limit = maxLineBytes): Splitter => { return out; }; + const endedDue = () => { + let due: number | undefined; + for (const state of states.values()) + if (ended.has(state.launchId) && pending(state)) + due = Math.min(due ?? Number.POSITIVE_INFINITY, state.lastChunk + endedLineGraceMillis); + return due; + }; + const flush = (now: number) => { const out: Array = []; for (const state of states.values()) flushLine(state, now, out); return out; }; - return { push, endLaunch, flushEnded, flush }; + return { push, endLaunch, flushEnded, endedDue, flush }; }; diff --git a/packages/stack/src/host/LogRecord.unit.test.ts b/packages/stack/src/host/LogRecord.unit.test.ts index 15781fecff..7552efe710 100644 --- a/packages/stack/src/host/LogRecord.unit.test.ts +++ b/packages/stack/src/host/LogRecord.unit.test.ts @@ -180,7 +180,6 @@ describe("splitter", () => { expect(waiting).toEqual([]); expect(lines(flushed)).toEqual(["idle partial"]); }); - it("reports a whole process of a launch whose chunks were dropped", () => { const splitter = makeSplitter(); push(splitter, chunk(1, 0, "startup\n", "stdout", 0), 10); @@ -210,6 +209,22 @@ describe("splitter", () => { ]); }); + it.each(["€", "😀"])( + "stamps a line with its first byte's time when %s spans one chunk per byte", + (character) => { + const splitter = makeSplitter(); + const encoded = bytes(`${character}\n`); + + const pushed = Array.from(encoded, (byte, seq) => + push(splitter, { ...chunk(1, seq, ""), bytes: new Uint8Array([byte]) }, 10 * (seq + 1)), + ); + + expect(pushed.at(-1)).toEqual([ + { kind: "stdout", timestamp: 10, launchId: 1, text: character, truncated: false }, + ]); + }, + ); + it("ends a startup command's unterminated line when the main process of the launch starts", () => { const splitter = makeSplitter(); push(splitter, chunk(1, 0, "migrations applied", "stdout", 0), 10); diff --git a/packages/stack/src/host/LogStore.integration.test.ts b/packages/stack/src/host/LogStore.integration.test.ts index e3ca8424c5..9b36630d9c 100644 --- a/packages/stack/src/host/LogStore.integration.test.ts +++ b/packages/stack/src/host/LogStore.integration.test.ts @@ -2,6 +2,7 @@ import { NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; import { Clock, + Deferred, Effect, Exit, Fiber, @@ -17,7 +18,7 @@ import { } from "effect"; import { TestClock } from "effect/testing"; import type { LaunchOutput } from "../runtime/Session.ts"; -import { segmentName, type LogRecord } from "./LogRecord.ts"; +import { endedLineGraceMillis, segmentName, type LogRecord } from "./LogRecord.ts"; import * as LogStore from "./LogStore.ts"; const encoder = new TextEncoder(); @@ -66,14 +67,44 @@ const fakeInstance = ( }; }); -/** Opens a store in its own scope so a test can close it like an owner stop. */ +/** Opens a store in a child of the test's scope so a test can close it like an owner stop. */ const openStore = (root: string, options: Omit = {}) => Effect.gen(function* () { - const scope = yield* Scope.make(); + const scope = yield* Scope.fork(yield* Scope.Scope); const store = yield* LogStore.make({ root, ...options }).pipe(Scope.provide(scope)); return { store, close: Scope.close(scope, Exit.void) }; }); +/** A file system whose segment writes run `writeAll` in place of the real file's. */ +const replaceWrites = ( + fs: FileSystem.FileSystem, + writeAll: ( + file: FileSystem.File, + buffer: Uint8Array, + ) => Effect.Effect, +): FileSystem.FileSystem => ({ + ...fs, + open: (target, options) => + fs.open(target, options).pipe( + Effect.map((file): FileSystem.File => ({ + [FileSystem.FileTypeId]: FileSystem.FileTypeId, + stat: file.stat, + seek: (offset, from) => file.seek(offset, from), + sync: file.sync, + read: (buffer) => file.read(buffer), + readAlloc: (size) => file.readAlloc(size), + truncate: (length) => file.truncate(length), + write: (buffer) => file.write(buffer), + writeAll: (buffer) => writeAll(file, buffer), + })), + ), +}); + +const lostCounts = (records: Iterable) => + Array.from(records).flatMap(({ kind, launchId, stream, count }) => + kind === "lost" ? [{ launchId, stream, count }] : [], + ); + /** Takes `count` items; `Queue.takeN` returns fewer once it has waited for the first. */ const takeExactly = (queue: Queue.Dequeue, count: number) => Effect.gen(function* () { @@ -216,6 +247,140 @@ describe("LogStore", () => { }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); + it.effect("writes a late partial line of an ended launch once it stays quiet for the grace", () => + Effect.gen(function* () { + const root = yield* tempRoot("log-store-quiet-"); + const { store } = yield* openStore(root); + const instance = yield* fakeInstance("quiet"); + yield* store.attach(instance); + const reader = yield* collect(store.read("quiet", { from: "oldest", follow: true })); + yield* instance.setLaunch(1); + yield* instance.publish(instance.chunk(1, "ending")); + yield* reader.take(1); + yield* instance.setLaunch(undefined); + yield* untilLast(reader, "ending"); + + yield* instance.publish(instance.chunk(1, "late"), instance.chunk(1, "written\n", "stderr")); + yield* untilLast(reader, "written"); + const beforeGrace = yield* Queue.size(reader.records); + yield* TestClock.adjust(endedLineGraceMillis); + const [late] = yield* reader.take(1); + + expect(beforeGrace).toBe(0); + expect(late).toMatchObject({ kind: "stdout", launchId: 1, text: "late" }); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("records each failed write as lost once, including a partial-line flush", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* tempRoot("log-store-failed-write-"); + const failures = yield* Queue.unbounded(); + const failOnce = new Set(["pending", "doomed"]); + const injected = replaceWrites(fs, (file, buffer) => { + const text = new TextDecoder().decode(buffer); + const word = [...failOnce].find((candidate) => text.includes(candidate)); + if (word === undefined) return file.writeAll(buffer); + failOnce.delete(word); + // Leaves part of the first record behind, like a write that fails part-way. + return file.writeAll(buffer.subarray(0, 10)).pipe( + Effect.andThen(Queue.offer(failures, word)), + Effect.andThen( + Effect.fail( + PlatformError.systemError({ + _tag: "Unknown", + module: "FileSystem", + method: "write", + }), + ), + ), + ); + }); + const { store } = yield* openStore(root, { rotateBytes: 1 }).pipe( + Effect.provideService(FileSystem.FileSystem, injected), + ); + const instance = yield* fakeInstance("failing"); + yield* store.attach(instance); + const reader = yield* collect(store.read("failing", { from: "oldest", follow: true })); + yield* instance.setLaunch(1); + yield* instance.publish(instance.chunk(1, "first\n")); + yield* untilLast(reader, "first"); + yield* instance.publish(instance.chunk(1, "pending"), instance.chunk(1, "sync\n", "stderr")); + yield* untilLast(reader, "sync"); + + yield* instance.setLaunch(undefined); + const flushFailure = yield* Queue.take(failures); + yield* TestClock.adjust(100); + yield* instance.publish({ ...instance.chunk(2, "doomed\n"), seq: 0 }); + const lineFailure = yield* Queue.take(failures); + yield* TestClock.adjust(200); + yield* instance.publish({ ...instance.chunk(2, "after\n"), seq: 1 }); + const followed = yield* untilLast(reader, "after"); + const offline = yield* LogStore.readStackLogs({ root }); + + const expectedLost = [ + { launchId: 1, stream: "stdout", count: 1 }, + { launchId: 2, stream: "stdout", count: 1 }, + ]; + expect([flushFailure, lineFailure]).toEqual(["pending", "doomed"]); + expect(lostCounts(followed)).toEqual(expectedLost); + expect(lostCounts(offline)).toEqual(expectedLost); + expect(texts(offline)).toEqual(["first", "sync", "after"]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("closes by the shutdown deadline while a segment write hangs", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* tempRoot("log-store-hung-write-"); + const writing = yield* Deferred.make(); + const injected = replaceWrites(fs, () => + Deferred.succeed(writing, undefined).pipe(Effect.andThen(Effect.never)), + ); + const opened = yield* openStore(root).pipe( + Effect.provideService(FileSystem.FileSystem, injected), + ); + const instance = yield* fakeInstance("hung"); + yield* opened.store.attach(instance); + yield* instance.publish(instance.chunk(1, "never written\n")); + yield* Deferred.await(writing); + + const closing = yield* Effect.forkChild(opened.close, { startImmediately: true }); + yield* TestClock.adjust(4_999); + const beforeDeadline = closing.pollUnsafe(); + yield* TestClock.adjust(1); + yield* Fiber.join(closing); + + expect(beforeDeadline).toBeUndefined(); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("reads past a partial record a write left at the end of an earlier segment", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* tempRoot("log-store-torn-"); + const directory = path.join(root, "auth", "torn"); + const torn = path.join(directory, segmentName(1)); + const content = + "1970-01-01T00:00:00.000Z stdout 1 | whole\n1970-01-01T00:00:00.000Z stdout 1 | to"; + yield* fs.makeDirectory(directory, { recursive: true }); + yield* fs.writeFileString(torn, content); + const { store } = yield* openStore(root); + const instance = yield* fakeInstance("torn"); + yield* store.attach(instance); + const reader = yield* collect(store.read("torn", { from: "oldest", follow: true })); + + yield* instance.publish(instance.chunk(2, "after\n")); + const followed = yield* untilLast(reader, "after"); + + expect(texts(followed)).toEqual(["whole", "after"]); + expect(texts(yield* LogStore.readStackLogs({ root }))).toEqual(["whole", "after"]); + expect(texts(yield* LogStore.readStackLogs({ root, tail: 3 }))).toEqual(["whole", "after"]); + expect(yield* fs.readFileString(torn)).toBe(content); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + it.effect("rotates segments while a follower reads and keeps its order", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -446,7 +611,41 @@ describe("LogStore", () => { }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); - it.effect("keeps sweeping older segments past one it cannot delete", () => + it.effect("tails the newest record when several newer segments hold older flushed lines", () => + Effect.gen(function* () { + const root = yield* tempRoot("log-store-late-flushes-"); + const { store } = yield* openStore(root, { rotateBytes: 1 }); + const instance = yield* fakeInstance("flushes"); + yield* store.attach(instance); + const progress = yield* collect(store.read("flushes", { from: "oldest", follow: true })); + yield* instance.setLaunch(1); + + // Partial lines of a later process part stay open while part 0 writes a newer line. + yield* TestClock.setTime(1_000); + yield* instance.publish( + { ...instance.chunk(1, "early stdout"), part: 1 }, + { ...instance.chunk(1, "early stderr", "stderr"), part: 1 }, + ); + yield* TestClock.setTime(3_000); + yield* instance.publish(instance.chunk(1, "latest\n")); + yield* untilLast(progress, "latest"); + yield* TestClock.setTime(4_000); + yield* instance.setLaunch(undefined); + yield* untilLast(progress, "early stderr"); + + const offline = yield* LogStore.readStackLogs({ root, tail: 1 }); + const live = yield* (yield* store.read("flushes", { + from: "oldest", + follow: false, + tail: 1, + })).pipe(Stream.runCollect); + + expect(texts(offline)).toEqual(["latest"]); + expect(texts(live)).toEqual(["latest"]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + + it.effect("sweeps past a segment it cannot delete and reports the deleted ones to a reader", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -481,8 +680,19 @@ describe("LogStore", () => { yield* store.read("stuck", { from: "end", follow: false }).pipe(Effect.asVoid); const segments = yield* fs.readDirectory(path.join(root, "auth", "stuck")); + const resumed = yield* (yield* store.read("stuck", { + from: { generation: 3, byteOffset: 0 }, + follow: false, + })).pipe(Stream.runCollect); + expect(segments).toContain(held); + expect(segments).not.toContain(segmentName(3)); expect(segments.length).toBeLessThanOrEqual(4); + const [gap] = resumed; + expect(gap?.kind).toBe("lost"); + expect(gap?.position).toBeUndefined(); + expect(gap?.resumeAt?.generation).toBeGreaterThan(3); + expect(texts(resumed).at(-1)).toBe("stuck line 9"); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); diff --git a/packages/stack/src/host/LogStore.ts b/packages/stack/src/host/LogStore.ts index e716b7e069..69c4eff33b 100644 --- a/packages/stack/src/host/LogStore.ts +++ b/packages/stack/src/host/LogStore.ts @@ -5,10 +5,12 @@ import { Duration, Effect, Exit, + Fiber, FileSystem, Option, Path, PubSub, + Queue, Ref, Schema, Scope, @@ -110,7 +112,10 @@ const readerPermits = 1024; const retryBaseMillis = 100; const retryMaxMillis = 30_000; const drainBatch = 4096; -const drainTimeout = Duration.seconds(5); +/** Draining output still queued at close stops here; the rest is recorded as `lost`. */ +const drainTimeout = Duration.seconds(4); +/** Detaching aborts filesystem work still pending here, so a hung write cannot block shutdown. */ +const shutdownTimeout = Duration.seconds(5); const encoder = new TextEncoder(); const decoder = new TextDecoder(); @@ -323,14 +328,16 @@ const makeReader = (fs: FileSystem.FileSystem, path: Path.Path) => { atEnd: true, partial, }); - const first = listed[0]; - if (first === undefined) return fresh ? atEnd(false) : relist(false); - if (cursor.generation < first) { - const resumeAt = { generation: first, byteOffset: 0 }; + /** Reports the records of the cursor's deleted segment as a gap before `generation`. */ + const gap = Effect.fnUntraced(function* (generation: number) { + const resumeAt = { generation, byteOffset: 0 }; const records: ReadonlyArray = cursor.generation === 0 ? [] : [{ kind: "lost", timestamp: yield* nowIso, resumeAt }]; return { records, cursor: resumeAt, listed, atEnd: false, partial: false } satisfies Step; - } + }); + const first = listed[0]; + if (first === undefined) return fresh ? atEnd(false) : relist(false); + if (cursor.generation < first) return yield* gap(first); const newer = listed.find((generation) => generation > cursor.generation); const moveOn = (generation: number): Step => ({ records: [], @@ -340,7 +347,7 @@ const makeReader = (fs: FileSystem.FileSystem, path: Path.Path) => { partial: false, }); if (!listed.includes(cursor.generation)) - return newer !== undefined ? moveOn(newer) : fresh ? atEnd(false) : relist(false); + return newer !== undefined ? yield* gap(newer) : fresh ? atEnd(false) : relist(false); const chunk = yield* readAt(directory, cursor, live); if (Option.isNone(chunk)) return relist(false); if (chunk.value.progressed) @@ -381,8 +388,8 @@ const makeReader = (fs: FileSystem.FileSystem, path: Path.Path) => { }); /** - * Reads the history after `cursor`. With `tail`, only the newest segments that hold the last - * `tail` records are read. + * Reads the history after `cursor`. With `tail`, every segment is read and only the last `tail` + * records are kept. */ const history = Effect.fnUntraced(function* ( directory: string, @@ -400,25 +407,17 @@ const makeReader = (fs: FileSystem.FileSystem, path: Path.Path) => { cursor: live === undefined ? cursor : yield* live.lock.withPermits(1)(Ref.get(live.end)), listed, }; - // A partial line is written when it ends, so a segment can hold a record older than records - // in the segment before it; one more segment is read past each stopping point. - const collected: Array = []; + // A partial line is written when it ends, so any segment can hold the newest records. + let records: ReadonlyArray = []; let end: LogPosition = cursor; - let first = true; - let past = 0; - for (const generation of listed.toReversed()) { + for (const [index, generation] of listed.toReversed().entries()) { const segment = yield* readSegment(directory, generation, live); - if (first) end = segment.position; - first = false; - const matching = segment.records.filter(keep); - collected.push(...matching); - const olderThanSince = - options.since !== undefined && segment.records.length > 0 && matching.length === 0; - past = collected.length >= tail || olderThanSince ? past + 1 : 0; - if (past > 1) break; + if (index === 0) end = segment.position; + records = [...records, ...segment.records.filter(keep)] + .toSorted(compareRecords) + .slice(-tail); } - const records = collected.toSorted(compareRecords); - return { records: records.slice(Math.max(0, records.length - tail)), cursor: end, listed }; + return { records, cursor: end, listed }; } const records: Array = []; let current = cursor; @@ -528,10 +527,14 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio readonly instanceId: string; readonly directory: string; readonly closed: Deferred.Deferred; + /** Completed at the shutdown deadline; filesystem work then fails at once. */ + readonly abort: Deferred.Deferred; readonly scope: Scope.Closeable; } const instances = yield* Ref.make>(new Map()); const closed = yield* Ref.make(false); + // Forked before the close finalizer is added, so the store's close detaches instances first. + const instancesScope = yield* Scope.fork(storeScope, "parallel"); const attached = (instanceId: string) => Ref.get(instances).pipe( Effect.flatMap((current) => { @@ -553,6 +556,25 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio yield* Scope.close(instance.scope, Exit.void); }); + /** Detaches instances together under one deadline that aborts their pending filesystem work. */ + const detachAll = Effect.fnUntraced(function* (handles: ReadonlyArray) { + if (handles.length === 0) return; + const deadline = yield* Effect.sleep(shutdownTimeout).pipe( + Effect.andThen( + Effect.forEach(handles, (handle) => Deferred.succeed(handle.abort, undefined)), + ), + Effect.andThen( + Effect.logWarning( + "Log persistence did not stop by the shutdown deadline; output not yet written is not recorded", + ), + ), + Effect.forkChild({ startImmediately: true }), + ); + yield* Effect.forEach(handles, detach, { concurrency: "unbounded", discard: true }).pipe( + Effect.ensuring(Fiber.interrupt(deadline)), + ); + }); + const attach = Effect.fn("LogStore.attach")(function* (instance: AttachedInstance) { if (yield* Ref.get(closed)) return; const directory = path.join(options.root, instance.service, instance.instanceId); @@ -567,7 +589,7 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio ), ); const latest = Option.getOrElse(existing, () => []).at(-1); - const scope = yield* Scope.fork(storeScope, "sequential"); + const scope = yield* Scope.fork(instancesScope, "sequential"); const handle: Attached = { service: instance.service, instanceId: instance.instanceId, @@ -581,6 +603,7 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio }), wake: yield* PubSub.sliding(1), closed: yield* Deferred.make(), + abort: yield* Deferred.make(), }; const splitter = makeSplitter(); let nextGeneration: number | undefined = Option.isSome(existing) @@ -594,14 +617,36 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio size: number; } | undefined; - /** Chunks not persisted, reported as `lost` by the next successful append. */ + /** Scopes of segments a failed or aborted write may have left with a partial record. */ + const retired: Array = []; + /** Output not persisted, reported as `lost` by the next successful append. */ const dropped = new Map>(); let broken: { readonly attempts: number; readonly retryAt: number } | undefined; + const aborted = new LogStoreError({ + operation: "write", + message: `Log persistence of ${instance.service} instance ${instance.instanceId} passed the shutdown deadline`, + }); + /** Fails at the shutdown deadline instead of waiting for filesystem work that hangs. */ + const abortable = (effect: Effect.Effect) => + Deferred.isDone(handle.abort).pipe( + Effect.flatMap((done) => + done + ? Effect.fail(aborted) + : effect.pipe( + Effect.raceFirst( + Deferred.await(handle.abort).pipe(Effect.andThen(Effect.fail(aborted))), + ), + ), + ), + ); + const closeFile = Effect.suspend(() => { - const current = file; + const scopes = [...retired.splice(0), ...(file === undefined ? [] : [file.scope])]; file = undefined; - return current === undefined ? Effect.void : Scope.close(current.scope, Exit.void); + return Effect.forEach(scopes, (current) => Scope.close(current, Exit.void), { + discard: true, + }); }); const enforceRetention = reader.segments(directory).pipe( @@ -675,7 +720,13 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio } yield* current.handle.writeAll(bytes).pipe( Effect.mapError(storeError("write")), - Effect.tapError(() => closeFile), + // The next record goes to a new segment, never after a partial one. + Effect.onError(() => + Effect.sync(() => { + retired.push(current.scope); + file = undefined; + }), + ), ); current.size += bytes.length; yield* Ref.set(handle.end, { generation: current.generation, byteOffset: current.size }); @@ -683,7 +734,7 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio }); /** Appends records, rotating before a record that would overflow the segment. */ - const append = (entries: ReadonlyArray) => + const append = (entries: ReadonlyArray, progress: { written: number }) => Effect.gen(function* () { let pieces: Array = []; let pending = 0; @@ -692,6 +743,7 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio const used = (file?.size ?? 0) + pending; if (used > 0 && used + bytes.length > rotateBytes) { if (pieces.length > 0) yield* writePieces(pieces); + progress.written += pieces.length; yield* openGeneration; pieces = []; pending = 0; @@ -700,40 +752,57 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio pending += bytes.length; } if (pieces.length > 0) yield* writePieces(pieces); + progress.written += pieces.length; }); - const countDropped = (chunks: ReadonlyArray) => { - for (const chunk of chunks) { - const key = `${chunk.launchId}:${chunk.stream}`; - dropped.set(key, { - kind: "lost", - timestamp: chunk.time, - launchId: chunk.launchId, - stream: chunk.stream, - count: (dropped.get(key)?.count ?? 0) + 1, - }); - } + const dropKey = (launchId: number, stream: LaunchOutput["stream"]) => `${launchId}:${stream}`; + const drop = ( + launchId: number, + stream: LaunchOutput["stream"], + time: number, + count: number, + ) => { + const previous = dropped.get(dropKey(launchId, stream)); + dropped.set(dropKey(launchId, stream), { + kind: "lost", + timestamp: Math.max(previous?.timestamp ?? time, time), + launchId, + stream, + count: (previous?.count ?? 0) + count, + }); + }; + const dropChunks = (chunks: ReadonlyArray) => { + for (const chunk of chunks) drop(chunk.launchId, chunk.stream, chunk.time, 1); + }; + /** Records unwritten records as `lost`: a line counts as one chunk, a gap with its count. */ + const dropRecords = (entries: ReadonlyArray) => { + for (const entry of entries) + if (entry.kind === "lost") drop(entry.launchId, entry.stream, entry.timestamp, entry.count); + else if (entry.kind !== "launch") drop(entry.launchId, entry.kind, entry.timestamp, 1); }; /** - * Appends records. After a failure, appends are skipped with backoff and their chunks are - * reported as `lost` once writing works again. + * Appends records after the pending `lost` markers. After a failure, appends are skipped with + * backoff, and every record not written is reported as `lost` once writing works again. */ - const persist = Effect.fnUntraced(function* ( - entries: ReadonlyArray, - chunks: ReadonlyArray, - ) { + const persist = Effect.fnUntraced(function* (entries: ReadonlyArray) { const now = yield* Clock.currentTimeMillis; - if (broken !== undefined && now < broken.retryAt) return countDropped(chunks); + if (broken !== undefined && now < broken.retryAt) return dropRecords(entries); if (entries.length === 0 && dropped.size === 0) return; - const written = yield* append([...dropped.values(), ...entries]).pipe(Effect.exit); + const markers = [...dropped.values()]; + const markersWritten = { written: 0 }; + const entriesWritten = { written: 0 }; + const written = yield* abortable( + append(markers, markersWritten).pipe(Effect.andThen(append(entries, entriesWritten))), + ).pipe(Effect.exit); + for (const marker of markers.slice(0, markersWritten.written)) + dropped.delete(dropKey(marker.launchId, marker.stream)); if (Exit.isSuccess(written)) { - dropped.clear(); broken = undefined; return; } - countDropped(chunks); - if (broken === undefined) + dropRecords(entries.slice(entriesWritten.written)); + if (broken === undefined && !(yield* Deferred.isDone(handle.abort))) yield* Effect.logError( `Unable to persist ${instance.service} logs of instance ${instance.instanceId}; retrying with a new segment`, written.cause, @@ -745,21 +814,51 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio }; }); + /** Persists flushed partial lines; only waiting for the writer lock is interruptible. */ + const persistFlushed = (flushed: (now: number) => ReadonlyArray) => + Clock.currentTimeMillis.pipe( + Effect.flatMap((now) => + handle.lock.withPermits(1)( + Effect.uninterruptible(Effect.suspend(() => persist(flushed(now)))), + ), + ), + ); + + /** Signals the late-line flusher that an ended launch holds a partial line. */ + const lateLines = yield* Queue.sliding(1); + const persistBatch = (chunks: ReadonlyArray) => Clock.currentTimeMillis.pipe( Effect.flatMap((now) => handle.lock.withPermits(1)( Effect.suspend(() => - persist( - [...chunks.flatMap((chunk) => splitter.push(chunk)), ...splitter.flushEnded(now)], - chunks, - ), + persist([ + ...chunks.flatMap((chunk) => splitter.push(chunk)), + ...splitter.flushEnded(now), + ]), ), ), ), + Effect.andThen( + Effect.suspend(() => + splitter.endedDue() === undefined ? Effect.void : Queue.offer(lateLines, undefined), + ), + ), Effect.uninterruptible, ); + /** Flushes late partial lines of ended launches once they stay idle for the grace. */ + const flushLate = Effect.forever( + Effect.gen(function* () { + yield* Queue.take(lateLines); + for (let due = splitter.endedDue(); due !== undefined; due = splitter.endedDue()) { + const now = yield* Clock.currentTimeMillis; + if (now < due) yield* Effect.sleep(Duration.millis(due - now)); + else yield* persistFlushed((at) => splitter.flushEnded(at)); + } + }), + ); + const flushLaunches = Effect.gen(function* () { const previous = yield* Ref.make(undefined); yield* instance.observation.pipe( @@ -770,13 +869,7 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio Effect.flatMap((prior) => prior === undefined ? Effect.void - : Clock.currentTimeMillis.pipe( - Effect.flatMap((now) => - handle.lock.withPermits(1)( - Effect.suspend(() => persist(splitter.endLaunch(prior, now), [])), - ), - ), - ), + : persistFlushed((now) => splitter.endLaunch(prior, now)), ), ), ), @@ -801,9 +894,9 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio Effect.flatMap((chunks) => handle.lock.withPermits(1)( Effect.suspend(() => { - countDropped(chunks); + dropChunks(chunks); broken = undefined; - return persist([], []); + return persist([]); }), ), ), @@ -814,15 +907,18 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio yield* Scope.addFinalizer( scope, drain.pipe( - Effect.andThen(Clock.currentTimeMillis), - Effect.flatMap((now) => - handle.lock.withPermits(1)(Effect.suspend(() => persist(splitter.flush(now), []))), + Effect.andThen(persistFlushed((now) => splitter.flush(now))), + // Closing outlives the deadline so aborted segments still release their handles. + Effect.andThen( + Effect.forkDetach(closeFile).pipe( + Effect.flatMap((fiber) => abortable(Fiber.join(fiber))), + ), ), Effect.ignore, - Effect.andThen(closeFile), ), ); - // A taken batch is always persisted, so interruption only lands while waiting for output. + // A taken batch is persisted unless the shutdown deadline aborts it, so interruption only + // lands while waiting for output. yield* Effect.forkIn( Effect.forever( Effect.uninterruptibleMask((restore) => @@ -832,6 +928,7 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio scope, ); yield* Effect.forkIn(flushLaunches, scope); + yield* Effect.forkIn(flushLate, scope); yield* Ref.update(instances, (current) => new Map(current).set(instance.instanceId, handle)); }); @@ -846,15 +943,15 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio readonly instanceId: string; }) { const directory = path.join(options.root, target.service, target.instanceId); - const instance = (yield* Ref.get(instances)).get(target.instanceId); - if (instance !== undefined) { - yield* Ref.update(instances, (current) => { + const instance = yield* Ref.modify( + instances, + (current): readonly [Attached | undefined, ReadonlyMap] => { const next = new Map(current); next.delete(target.instanceId); - return next; - }); - yield* detach(instance); - } + return [current.get(target.instanceId), next]; + }, + ); + if (instance !== undefined) yield* detachAll([instance]); yield* fs .remove(directory, { recursive: true, force: true }) .pipe(retryShared, Effect.mapError(removeError(directory))); @@ -879,7 +976,7 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio const close = Effect.gen(function* () { yield* Ref.set(closed, true); const current = yield* Ref.getAndSet(instances, new Map()); - yield* Effect.forEach(current.values(), detach, { discard: true }); + yield* detachAll([...current.values()]); }).pipe(Effect.withSpan("LogStore.close")); yield* Effect.addFinalizer(() => close); diff --git a/packages/stack/src/internal/sharing-violation.ts b/packages/stack/src/internal/sharing-violation.ts index 4eff999c3d..b7dd11fcac 100644 --- a/packages/stack/src/internal/sharing-violation.ts +++ b/packages/stack/src/internal/sharing-violation.ts @@ -1,11 +1,14 @@ import { Duration, Effect, Predicate, Schedule } from "effect"; -/** The errno code a platform error carries on its cause, when it has one. */ +/** The errno code a platform error carries on its cause, also when another error wraps it. */ export const errorCode = (error: unknown): string | undefined => { - if (!Predicate.hasProperty(error, "cause")) return undefined; - return Predicate.hasProperty(error.cause, "code") && typeof error.cause.code === "string" - ? error.cause.code - : undefined; + let current = error; + for (let depth = 0; depth < 4 && Predicate.hasProperty(current, "cause"); depth++) { + current = current.cause; + if (Predicate.hasProperty(current, "code") && typeof current.code === "string") + return current.code; + } + return undefined; }; /** Windows reports a file that another process holds or replaces as a transient sharing violation. */ diff --git a/packages/stack/src/internal/sharing-violation.unit.test.ts b/packages/stack/src/internal/sharing-violation.unit.test.ts new file mode 100644 index 0000000000..b2097f0b49 --- /dev/null +++ b/packages/stack/src/internal/sharing-violation.unit.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from "@effect/vitest"; +import { Effect, Fiber, PlatformError } from "effect"; +import { TestClock } from "effect/testing"; +import { LogStoreError } from "../host/LogStore.ts"; +import { retrySharingViolation } from "./sharing-violation.ts"; + +const wrappedBusy = new LogStoreError({ + operation: "list", + message: "resource busy", + cause: PlatformError.systemError({ + _tag: "Busy", + module: "FileSystem", + method: "readDirectory", + cause: Object.assign(new Error("resource busy"), { code: "EBUSY" }), + }), +}); + +/** Fails with `wrappedBusy` on its first `failures` attempts. */ +const flaky = (failures: number) => { + let attempts = 0; + return { + attempts: () => attempts, + effect: Effect.suspend(() => + attempts++ < failures ? Effect.fail(wrappedBusy) : Effect.succeed("listed"), + ), + }; +}; + +describe("retrySharingViolation", () => { + it.effect("retries a sharing violation wrapped in another error on Windows", () => + Effect.gen(function* () { + const operation = flaky(2); + + const fiber = yield* operation.effect.pipe(retrySharingViolation("win32"), Effect.forkChild); + yield* TestClock.adjust("1 second"); + const result = yield* Fiber.join(fiber); + + expect(result).toBe("listed"); + expect(operation.attempts()).toBe(3); + }), + ); + + it.effect("fails a wrapped sharing violation at once on other platforms", () => + Effect.gen(function* () { + const operation = flaky(1); + + const error = yield* operation.effect.pipe(retrySharingViolation("linux"), Effect.flip); + + expect(error).toBe(wrappedBusy); + expect(operation.attempts()).toBe(1); + }), + ); +}); From 2f8397f21e45fb84b127078c1c4b3078e0948074 Mon Sep 17 00:00:00 2001 From: avallete Date: Thu, 1 Oct 2026 13:46:23 +0200 Subject: [PATCH 06/35] fix(stack): keep late log lines whole and report skipped log segments - Flush a late partial line only after every chunk published before its grace deadline is processed, comparing publish times. - Report a lost marker when a reader finishes a segment and the next retained generation is not contiguous. - Treat stack logs --since as a value-consuming flag. Co-Authored-By: Claude Opus 5.5 --- .../src/command-internal/db-target-flags.ts | 1 + .../src/host/LogStore.integration.test.ts | 62 +++++++++++++++- packages/stack/src/host/LogStore.ts | 72 +++++++++++++------ 3 files changed, 113 insertions(+), 22 deletions(-) diff --git a/apps/cli/src/command-internal/db-target-flags.ts b/apps/cli/src/command-internal/db-target-flags.ts index 59a2a106ed..035d2546fe 100644 --- a/apps/cli/src/command-internal/db-target-flags.ts +++ b/apps/cli/src/command-internal/db-target-flags.ts @@ -139,6 +139,7 @@ export const VALUE_CONSUMING_LONG_FLAGS = new Set([ // experimental stack flags "capability", "service", + "since", "stack", "stack-id", "preparation", diff --git a/packages/stack/src/host/LogStore.integration.test.ts b/packages/stack/src/host/LogStore.integration.test.ts index 9b36630d9c..961550b8d9 100644 --- a/packages/stack/src/host/LogStore.integration.test.ts +++ b/packages/stack/src/host/LogStore.integration.test.ts @@ -18,7 +18,12 @@ import { } from "effect"; import { TestClock } from "effect/testing"; import type { LaunchOutput } from "../runtime/Session.ts"; -import { endedLineGraceMillis, segmentName, type LogRecord } from "./LogRecord.ts"; +import { + endedLineGraceMillis, + segmentGeneration, + segmentName, + type LogRecord, +} from "./LogRecord.ts"; import * as LogStore from "./LogStore.ts"; const encoder = new TextEncoder(); @@ -271,6 +276,47 @@ describe("LogStore", () => { }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); + it.effect("keeps a late line whole when its newline was published before the grace ran out", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* tempRoot("log-store-late-newline-"); + const blocked = yield* Deferred.make(); + const release = yield* Deferred.make(); + const injected = replaceWrites(fs, (file, buffer) => + new TextDecoder().decode(buffer).includes("busy") + ? Deferred.succeed(blocked, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.andThen(file.writeAll(buffer)), + ) + : file.writeAll(buffer), + ); + const { store } = yield* openStore(root).pipe( + Effect.provideService(FileSystem.FileSystem, injected), + ); + const instance = yield* fakeInstance("newline"); + yield* store.attach(instance); + const reader = yield* collect(store.read("newline", { from: "oldest", follow: true })); + yield* instance.setLaunch(1); + yield* instance.publish(instance.chunk(1, "ending")); + yield* reader.take(1); + yield* instance.setLaunch(undefined); + yield* untilLast(reader, "ending"); + + // The writer is busy with `busy` while the newline is published, so it stays queued. + yield* instance.publish(instance.chunk(1, "late "), instance.chunk(1, "busy\n", "stderr")); + yield* Deferred.await(blocked); + yield* TestClock.adjust(endedLineGraceMillis / 2); + yield* instance.publish(instance.chunk(1, "line end\n")); + yield* TestClock.adjust(endedLineGraceMillis / 2); + yield* Deferred.succeed(release, undefined); + const records: Array = []; + while (!records.some((record) => record.text?.endsWith("line end") === true)) + records.push(...(yield* reader.take(1))); + + expect(texts(records)).toEqual(["busy", "late line end"]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + it.effect("records each failed write as lost once, including a partial-line flush", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -684,8 +730,17 @@ describe("LogStore", () => { from: { generation: 3, byteOffset: 0 }, follow: false, })).pipe(Stream.runCollect); + const fromHeld = yield* (yield* store.read("stuck", { + from: { generation: 1, byteOffset: 0 }, + follow: false, + })).pipe(Stream.runCollect); + const afterHeld = segments + .map(segmentGeneration) + .filter((generation) => generation !== undefined && generation > 1) + .toSorted((left, right) => (left ?? 0) - (right ?? 0))[0]; expect(segments).toContain(held); + expect(segments).not.toContain(segmentName(2)); expect(segments).not.toContain(segmentName(3)); expect(segments.length).toBeLessThanOrEqual(4); const [gap] = resumed; @@ -693,6 +748,11 @@ describe("LogStore", () => { expect(gap?.position).toBeUndefined(); expect(gap?.resumeAt?.generation).toBeGreaterThan(3); expect(texts(resumed).at(-1)).toBe("stuck line 9"); + const heldGaps = Array.from(fromHeld).filter((record) => record.kind === "lost"); + expect(heldGaps).toHaveLength(1); + expect(heldGaps[0]?.resumeAt).toEqual({ generation: afterHeld, byteOffset: 0 }); + expect(fromHeld[0]?.kind).toBe("launch"); + expect(texts(fromHeld).at(-1)).toBe("stuck line 9"); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); diff --git a/packages/stack/src/host/LogStore.ts b/packages/stack/src/host/LogStore.ts index 69c4eff33b..fd6c083ecb 100644 --- a/packages/stack/src/host/LogStore.ts +++ b/packages/stack/src/host/LogStore.ts @@ -328,7 +328,7 @@ const makeReader = (fs: FileSystem.FileSystem, path: Path.Path) => { atEnd: true, partial, }); - /** Reports the records of the cursor's deleted segment as a gap before `generation`. */ + /** Reports segments deleted before `generation` as a gap that resumes there. */ const gap = Effect.fnUntraced(function* (generation: number) { const resumeAt = { generation, byteOffset: 0 }; const records: ReadonlyArray = @@ -359,7 +359,9 @@ const makeReader = (fs: FileSystem.FileSystem, path: Path.Path) => { partial: false, } satisfies Step; // A segment is closed once a newer one exists, so its incomplete tail never completes. - if (newer !== undefined) return moveOn(newer); + // Generations are contiguous, so a hole before the next one is a deleted segment. + if (newer !== undefined) + return newer === cursor.generation + 1 ? moveOn(newer) : yield* gap(newer); if (fresh) return atEnd(chunk.value.partial); const writing = live === undefined ? undefined : yield* Ref.get(live.end); return writing !== undefined && writing.generation <= cursor.generation @@ -827,25 +829,48 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio /** Signals the late-line flusher that an ended launch holds a partial line. */ const lateLines = yield* Queue.sliding(1); + /** The writer took a batch it has not persisted yet. */ + let holding = false; + + /** + * Persists a batch, then the late partial lines its publish times prove idle: every chunk + * published before the batch's last one has been split. + */ const persistBatch = (chunks: ReadonlyArray) => - Clock.currentTimeMillis.pipe( - Effect.flatMap((now) => - handle.lock.withPermits(1)( - Effect.suspend(() => - persist([ - ...chunks.flatMap((chunk) => splitter.push(chunk)), - ...splitter.flushEnded(now), - ]), - ), - ), - ), - Effect.andThen( - Effect.suspend(() => - splitter.endedDue() === undefined ? Effect.void : Queue.offer(lateLines, undefined), + handle.lock + .withPermits(1)( + Effect.suspend(() => { + const entries = chunks.flatMap((chunk) => splitter.push(chunk)); + const published = chunks.reduce((latest, chunk) => Math.max(latest, chunk.time), 0); + return persist([...entries, ...splitter.flushEnded(published)]); + }), + ) + .pipe( + Effect.andThen( + Effect.suspend(() => { + holding = false; + return splitter.endedDue() === undefined + ? Effect.void + : Queue.offer(lateLines, undefined); + }), ), - ), - Effect.uninterruptible, - ); + Effect.uninterruptible, + ); + + /** + * Flushes due late partial lines unless published output is still unsplit, whose batch then + * signals again; `false` when it deferred. + */ + const flushIdle = handle.lock.withPermits(1)( + Effect.uninterruptible( + Effect.gen(function* () { + if (holding || (yield* PubSub.remaining(subscription)) > 0) return false; + const now = yield* Clock.currentTimeMillis; + yield* persist(splitter.flushEnded(now)); + return true; + }), + ), + ); /** Flushes late partial lines of ended launches once they stay idle for the grace. */ const flushLate = Effect.forever( @@ -854,7 +879,7 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio for (let due = splitter.endedDue(); due !== undefined; due = splitter.endedDue()) { const now = yield* Clock.currentTimeMillis; if (now < due) yield* Effect.sleep(Duration.millis(due - now)); - else yield* persistFlushed((at) => splitter.flushEnded(at)); + else if (!(yield* flushIdle)) break; } }), ); @@ -922,7 +947,12 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio yield* Effect.forkIn( Effect.forever( Effect.uninterruptibleMask((restore) => - restore(PubSub.takeAll(subscription)).pipe(Effect.flatMap(persistBatch)), + restore(PubSub.takeAll(subscription)).pipe( + Effect.flatMap((chunks) => { + holding = true; + return persistBatch(chunks); + }), + ), ), ), scope, From 4cb9cd450e367bbd70df7be779c0303b0b645f09 Mon Sep 17 00:00:00 2001 From: avallete Date: Thu, 1 Oct 2026 13:44:07 +0200 Subject: [PATCH 07/35] feat(stack): log gateway requests and ship them to Studio's API Gateway - The shared API proxy records one access line per request and websocket upgrade (nginx combined format plus duration, credentials in the query string redacted), persisted as a gateway log stream. - Gateway lines ship to the cloudflare.logs.prod source with the Kong request and response metadata Studio's API Gateway page reads. - supabase stack logs includes gateway lines and selects them with --service gateway. Co-Authored-By: Claude Opus 5.5 --- apps/cli/docs/stack-commands.md | 15 +- .../db-config.integration.test.ts | 1 + .../commands/db/dump/dump.integration.test.ts | 1 + .../db/reset/reset.integration.test.ts | 1 + .../generate/generate.integration.test.ts | 1 + .../declarative/sync/sync.integration.test.ts | 1 + .../db/start/start.integration.test.ts | 1 + .../experimental/stack/logs/SIDE_EFFECTS.md | 8 +- .../experimental/stack/logs/logs.command.ts | 2 +- .../experimental/stack/logs/logs.handler.ts | 34 ++- .../stack/logs/logs.integration.test.ts | 79 +++++ .../stack/prepare/prepare.integration.test.ts | 1 + .../experimental/stack/start/SIDE_EFFECTS.md | 16 +- .../start-export-pointer.integration.test.ts | 1 + .../stack/start/start.integration.test.ts | 1 + .../stack/status/status.integration.test.ts | 1 + .../serve/serve.stack.integration.test.ts | 1 + apps/cli/tests/helpers/storage.ts | 1 + packages/stack/ARCHITECTURE.md | 10 +- .../stack/src/HttpProxy.integration.test.ts | 258 +++++++++++++++- packages/stack/src/HttpProxy.ts | 282 ++++++++++++++---- packages/stack/src/Network.ts | 20 +- .../src/Owner.analytics.integration.test.ts | 10 + .../stack/src/Owner.logs.integration.test.ts | 56 ++++ packages/stack/src/Owner.ts | 46 ++- packages/stack/src/effect.ts | 30 +- packages/stack/src/host/GatewayLog.ts | 63 ++++ .../src/host/LogForwarder.integration.test.ts | 52 +++- packages/stack/src/host/LogForwarder.ts | 27 +- packages/stack/src/host/LogflareEvents.ts | 61 +++- .../src/host/LogflareEvents.unit.test.ts | 45 +++ 31 files changed, 998 insertions(+), 128 deletions(-) create mode 100644 packages/stack/src/host/GatewayLog.ts diff --git a/apps/cli/docs/stack-commands.md b/apps/cli/docs/stack-commands.md index f6d4e42511..05933fc4c0 100644 --- a/apps/cli/docs/stack-commands.md +++ b/apps/cli/docs/stack-commands.md @@ -237,13 +237,14 @@ lint transaction (always rolled back). It does not launch a client binary. ## Reading stack logs -`supabase stack logs` prints the retained stdout/stderr of composition members and -exits; `-f/--follow` then streams new lines until interrupted. Select `--stack ` -or `--stack-id `; the repeatable `--service ` can include -standalone services too. History is read from the persisted log files, so it works -while the stack is stopped; `--follow` requires a running owner and fails before -printing anything without one. Neither mode starts an owner or service, and Ctrl-C -leaves services running. +`supabase stack logs` prints the retained stdout/stderr of composition members, plus +the `gateway` request lines of the shared API port, and exits; `-f/--follow` then +streams new lines until interrupted. Select `--stack ` or `--stack-id `; the +repeatable `--service ` can include standalone services too, and +`--service gateway` reads only the request lines. History is read from the persisted +log files, so it works while the stack is stopped; `--follow` requires a running owner +and fails before printing anything without one. Neither mode starts an owner or +service, and Ctrl-C leaves services running. `--tail N` (default 200) keeps the newest lines across the selected services and `--since` takes a duration (`10m`, `1h30m`), an ISO-8601 time, or `start` for each diff --git a/apps/cli/src/command-internal/db-config.integration.test.ts b/apps/cli/src/command-internal/db-config.integration.test.ts index 1002a2b5d7..ad1e3d4390 100644 --- a/apps/cli/src/command-internal/db-config.integration.test.ts +++ b/apps/cli/src/command-internal/db-config.integration.test.ts @@ -489,6 +489,7 @@ describe("dbConfigResolver (db-url under the stack backend)", () => { }, stop: unused, destroy: unused, + gateway: { readLogs: () => Stream.die("unused") }, commands: { run: () => unused }, }; return Layer.succeed(StackApi, { diff --git a/apps/cli/src/commands/db/dump/dump.integration.test.ts b/apps/cli/src/commands/db/dump/dump.integration.test.ts index e020809e10..642203185b 100644 --- a/apps/cli/src/commands/db/dump/dump.integration.test.ts +++ b/apps/cli/src/commands/db/dump/dump.integration.test.ts @@ -153,6 +153,7 @@ const managedDumpStackApi = (runtime: "native" | "docker") => { }, stop: Effect.void, destroy: Effect.succeed({ runtimeCleanup: "complete" as const }), + gateway: { readLogs: () => Stream.die("unused") }, commands: { run: runCommand }, } satisfies Stack; return Layer.succeed(StackApi, { diff --git a/apps/cli/src/commands/db/reset/reset.integration.test.ts b/apps/cli/src/commands/db/reset/reset.integration.test.ts index 1f73deb1b8..15f5c5b81f 100644 --- a/apps/cli/src/commands/db/reset/reset.integration.test.ts +++ b/apps/cli/src/commands/db/reset/reset.integration.test.ts @@ -808,6 +808,7 @@ function mockResetStackApi(opts: { }, stop: Effect.die("unused"), destroy: Effect.die("unused"), + gateway: { readLogs: () => Stream.die("unused") }, commands: { run: () => Effect.die("unused") }, }; return { diff --git a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts index f6838085f6..cfeca95094 100644 --- a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts @@ -164,6 +164,7 @@ function generateStackApi(workdir: string) { }, stop: unusedStack, destroy: unusedStack, + gateway: { readLogs: () => Stream.die("unused") }, commands: { run: unusedStackFn }, }; const identity = { projectRoot: workdir, branchContext: "main", stackName: "default" }; diff --git a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts index c7035c2c4f..bb222990b2 100644 --- a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts @@ -166,6 +166,7 @@ function syncStackApi(workdir: string, port: number) { }, stop: unusedSync, destroy: unusedSync, + gateway: { readLogs: () => Stream.die("unused") }, commands: { run: unusedSyncFn }, }; const identity = { projectRoot: workdir, branchContext: "main", stackName: "default" }; diff --git a/apps/cli/src/commands/db/start/start.integration.test.ts b/apps/cli/src/commands/db/start/start.integration.test.ts index 48f591f4dc..fdeca26575 100644 --- a/apps/cli/src/commands/db/start/start.integration.test.ts +++ b/apps/cli/src/commands/db/start/start.integration.test.ts @@ -1733,6 +1733,7 @@ describe("db start stack backend", () => { }, stop: Effect.void, destroy: Effect.succeed({ runtimeCleanup: "complete" as const }), + gateway: { readLogs: () => Stream.die("unused") }, commands: { run: () => Effect.die("unused") }, }; return { stack, state }; diff --git a/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md index 8cd7e8e4cf..0e67af126f 100644 --- a/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md @@ -8,9 +8,11 @@ starts/stops a service. ## Selection and files Select the current project/branch/name, `--stack `, or `--stack-id `. -The selectors are mutually exclusive. By default, only composition members are -included. `--service ` is repeatable and can also select -standalone instances; a value that matches no saved instance fails with status 1. +The selectors are mutually exclusive. By default, composition members are +included, plus `gateway` (the shared API port's request lines) once the stack has +claimed that port. `--service ` is repeatable and can also select +standalone instances or `gateway`; a value that matches no saved instance, or +`gateway` without the shared API port, fails with status 1. A missing stack fails with status 1. Reads saved definitions under `/stacks//` and diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.command.ts b/apps/cli/src/commands/experimental/stack/logs/logs.command.ts index 0e3d972b62..96ae1d2583 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.command.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.command.ts @@ -17,7 +17,7 @@ const config = { ), service: Flag.string("service").pipe( Flag.withDescription( - "Read one service kind or instance ID; repeat to select several. Defaults to composition members.", + "Read one service kind, instance ID, or gateway (shared API requests); repeat to select several. Defaults to composition members and gateway.", ), Flag.atLeast(0), ), diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.handler.ts b/apps/cli/src/commands/experimental/stack/logs/logs.handler.ts index cc91d681a4..975a0d9f2b 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.handler.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.handler.ts @@ -1,5 +1,10 @@ import { Clock, Effect, Option, Path, Stream } from "effect"; -import { streamStackLogs, type SavedStack, type StackLogRecord } from "@supabase/stack/effect"; +import { + gatewayLog, + streamStackLogs, + type SavedStack, + type StackLogRecord, +} from "@supabase/stack/effect"; import { Output } from "../../../../shared/output/output.service.ts"; import { OutputFlag } from "../../../../command-internal/global-flags.ts"; import { dim } from "../../../../command-internal/colors.ts"; @@ -48,9 +53,16 @@ const select = ( id, service: creation.service, }); + // The owner records the shared API listener's requests once the stack has claimed its port. + const gateway: ReadonlyArray = definition.ports.some(({ key }) => key === "api") + ? [{ id: gatewayLog.instanceId, service: gatewayLog.service }] + : []; if (requested.length === 0) { const members = new Set(definition.composition.members.map(({ id }) => id)); - const selected = definition.instances.filter(({ id }) => members.has(id)).map(subject); + const selected = [ + ...definition.instances.filter(({ id }) => members.has(id)).map(subject), + ...gateway, + ]; return selected.length === 0 ? Effect.fail( new StackCommandLogsError({ @@ -63,17 +75,20 @@ const select = ( } const unmatched = requested.find( (value) => - !definition.instances.some(({ id, creation }) => id === value || creation.service === value), + !definition.instances.some( + ({ id, creation }) => id === value || creation.service === value, + ) && !gateway.some(({ service }) => service === value), ); if (unmatched !== undefined) return Effect.fail( new StackCommandLogsError({ reason: "flags", message: `No service matches ${unmatched}.` }), ); - return Effect.succeed( - definition.instances + return Effect.succeed([ + ...definition.instances .filter(({ id, creation }) => requested.includes(id) || requested.includes(creation.service)) .map(subject), - ); + ...gateway.filter(({ service }) => requested.includes(service)), + ]); }; export const stackLogs = Effect.fn("experimental.stack.logs")(function* (flags: StackLogsFlags) { @@ -177,13 +192,14 @@ export const stackLogs = Effect.fn("experimental.stack.logs")(function* (flags: ]), ); const streams = selected.flatMap(({ id, service }) => { - const handle = handles.get(id); - if (handle === undefined) return []; + const readLogs = + id === gatewayLog.instanceId ? stack.gateway.readLogs : handles.get(id)?.readLogs; + if (readLogs === undefined) return []; const from = printed.get(id); // Without history, the owner pins the start at its end under the writer's lock. const start = flags.tail === 0 ? { tail: 0 } : from === undefined ? {} : { from }; return [ - handle.readLogs({ follow: true, ...start, ...sinceTime }).pipe( + readLogs({ follow: true, ...start, ...sinceTime }).pipe( Stream.filter((record) => isAfter(record, from)), Stream.map((record): StackLogRecord => ({ ...record, service, instanceId: id })), ), diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts b/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts index baedd7cdbf..b3221de3ba 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts @@ -96,6 +96,8 @@ const fixture = Effect.fn("StackLogsTest.fixture")(function* (options: { readonly stateRoot: string; readonly stackId: string; }) => ReadonlyArray>; + readonly ports?: SavedStack["ports"]; + readonly gatewayLogs?: (options?: ReadLogsOptions) => Stream.Stream; }) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; @@ -113,6 +115,7 @@ const fixture = Effect.fn("StackLogsTest.fixture")(function* (options: { const definition: SavedStack = { ...found.value.definition, instances: options.instances, + ports: options.ports ?? [], composition: { members: options.members.map((id) => ({ id, activation: "eager" as const })), dependencies: [], @@ -147,6 +150,7 @@ const fixture = Effect.fn("StackLogsTest.fixture")(function* (options: { ...stack.services, list: Effect.succeed(options.handles?.({ stateRoot, stackId: stack.id }) ?? []), }, + gateway: { readLogs: options.gatewayLogs ?? stack.gateway.readLogs }, }), ), }), @@ -435,6 +439,81 @@ describe("stack logs", () => { }).pipe(Effect.scoped, Effect.provide(live)), ); + const apiPort = [{ key: "api", host: "127.0.0.1", port: 54321 }]; + const request = + '127.0.0.1 - - [29/Sep/2026:10:00:00 +0000] "GET /rest/v1/ HTTP/1.1" 200 2 "-" "curl/8.7.1" 3ms'; + + it.live("includes the gateway's requests by default and selects them with --service", () => + Effect.gen(function* () { + const f = yield* fixture({ + instances: [mail("mail-a")], + members: ["mail-a"], + ports: apiPort, + }); + yield* f.writeSegment("mail", "mail-a", [launch(t0), line(t0 + 1, "stdout", "mail up")]); + yield* f.writeSegment("gateway", "gateway", [launch(t0), line(t0 + 2, "stdout", request)]); + + const all = yield* f.run({}); + const gateway = yield* f.run({ service: ["gateway"] }, "stream-json"); + + expect(lines(all.output.stdoutText).map((value) => value.replace(/\d{2}:\S+ /u, ""))).toEqual( + [ + "gateway | --- launch 1 ---", + "mail | --- launch 1 ---", + "mail | mail up", + `gateway | ${request}`, + ], + ); + expect(eventLines(gateway.output.events)).toEqual([request]); + expect(gateway.output.events.at(-1)).toMatchObject({ + service: "gateway", + instance_id: "gateway", + }); + }).pipe(Effect.scoped, Effect.provide(live)), + ); + + it.live("rejects --service gateway when the stack has no shared API listener", () => + Effect.gen(function* () { + const f = yield* fixture({ instances: [mail("mail-a")], members: ["mail-a"] }); + yield* f.writeSegment("mail", "mail-a", [launch(t0), line(t0 + 1, "stdout", "mail up")]); + + const selected = yield* f.run({ service: ["gateway"] }); + const all = yield* f.run({}, "stream-json"); + + expect(failure(selected.exit)).toMatchObject({ + reason: "flags", + message: "No service matches gateway.", + }); + expect(eventLines(all.output.events)).toEqual(["mail up"]); + }).pipe(Effect.scoped, Effect.provide(live)), + ); + + it.live("follows the gateway's new requests through the owner", () => + Effect.gen(function* () { + const f = yield* fixture({ + instances: [], + members: [], + ports: apiPort, + running: true, + gatewayLogs: () => + Stream.make({ + kind: "stdout" as const, + timestamp: iso(t0 + 100), + launchId: 1, + text: request, + position: { generation: 1, byteOffset: 1_000 }, + }), + }); + + const { exit, output } = yield* f.run({ follow: true, tail: 0 }, "stream-json"); + + expect(Exit.isSuccess(exit)).toBe(true); + expect(output.events).toEqual([ + expect.objectContaining({ service: "gateway", line: request, source: "live" }), + ]); + }).pipe(Effect.scoped, Effect.provide(live)), + ); + it.live("interrupts a follow and releases its subscriptions", () => Effect.gen(function* () { const subscribed = yield* Deferred.make(); diff --git a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts index 352518aadb..c377fd3316 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts @@ -157,6 +157,7 @@ const makeFixture = (root: string, options: FixtureOptions = {}) => { }, stop: Effect.void, destroy: Effect.succeed({ runtimeCleanup: "complete" as const }), + gateway: { readLogs: () => Stream.die("unused") }, commands: { run: () => Effect.die("unused") }, }; const output = mockOutput(); diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index 0636d9e740..d55872ede0 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -41,7 +41,10 @@ The owner persists each service's output under `$SUPABASE_HOME/stacks/ most about 10 MiB (plus the segment being written) per service instance. Destroying an instance or the stack deletes those logs; stopping the stack and resetting database data keep them. PostgREST runs with `PGRST_LOG_LEVEL=info`, so every request line, query string included, is -persisted and shipped to Analytics. +persisted and shipped to Analytics. The shared API port also records one nginx combined line plus +duration (`... "GET /rest/v1/todos?select=* HTTP/1.1" 200 126 "-" "curl/8.7.1" 12ms`) per request +and WebSocket upgrade under `logs/gateway/gateway/`, with the same retention; `apikey`, +`access_token`, and `token` query values are written as `redacted`. When an owner starts a stack saved with a Vector instance, it removes that instance, its composition members, dependencies and port claims from `state.json`, and its stack-owned Vector config files under `data//runtime/vector/`; its containers go with the stack's container sweep. @@ -89,11 +92,12 @@ Studio requires REST; excluding REST while keeping Studio fails before stopping ## Service logs in Analytics -The owner ships the persisted Auth, REST, Realtime, Storage, Functions, and database output lines -(not launch or lost markers) to Analytics' `POST /api/logs` ingest endpoint on its direct backend, -using the Analytics API key and the legacy Logflare source names (`gotrue.logs.prod`, -`postgREST.logs.prod`, `realtime.logs.prod`, `storage.logs.prod.2`, `deno-relay-logs`, -`postgres.logs`) with the legacy per-service field remaps. This applies to the Docker, Podman, and +The owner ships the persisted Auth, REST, Realtime, Storage, Functions, database, and gateway +output lines (not launch or lost markers) to Analytics' `POST /api/logs` ingest endpoint on its +direct backend, using the Analytics API key and the legacy Logflare source names +(`gotrue.logs.prod`, `postgREST.logs.prod`, `realtime.logs.prod`, `storage.logs.prod.2`, +`deno-relay-logs`, `postgres.logs`, and `cloudflare.logs.prod` for Studio's API Gateway page) with +the legacy per-service field remaps. This applies to the Docker, Podman, and native runtimes. Shipping runs only while the composed Analytics service is running and healthy; each instance keeps its position in `logs///cursor.json`, so lines written while Analytics is stopped, starting, or unhealthy are shipped with their original timestamps once diff --git a/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts index b0b01e52d0..479a6963e5 100644 --- a/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts @@ -112,6 +112,7 @@ function makeDatabaseStack(sqlPort: number, credentials: StackCredentials): Stac }, stop: Effect.die("unused"), destroy: Effect.die("unused"), + gateway: { readLogs: () => Stream.die("unused") }, commands: { run: () => Effect.die("unused") }, } satisfies Stack; } diff --git a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts index 91a1f0a761..5b0d9379a7 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts @@ -352,6 +352,7 @@ const fakeStack = (compositionStart?: Stack["composition"]["start"]) => { hostDestroyed += 1; return { runtimeCleanup: "complete" as const }; }), + gateway: { readLogs: () => Stream.die("unused") }, commands: { run: () => Effect.die("command not used") }, }; return { diff --git a/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts b/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts index 3d2584df86..2abbb751c0 100644 --- a/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts @@ -176,6 +176,7 @@ const makeStack = ( }, stop: Effect.die("unused"), destroy: Effect.die("unused"), + gateway: { readLogs: () => Stream.die("unused") }, commands: { run: (_tool, _options) => Effect.die("unused"), }, diff --git a/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts b/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts index 16c710780e..e948fa0ac2 100644 --- a/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts +++ b/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts @@ -287,6 +287,7 @@ const fixture = ( }, stop: Effect.die("unused"), destroy: Effect.die("unused"), + gateway: { readLogs: () => Stream.die("unused") }, commands: { run: () => Effect.die("unused") }, } satisfies Stack; const identity = { projectRoot: "/project", branchContext: "main", stackName: "default" }; diff --git a/apps/cli/tests/helpers/storage.ts b/apps/cli/tests/helpers/storage.ts index 95cb7d9854..3e5b651316 100644 --- a/apps/cli/tests/helpers/storage.ts +++ b/apps/cli/tests/helpers/storage.ts @@ -263,6 +263,7 @@ export function buildStorageStackApi( }, stop: Effect.die("unused"), destroy: Effect.die("unused"), + gateway: { readLogs: () => Stream.die("unused") }, commands: { run: () => Effect.die("unused") }, }; const definition = { diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 0d558590ce..34fe79f1f0 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -662,8 +662,16 @@ follow can resume at a record position, and a reader that finds its segment dele instances no longer saved, so a failed deletion is retried; destroying the stack removes `logs/`; resetting database data keeps them. +The shared API listener is the stack's gateway. Each completed request, and each WebSocket upgrade +once its handshake status is known, becomes one nginx combined line plus duration, with `apikey`, +`access_token` and `token` query values redacted. The proxy hands it to a bounded sliding buffer +after the response settles, so logging never delays a response or holds a target's activity. +The owner persists these lines as the `gateway` stream, `logs/gateway/gateway/`, with one launch +per owner run; it is not a service instance, so orphan cleanup keeps it. + While the composed Analytics instance is running and healthy, the owner ships the persisted -stdout/stderr records of Auth, REST, Realtime, Storage, Functions and database instances to its +stdout/stderr records of Auth, REST, Realtime, Storage, Functions and database instances, and of +the gateway stream as Studio's API Gateway source, to its direct backend (never the proxy, so shipping neither wakes it nor counts as activity), in bodies of at most 256 events and 1 MiB. Each instance reads from `cursor.json` in its logs directory, the position of its last shipped record, written atomically after each body settles: a missing cursor diff --git a/packages/stack/src/HttpProxy.integration.test.ts b/packages/stack/src/HttpProxy.integration.test.ts index 8e6398d412..42a2bea3bd 100644 --- a/packages/stack/src/HttpProxy.integration.test.ts +++ b/packages/stack/src/HttpProxy.integration.test.ts @@ -1,6 +1,6 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { expect, it } from "@effect/vitest"; -import { Data, Deferred, Effect, Fiber, Layer } from "effect"; +import { Data, Deferred, Effect, Fiber, Layer, Queue } from "effect"; import { HttpClient, HttpClientRequest } from "effect/unstable/http"; import { createServer, type Server, type ServerResponse } from "node:http"; // oxlint-disable-line effecttsgo/node-builtin-import -- raw server fixture. import { createServer as createTcpServer, Socket, type Server as NetServer } from "node:net"; // oxlint-disable-line effecttsgo/node-builtin-import -- raw disconnect fixture. @@ -8,7 +8,7 @@ import { createServer as createTcpServer, Socket, type Server as NetServer } fro import { WebSocket, WebSocketServer } from "ws"; import { captureLogs } from "../tests/logs.ts"; import { ProxyError } from "./Proxy.ts"; -import { makeHttpProxy, type HttpRoute } from "./HttpProxy.ts"; +import { makeHttpProxy, type HttpAccess, type HttpRoute } from "./HttpProxy.ts"; const listen = (server: Server | NetServer, options?: { readonly beforeClose?: () => void }) => Effect.acquireRelease( @@ -1027,3 +1027,257 @@ it.live("releases a waiting WebSocket target quietly when its client resets", () }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, captureErrors(logs)))); }); + +/** Opens a raw client connection that has written `text`. */ +const rawClient = (port: number, text: string) => + Effect.acquireRelease( + Effect.callback((resume) => { + const socket = new Socket(); + socket.on("error", () => undefined); + socket.connect(port, "127.0.0.1", () => { + socket.write(text); + resume(Effect.succeed(socket)); + }); + }), + (socket) => Effect.sync(() => socket.destroy()), + ); + +const upgradeRequest = (path: string) => + `GET ${path} HTTP/1.1\r\nHost: localhost\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n`; + +/** Sends a raw upgrade request and waits until the proxy closes the connection. */ +const rawUpgrade = (port: number, path: string) => + rawClient(port, upgradeRequest(path)).pipe( + Effect.flatMap((socket) => + Effect.callback((resume) => { + socket.once("close", () => resume(Effect.void)); + if (socket.destroyed) resume(Effect.void); + }), + ), + Effect.timeout("5 seconds"), + ); + +it.live("records each request once with the status the client was sent", () => { + const logs: Array = []; + return Effect.scoped( + Effect.gen(function* () { + const backend = createServer((request, response) => { + request.resume(); + if (request.url?.startsWith("/ok")) response.end("hello"); + else { + response.statusCode = 404; + response.end("nope"); + } + }); + const backendAddress = yield* listen(backend); + const accesses = yield* Queue.unbounded(); + const proxy = yield* makeHttpProxy({ + host: "127.0.0.1", + port: 0, + onAccess: (access) => Queue.offer(accesses, access), + }); + yield* proxy.setRoutes([ + { id: "api", prefix: "/api", upstreamPrefix: "/", target: Effect.succeed(backendAddress) }, + { + id: "down", + prefix: "/down", + target: Effect.fail(new ProxyError({ message: "wake failed" })), + }, + ]); + const get = (path: string, headers: Readonly> = {}) => + request(proxy.port, path, new Uint8Array(), headers, "GET").pipe( + Effect.map(({ status }) => status), + ); + + const statuses = [ + yield* get("/api/ok?select=*&apikey=sb_secret_x&Access_Token=jwt", { + "user-agent": "proxy-test/1", + referer: "http://studio.test/logs", + }), + yield* get("/api/missing"), + yield* get("/elsewhere"), + yield* get("/down/thing?token=t"), + ]; + const recorded = yield* Queue.takeN(accesses, 4); + + expect(statuses).toEqual([200, 404, 404, 502]); + expect(recorded.toSorted((left, right) => left.time - right.time)).toEqual([ + { + time: expect.any(Number), + client: "127.0.0.1", + method: "GET", + target: "/api/ok?select=*&apikey=redacted&Access_Token=redacted", + protocol: "HTTP/1.1", + status: 200, + bytes: 5, + referer: "http://studio.test/logs", + userAgent: "proxy-test/1", + durationMillis: expect.any(Number), + }, + expect.objectContaining({ target: "/api/missing", status: 404, bytes: 4 }), + expect.objectContaining({ target: "/elsewhere", status: 404, bytes: 9 }), + expect.objectContaining({ target: "/down/thing?token=redacted", status: 502, bytes: 11 }), + ]); + expect(yield* Queue.size(accesses)).toBe(0); + }), + ).pipe( + Effect.provide( + Layer.mergeAll(NodeHttpClient.layerNodeHttp, NodeServices.layer, captureErrors(logs)), + ), + ); +}); + +it.live("records WebSocket upgrades at the handshake with the status sent to the client", () => { + const logs: Array = []; + return Effect.scoped( + Effect.gen(function* () { + const backend = createServer(); + const sockets = new WebSocketServer({ server: backend }); + const backendAddress = yield* listen(backend, { beforeClose: () => sockets.close() }); + const upgradeReceived = yield* Deferred.make(); + const silent = createTcpServer((connection) => { + connection.on("error", () => undefined); + connection.once("data", () => Deferred.doneUnsafe(upgradeReceived, Effect.void)); + }); + const silentAddress = yield* listen(silent); + const accesses = yield* Queue.unbounded(); + const proxy = yield* makeHttpProxy({ + host: "127.0.0.1", + port: 0, + onAccess: (access) => Queue.offer(accesses, access), + }); + yield* proxy.setRoutes([ + { id: "ws", prefix: "/socket", target: Effect.succeed(backendAddress) }, + { id: "silent", prefix: "/silent", target: Effect.succeed(silentAddress) }, + { + id: "down", + prefix: "/down", + target: Effect.fail(new ProxyError({ message: "wake failed" })), + }, + ]); + const client = yield* Effect.acquireRelease( + Effect.callback((resume) => { + const socket = new WebSocket( + `ws://127.0.0.1:${proxy.port}/socket?apikey=sb_publishable_x&vsn=2.0.0`, + ); + socket.once("open", () => resume(Effect.succeed(socket))); + socket.once("error", (cause) => + resume(Effect.fail(new HttpProxyTestError({ message: cause.message, cause }))), + ); + }), + (socket) => Effect.sync(() => socket.terminate()), + ).pipe(Effect.timeout("5 seconds")); + + const opened = yield* Queue.take(accesses); + + expect(client.readyState).toBe(WebSocket.OPEN); + expect(opened).toMatchObject({ + method: "GET", + target: "/socket?apikey=redacted&vsn=2.0.0", + status: 101, + }); + expect(opened.bytes).toBeUndefined(); + + yield* rawUpgrade(proxy.port, "/elsewhere"); + expect(yield* Queue.take(accesses)).toMatchObject({ target: "/elsewhere", status: 404 }); + yield* rawUpgrade(proxy.port, "/down"); + expect(yield* Queue.take(accesses)).toMatchObject({ target: "/down", status: 502 }); + + const leaving = yield* rawClient(proxy.port, upgradeRequest("/silent")); + yield* Deferred.await(upgradeReceived); + leaving.destroy(); + expect(yield* Queue.take(accesses)).toMatchObject({ target: "/silent", status: 499 }); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, captureErrors(logs)))); +}); + +it.live("records no body bytes for a client that left before its response completed", () => + Effect.scoped( + Effect.gen(function* () { + const backend = createServer((request, response) => { + request.resume(); + response.writeHead(200); + response.write("first chunk"); + }); + const backendAddress = yield* listen(backend, { + beforeClose: () => backend.closeAllConnections(), + }); + const acquiring = yield* Deferred.make(); + const accesses = yield* Queue.unbounded(); + const proxy = yield* makeHttpProxy({ + host: "127.0.0.1", + port: 0, + onAccess: (access) => Queue.offer(accesses, access), + }); + yield* proxy.setRoutes([ + { + id: "waking", + prefix: "/waking", + target: Deferred.succeed(acquiring, undefined).pipe(Effect.andThen(Effect.never)), + }, + { id: "streaming", prefix: "/streaming", target: Effect.succeed(backendAddress) }, + ]); + + const waiting = yield* rawClient( + proxy.port, + "GET /waking HTTP/1.1\r\nHost: localhost\r\n\r\n", + ); + yield* Deferred.await(acquiring); + waiting.destroy(); + const beforeHeaders = yield* Queue.take(accesses).pipe(Effect.timeout("5 seconds")); + const reading = yield* rawClient( + proxy.port, + "GET /streaming HTTP/1.1\r\nHost: localhost\r\n\r\n", + ); + yield* Effect.callback((resume) => { + reading.once("data", () => resume(Effect.void)); + }); + reading.destroy(); + const midBody = yield* Queue.take(accesses).pipe(Effect.timeout("5 seconds")); + + expect(beforeHeaders).toMatchObject({ target: "/waking", status: 499 }); + expect(beforeHeaders.bytes).toBeUndefined(); + expect(midBody).toMatchObject({ target: "/streaming", status: 200 }); + expect(midBody.bytes).toBeUndefined(); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); + +it.live("answers and releases targets while the access sink is stalled", () => + Effect.scoped( + Effect.gen(function* () { + const backend = createServer((request, response) => { + request.resume(); + response.end("ok"); + }); + const backendAddress = yield* listen(backend); + const released = yield* Queue.unbounded(); + const proxy = yield* makeHttpProxy({ + host: "127.0.0.1", + port: 0, + onAccess: () => Effect.never, + }); + yield* proxy.setRoutes([ + { + id: "api", + prefix: "/api", + target: Effect.acquireRelease(Effect.succeed(backendAddress), () => + Queue.offer(released, undefined), + ), + }, + ]); + + const statuses = yield* Effect.forEach( + Array.from({ length: 20 }, (_, index) => `/api/${index}`), + (path) => + request(proxy.port, path, new Uint8Array(), {}, "GET").pipe( + Effect.map(({ status }) => status), + ), + { concurrency: 5 }, + ).pipe(Effect.timeout("10 seconds")); + + expect(statuses).toEqual(Array.from({ length: 20 }, () => 200)); + expect(yield* Queue.takeN(released, 20).pipe(Effect.timeout("5 seconds"))).toHaveLength(20); + }), + ).pipe(Effect.provide(Layer.merge(NodeHttpClient.layerNodeHttp, NodeServices.layer))), +); diff --git a/packages/stack/src/HttpProxy.ts b/packages/stack/src/HttpProxy.ts index d2ae57d119..2af6364346 100644 --- a/packages/stack/src/HttpProxy.ts +++ b/packages/stack/src/HttpProxy.ts @@ -1,4 +1,4 @@ -import { Data, Effect, FiberSet, Ref, Scope } from "effect"; +import { Clock, Data, Deferred, Effect, Fiber, FiberSet, Ref, Scope } from "effect"; import { PortError } from "./Ports.ts"; import type { BackendAddress, ProxyError } from "./Proxy.ts"; import { @@ -41,6 +41,32 @@ interface HttpRouteKeyRewrite { }; } +/** One request or WebSocket upgrade the proxy completed. */ +export interface HttpAccess { + /** Epoch milliseconds when the request arrived. */ + readonly time: number; + readonly client: string; + readonly method: string; + /** The request path and query, with credential query values redacted. */ + readonly target: string; + readonly protocol: string; + /** + * The recorded outcome: the status sent, or 499 when the client left before a response. An + * upgrade records the upstream handshake status; without one, 404 for no route and 502 for a + * failure, while the client sees its socket reset. + */ + readonly status: number; + /** Body bytes of a response that finished; absent when it was cut short, and for upgrades. */ + readonly bytes?: number; + readonly referer?: string; + readonly userAgent?: string; + /** Until the response finished; for an upgrade, until the upstream handshake answered. */ + readonly durationMillis: number; +} + +/** Receives each completed request after its response; it must not block. */ +export type HttpAccessSink = (access: HttpAccess) => Effect.Effect; + export interface HttpProxy { readonly host: string; readonly port: number; @@ -125,6 +151,73 @@ const decodeQuery = (value: string) => { } }; +const credentialParameters = new Set(["apikey", "access_token", "token"]); + +const redactCredentials = (url: string) => { + const queryAt = url.indexOf("?"); + if (queryAt < 0) return url; + const query = url + .slice(queryAt + 1) + .split("&") + .map((parameter) => { + const separator = parameter.indexOf("="); + if (separator < 0) return parameter; + const name = parameter.slice(0, separator); + return credentialParameters.has(decodeQuery(name).toLowerCase()) + ? `${name}=redacted` + : parameter; + }) + .join("&"); + return `${url.slice(0, queryAt)}?${query}`; +}; + +/** Captures a request's access fields while its socket is open; completes them once it settles. */ +const accessFor = (request: IncomingMessage, time: number) => { + const referer = headerValue(request.headers.referer); + const userAgent = headerValue(request.headers["user-agent"]); + const fields = { + time, + client: request.socket.remoteAddress ?? "-", + method: request.method ?? "GET", + target: redactCredentials(request.url ?? "/"), + protocol: `HTTP/${request.httpVersion}`, + ...(referer === undefined ? {} : { referer }), + ...(userAgent === undefined ? {} : { userAgent }), + }; + return (ended: number, status: number, bytes?: number): HttpAccess => ({ + ...fields, + status, + ...(bytes === undefined ? {} : { bytes }), + durationMillis: Math.max(0, ended - time), + }); +}; + +/** What one request's client was sent. */ +interface Sent { + /** Body bytes handed to the response. */ + bytes: number; + /** The client went away before the response completed. */ + clientLeft: boolean; +} + +const respond = (response: ServerResponse, sent: Sent, status: number, body?: string) => { + response.statusCode = status; + sent.bytes = body === undefined ? 0 : Buffer.byteLength(body); + response.end(body); +}; + +const responseSettled = (response: ServerResponse) => + Effect.callback((resume) => { + const onDone = () => resume(Effect.void); + response.once("finish", onDone); + response.once("close", onDone); + if (response.writableFinished || response.destroyed) onDone(); + return Effect.sync(() => { + response.off("finish", onDone); + response.off("close", onDone); + }); + }); + const queryValueFor = (value: string, keys: HttpRouteKeyRewrite["keys"]) => value === keys.secretKey ? keys.serviceRoleKey @@ -247,6 +340,7 @@ const forward = Effect.fn("HttpProxy.forward")( route: HttpRoute, backend: BackendAddress, agent: Agent | false, + sent: Sent, ) => Effect.callback((resume) => { let outgoing: ReturnType | undefined; @@ -296,6 +390,9 @@ const forward = Effect.fn("HttpProxy.forward")( incoming = value; value.once("aborted", onError); value.on("error", onError); + value.on("data", (chunk: Buffer) => { + sent.bytes += chunk.length; + }); response.once("finish", onFinish); setCors(response, request); response.statusCode = value.statusCode ?? 502; @@ -323,22 +420,43 @@ const forward = Effect.fn("HttpProxy.forward")( ); const proxyRequest = Effect.fn("HttpProxy.proxyRequest")( - (request: IncomingMessage, response: ServerResponse, route: HttpRoute, agent: Agent) => + ( + request: IncomingMessage, + response: ServerResponse, + route: HttpRoute, + agent: Agent, + sent: Sent, + ) => Effect.gen(function* () { const backend = yield* Effect.raceFirst(route.target, disconnected(request, response)); - yield* forward(request, response, route, backend, isReplayable(request) ? agent : false).pipe( + yield* forward( + request, + response, + route, + backend, + isReplayable(request) ? agent : false, + sent, + ).pipe( Effect.catchIf(isRetryable(request, response), (error) => Effect.logWarning( `Route ${route.id} ${request.method ?? "GET"} upstream failed before responding, retrying`, error, - ).pipe(Effect.andThen(forward(request, response, route, backend, false))), + ).pipe(Effect.andThen(forward(request, response, route, backend, false, sent))), ), ); }), ); +const statusLine = /^HTTP\/\d(?:\.\d)? (\d{3})\b/u; + const upgrade = Effect.fn("HttpProxy.upgrade")( - (request: IncomingMessage, client: Duplex, head: Buffer, route: HttpRoute) => + ( + request: IncomingMessage, + client: Duplex, + head: Buffer, + route: HttpRoute, + handshake: Deferred.Deferred, + ) => Effect.gen(function* () { const backend = yield* Effect.raceFirst( route.target, @@ -352,12 +470,14 @@ const upgrade = Effect.fn("HttpProxy.upgrade")( const upstream = yield* connectInterruptibly(backend); yield* Effect.callback((resume) => { let settled = false; + let answer = ""; const cleanup = () => { - client.off("close", onClose); + client.off("close", onClientClose); upstream.off("close", onClose); client.off("end", onClientEnd); upstream.off("end", onUpstreamEnd); + upstream.off("data", onAnswer); }; const finish = (result: Effect.Effect) => { if (settled) return; @@ -373,10 +493,21 @@ const upgrade = Effect.fn("HttpProxy.upgrade")( const onError = (cause: Error) => abandon(Effect.fail(errorFor(cause))); const onClientGone = () => abandon(Effect.fail(new HttpProxyDisconnected())); const onClose = () => abandon(Effect.void); + // A client leaving before the upstream answered never received a response. + const onClientClose = () => (Deferred.isDoneUnsafe(handshake) ? onClose() : onClientGone()); const onClientEnd = () => upstream.end(); const onUpstreamEnd = () => client.end(); + // Reads the handshake status off the bytes relayed to the client. + const onAnswer = (chunk: Buffer) => { + answer += chunk.toString("latin1"); + if (!answer.includes("\r\n") && answer.length < 64) return; + upstream.off("data", onAnswer); + const status = statusLine.exec(answer)?.[1]; + if (status !== undefined) Deferred.doneUnsafe(handshake, Effect.succeed(Number(status))); + }; + upstream.on("data", onAnswer); client.on("error", onClientGone); - client.once("close", onClose); + client.once("close", onClientClose); upstream.on("error", onError); upstream.once("close", onClose); client.once("end", onClientEnd); @@ -404,6 +535,7 @@ const upgrade = Effect.fn("HttpProxy.upgrade")( export const makeHttpProxy = (options: { readonly host: string; readonly port: number; + readonly onAccess?: HttpAccessSink; }): Effect.Effect => Effect.gen(function* () { const routes = yield* Ref.make>([]); @@ -415,42 +547,54 @@ export const makeHttpProxy = (options: { ); const runRequest = yield* FiberSet.makeRuntime(); const sockets = new Set(); + const onAccess = options.onAccess; const server = createServer((request, response) => { runRequest( - Effect.scoped( - Effect.gen(function* () { - const route = yield* Ref.get(routes).pipe( - Effect.map((current) => routeFor(request.url ?? "/", current)), - ); - setCors(response, request); - if (request.method === "OPTIONS") { - response.statusCode = 204; - response.end(); - } else if (route === undefined) { - response.statusCode = 404; - response.end("Not Found"); - } else { - yield* proxyRequest(request, response, route, agent).pipe( - Effect.tapError((cause) => - cause._tag === "HttpProxyDisconnected" - ? Effect.void - : Effect.logError(`Route ${route.id} request failed`, cause), - ), - Effect.catch(() => - Effect.sync(() => { - if (response.destroyed) return; - if (response.headersSent) response.destroy(); - else { - setCors(response, request); - response.statusCode = 502; - response.end("Bad Gateway"); - } - }), - ), + Effect.gen(function* () { + const complete = accessFor(request, yield* Clock.currentTimeMillis); + const sent: Sent = { bytes: 0, clientLeft: false }; + // The access record waits outside this scope, so it never holds the target's activity. + yield* Effect.scoped( + Effect.gen(function* () { + const route = yield* Ref.get(routes).pipe( + Effect.map((current) => routeFor(request.url ?? "/", current)), ); - } - }), - ), + setCors(response, request); + if (request.method === "OPTIONS") respond(response, sent, 204); + else if (route === undefined) respond(response, sent, 404, "Not Found"); + else { + yield* proxyRequest(request, response, route, agent, sent).pipe( + Effect.tapError((cause) => + cause._tag === "HttpProxyDisconnected" + ? Effect.void + : Effect.logError(`Route ${route.id} request failed`, cause), + ), + Effect.catch((cause) => + Effect.sync(() => { + if (cause._tag === "HttpProxyDisconnected") sent.clientLeft = true; + if (response.destroyed) return; + if (response.headersSent || sent.clientLeft) response.destroy(); + else { + setCors(response, request); + respond(response, sent, 502, "Bad Gateway"); + } + }), + ), + ); + } + }), + ); + if (onAccess === undefined) return; + yield* responseSettled(response); + const delivered = response.writableFinished && !sent.clientLeft; + yield* onAccess( + complete( + yield* Clock.currentTimeMillis, + response.headersSent ? response.statusCode : 499, + delivered ? sent.bytes : undefined, + ), + ); + }), ); }); server.on("connection", (socket) => { @@ -460,23 +604,47 @@ export const makeHttpProxy = (options: { server.on("upgrade", (request, socket, head) => { socket.on("error", () => socket.destroy()); runRequest( - Effect.scoped( - Effect.gen(function* () { - const route = yield* Ref.get(routes).pipe( - Effect.map((current) => routeFor(request.url ?? "/", current)), - ); - if (route === undefined) socket.destroy(); - else - yield* upgrade(request, socket, head, route).pipe( - Effect.tapError((cause) => - cause._tag === "HttpProxyDisconnected" - ? Effect.void - : Effect.logError(`Route ${route.id} upgrade failed`, cause), - ), - Effect.catch(() => Effect.sync(() => socket.destroy())), - ); - }), - ), + Effect.gen(function* () { + const complete = accessFor(request, yield* Clock.currentTimeMillis); + // Completed by the upstream's handshake status, otherwise by the upgrade's outcome. + const handshake = yield* Deferred.make(); + const recorded = + onAccess === undefined + ? undefined + : yield* Deferred.await(handshake).pipe( + Effect.flatMap((status) => + Clock.currentTimeMillis.pipe( + Effect.flatMap((ended) => onAccess(complete(ended, status))), + ), + ), + Effect.forkChild({ startImmediately: true }), + ); + const route = yield* Ref.get(routes).pipe( + Effect.map((current) => routeFor(request.url ?? "/", current)), + ); + const outcome = + route === undefined + ? yield* Effect.sync(() => { + socket.destroy(); + return 404; + }) + : yield* Effect.scoped(upgrade(request, socket, head, route, handshake)).pipe( + Effect.as(502), + Effect.tapError((cause) => + cause._tag === "HttpProxyDisconnected" + ? Effect.void + : Effect.logError(`Route ${route.id} upgrade failed`, cause), + ), + Effect.catch((cause) => + Effect.sync(() => { + socket.destroy(); + return cause._tag === "HttpProxyDisconnected" ? 499 : 502; + }), + ), + ); + yield* Deferred.succeed(handshake, outcome); + if (recorded !== undefined) yield* Fiber.join(recorded); + }), ); }); yield* Effect.acquireRelease( diff --git a/packages/stack/src/Network.ts b/packages/stack/src/Network.ts index bb80763509..7f13dbaafc 100644 --- a/packages/stack/src/Network.ts +++ b/packages/stack/src/Network.ts @@ -3,7 +3,7 @@ import { DOCKER_HOST_ALIAS } from "./runtime/Container.ts"; import * as State from "./State.ts"; import { makePorts, PortError } from "./Ports.ts"; import { bindTcp, serveTcp, type BackendAddress, type ProxyError } from "./Proxy.ts"; -import { makeHttpProxy, type HttpProxy, type HttpRoute } from "./HttpProxy.ts"; +import { makeHttpProxy, type HttpAccessSink, type HttpProxy, type HttpRoute } from "./HttpProxy.ts"; export type NetworkRuntime = "native" | "docker" | "podman"; @@ -70,6 +70,7 @@ const makeNetwork = (options: { readonly stackId: string; readonly runtime: NetworkRuntime; readonly state: State.Interface; + readonly onAccess?: HttpAccessSink; }) => Effect.gen(function* () { const ports = yield* makePorts(options.state).pipe( @@ -174,7 +175,15 @@ const makeNetwork = (options: { }); return { proxy: current.proxy }; } - return { proxy: yield* makeHttpProxy({ host, port }) }; + return { + proxy: yield* makeHttpProxy({ + host, + port, + ...(options.onAccess === undefined + ? {} + : { onAccess: options.onAccess }), + }), + }; } if (endpoint.protocol === "http") { const proxy = yield* makeHttpProxy({ host, port }); @@ -341,7 +350,12 @@ const makeNetwork = (options: { return { register, release: release() } satisfies Interface; }); -export const layer = (options: { readonly stackId: string; readonly runtime: NetworkRuntime }) => +export const layer = (options: { + readonly stackId: string; + readonly runtime: NetworkRuntime; + /** Receives the shared API listener's completed requests. */ + readonly onAccess?: HttpAccessSink; +}) => Layer.effect( Service, Effect.gen(function* () { diff --git a/packages/stack/src/Owner.analytics.integration.test.ts b/packages/stack/src/Owner.analytics.integration.test.ts index 61dfe7c031..9b71be5a8d 100644 --- a/packages/stack/src/Owner.analytics.integration.test.ts +++ b/packages/stack/src/Owner.analytics.integration.test.ts @@ -201,6 +201,16 @@ it.live( timestamp: expect.any(String), }, ]); + expect( + yield* awaitStored( + "cloudflare.logs.prod", + `body->'metadata'->'request'->>'path' LIKE '%${restPath}' AND body->'metadata'->'response'->>'status_code' = '404'`, + ), + ).toEqual([ + expect.objectContaining({ + message: expect.stringContaining(`${restPath} HTTP/1.1" 404 `), + }), + ]); yield* Fiber.interrupt(keeper); const noise = yield* Effect.forkScoped( diff --git a/packages/stack/src/Owner.logs.integration.test.ts b/packages/stack/src/Owner.logs.integration.test.ts index 684eebfbc6..cd7ccafc0e 100644 --- a/packages/stack/src/Owner.logs.integration.test.ts +++ b/packages/stack/src/Owner.logs.integration.test.ts @@ -13,6 +13,7 @@ import { Scope, Stream, } from "effect"; +import { HttpClient } from "effect/unstable/http"; import { tmpdir } from "node:os"; import { ownerFor } from "../tests/owner-rpc.ts"; import type { LogRecord } from "./host/LogRecord.ts"; @@ -147,6 +148,61 @@ describe("owner persisted logs", () => { ).pipe(Effect.provide(services)), ); + it.live("keeps shared API requests as gateway logs across owner restarts until destroy", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const client = yield* HttpClient.HttpClient; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "owner-logs-gateway-" }); + const firstScope = yield* Scope.make(); + const first = yield* openOwner("owner-logs-gateway-", "native", { root }).pipe( + Scope.provide(firstScope), + ); + const rest = yield* first.rpc.createService({ + service: "rest", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + yield* first.rpc.configureComposition({ + members: [{ id: rest.id, activation: "lazy" }], + dependencies: [], + }); + yield* first.rpc.startComposition(); + const api = (yield* first.state.read(first.stack.id))?.ports.find( + ({ key }) => key === "api", + ); + if (api === undefined) return yield* Effect.die("the shared API port was not claimed"); + + const response = yield* client.get(`http://127.0.0.1:${api.port}/unrouted?apikey=k`); + const [recorded] = yield* firstOutput(first.rpc.readLogs({ id: "gateway", follow: true })); + + expect(response.status).toBe(404); + expect(recorded?.text).toMatch( + /^127\.0\.0\.1 - - \[[^\]]+\] "GET \/unrouted\?apikey=redacted HTTP\/1\.1" 404 9 "-" "[^"]*" \d+ms$/u, + ); + const directory = path.join(first.logsRoot, "gateway", "gateway"); + expect(yield* fs.readDirectory(directory)).toEqual(["0000000001.log"]); + yield* Scope.close(firstScope, Exit.void); + + const state = Context.get( + yield* Layer.build(State.layer({ root: first.stateRoot })), + State.Service, + ); + const saved = yield* state.read(first.stack.id); + if (saved === undefined) return yield* Effect.die("the stopped stack was not saved"); + const second = yield* ownerFor({ saved, state, root: `${root}/data`, cacheRoot }); + const history = Array.from( + yield* second.rpc.readLogs({ id: "gateway", follow: false }).pipe(Stream.runCollect), + ); + + expect(history).toContainEqual(recorded); + yield* second.namespace.destroy; + expect(yield* fs.exists(first.logsRoot)).toBe(false); + }), + ).pipe(Effect.provide(services)), + ); + it.live("resumes a follow at a record position without replaying earlier records", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index 4ffb5dbe11..877e1308c5 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -67,6 +67,7 @@ import { stackError, type OwnerRpc } from "./Rpc.ts"; import * as State from "./State.ts"; import type { SavedStack, StackCredentials, StackKeysInput } from "./State.ts"; import { makeDockerHelperRegistry } from "./storage/DockerHelperRegistry.ts"; +import * as GatewayLog from "./host/GatewayLog.ts"; import * as LogForwarder from "./host/LogForwarder.ts"; import * as LogStore from "./host/LogStore.ts"; @@ -171,7 +172,10 @@ const withoutInstance = (current: SavedStack, id: string): SavedStack => const drainingBlocks: ReadonlyArray = ["start", "arm", "restart", "storage"]; -const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { +const makeOwner = Effect.fn("Owner.make")(function* ( + options: OwnerOptions, + gateway: GatewayLog.GatewayLog, +) { const services = yield* Effect.context< | FileSystem.FileSystem | Path.Path @@ -193,6 +197,15 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { composition: orchestrator.composition, logs: logStore, }).pipe(Effect.provideContext(services)); + yield* logStore.attach({ + ...GatewayLog.gatewayLog, + logs: gateway.logs, + observation: gateway.observation, + }); + yield* forwarder.attach({ + id: GatewayLog.gatewayLog.instanceId, + service: GatewayLog.gatewayLog.service, + }); const definitionGate = yield* Semaphore.make(1); const draining = yield* Ref.make(false); const { id: stackId, runtime } = options.saved; @@ -753,17 +766,24 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { }); export const layer = (options: Omit) => - Layer.effect( - Service, - Effect.gen(function* () { - const state = yield* State.Service; - return Service.of(yield* makeOwner({ ...options, state })); - }), - ).pipe( - Layer.provide( - Network.layer({ - stackId: options.saved.id, - runtime: options.saved.runtime, - }), + Layer.unwrap( + GatewayLog.make.pipe( + Effect.map((gateway) => + Layer.effect( + Service, + Effect.gen(function* () { + const state = yield* State.Service; + return Service.of(yield* makeOwner({ ...options, state }, gateway)); + }), + ).pipe( + Layer.provide( + Network.layer({ + stackId: options.saved.id, + runtime: options.saved.runtime, + onAccess: gateway.record, + }), + ), + ), + ), ), ); diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index b8ab3d857d..94ff330d83 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -50,6 +50,7 @@ import { sinceMillis, streamStackLogs as streamPersistedLogs, } from "./host/LogStore.ts"; +import { gatewayLog } from "./host/GatewayLog.ts"; import type { LogPosition, LogRecord, StackLogRecord } from "./host/LogRecord.ts"; import { reclaimStack } from "./Sweep.ts"; import { @@ -69,6 +70,7 @@ import type { export { initialization, postgres } from "./Commands.ts"; export { resolveNativePostgresUser } from "./runtime/postgres-user.ts"; export { apiRoute } from "./host/Endpoints.ts"; +export { gatewayLog }; export { StackError } from "./Rpc.ts"; export type { ServiceCreation } from "./services/Catalog.ts"; /** A service creation as `services.create` accepts it, before stack credentials fill its inputs. */ @@ -279,6 +281,10 @@ export interface Stack { readonly stop: Effect.Effect, StackError>; readonly restart: Effect.Effect, StackError>; }; + /** The shared API listener's access log, read through the live owner like an instance's. */ + readonly gateway: { + readonly readLogs: (options?: ReadLogsOptions) => Stream.Stream; + }; readonly stop: Effect.Effect; readonly destroy: Effect.Effect; readonly commands: { @@ -640,6 +646,18 @@ const makeHandle = Effect.fn("Stack.makeHandle")(function* ( const snapshotScope = (options: DatabaseSnapshotOptions | undefined) => options?.scope === undefined ? {} : { scope: options.scope }; + const readLogs = + (id: string) => + (options?: ReadLogsOptions): Stream.Stream => + stream("readLogs", (rpc) => + rpc.readLogs({ + id, + follow: options?.follow ?? false, + ...(options?.from === undefined ? {} : { from: options.from }), + ...(options?.since === undefined ? {} : { since: options.since }), + ...(options?.tail === undefined ? {} : { tail: options.tail }), + }), + ); const common = (id: string, service: K): ServiceInstance => ({ id, service, @@ -661,16 +679,7 @@ const makeHandle = Effect.fn("Stack.makeHandle")(function* ( prepare: call("prepare", (rpc) => rpc.prepareService({ id })), status: call("status", (rpc) => rpc.status({ id }), "attach"), followStatus: stream("followStatus", (rpc) => rpc.followStatus({ id })), - readLogs: (options) => - stream("readLogs", (rpc) => - rpc.readLogs({ - id, - follow: options?.follow ?? false, - ...(options?.from === undefined ? {} : { from: options.from }), - ...(options?.since === undefined ? {} : { since: options.since }), - ...(options?.tail === undefined ? {} : { tail: options.tail }), - }), - ), + readLogs: readLogs(id), credentials: (options) => call("credentials", (rpc) => rpc.credentials({ id, from: options?.from ?? "host" })), }); @@ -915,6 +924,7 @@ const makeHandle = Effect.fn("Stack.makeHandle")(function* ( stop: whileRunning("stopComposition", (rpc) => rpc.stopComposition(), []), restart: call("restartComposition", (rpc) => rpc.restartComposition()), }, + gateway: { readLogs: readLogs(gatewayLog.instanceId) }, stop: shutdown(false).pipe(Effect.asVoid), destroy: shutdown(true), commands: { run }, diff --git a/packages/stack/src/host/GatewayLog.ts b/packages/stack/src/host/GatewayLog.ts new file mode 100644 index 0000000000..91e98d5713 --- /dev/null +++ b/packages/stack/src/host/GatewayLog.ts @@ -0,0 +1,63 @@ +import { DateTime, Effect, PubSub, Stream } from "effect"; +import type { HttpAccess, HttpAccessSink } from "../HttpProxy.ts"; +import { launchOutputPublisher, type LaunchOutput } from "../runtime/Session.ts"; +import type { CatalogLogs } from "../services/Recipe.ts"; + +/** The log stream of the shared API listener's access records, one per owner and stack. */ +export const gatewayLog = { service: "gateway", instanceId: "gateway" } as const; + +/** Bounds records not yet persisted; the oldest are dropped and reported as lost. */ +const bufferedRecords = 4096; + +const monthNames = [ + "Jan", + "Feb", + "Mar", + "Apr", + "May", + "Jun", + "Jul", + "Aug", + "Sep", + "Oct", + "Nov", + "Dec", +]; +const two = (value: number) => String(value).padStart(2, "0"); + +/** Formats epoch milliseconds as nginx's `$time_local` in UTC. */ +const nginxTime = (millis: number) => { + const parts = DateTime.toPartsUtc(DateTime.makeUnsafe(millis)); + return `${two(parts.day)}/${monthNames[parts.month - 1]}/${parts.year}:${two(parts.hour)}:${two(parts.minute)}:${two(parts.second)} +0000`; +}; + +/** Escapes quotes, backslashes and control characters like nginx's default log escaping. */ +const escape = (value: string) => + value.replace( + // oxlint-disable-next-line no-control-regex -- control characters are what this escapes. + /["\\\u0000-\u001f\u007f]/gu, + (character) => `\\x${character.charCodeAt(0).toString(16).padStart(2, "0")}`, + ); + +/** Formats an access record as an nginx combined log line followed by its duration. */ +export const formatAccess = (access: HttpAccess) => + `${access.client} - - [${nginxTime(access.time)}] "${escape(`${access.method} ${access.target} ${access.protocol}`)}" ${access.status} ${access.bytes ?? "-"} "${escape(access.referer ?? "-")}" "${escape(access.userAgent ?? "-")}" ${access.durationMillis}ms`; + +const encoder = new TextEncoder(); + +/** The gateway stream's output, its access sink, and an observation of one launch per owner run. */ +export interface GatewayLog { + readonly logs: CatalogLogs; + readonly record: HttpAccessSink; + readonly observation: Stream.Stream<{ readonly launchId: number }>; +} + +export const make = Effect.gen(function* () { + const output = yield* PubSub.sliding(bufferedRecords); + const publish = yield* (yield* launchOutputPublisher(output, 1)).part; + return { + logs: PubSub.subscribe(output), + record: (access) => publish("stdout", encoder.encode(`${formatAccess(access)}\n`)), + observation: Stream.make({ launchId: 1 }).pipe(Stream.concat(Stream.never)), + } satisfies GatewayLog; +}); diff --git a/packages/stack/src/host/LogForwarder.integration.test.ts b/packages/stack/src/host/LogForwarder.integration.test.ts index 6b6ae7e775..0c382e20fc 100644 --- a/packages/stack/src/host/LogForwarder.integration.test.ts +++ b/packages/stack/src/host/LogForwarder.integration.test.ts @@ -18,6 +18,7 @@ import { import type { ServiceObservation } from "../Service.ts"; import type { LaunchOutput } from "../runtime/Session.ts"; import { CatalogError } from "../services/Recipe.ts"; +import * as GatewayLog from "./GatewayLog.ts"; import type { LogRecord } from "./LogRecord.ts"; import * as LogForwarder from "./LogForwarder.ts"; import * as LogStore from "./LogStore.ts"; @@ -190,10 +191,12 @@ const composition = Effect.succeed({ /** Starts a forwarder in its own scope so a test can stop it like an owner. */ const startForwarder = ( store: LogStore.Interface, - instances: ReadonlyArray, + instances: ReadonlyArray, ) => Effect.gen(function* () { const scope = yield* Scope.make(); + // Stops before the test's store and temp directory close, so no cursor write races removal. + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); const forwarder = yield* LogForwarder.make({ composition, logs: store }).pipe( Scope.provide(scope), ); @@ -389,4 +392,51 @@ describe("LogForwarder", () => { ); }).pipe(Effect.scoped, Effect.provide(layer)), ); + + it.live("ships gateway access lines to the API Gateway source", () => + Effect.gen(function* () { + const { store, sink, analytics } = yield* fixture(); + const gateway = yield* GatewayLog.make; + yield* store.attach({ + ...GatewayLog.gatewayLog, + logs: gateway.logs, + observation: gateway.observation, + }); + yield* analytics.set(true); + yield* startForwarder(store, [ + analytics.instance, + { id: GatewayLog.gatewayLog.instanceId, service: GatewayLog.gatewayLog.service }, + ]); + + yield* gateway.record({ + time: Date.parse("2026-10-01T09:25:23.000Z"), + client: "127.0.0.1", + method: "POST", + target: "/auth/v1/token?grant_type=password", + protocol: "HTTP/1.1", + status: 400, + bytes: 60, + durationMillis: 3, + }); + const shipped = yield* sink.next; + + expect(shipped.url).toBe("/api/logs?source_name=cloudflare.logs.prod"); + expect(shipped.events).toEqual([ + expect.objectContaining({ + appname: "gateway", + timestamp: "2026-10-01T09:25:23.000Z", + metadata: { + request: { + method: "POST", + path: "/auth/v1/token", + search: "?grant_type=password", + protocol: "HTTP/1.1", + headers: { cf_connecting_ip: "127.0.0.1" }, + }, + response: { status_code: 400 }, + }, + }), + ]); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); }); diff --git a/packages/stack/src/host/LogForwarder.ts b/packages/stack/src/host/LogForwarder.ts index 9f81111cca..8bc2fbbfb3 100644 --- a/packages/stack/src/host/LogForwarder.ts +++ b/packages/stack/src/host/LogForwarder.ts @@ -27,6 +27,7 @@ import { type LogflareEvent, type ShippedService, } from "./LogflareEvents.ts"; +import type { gatewayLog } from "./GatewayLog.ts"; import { LogPosition, type LogRecord } from "./LogRecord.ts"; import type * as LogStore from "./LogStore.ts"; @@ -39,9 +40,15 @@ export interface ForwardedInstance { readonly observation: Stream.Stream>; } +/** An owner log stream without a service instance; it ships while the owner runs. */ +export interface ForwardedStream { + readonly id: string; + readonly service: typeof gatewayLog.service; +} + interface Interface { /** Ships a shipped service's persisted logs, or tracks an Analytics instance as the target. */ - readonly attach: (instance: ForwardedInstance) => Effect.Effect; + readonly attach: (instance: ForwardedInstance | ForwardedStream) => Effect.Effect; /** Re-selects the shipping target after the composition changes. */ readonly rebind: Effect.Effect; /** Emits whether records are currently shipped; the current value first. */ @@ -372,25 +379,29 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar Stream.runDrain, ); - const forward = (instance: ForwardedInstance, service: ShippedService) => + const forward = (id: string, service: ShippedService, until: Effect.Effect) => Effect.gen(function* () { const current = yield* serving; // A stopped target pauses shipping; the cursor keeps the position to resume from. - yield* session(instance.id, service, current).pipe( + yield* session(id, service, current).pipe( Effect.catch((error) => (error._tag === "StaleTarget" ? Effect.void - : Effect.logWarning(`Log shipping of ${instance.id} paused`, error) + : Effect.logWarning(`Log shipping of ${id} paused`, error) ).pipe(Effect.andThen(retargeted(current))), ), Effect.raceFirst(retargeted(current)), ); - }).pipe(Effect.forever, Effect.raceFirst(unregistered(instance))); + }).pipe(Effect.forever, Effect.raceFirst(until)); - const attach = Effect.fn("LogForwarder.attach")(function* (instance: ForwardedInstance) { - if (instance.service === "analytics") yield* Effect.forkIn(trackTarget(instance), scope); + const attach = Effect.fn("LogForwarder.attach")(function* ( + instance: ForwardedInstance | ForwardedStream, + ) { + if (instance.service === "gateway") + yield* Effect.forkIn(forward(instance.id, instance.service, Effect.never), scope); + else if (instance.service === "analytics") yield* Effect.forkIn(trackTarget(instance), scope); else if (isShippedService(instance.service)) - yield* Effect.forkIn(forward(instance, instance.service), scope); + yield* Effect.forkIn(forward(instance.id, instance.service, unregistered(instance)), scope); }); return { diff --git a/packages/stack/src/host/LogflareEvents.ts b/packages/stack/src/host/LogflareEvents.ts index 432200106e..fe4846ae7e 100644 --- a/packages/stack/src/host/LogflareEvents.ts +++ b/packages/stack/src/host/LogflareEvents.ts @@ -1,7 +1,11 @@ import { DateTime, Option } from "effect"; import type { ServiceCreation } from "../services/Catalog.ts"; +import type { gatewayLog } from "./GatewayLog.ts"; -/** Logflare source per shipped service kind; Studio's Logs pages query these names. */ +/** A service kind, or the owner's `gateway` stream of shared API listener requests. */ +export type LogService = ServiceCreation["service"] | typeof gatewayLog.service; + +/** Logflare source per shipped log service; Studio's Logs pages query these names. */ export const logflareSources = { auth: "gotrue.logs.prod", rest: "postgREST.logs.prod", @@ -9,11 +13,12 @@ export const logflareSources = { storage: "storage.logs.prod.2", functions: "deno-relay-logs", database: "postgres.logs", -} as const satisfies Partial>; + gateway: "cloudflare.logs.prod", +} as const satisfies Partial>; export type ShippedService = keyof typeof logflareSources; -export const isShippedService = (service: ServiceCreation["service"]): service is ShippedService => +export const isShippedService = (service: LogService): service is ShippedService => Object.hasOwn(logflareSources, service); /** One ingest event in the shape Studio's local log queries expect. */ @@ -53,8 +58,8 @@ const months: Record = { dec: 11, }; -/** Parses PostgREST's `%d/%b/%Y:%H:%M:%S %z` prefix into an ISO-8601 timestamp. */ -const parsePostgrestTime = (text: string): string | undefined => { +/** Parses the `%d/%b/%Y:%H:%M:%S %z` time of PostgREST and nginx logs into ISO-8601. */ +const parseLogTime = (text: string): string | undefined => { const match = /^(\d{2})\/([A-Za-z]{3})\/(\d{4}):(\d{2}):(\d{2}):(\d{2}) ([+-])(\d{2})(\d{2})$/u.exec(text); if (match === null) return undefined; @@ -78,6 +83,19 @@ const parsePostgrestTime = (text: string): string | undefined => { /** The time and request of PostgREST's Apache combined request line. */ const postgrestRequest = /^\S+ \S+ \S+ \[([^\]]+)\] "([A-Z]+) (\S+) ([^"\s]+)" (\d{3}) /u; +/** The gateway's nginx combined line with its trailing duration. */ +const gatewayRequest = + /^(\S+) \S+ \S+ \[([^\]]+)\] "(\S+) (\S+) (\S+)" (\d{3}) (?:\d+|-) "([^"]*)" "([^"]*)" \d+ms$/u; + +const unescapeLogValue = (value: string) => + value.replace(/\\x([0-9a-f]{2})/giu, (_, hex: string) => + String.fromCharCode(Number.parseInt(hex, 16)), + ); + +/** A quoted combined-log field, absent when nginx wrote `-`. */ +const logValue = (value: string | undefined) => + value === undefined || value === "-" ? undefined : unescapeLogValue(value); + const withoutProject = ({ project: _project, ...event }: LogflareEvent): LogflareEvent => event; const remaps: Record LogflareEvent> = { @@ -89,7 +107,7 @@ const remaps: Record LogflareEvent> = }, rest: (event) => { const request = postgrestRequest.exec(event.event_message); - const requestTime = request === null ? undefined : parsePostgrestTime(request[1] ?? ""); + const requestTime = request === null ? undefined : parseLogTime(request[1] ?? ""); if (request !== null && requestTime !== undefined) return { ...event, @@ -104,7 +122,7 @@ const remaps: Record LogflareEvent> = }, }; const match = /^(.*): (.*)$/u.exec(event.event_message); - const timestamp = match === null ? undefined : parsePostgrestTime(match[1] ?? ""); + const timestamp = match === null ? undefined : parseLogTime(match[1] ?? ""); return match === null || timestamp === undefined ? event : { @@ -154,6 +172,35 @@ const remaps: Record LogflareEvent> = }, }; }, + gateway: (event) => { + const match = gatewayRequest.exec(event.event_message); + const timestamp = match === null ? undefined : parseLogTime(match[2] ?? ""); + if (match === null || timestamp === undefined) return event; + const [, client, , method, target = "", protocol, status, referer, userAgent] = match; + const decoded = unescapeLogValue(target); + const queryAt = decoded.indexOf("?"); + const refererHeader = logValue(referer); + const userAgentHeader = logValue(userAgent); + return { + ...event, + timestamp, + metadata: { + ...event.metadata, + request: { + method, + path: queryAt < 0 ? decoded : decoded.slice(0, queryAt), + ...(queryAt < 0 ? {} : { search: decoded.slice(queryAt) }), + protocol, + headers: { + cf_connecting_ip: client, + ...(refererHeader === undefined ? {} : { referer: refererHeader }), + ...(userAgentHeader === undefined ? {} : { user_agent: userAgentHeader }), + }, + }, + response: { status_code: Number(status) }, + }, + }; + }, }; /** Builds the Logflare event for one service log line received at `timestamp`. */ diff --git a/packages/stack/src/host/LogflareEvents.unit.test.ts b/packages/stack/src/host/LogflareEvents.unit.test.ts index ebbf816ad6..ae7b955152 100644 --- a/packages/stack/src/host/LogflareEvents.unit.test.ts +++ b/packages/stack/src/host/LogflareEvents.unit.test.ts @@ -1,4 +1,5 @@ import { expect, it } from "@effect/vitest"; +import { formatAccess } from "./GatewayLog.ts"; import { logflareEvent } from "./LogflareEvents.ts"; const received = "2026-09-28T10:00:00.000Z"; @@ -112,3 +113,47 @@ it("derives the Postgres severity from the last level marker and defaults to LOG error_severity: "LOG", }); }); + +it("ships a gateway access line as an API Gateway request stamped with its request time", () => { + const line = formatAccess({ + time: Date.parse("2026-10-01T09:25:23.456Z"), + client: "127.0.0.1", + method: "GET", + target: '/rest/v1/todos?select=*&q="x"', + protocol: "HTTP/1.1", + status: 200, + bytes: 126, + userAgent: "curl/8.7.1", + durationMillis: 12, + }); + + expect(line).toBe( + '127.0.0.1 - - [01/Oct/2026:09:25:23 +0000] "GET /rest/v1/todos?select=*&q=\\x22x\\x22 HTTP/1.1" 200 126 "-" "curl/8.7.1" 12ms', + ); + expect(logflareEvent("gateway", received, line)).toEqual({ + project: "default", + appname: "gateway", + event_message: line, + timestamp: "2026-10-01T09:25:23.000Z", + metadata: { + request: { + method: "GET", + path: "/rest/v1/todos", + search: '?select=*&q="x"', + protocol: "HTTP/1.1", + headers: { cf_connecting_ip: "127.0.0.1", user_agent: "curl/8.7.1" }, + }, + response: { status_code: 200 }, + }, + }); +}); + +it("passes a malformed gateway line through without request metadata", () => { + expect(logflareEvent("gateway", received, "not an access line")).toEqual({ + project: "default", + appname: "gateway", + event_message: "not an access line", + timestamp: received, + metadata: {}, + }); +}); From 0b33f5c2663fee51238d89f09f12e683ea0d2dff Mon Sep 17 00:00:00 2001 From: avallete Date: Thu, 1 Oct 2026 14:27:59 +0200 Subject: [PATCH 08/35] fix(stack): redact referer credentials and settle reset gateway requests - Redact apikey, access_token and token in the Referer's query and in fragments, as for the request target. - Settle a forwarded request when the client closes after the response ended but before it finished, so it records once and releases its target. Co-Authored-By: Claude Opus 5.5 --- .../stack/src/HttpProxy.integration.test.ts | 71 +++++++++++++++++-- packages/stack/src/HttpProxy.ts | 29 +++++--- 2 files changed, 85 insertions(+), 15 deletions(-) diff --git a/packages/stack/src/HttpProxy.integration.test.ts b/packages/stack/src/HttpProxy.integration.test.ts index 42a2bea3bd..325c874c30 100644 --- a/packages/stack/src/HttpProxy.integration.test.ts +++ b/packages/stack/src/HttpProxy.integration.test.ts @@ -1092,10 +1092,10 @@ it.live("records each request once with the status the client was sent", () => { const statuses = [ yield* get("/api/ok?select=*&apikey=sb_secret_x&Access_Token=jwt", { "user-agent": "proxy-test/1", - referer: "http://studio.test/logs", + referer: "http://127.0.0.1:54321/x?token=t1&select=*#access_token=frag&type=bearer", }), - yield* get("/api/missing"), - yield* get("/elsewhere"), + yield* get("/api/missing", { referer: "http://127.0.0.1:54321/y?apikey=sb_publishable_x" }), + yield* get("/elsewhere", { referer: "/relative?Access_Token=jwt" }), yield* get("/down/thing?token=t"), ]; const recorded = yield* Queue.takeN(accesses, 4); @@ -1110,12 +1110,23 @@ it.live("records each request once with the status the client was sent", () => { protocol: "HTTP/1.1", status: 200, bytes: 5, - referer: "http://studio.test/logs", + referer: + "http://127.0.0.1:54321/x?token=redacted&select=*#access_token=redacted&type=bearer", userAgent: "proxy-test/1", durationMillis: expect.any(Number), }, - expect.objectContaining({ target: "/api/missing", status: 404, bytes: 4 }), - expect.objectContaining({ target: "/elsewhere", status: 404, bytes: 9 }), + expect.objectContaining({ + target: "/api/missing", + status: 404, + bytes: 4, + referer: "http://127.0.0.1:54321/y?apikey=redacted", + }), + expect.objectContaining({ + target: "/elsewhere", + status: 404, + bytes: 9, + referer: "/relative?Access_Token=redacted", + }), expect.objectContaining({ target: "/down/thing?token=redacted", status: 502, bytes: 11 }), ]); expect(yield* Queue.size(accesses)).toBe(0); @@ -1281,3 +1292,51 @@ it.live("answers and releases targets while the access sink is stalled", () => }), ).pipe(Effect.provide(Layer.merge(NodeHttpClient.layerNodeHttp, NodeServices.layer))), ); + +it.live("records and releases a request whose client resets after the full body was sent", () => + Effect.scoped( + Effect.gen(function* () { + const bodySent = yield* Deferred.make(); + const body = new Uint8Array(256 * 1024).fill(65); + const backend = createServer((request, response) => { + request.resume(); + response.once("finish", () => Deferred.doneUnsafe(bodySent, Effect.void)); + response.end(body); + }); + const backendAddress = yield* listen(backend, { + beforeClose: () => backend.closeAllConnections(), + }); + const released = yield* Deferred.make(); + const accesses = yield* Queue.unbounded(); + const proxy = yield* makeHttpProxy({ + host: "127.0.0.1", + port: 0, + onAccess: (access) => Queue.offer(accesses, access), + }); + yield* proxy.setRoutes([ + { + id: "api", + prefix: "/api", + target: Effect.acquireRelease(Effect.succeed(backendAddress), () => + Deferred.succeed(released, undefined), + ), + }, + ]); + + // The client never reads; socket buffers decide how much of the body the proxy flushed + // before the reset, so the record holds either the whole body or no byte count. + const client = yield* rawClient( + proxy.port, + "GET /api/full HTTP/1.1\r\nHost: localhost\r\n\r\n", + ); + yield* Deferred.await(bodySent).pipe(Effect.timeout("5 seconds")); + client.resetAndDestroy(); + const recorded = yield* Queue.take(accesses).pipe(Effect.timeout("5 seconds")); + yield* Deferred.await(released).pipe(Effect.timeout("5 seconds")); + + expect(recorded).toMatchObject({ target: "/api/full", status: 200 }); + expect([undefined, body.length]).toContain(recorded.bytes); + expect(yield* Queue.size(accesses)).toBe(0); + }), + ).pipe(Effect.provide(NodeServices.layer)), +); diff --git a/packages/stack/src/HttpProxy.ts b/packages/stack/src/HttpProxy.ts index 2af6364346..ee41177dc3 100644 --- a/packages/stack/src/HttpProxy.ts +++ b/packages/stack/src/HttpProxy.ts @@ -47,7 +47,7 @@ export interface HttpAccess { readonly time: number; readonly client: string; readonly method: string; - /** The request path and query, with credential query values redacted. */ + /** The request path and query, with credential query and fragment values redacted. */ readonly target: string; readonly protocol: string; /** @@ -58,6 +58,7 @@ export interface HttpAccess { readonly status: number; /** Body bytes of a response that finished; absent when it was cut short, and for upgrades. */ readonly bytes?: number; + /** The Referer header, with credential query and fragment values redacted. */ readonly referer?: string; readonly userAgent?: string; /** Until the response finished; for an upgrade, until the upstream handshake answered. */ @@ -153,11 +154,8 @@ const decodeQuery = (value: string) => { const credentialParameters = new Set(["apikey", "access_token", "token"]); -const redactCredentials = (url: string) => { - const queryAt = url.indexOf("?"); - if (queryAt < 0) return url; - const query = url - .slice(queryAt + 1) +const redactPairs = (pairs: string) => + pairs .split("&") .map((parameter) => { const separator = parameter.indexOf("="); @@ -168,7 +166,19 @@ const redactCredentials = (url: string) => { : parameter; }) .join("&"); - return `${url.slice(0, queryAt)}?${query}`; + +/** + * Redacts credential values in a URL's query and fragment, where OAuth implicit grants put + * `access_token`. Edits the text in place, so relative and unparsable URLs work and the rest of + * the URL keeps its original encoding. + */ +const redactCredentials = (url: string) => { + const hashAt = url.indexOf("#"); + const beforeHash = hashAt < 0 ? url : url.slice(0, hashAt); + const queryAt = beforeHash.indexOf("?"); + const query = queryAt < 0 ? "" : `?${redactPairs(beforeHash.slice(queryAt + 1))}`; + const fragment = hashAt < 0 ? "" : `#${redactPairs(url.slice(hashAt + 1))}`; + return `${queryAt < 0 ? beforeHash : beforeHash.slice(0, queryAt)}${query}${fragment}`; }; /** Captures a request's access fields while its socket is open; completes them once it settles. */ @@ -181,7 +191,7 @@ const accessFor = (request: IncomingMessage, time: number) => { method: request.method ?? "GET", target: redactCredentials(request.url ?? "/"), protocol: `HTTP/${request.httpVersion}`, - ...(referer === undefined ? {} : { referer }), + ...(referer === undefined ? {} : { referer: redactCredentials(referer) }), ...(userAgent === undefined ? {} : { userAgent }), }; return (ended: number, status: number, bytes?: number): HttpAccess => ({ @@ -372,8 +382,9 @@ const forward = Effect.fn("HttpProxy.forward")( abandon(Effect.fail(errorFor(cause, incoming !== undefined))); const onClientGone = () => abandon(Effect.fail(new HttpProxyDisconnected())); const onFinish = () => finish(Effect.void); + // A close after `end()` but before `finish` means the client reset with writes still queued. const onResponseClose = () => { - if (!response.writableEnded) onClientGone(); + if (!response.writableFinished) onClientGone(); }; outgoing = upstreamRequest( { From 268e8f12122c088a5c639618059abf757bf80ba8 Mon Sep 17 00:00:00 2001 From: avallete Date: Thu, 1 Oct 2026 14:31:23 +0200 Subject: [PATCH 09/35] test(stack): exercise the late-line grace race deterministically Hold the newline chunk behind a blocked write while the grace flusher wakes past its deadline, so the test fails if the flush skips queued output. Co-Authored-By: Claude Opus 5.5 --- .../src/host/LogStore.integration.test.ts | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/packages/stack/src/host/LogStore.integration.test.ts b/packages/stack/src/host/LogStore.integration.test.ts index 961550b8d9..69fb38494d 100644 --- a/packages/stack/src/host/LogStore.integration.test.ts +++ b/packages/stack/src/host/LogStore.integration.test.ts @@ -290,11 +290,28 @@ describe("LogStore", () => { ) : file.writeAll(buffer), ); + const armed = yield* Deferred.make(); + const woke = yield* Deferred.make(); + const clock = yield* TestClock.testClockWith(Effect.succeed); + // The store's only sleep before close is the grace flusher's wait for the deadline. + const graceClock: Clock.Clock = { + currentTimeMillisUnsafe: () => clock.currentTimeMillisUnsafe(), + currentTimeMillis: clock.currentTimeMillis, + currentTimeNanosUnsafe: () => clock.currentTimeNanosUnsafe(), + currentTimeNanos: clock.currentTimeNanos, + monotonicTimeNanosUnsafe: () => clock.monotonicTimeNanosUnsafe(), + monotonicTimeNanos: clock.monotonicTimeNanos, + sleep: (duration) => + Deferred.succeed(armed, undefined).pipe( + Effect.andThen(clock.sleep(duration)), + Effect.andThen(Deferred.succeed(woke, undefined)), + ), + }; const { store } = yield* openStore(root).pipe( Effect.provideService(FileSystem.FileSystem, injected), ); const instance = yield* fakeInstance("newline"); - yield* store.attach(instance); + yield* store.attach(instance).pipe(Effect.provideService(Clock.Clock, graceClock)); const reader = yield* collect(store.read("newline", { from: "oldest", follow: true })); yield* instance.setLaunch(1); yield* instance.publish(instance.chunk(1, "ending")); @@ -302,12 +319,14 @@ describe("LogStore", () => { yield* instance.setLaunch(undefined); yield* untilLast(reader, "ending"); - // The writer is busy with `busy` while the newline is published, so it stays queued. - yield* instance.publish(instance.chunk(1, "late "), instance.chunk(1, "busy\n", "stderr")); + yield* instance.publish(instance.chunk(1, "late ")); + yield* Deferred.await(armed); + yield* instance.publish(instance.chunk(1, "busy\n", "stderr")); yield* Deferred.await(blocked); yield* TestClock.adjust(endedLineGraceMillis / 2); yield* instance.publish(instance.chunk(1, "line end\n")); yield* TestClock.adjust(endedLineGraceMillis / 2); + yield* Deferred.await(woke); yield* Deferred.succeed(release, undefined); const records: Array = []; while (!records.some((record) => record.text?.endsWith("line end") === true)) From f8afda812dd77603ced8eb56881a586d452aa642 Mon Sep 17 00:00:00 2001 From: avallete Date: Thu, 1 Oct 2026 16:33:24 +0200 Subject: [PATCH 10/35] fix(stack): redact OAuth codes and tokens in gateway access lines Auth redirect and verify URLs carry PKCE codes, OTP token hashes and refresh, ID and provider tokens; redact them like API keys. Co-Authored-By: Claude Opus 5.5 --- .../experimental/stack/start/SIDE_EFFECTS.md | 5 +++-- packages/stack/ARCHITECTURE.md | 5 +++-- packages/stack/src/HttpProxy.integration.test.ts | 10 +++++++--- packages/stack/src/HttpProxy.ts | 13 ++++++++++++- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index d55872ede0..0f480d6be8 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -43,8 +43,9 @@ the stack deletes those logs; stopping the stack and resetting database data kee PostgREST runs with `PGRST_LOG_LEVEL=info`, so every request line, query string included, is persisted and shipped to Analytics. The shared API port also records one nginx combined line plus duration (`... "GET /rest/v1/todos?select=* HTTP/1.1" 200 126 "-" "curl/8.7.1" 12ms`) per request -and WebSocket upgrade under `logs/gateway/gateway/`, with the same retention; `apikey`, -`access_token`, and `token` query values are written as `redacted`. +and WebSocket upgrade under `logs/gateway/gateway/`, with the same retention; credential query and +fragment values (`apikey`, `token`, `token_hash`, `code`, and access, refresh, ID, and provider +tokens), in the request target and the Referer, are written as `redacted`. When an owner starts a stack saved with a Vector instance, it removes that instance, its composition members, dependencies and port claims from `state.json`, and its stack-owned Vector config files under `data//runtime/vector/`; its containers go with the stack's container sweep. diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 34fe79f1f0..415ca25de2 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -663,8 +663,9 @@ instances no longer saved, so a failed deletion is retried; destroying the stack resetting database data keeps them. The shared API listener is the stack's gateway. Each completed request, and each WebSocket upgrade -once its handshake status is known, becomes one nginx combined line plus duration, with `apikey`, -`access_token` and `token` query values redacted. The proxy hands it to a bounded sliding buffer +once its handshake status is known, becomes one nginx combined line plus duration, with credential +query and fragment values (API keys, tokens, token hashes, PKCE codes) redacted in the target and +the Referer. The proxy hands it to a bounded sliding buffer after the response settles, so logging never delays a response or holds a target's activity. The owner persists these lines as the `gateway` stream, `logs/gateway/gateway/`, with one launch per owner run; it is not a service instance, so orphan cleanup keeps it. diff --git a/packages/stack/src/HttpProxy.integration.test.ts b/packages/stack/src/HttpProxy.integration.test.ts index 325c874c30..941300fa15 100644 --- a/packages/stack/src/HttpProxy.integration.test.ts +++ b/packages/stack/src/HttpProxy.integration.test.ts @@ -1094,7 +1094,10 @@ it.live("records each request once with the status the client was sent", () => { "user-agent": "proxy-test/1", referer: "http://127.0.0.1:54321/x?token=t1&select=*#access_token=frag&type=bearer", }), - yield* get("/api/missing", { referer: "http://127.0.0.1:54321/y?apikey=sb_publishable_x" }), + yield* get("/api/missing?code=pkce&state=s", { + referer: + "http://127.0.0.1:54321/y?apikey=sb_publishable_x#refresh_token=r&provider_token=p", + }), yield* get("/elsewhere", { referer: "/relative?Access_Token=jwt" }), yield* get("/down/thing?token=t"), ]; @@ -1116,10 +1119,11 @@ it.live("records each request once with the status the client was sent", () => { durationMillis: expect.any(Number), }, expect.objectContaining({ - target: "/api/missing", + target: "/api/missing?code=redacted&state=s", status: 404, bytes: 4, - referer: "http://127.0.0.1:54321/y?apikey=redacted", + referer: + "http://127.0.0.1:54321/y?apikey=redacted#refresh_token=redacted&provider_token=redacted", }), expect.objectContaining({ target: "/elsewhere", diff --git a/packages/stack/src/HttpProxy.ts b/packages/stack/src/HttpProxy.ts index ee41177dc3..f8fc1c33b8 100644 --- a/packages/stack/src/HttpProxy.ts +++ b/packages/stack/src/HttpProxy.ts @@ -152,7 +152,18 @@ const decodeQuery = (value: string) => { } }; -const credentialParameters = new Set(["apikey", "access_token", "token"]); +// Auth puts PKCE codes, OTP token hashes and OAuth tokens in redirect and verify URLs. +const credentialParameters = new Set([ + "apikey", + "access_token", + "token", + "token_hash", + "code", + "refresh_token", + "id_token", + "provider_token", + "provider_refresh_token", +]); const redactPairs = (pairs: string) => pairs From d5254148f2ccae54c7e16789bb0df83e183dfbd7 Mon Sep 17 00:00:00 2001 From: avallete Date: Thu, 1 Oct 2026 16:47:28 +0200 Subject: [PATCH 11/35] fix(stack): tighten gateway access timing, upgrade status and redaction - Measure request duration when the response settles, before target cleanup, and skip interim 1xx answers when recording upgrade status. - Redact parameters whose value carries nested credentials, and URL userinfo in logged URLs. - Make the reset, restart and raw-client test fixtures deterministic and leak-free. Co-Authored-By: Claude Opus 5.5 --- .../experimental/stack/start/SIDE_EFFECTS.md | 3 +- .../stack/src/HttpProxy.integration.test.ts | 74 ++++++++++++++++--- packages/stack/src/HttpProxy.ts | 63 ++++++++++++---- .../stack/src/Owner.logs.integration.test.ts | 2 + 4 files changed, 118 insertions(+), 24 deletions(-) diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index 0f480d6be8..5c3c5c40de 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -45,7 +45,8 @@ persisted and shipped to Analytics. The shared API port also records one nginx c duration (`... "GET /rest/v1/todos?select=* HTTP/1.1" 200 126 "-" "curl/8.7.1" 12ms`) per request and WebSocket upgrade under `logs/gateway/gateway/`, with the same retention; credential query and fragment values (`apikey`, `token`, `token_hash`, `code`, and access, refresh, ID, and provider -tokens), in the request target and the Referer, are written as `redacted`. +tokens), in the request target and the Referer, are written as `redacted`, as are values that +themselves carry such a pair (a `redirect_to` URL with a token) and URL userinfo. When an owner starts a stack saved with a Vector instance, it removes that instance, its composition members, dependencies and port claims from `state.json`, and its stack-owned Vector config files under `data//runtime/vector/`; its containers go with the stack's container sweep. diff --git a/packages/stack/src/HttpProxy.integration.test.ts b/packages/stack/src/HttpProxy.integration.test.ts index 941300fa15..ad7a960709 100644 --- a/packages/stack/src/HttpProxy.integration.test.ts +++ b/packages/stack/src/HttpProxy.integration.test.ts @@ -1031,13 +1031,21 @@ it.live("releases a waiting WebSocket target quietly when its client resets", () /** Opens a raw client connection that has written `text`. */ const rawClient = (port: number, text: string) => Effect.acquireRelease( - Effect.callback((resume) => { + Effect.callback((resume) => { const socket = new Socket(); - socket.on("error", () => undefined); + const onConnectError = (cause: Error) => { + socket.destroy(); + resume(Effect.fail(new HttpProxyTestError({ message: cause.message, cause }))); + }; + socket.once("error", onConnectError); socket.connect(port, "127.0.0.1", () => { + socket.off("error", onConnectError); + // Tests reset these connections on purpose. + socket.on("error", () => undefined); socket.write(text); resume(Effect.succeed(socket)); }); + return Effect.sync(() => socket.destroy()); }), (socket) => Effect.sync(() => socket.destroy()), ); @@ -1051,6 +1059,8 @@ const rawUpgrade = (port: number, path: string) => Effect.flatMap((socket) => Effect.callback((resume) => { socket.once("close", () => resume(Effect.void)); + // Drains any answer so an upstream's graceful end reaches the client as a close. + socket.resume(); if (socket.destroyed) resume(Effect.void); }), ), @@ -1098,8 +1108,11 @@ it.live("records each request once with the status the client was sent", () => { referer: "http://127.0.0.1:54321/y?apikey=sb_publishable_x#refresh_token=r&provider_token=p", }), - yield* get("/elsewhere", { referer: "/relative?Access_Token=jwt" }), - yield* get("/down/thing?token=t"), + yield* get( + "/elsewhere?redirect_to=https%3A%2F%2Fclient%2Fcb%3Faccess_token%3DJWT&next=http%3A%2F%2Flocalhost%3A3000%2F", + { referer: "https://user:password@studio.test/relative?Access_Token=jwt" }, + ), + yield* get("/down/thing?token=t&redirect_to=https://client/cb?access_token=JWT"), ]; const recorded = yield* Queue.takeN(accesses, 4); @@ -1126,12 +1139,16 @@ it.live("records each request once with the status the client was sent", () => { "http://127.0.0.1:54321/y?apikey=redacted#refresh_token=redacted&provider_token=redacted", }), expect.objectContaining({ - target: "/elsewhere", + target: "/elsewhere?redirect_to=redacted&next=http%3A%2F%2Flocalhost%3A3000%2F", status: 404, bytes: 9, - referer: "/relative?Access_Token=redacted", + referer: "https://redacted@studio.test/relative?Access_Token=redacted", + }), + expect.objectContaining({ + target: "/down/thing?token=redacted&redirect_to=redacted", + status: 502, + bytes: 11, }), - expect.objectContaining({ target: "/down/thing?token=redacted", status: 502, bytes: 11 }), ]); expect(yield* Queue.size(accesses)).toBe(0); }), @@ -1155,6 +1172,24 @@ it.live("records WebSocket upgrades at the handshake with the status sent to the connection.once("data", () => Deferred.doneUnsafe(upgradeReceived, Effect.void)); }); const silentAddress = yield* listen(silent); + // Answers with an interim 1xx before the final handshake status, in separate writes. + const interim = createTcpServer((connection) => { + connection.on("error", () => undefined); + connection.once("data", (chunk: Buffer) => { + const upgrading = chunk.toString("latin1").startsWith("GET /interim/upgrade "); + connection.write( + upgrading + ? "HTTP/1.1 103 Early Hints\r\nLink: ; rel=preload\r\n\r\n" + : "HTTP/1.1 100 Continue\r\n\r\n", + ); + connection.end( + upgrading + ? "HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n" + : "HTTP/1.1 403 Forbidden\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ); + }); + }); + const interimAddress = yield* listen(interim); const accesses = yield* Queue.unbounded(); const proxy = yield* makeHttpProxy({ host: "127.0.0.1", @@ -1164,6 +1199,7 @@ it.live("records WebSocket upgrades at the handshake with the status sent to the yield* proxy.setRoutes([ { id: "ws", prefix: "/socket", target: Effect.succeed(backendAddress) }, { id: "silent", prefix: "/silent", target: Effect.succeed(silentAddress) }, + { id: "interim", prefix: "/interim", target: Effect.succeed(interimAddress) }, { id: "down", prefix: "/down", @@ -1197,6 +1233,16 @@ it.live("records WebSocket upgrades at the handshake with the status sent to the expect(yield* Queue.take(accesses)).toMatchObject({ target: "/elsewhere", status: 404 }); yield* rawUpgrade(proxy.port, "/down"); expect(yield* Queue.take(accesses)).toMatchObject({ target: "/down", status: 502 }); + yield* rawUpgrade(proxy.port, "/interim/forbidden"); + expect(yield* Queue.take(accesses)).toMatchObject({ + target: "/interim/forbidden", + status: 403, + }); + yield* rawUpgrade(proxy.port, "/interim/upgrade"); + expect(yield* Queue.take(accesses)).toMatchObject({ + target: "/interim/upgrade", + status: 101, + }); const leaving = yield* rawClient(proxy.port, upgradeRequest("/silent")); yield* Deferred.await(upgradeReceived); @@ -1327,13 +1373,21 @@ it.live("records and releases a request whose client resets after the full body }, ]); - // The client never reads; socket buffers decide how much of the body the proxy flushed - // before the reset, so the record holds either the whole body or no byte count. + // The client stops reading after its first bytes; socket buffers decide how much of the + // body the proxy flushed before the reset, so the record holds the whole body or no count. const client = yield* rawClient( proxy.port, "GET /api/full HTTP/1.1\r\nHost: localhost\r\n\r\n", ); - yield* Deferred.await(bodySent).pipe(Effect.timeout("5 seconds")); + const firstBytes = Effect.callback((resume) => { + client.once("data", () => { + client.pause(); + resume(Effect.void); + }); + }); + yield* Effect.all([firstBytes, Deferred.await(bodySent)], { concurrency: "unbounded" }).pipe( + Effect.timeout("5 seconds"), + ); client.resetAndDestroy(); const recorded = yield* Queue.take(accesses).pipe(Effect.timeout("5 seconds")); yield* Deferred.await(released).pipe(Effect.timeout("5 seconds")); diff --git a/packages/stack/src/HttpProxy.ts b/packages/stack/src/HttpProxy.ts index f8fc1c33b8..e0a0a2b634 100644 --- a/packages/stack/src/HttpProxy.ts +++ b/packages/stack/src/HttpProxy.ts @@ -165,6 +165,9 @@ const credentialParameters = new Set([ "provider_refresh_token", ]); +/** A credential pair inside a decoded value, such as a `redirect_to` URL carrying a token. */ +const nestedCredential = new RegExp(`(?:^|[?&#])(?:${[...credentialParameters].join("|")})=`, "iu"); + const redactPairs = (pairs: string) => pairs .split("&") @@ -172,24 +175,31 @@ const redactPairs = (pairs: string) => const separator = parameter.indexOf("="); if (separator < 0) return parameter; const name = parameter.slice(0, separator); - return credentialParameters.has(decodeQuery(name).toLowerCase()) + return credentialParameters.has(decodeQuery(name).toLowerCase()) || + nestedCredential.test(decodeQuery(parameter.slice(separator + 1))) ? `${name}=redacted` : parameter; }) .join("&"); +const userinfo = /^([a-z][a-z\d+.-]*:\/\/)[^/?#@]*@/iu; + /** - * Redacts credential values in a URL's query and fragment, where OAuth implicit grants put - * `access_token`. Edits the text in place, so relative and unparsable URLs work and the rest of - * the URL keeps its original encoding. + * Redacts an absolute URL's userinfo and credential values in its query and fragment, where + * OAuth implicit grants put `access_token`. Edits the text in place, so relative and unparsable + * URLs work and the rest of the URL keeps its original encoding. */ const redactCredentials = (url: string) => { const hashAt = url.indexOf("#"); const beforeHash = hashAt < 0 ? url : url.slice(0, hashAt); const queryAt = beforeHash.indexOf("?"); + const base = (queryAt < 0 ? beforeHash : beforeHash.slice(0, queryAt)).replace( + userinfo, + "$1redacted@", + ); const query = queryAt < 0 ? "" : `?${redactPairs(beforeHash.slice(queryAt + 1))}`; const fragment = hashAt < 0 ? "" : `#${redactPairs(url.slice(hashAt + 1))}`; - return `${queryAt < 0 ? beforeHash : beforeHash.slice(0, queryAt)}${query}${fragment}`; + return `${base}${query}${fragment}`; }; /** Captures a request's access fields while its socket is open; completes them once it settles. */ @@ -470,6 +480,8 @@ const proxyRequest = Effect.fn("HttpProxy.proxyRequest")( ); const statusLine = /^HTTP\/\d(?:\.\d)? (\d{3})\b/u; +/** Upstream bytes read for the handshake status before giving up on finding one. */ +const answerLimit = 8 * 1024; const upgrade = Effect.fn("HttpProxy.upgrade")( ( @@ -519,13 +531,30 @@ const upgrade = Effect.fn("HttpProxy.upgrade")( const onClientClose = () => (Deferred.isDoneUnsafe(handshake) ? onClose() : onClientGone()); const onClientEnd = () => upstream.end(); const onUpstreamEnd = () => client.end(); - // Reads the handshake status off the bytes relayed to the client. + // Reads the final handshake status off the bytes relayed to the client, skipping interim + // 1xx responses such as 100 Continue (RFC 9110 section 15.2). const onAnswer = (chunk: Buffer) => { answer += chunk.toString("latin1"); - if (!answer.includes("\r\n") && answer.length < 64) return; - upstream.off("data", onAnswer); - const status = statusLine.exec(answer)?.[1]; - if (status !== undefined) Deferred.doneUnsafe(handshake, Effect.succeed(Number(status))); + while (true) { + const lineEnd = answer.indexOf("\r\n"); + if (lineEnd < 0) { + if (answer.length >= answerLimit) upstream.off("data", onAnswer); + return; + } + const status = Number(statusLine.exec(answer.slice(0, lineEnd))?.[1] ?? Number.NaN); + if (status >= 100 && status < 200 && status !== 101) { + const headEnd = answer.indexOf("\r\n\r\n"); + if (headEnd < 0) { + if (answer.length >= answerLimit) upstream.off("data", onAnswer); + return; + } + answer = answer.slice(headEnd + 4); + continue; + } + upstream.off("data", onAnswer); + if (!Number.isNaN(status)) Deferred.doneUnsafe(handshake, Effect.succeed(status)); + return; + } }; upstream.on("data", onAnswer); client.on("error", onClientGone); @@ -575,6 +604,14 @@ export const makeHttpProxy = (options: { Effect.gen(function* () { const complete = accessFor(request, yield* Clock.currentTimeMillis); const sent: Sent = { bytes: 0, clientLeft: false }; + // Timed when the response settles, before the target's release runs. + const settled = + onAccess === undefined + ? undefined + : yield* responseSettled(response).pipe( + Effect.andThen(Clock.currentTimeMillis), + Effect.forkChild({ startImmediately: true }), + ); // The access record waits outside this scope, so it never holds the target's activity. yield* Effect.scoped( Effect.gen(function* () { @@ -606,12 +643,12 @@ export const makeHttpProxy = (options: { } }), ); - if (onAccess === undefined) return; - yield* responseSettled(response); + if (onAccess === undefined || settled === undefined) return; + const ended = yield* Fiber.join(settled); const delivered = response.writableFinished && !sent.clientLeft; yield* onAccess( complete( - yield* Clock.currentTimeMillis, + ended, response.headersSent ? response.statusCode : 499, delivered ? sent.bytes : undefined, ), diff --git a/packages/stack/src/Owner.logs.integration.test.ts b/packages/stack/src/Owner.logs.integration.test.ts index cd7ccafc0e..3209ef069b 100644 --- a/packages/stack/src/Owner.logs.integration.test.ts +++ b/packages/stack/src/Owner.logs.integration.test.ts @@ -156,6 +156,8 @@ describe("owner persisted logs", () => { const client = yield* HttpClient.HttpClient; const root = yield* fs.makeTempDirectoryScoped({ prefix: "owner-logs-gateway-" }); const firstScope = yield* Scope.make(); + // Closes the first owner if the test fails before the restart closes it. + yield* Effect.addFinalizer(() => Scope.close(firstScope, Exit.void)); const first = yield* openOwner("owner-logs-gateway-", "native", { root }).pipe( Scope.provide(firstScope), ); From 12bfb02623512ce669e70b2c16cf8788b320f344 Mon Sep 17 00:00:00 2001 From: avallete Date: Thu, 1 Oct 2026 16:58:17 +0200 Subject: [PATCH 12/35] fix(stack): redact userinfo in URLs nested in gateway query values Co-Authored-By: Claude Opus 5.5 --- packages/stack/src/HttpProxy.integration.test.ts | 5 +++-- packages/stack/src/HttpProxy.ts | 9 ++++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/stack/src/HttpProxy.integration.test.ts b/packages/stack/src/HttpProxy.integration.test.ts index ad7a960709..eda9029e4c 100644 --- a/packages/stack/src/HttpProxy.integration.test.ts +++ b/packages/stack/src/HttpProxy.integration.test.ts @@ -1109,7 +1109,7 @@ it.live("records each request once with the status the client was sent", () => { "http://127.0.0.1:54321/y?apikey=sb_publishable_x#refresh_token=r&provider_token=p", }), yield* get( - "/elsewhere?redirect_to=https%3A%2F%2Fclient%2Fcb%3Faccess_token%3DJWT&next=http%3A%2F%2Flocalhost%3A3000%2F", + "/elsewhere?redirect_to=https%3A%2F%2Fclient%2Fcb%3Faccess_token%3DJWT&return_to=https%3A%2F%2Fuser%3Asecret%40client%2Fcb&next=http%3A%2F%2Flocalhost%3A3000%2F", { referer: "https://user:password@studio.test/relative?Access_Token=jwt" }, ), yield* get("/down/thing?token=t&redirect_to=https://client/cb?access_token=JWT"), @@ -1139,7 +1139,8 @@ it.live("records each request once with the status the client was sent", () => { "http://127.0.0.1:54321/y?apikey=redacted#refresh_token=redacted&provider_token=redacted", }), expect.objectContaining({ - target: "/elsewhere?redirect_to=redacted&next=http%3A%2F%2Flocalhost%3A3000%2F", + target: + "/elsewhere?redirect_to=redacted&return_to=redacted&next=http%3A%2F%2Flocalhost%3A3000%2F", status: 404, bytes: 9, referer: "https://redacted@studio.test/relative?Access_Token=redacted", diff --git a/packages/stack/src/HttpProxy.ts b/packages/stack/src/HttpProxy.ts index e0a0a2b634..afc8a4b300 100644 --- a/packages/stack/src/HttpProxy.ts +++ b/packages/stack/src/HttpProxy.ts @@ -168,6 +168,9 @@ const credentialParameters = new Set([ /** A credential pair inside a decoded value, such as a `redirect_to` URL carrying a token. */ const nestedCredential = new RegExp(`(?:^|[?&#])(?:${[...credentialParameters].join("|")})=`, "iu"); +const userinfo = /^([a-z][a-z\d+.-]*:\/\/)[^/?#@]*@/iu; +const nestedUserinfo = /[a-z][a-z\d+.-]*:\/\/[^/?#@]*@/iu; + const redactPairs = (pairs: string) => pairs .split("&") @@ -175,15 +178,15 @@ const redactPairs = (pairs: string) => const separator = parameter.indexOf("="); if (separator < 0) return parameter; const name = parameter.slice(0, separator); + const value = decodeQuery(parameter.slice(separator + 1)); return credentialParameters.has(decodeQuery(name).toLowerCase()) || - nestedCredential.test(decodeQuery(parameter.slice(separator + 1))) + nestedCredential.test(value) || + nestedUserinfo.test(value) ? `${name}=redacted` : parameter; }) .join("&"); -const userinfo = /^([a-z][a-z\d+.-]*:\/\/)[^/?#@]*@/iu; - /** * Redacts an absolute URL's userinfo and credential values in its query and fragment, where * OAuth implicit grants put `access_token`. Edits the text in place, so relative and unparsable From 04ed9439e2e2c3f3c230dd780691a12d780af755 Mon Sep 17 00:00:00 2001 From: avallete Date: Thu, 1 Oct 2026 17:08:13 +0200 Subject: [PATCH 13/35] fix(stack): detect credentials in multiply-encoded gateway query values Co-Authored-By: Claude Opus 5.5 --- packages/stack/src/HttpProxy.integration.test.ts | 6 ++++-- packages/stack/src/HttpProxy.ts | 13 ++++++++++++- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/packages/stack/src/HttpProxy.integration.test.ts b/packages/stack/src/HttpProxy.integration.test.ts index eda9029e4c..d8f4adb4ff 100644 --- a/packages/stack/src/HttpProxy.integration.test.ts +++ b/packages/stack/src/HttpProxy.integration.test.ts @@ -1112,7 +1112,9 @@ it.live("records each request once with the status the client was sent", () => { "/elsewhere?redirect_to=https%3A%2F%2Fclient%2Fcb%3Faccess_token%3DJWT&return_to=https%3A%2F%2Fuser%3Asecret%40client%2Fcb&next=http%3A%2F%2Flocalhost%3A3000%2F", { referer: "https://user:password@studio.test/relative?Access_Token=jwt" }, ), - yield* get("/down/thing?token=t&redirect_to=https://client/cb?access_token=JWT"), + yield* get( + "/down/thing?token=t&redirect_to=https://client/cb?access_token=JWT&back=https%253A%252F%252Fclient%252Fcb%253Faccess_token%253DJWT", + ), ]; const recorded = yield* Queue.takeN(accesses, 4); @@ -1146,7 +1148,7 @@ it.live("records each request once with the status the client was sent", () => { referer: "https://redacted@studio.test/relative?Access_Token=redacted", }), expect.objectContaining({ - target: "/down/thing?token=redacted&redirect_to=redacted", + target: "/down/thing?token=redacted&redirect_to=redacted&back=redacted", status: 502, bytes: 11, }), diff --git a/packages/stack/src/HttpProxy.ts b/packages/stack/src/HttpProxy.ts index afc8a4b300..657c24961e 100644 --- a/packages/stack/src/HttpProxy.ts +++ b/packages/stack/src/HttpProxy.ts @@ -152,6 +152,17 @@ const decodeQuery = (value: string) => { } }; +/** Decodes repeatedly so a nested URL encoded more than once still exposes its delimiters. */ +const decodeNested = (value: string) => { + let current = value; + for (let pass = 0; pass < 4; pass++) { + const next = decodeQuery(current); + if (next === current) break; + current = next; + } + return current; +}; + // Auth puts PKCE codes, OTP token hashes and OAuth tokens in redirect and verify URLs. const credentialParameters = new Set([ "apikey", @@ -178,7 +189,7 @@ const redactPairs = (pairs: string) => const separator = parameter.indexOf("="); if (separator < 0) return parameter; const name = parameter.slice(0, separator); - const value = decodeQuery(parameter.slice(separator + 1)); + const value = decodeNested(parameter.slice(separator + 1)); return credentialParameters.has(decodeQuery(name).toLowerCase()) || nestedCredential.test(value) || nestedUserinfo.test(value) From 0cb3e904474850db8ac83ac24600254f8eedbbe1 Mon Sep 17 00:00:00 2001 From: avallete Date: Thu, 1 Oct 2026 18:32:44 +0200 Subject: [PATCH 14/35] refactor(stack): simplify gateway access recording and redaction - Build gateway access records only on the proxy that logs them. - Collapse credential redaction into one pattern and move its cases to a unit table. - Share the month table, rename the log escaper, read upgrade answers in one loop, and trim a forwarder assertion the remap test already pins. Co-Authored-By: Claude Opus 5.5 --- packages/stack/ARCHITECTURE.md | 7 +- .../stack/src/HttpProxy.integration.test.ts | 167 ++++++++---------- packages/stack/src/HttpProxy.ts | 98 +++++----- packages/stack/src/HttpProxy.unit.test.ts | 71 ++++++++ packages/stack/src/Network.ts | 4 +- packages/stack/src/host/GatewayLog.ts | 19 +- .../src/host/LogForwarder.integration.test.ts | 17 +- packages/stack/src/host/LogForwarder.ts | 18 +- packages/stack/src/host/LogflareEvents.ts | 35 ++-- 9 files changed, 235 insertions(+), 201 deletions(-) create mode 100644 packages/stack/src/HttpProxy.unit.test.ts diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 415ca25de2..98b1677332 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -664,9 +664,10 @@ resetting database data keeps them. The shared API listener is the stack's gateway. Each completed request, and each WebSocket upgrade once its handshake status is known, becomes one nginx combined line plus duration, with credential -query and fragment values (API keys, tokens, token hashes, PKCE codes) redacted in the target and -the Referer. The proxy hands it to a bounded sliding buffer -after the response settles, so logging never delays a response or holds a target's activity. +query and fragment values (API keys, tokens, token hashes, PKCE codes), values that nest one or a +URL with userinfo, and URL userinfo redacted in the target and the Referer. The proxy hands it to +a bounded sliding buffer after the response settles, so logging never delays a response or holds +a target's activity. The owner persists these lines as the `gateway` stream, `logs/gateway/gateway/`, with one launch per owner run; it is not a service instance, so orphan cleanup keeps it. diff --git a/packages/stack/src/HttpProxy.integration.test.ts b/packages/stack/src/HttpProxy.integration.test.ts index d8f4adb4ff..c9d6e8b9e9 100644 --- a/packages/stack/src/HttpProxy.integration.test.ts +++ b/packages/stack/src/HttpProxy.integration.test.ts @@ -1067,100 +1067,83 @@ const rawUpgrade = (port: number, path: string) => Effect.timeout("5 seconds"), ); -it.live("records each request once with the status the client was sent", () => { - const logs: Array = []; - return Effect.scoped( - Effect.gen(function* () { - const backend = createServer((request, response) => { - request.resume(); - if (request.url?.startsWith("/ok")) response.end("hello"); - else { - response.statusCode = 404; - response.end("nope"); - } - }); - const backendAddress = yield* listen(backend); - const accesses = yield* Queue.unbounded(); - const proxy = yield* makeHttpProxy({ - host: "127.0.0.1", - port: 0, - onAccess: (access) => Queue.offer(accesses, access), - }); - yield* proxy.setRoutes([ - { id: "api", prefix: "/api", upstreamPrefix: "/", target: Effect.succeed(backendAddress) }, - { - id: "down", - prefix: "/down", - target: Effect.fail(new ProxyError({ message: "wake failed" })), - }, - ]); - const get = (path: string, headers: Readonly> = {}) => - request(proxy.port, path, new Uint8Array(), headers, "GET").pipe( - Effect.map(({ status }) => status), - ); +it.live( + "records each request once with its sent status, body bytes and redacted credentials", + () => { + const logs: Array = []; + return Effect.scoped( + Effect.gen(function* () { + const backend = createServer((request, response) => { + request.resume(); + if (request.url?.startsWith("/ok")) response.end("hello"); + else { + response.statusCode = 404; + response.end("nope"); + } + }); + const backendAddress = yield* listen(backend); + const accesses = yield* Queue.unbounded(); + const proxy = yield* makeHttpProxy({ + host: "127.0.0.1", + port: 0, + onAccess: (access) => Queue.offer(accesses, access), + }); + yield* proxy.setRoutes([ + { + id: "api", + prefix: "/api", + upstreamPrefix: "/", + target: Effect.succeed(backendAddress), + }, + { + id: "down", + prefix: "/down", + target: Effect.fail(new ProxyError({ message: "wake failed" })), + }, + ]); + const get = (path: string, headers: Readonly> = {}) => + request(proxy.port, path, new Uint8Array(), headers, "GET").pipe( + Effect.map(({ status }) => status), + ); - const statuses = [ - yield* get("/api/ok?select=*&apikey=sb_secret_x&Access_Token=jwt", { - "user-agent": "proxy-test/1", - referer: "http://127.0.0.1:54321/x?token=t1&select=*#access_token=frag&type=bearer", - }), - yield* get("/api/missing?code=pkce&state=s", { - referer: - "http://127.0.0.1:54321/y?apikey=sb_publishable_x#refresh_token=r&provider_token=p", - }), - yield* get( - "/elsewhere?redirect_to=https%3A%2F%2Fclient%2Fcb%3Faccess_token%3DJWT&return_to=https%3A%2F%2Fuser%3Asecret%40client%2Fcb&next=http%3A%2F%2Flocalhost%3A3000%2F", - { referer: "https://user:password@studio.test/relative?Access_Token=jwt" }, - ), - yield* get( - "/down/thing?token=t&redirect_to=https://client/cb?access_token=JWT&back=https%253A%252F%252Fclient%252Fcb%253Faccess_token%253DJWT", - ), - ]; - const recorded = yield* Queue.takeN(accesses, 4); + const statuses = [ + yield* get("/api/ok?select=*&apikey=sb_secret_x", { + "user-agent": "proxy-test/1", + referer: "http://127.0.0.1:54321/x?token=t1&select=*", + }), + yield* get("/api/missing"), + yield* get("/elsewhere"), + yield* get("/down/thing"), + ]; + const recorded = yield* Queue.takeN(accesses, 4); - expect(statuses).toEqual([200, 404, 404, 502]); - expect(recorded.toSorted((left, right) => left.time - right.time)).toEqual([ - { - time: expect.any(Number), - client: "127.0.0.1", - method: "GET", - target: "/api/ok?select=*&apikey=redacted&Access_Token=redacted", - protocol: "HTTP/1.1", - status: 200, - bytes: 5, - referer: - "http://127.0.0.1:54321/x?token=redacted&select=*#access_token=redacted&type=bearer", - userAgent: "proxy-test/1", - durationMillis: expect.any(Number), - }, - expect.objectContaining({ - target: "/api/missing?code=redacted&state=s", - status: 404, - bytes: 4, - referer: - "http://127.0.0.1:54321/y?apikey=redacted#refresh_token=redacted&provider_token=redacted", - }), - expect.objectContaining({ - target: - "/elsewhere?redirect_to=redacted&return_to=redacted&next=http%3A%2F%2Flocalhost%3A3000%2F", - status: 404, - bytes: 9, - referer: "https://redacted@studio.test/relative?Access_Token=redacted", - }), - expect.objectContaining({ - target: "/down/thing?token=redacted&redirect_to=redacted&back=redacted", - status: 502, - bytes: 11, - }), - ]); - expect(yield* Queue.size(accesses)).toBe(0); - }), - ).pipe( - Effect.provide( - Layer.mergeAll(NodeHttpClient.layerNodeHttp, NodeServices.layer, captureErrors(logs)), - ), - ); -}); + expect(statuses).toEqual([200, 404, 404, 502]); + expect(recorded.toSorted((left, right) => left.time - right.time)).toEqual([ + { + time: expect.any(Number), + client: "127.0.0.1", + method: "GET", + target: "/api/ok?select=*&apikey=redacted", + protocol: "HTTP/1.1", + status: 200, + bytes: 5, + referer: "http://127.0.0.1:54321/x?token=redacted&select=*", + userAgent: "proxy-test/1", + durationMillis: expect.any(Number), + }, + expect.objectContaining({ target: "/api/missing", status: 404, bytes: 4 }), + expect.objectContaining({ target: "/elsewhere", status: 404, bytes: 9 }), + expect.objectContaining({ target: "/down/thing", status: 502, bytes: 11 }), + ]); + expect(yield* Queue.size(accesses)).toBe(0); + }), + ).pipe( + Effect.provide( + Layer.mergeAll(NodeHttpClient.layerNodeHttp, NodeServices.layer, captureErrors(logs)), + ), + ); + }, +); it.live("records WebSocket upgrades at the handshake with the status sent to the client", () => { const logs: Array = []; diff --git a/packages/stack/src/HttpProxy.ts b/packages/stack/src/HttpProxy.ts index 657c24961e..3c6ec89959 100644 --- a/packages/stack/src/HttpProxy.ts +++ b/packages/stack/src/HttpProxy.ts @@ -164,7 +164,7 @@ const decodeNested = (value: string) => { }; // Auth puts PKCE codes, OTP token hashes and OAuth tokens in redirect and verify URLs. -const credentialParameters = new Set([ +const credentialParameters = [ "apikey", "access_token", "token", @@ -174,36 +174,36 @@ const credentialParameters = new Set([ "id_token", "provider_token", "provider_refresh_token", -]); - -/** A credential pair inside a decoded value, such as a `redirect_to` URL carrying a token. */ -const nestedCredential = new RegExp(`(?:^|[?&#])(?:${[...credentialParameters].join("|")})=`, "iu"); +]; +const scheme = String.raw`[a-z][a-z\d+.-]*`; -const userinfo = /^([a-z][a-z\d+.-]*:\/\/)[^/?#@]*@/iu; -const nestedUserinfo = /[a-z][a-z\d+.-]*:\/\/[^/?#@]*@/iu; +/** + * A decoded parameter that is a credential pair, nests one (a `redirect_to` URL carrying a + * token), or nests a URL with userinfo. + */ +const sensitive = new RegExp( + String.raw`(?:^|[?&#=])(?:${credentialParameters.join("|")})=|${scheme}:\/\/[^/?#@]*@`, + "iu", +); +const userinfo = new RegExp(String.raw`^(${scheme}:\/\/)[^/?#@]*@`, "iu"); const redactPairs = (pairs: string) => pairs .split("&") .map((parameter) => { const separator = parameter.indexOf("="); - if (separator < 0) return parameter; - const name = parameter.slice(0, separator); - const value = decodeNested(parameter.slice(separator + 1)); - return credentialParameters.has(decodeQuery(name).toLowerCase()) || - nestedCredential.test(value) || - nestedUserinfo.test(value) - ? `${name}=redacted` + return separator >= 0 && sensitive.test(decodeNested(parameter)) + ? `${parameter.slice(0, separator)}=redacted` : parameter; }) .join("&"); /** * Redacts an absolute URL's userinfo and credential values in its query and fragment, where - * OAuth implicit grants put `access_token`. Edits the text in place, so relative and unparsable - * URLs work and the rest of the URL keeps its original encoding. + * OAuth implicit grants put `access_token` for non-browser clients. Edits the text in place, so + * relative and unparsable URLs work and the rest of the URL keeps its original encoding. */ -const redactCredentials = (url: string) => { +export const redactCredentials = (url: string) => { const hashAt = url.indexOf("#"); const beforeHash = hashAt < 0 ? url : url.slice(0, hashAt); const queryAt = beforeHash.indexOf("?"); @@ -549,19 +549,13 @@ const upgrade = Effect.fn("HttpProxy.upgrade")( // 1xx responses such as 100 Continue (RFC 9110 section 15.2). const onAnswer = (chunk: Buffer) => { answer += chunk.toString("latin1"); - while (true) { - const lineEnd = answer.indexOf("\r\n"); - if (lineEnd < 0) { - if (answer.length >= answerLimit) upstream.off("data", onAnswer); - return; - } - const status = Number(statusLine.exec(answer.slice(0, lineEnd))?.[1] ?? Number.NaN); + for ( + let headEnd = answer.indexOf("\r\n\r\n"); + headEnd >= 0; + headEnd = answer.indexOf("\r\n\r\n") + ) { + const status = Number(statusLine.exec(answer)?.[1] ?? Number.NaN); if (status >= 100 && status < 200 && status !== 101) { - const headEnd = answer.indexOf("\r\n\r\n"); - if (headEnd < 0) { - if (answer.length >= answerLimit) upstream.off("data", onAnswer); - return; - } answer = answer.slice(headEnd + 4); continue; } @@ -569,6 +563,7 @@ const upgrade = Effect.fn("HttpProxy.upgrade")( if (!Number.isNaN(status)) Deferred.doneUnsafe(handshake, Effect.succeed(status)); return; } + if (answer.length >= answerLimit) upstream.off("data", onAnswer); }; upstream.on("data", onAnswer); client.on("error", onClientGone); @@ -600,7 +595,7 @@ const upgrade = Effect.fn("HttpProxy.upgrade")( export const makeHttpProxy = (options: { readonly host: string; readonly port: number; - readonly onAccess?: HttpAccessSink; + readonly onAccess?: HttpAccessSink | undefined; }): Effect.Effect => Effect.gen(function* () { const routes = yield* Ref.make>([]); @@ -616,16 +611,19 @@ export const makeHttpProxy = (options: { const server = createServer((request, response) => { runRequest( Effect.gen(function* () { - const complete = accessFor(request, yield* Clock.currentTimeMillis); const sent: Sent = { bytes: 0, clientLeft: false }; - // Timed when the response settles, before the target's release runs. - const settled = + const access = onAccess === undefined ? undefined - : yield* responseSettled(response).pipe( - Effect.andThen(Clock.currentTimeMillis), - Effect.forkChild({ startImmediately: true }), - ); + : { + complete: accessFor(request, yield* Clock.currentTimeMillis), + // Timed when the response settles, before the target's release runs. + settled: yield* responseSettled(response).pipe( + Effect.andThen(Clock.currentTimeMillis), + Effect.forkChild({ startImmediately: true }), + ), + record: onAccess, + }; // The access record waits outside this scope, so it never holds the target's activity. yield* Effect.scoped( Effect.gen(function* () { @@ -657,11 +655,11 @@ export const makeHttpProxy = (options: { } }), ); - if (onAccess === undefined || settled === undefined) return; - const ended = yield* Fiber.join(settled); + if (access === undefined) return; + const ended = yield* Fiber.join(access.settled); const delivered = response.writableFinished && !sent.clientLeft; - yield* onAccess( - complete( + yield* access.record( + access.complete( ended, response.headersSent ? response.statusCode : 499, delivered ? sent.bytes : undefined, @@ -678,20 +676,22 @@ export const makeHttpProxy = (options: { socket.on("error", () => socket.destroy()); runRequest( Effect.gen(function* () { - const complete = accessFor(request, yield* Clock.currentTimeMillis); // Completed by the upstream's handshake status, otherwise by the upgrade's outcome. const handshake = yield* Deferred.make(); const recorded = onAccess === undefined ? undefined - : yield* Deferred.await(handshake).pipe( - Effect.flatMap((status) => - Clock.currentTimeMillis.pipe( - Effect.flatMap((ended) => onAccess(complete(ended, status))), + : yield* Effect.gen(function* () { + const complete = accessFor(request, yield* Clock.currentTimeMillis); + return yield* Deferred.await(handshake).pipe( + Effect.flatMap((status) => + Clock.currentTimeMillis.pipe( + Effect.flatMap((ended) => onAccess(complete(ended, status))), + ), ), - ), - Effect.forkChild({ startImmediately: true }), - ); + Effect.forkChild({ startImmediately: true }), + ); + }); const route = yield* Ref.get(routes).pipe( Effect.map((current) => routeFor(request.url ?? "/", current)), ); diff --git a/packages/stack/src/HttpProxy.unit.test.ts b/packages/stack/src/HttpProxy.unit.test.ts new file mode 100644 index 0000000000..414cfc6025 --- /dev/null +++ b/packages/stack/src/HttpProxy.unit.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "@effect/vitest"; +import { redactCredentials } from "./HttpProxy.ts"; + +describe("redactCredentials", () => { + it.each([ + { + case: "redacts credential query values, any case", + url: "/api/ok?select=*&apikey=sb_secret_x&Access_Token=jwt", + redacted: "/api/ok?select=*&apikey=redacted&Access_Token=redacted", + }, + { + case: "redacts Auth codes and OTP token hashes", + url: "/auth/v1/verify?code=pkce&token_hash=h&type=signup", + redacted: "/auth/v1/verify?code=redacted&token_hash=redacted&type=signup", + }, + { + case: "redacts fragment credentials of an absolute URL", + url: "http://127.0.0.1:54321/x?token=t1&select=*#access_token=frag&type=bearer", + redacted: + "http://127.0.0.1:54321/x?token=redacted&select=*#access_token=redacted&type=bearer", + }, + { + case: "redacts refresh and provider tokens in a fragment", + url: "http://127.0.0.1:54321/y?apikey=k#refresh_token=r&provider_token=p&provider_refresh_token=q", + redacted: + "http://127.0.0.1:54321/y?apikey=redacted#refresh_token=redacted&provider_token=redacted&provider_refresh_token=redacted", + }, + { + case: "redacts an encoded nested URL carrying a token", + url: "/elsewhere?redirect_to=https%3A%2F%2Fclient%2Fcb%3Faccess_token%3DJWT", + redacted: "/elsewhere?redirect_to=redacted", + }, + { + case: "redacts a raw nested URL carrying a token", + url: "/down?redirect_to=https://client/cb?access_token=JWT", + redacted: "/down?redirect_to=redacted", + }, + { + case: "redacts a doubly encoded nested URL carrying a token", + url: "/down?back=https%253A%252F%252Fclient%252Fcb%253Faccess_token%253DJWT", + redacted: "/down?back=redacted", + }, + { + case: "redacts a nested URL with userinfo", + url: "/elsewhere?return_to=https%3A%2F%2Fuser%3Asecret%40client%2Fcb", + redacted: "/elsewhere?return_to=redacted", + }, + { + case: "redacts userinfo of an absolute URL", + url: "https://user:password@studio.test/relative?Access_Token=jwt", + redacted: "https://redacted@studio.test/relative?Access_Token=redacted", + }, + { + case: "redacts a value that embeds a credential pair", + url: "/api?data=a=token=b", + redacted: "/api?data=redacted", + }, + { + case: "keeps a nested URL without credentials, with its encoding", + url: "/elsewhere?next=http%3A%2F%2Flocalhost%3A3000%2F&country_code=FR", + redacted: "/elsewhere?next=http%3A%2F%2Flocalhost%3A3000%2F&country_code=FR", + }, + { + case: "keeps a path without a query", + url: "/rest/v1/todos", + redacted: "/rest/v1/todos", + }, + ])("$case", ({ url, redacted }) => { + expect(redactCredentials(url)).toBe(redacted); + }); +}); diff --git a/packages/stack/src/Network.ts b/packages/stack/src/Network.ts index 7f13dbaafc..d5b2ca44ad 100644 --- a/packages/stack/src/Network.ts +++ b/packages/stack/src/Network.ts @@ -179,9 +179,7 @@ const makeNetwork = (options: { proxy: yield* makeHttpProxy({ host, port, - ...(options.onAccess === undefined - ? {} - : { onAccess: options.onAccess }), + onAccess: options.onAccess, }), }; } diff --git a/packages/stack/src/host/GatewayLog.ts b/packages/stack/src/host/GatewayLog.ts index 91e98d5713..b833be862a 100644 --- a/packages/stack/src/host/GatewayLog.ts +++ b/packages/stack/src/host/GatewayLog.ts @@ -2,6 +2,7 @@ import { DateTime, Effect, PubSub, Stream } from "effect"; import type { HttpAccess, HttpAccessSink } from "../HttpProxy.ts"; import { launchOutputPublisher, type LaunchOutput } from "../runtime/Session.ts"; import type { CatalogLogs } from "../services/Recipe.ts"; +import { monthNames } from "./LogflareEvents.ts"; /** The log stream of the shared API listener's access records, one per owner and stack. */ export const gatewayLog = { service: "gateway", instanceId: "gateway" } as const; @@ -9,20 +10,6 @@ export const gatewayLog = { service: "gateway", instanceId: "gateway" } as const /** Bounds records not yet persisted; the oldest are dropped and reported as lost. */ const bufferedRecords = 4096; -const monthNames = [ - "Jan", - "Feb", - "Mar", - "Apr", - "May", - "Jun", - "Jul", - "Aug", - "Sep", - "Oct", - "Nov", - "Dec", -]; const two = (value: number) => String(value).padStart(2, "0"); /** Formats epoch milliseconds as nginx's `$time_local` in UTC. */ @@ -32,7 +19,7 @@ const nginxTime = (millis: number) => { }; /** Escapes quotes, backslashes and control characters like nginx's default log escaping. */ -const escape = (value: string) => +const escapeLogValue = (value: string) => value.replace( // oxlint-disable-next-line no-control-regex -- control characters are what this escapes. /["\\\u0000-\u001f\u007f]/gu, @@ -41,7 +28,7 @@ const escape = (value: string) => /** Formats an access record as an nginx combined log line followed by its duration. */ export const formatAccess = (access: HttpAccess) => - `${access.client} - - [${nginxTime(access.time)}] "${escape(`${access.method} ${access.target} ${access.protocol}`)}" ${access.status} ${access.bytes ?? "-"} "${escape(access.referer ?? "-")}" "${escape(access.userAgent ?? "-")}" ${access.durationMillis}ms`; + `${access.client} - - [${nginxTime(access.time)}] "${escapeLogValue(`${access.method} ${access.target} ${access.protocol}`)}" ${access.status} ${access.bytes ?? "-"} "${escapeLogValue(access.referer ?? "-")}" "${escapeLogValue(access.userAgent ?? "-")}" ${access.durationMillis}ms`; const encoder = new TextEncoder(); diff --git a/packages/stack/src/host/LogForwarder.integration.test.ts b/packages/stack/src/host/LogForwarder.integration.test.ts index 0c382e20fc..2845fbd435 100644 --- a/packages/stack/src/host/LogForwarder.integration.test.ts +++ b/packages/stack/src/host/LogForwarder.integration.test.ts @@ -421,22 +421,7 @@ describe("LogForwarder", () => { const shipped = yield* sink.next; expect(shipped.url).toBe("/api/logs?source_name=cloudflare.logs.prod"); - expect(shipped.events).toEqual([ - expect.objectContaining({ - appname: "gateway", - timestamp: "2026-10-01T09:25:23.000Z", - metadata: { - request: { - method: "POST", - path: "/auth/v1/token", - search: "?grant_type=password", - protocol: "HTTP/1.1", - headers: { cf_connecting_ip: "127.0.0.1" }, - }, - response: { status_code: 400 }, - }, - }), - ]); + expect(shipped.events).toEqual([expect.objectContaining({ appname: "gateway" })]); }).pipe(Effect.scoped, Effect.provide(layer)), ); }); diff --git a/packages/stack/src/host/LogForwarder.ts b/packages/stack/src/host/LogForwarder.ts index 8bc2fbbfb3..991b799387 100644 --- a/packages/stack/src/host/LogForwarder.ts +++ b/packages/stack/src/host/LogForwarder.ts @@ -47,7 +47,10 @@ export interface ForwardedStream { } interface Interface { - /** Ships a shipped service's persisted logs, or tracks an Analytics instance as the target. */ + /** + * Ships the persisted logs of a shipped service or an owner stream, or tracks an Analytics + * instance as the target. + */ readonly attach: (instance: ForwardedInstance | ForwardedStream) => Effect.Effect; /** Re-selects the shipping target after the composition changes. */ readonly rebind: Effect.Effect; @@ -397,11 +400,16 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar const attach = Effect.fn("LogForwarder.attach")(function* ( instance: ForwardedInstance | ForwardedStream, ) { - if (instance.service === "gateway") - yield* Effect.forkIn(forward(instance.id, instance.service, Effect.never), scope); - else if (instance.service === "analytics") yield* Effect.forkIn(trackTarget(instance), scope); + if (instance.service === "analytics") yield* Effect.forkIn(trackTarget(instance), scope); else if (isShippedService(instance.service)) - yield* Effect.forkIn(forward(instance.id, instance.service, unregistered(instance)), scope); + yield* Effect.forkIn( + forward( + instance.id, + instance.service, + "observation" in instance ? unregistered(instance) : Effect.never, + ), + scope, + ); }); return { diff --git a/packages/stack/src/host/LogflareEvents.ts b/packages/stack/src/host/LogflareEvents.ts index fe4846ae7e..21f6d1a109 100644 --- a/packages/stack/src/host/LogflareEvents.ts +++ b/packages/stack/src/host/LogflareEvents.ts @@ -3,7 +3,7 @@ import type { ServiceCreation } from "../services/Catalog.ts"; import type { gatewayLog } from "./GatewayLog.ts"; /** A service kind, or the owner's `gateway` stream of shared API listener requests. */ -export type LogService = ServiceCreation["service"] | typeof gatewayLog.service; +type LogService = ServiceCreation["service"] | typeof gatewayLog.service; /** Logflare source per shipped log service; Studio's Logs pages query these names. */ export const logflareSources = { @@ -43,20 +43,21 @@ const parseJsonObject = (text: string): Record | undefined => { } }; -const months: Record = { - jan: 0, - feb: 1, - mar: 2, - apr: 3, - may: 4, - jun: 5, - jul: 6, - aug: 7, - sep: 8, - oct: 9, - nov: 10, - dec: 11, -}; +/** The `%b` month abbreviations of PostgREST and nginx log times, January first. */ +export const monthNames = [ + "Jan", + "Feb", + "Mar", + "Apr", + "May", + "Jun", + "Jul", + "Aug", + "Sep", + "Oct", + "Nov", + "Dec", +] as const; /** Parses the `%d/%b/%Y:%H:%M:%S %z` time of PostgREST and nginx logs into ISO-8601. */ const parseLogTime = (text: string): string | undefined => { @@ -64,8 +65,8 @@ const parseLogTime = (text: string): string | undefined => { /^(\d{2})\/([A-Za-z]{3})\/(\d{4}):(\d{2}):(\d{2}):(\d{2}) ([+-])(\d{2})(\d{2})$/u.exec(text); if (match === null) return undefined; const [, day, month, year, hour, minute, second, sign, zoneHours, zoneMinutes] = match; - const monthIndex = months[month?.toLowerCase() ?? ""]; - if (monthIndex === undefined) return undefined; + const monthIndex = monthNames.findIndex((name) => name.toLowerCase() === month?.toLowerCase()); + if (monthIndex < 0) return undefined; const utc = Date.UTC( Number(year), monthIndex, From d53b8eb22dc812e7e2af697e5671711e38231b3c Mon Sep 17 00:00:00 2001 From: avallete Date: Thu, 1 Oct 2026 19:14:50 +0200 Subject: [PATCH 15/35] fix(stack): keep latest-launch logs and clean up log store edges - Keep launch ids increasing per instance across owner restarts so stack logs --since start always selects the latest launch. - Migrate saved Vector stacks without taking the registry lock when there is nothing to migrate, and remove the rendered Vector config. - Fail stack logs --follow when the running stack serves none of the selected instances, and warn about skipped ones. - Detach an instance from log shipping, after its cursor write lands, before its logs are removed. - Remove readStackLogs, the end read option and unused platform options; reuse existing helpers and trim redundant tests. Co-Authored-By: Claude Opus 5.5 --- apps/cli/src/command-internal/colors.ts | 4 +- .../experimental/stack/logs/SIDE_EFFECTS.md | 9 +- .../experimental/stack/logs/logs.format.ts | 40 ++++++--- .../stack/logs/logs.format.unit.test.ts | 78 ++++++++-------- .../experimental/stack/logs/logs.handler.ts | 14 +++ .../stack/logs/logs.integration.test.ts | 88 ++++++++++++++----- packages/stack/ARCHITECTURE.md | 21 ++--- packages/stack/README.md | 2 +- .../stack/src/Owner.logs.integration.test.ts | 68 ++++++++++---- packages/stack/src/Owner.ts | 23 +++-- packages/stack/src/Rpc.ts | 2 +- packages/stack/src/Service.ts | 4 +- packages/stack/src/State.integration.test.ts | 1 - packages/stack/src/State.ts | 46 ++++++---- ...State.vector-migration.integration.test.ts | 28 +++++- packages/stack/src/effect.ts | 31 +------ .../src/host/LogForwarder.integration.test.ts | 86 ++++++++++++++++-- packages/stack/src/host/LogForwarder.ts | 46 ++++++---- packages/stack/src/host/LogRecord.ts | 8 +- .../stack/src/host/LogRecord.unit.test.ts | 10 --- .../src/host/LogStore.integration.test.ts | 88 +++++++++---------- packages/stack/src/host/LogStore.ts | 87 +++++------------- packages/stack/src/host/LogflareEvents.ts | 7 +- packages/stack/src/index.ts | 6 -- packages/stack/src/services/Database.ts | 5 +- packages/stack/src/services/ProcessRecipe.ts | 8 +- packages/stack/src/services/Recipe.ts | 4 +- 27 files changed, 483 insertions(+), 331 deletions(-) diff --git a/apps/cli/src/command-internal/colors.ts b/apps/cli/src/command-internal/colors.ts index 1cce53c10f..b1b6933cf1 100644 --- a/apps/cli/src/command-internal/colors.ts +++ b/apps/cli/src/command-internal/colors.ts @@ -78,12 +78,12 @@ export function gray(text: string, stream: ColorStream = process.stderr): string return supportsColor(stream) ? styleText("gray", text, { validateStream: false }) : text; } -/** Renders in bright magenta. */ +/** Renders in magenta. */ export function magenta(text: string, stream: ColorStream = process.stderr): string { return supportsColor(stream) ? styleText("magenta", text, { validateStream: false }) : text; } -/** Renders in bright blue. */ +/** Renders in blue. */ export function blue(text: string, stream: ColorStream = process.stderr): string { return supportsColor(stream) ? styleText("blue", text, { validateStream: false }) : text; } diff --git a/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md index 8cd7e8e4cf..0f5d9d6fc0 100644 --- a/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md @@ -26,8 +26,9 @@ service configuration, artifacts, logs, or data are changed. `--tail N` (default 200, 0 prints none) keeps the newest N output lines across the selected services, with the `launch` and `lost` markers between them. `--since` accepts a duration before now (`30s`, `10m`, `1h30m`, `2d`), an -ISO-8601 time, or `start`, which keeps each instance's records from its latest -launch (all of them when retention removed that launch record). Records are +ISO-8601 time, or `start`, which keeps the records of each instance's highest +launch id, which keeps increasing across owner restarts (all of them when +retention removed that launch record). Records are ordered by timestamp, service, instance, and file position. Timestamps are the owner's clock at each line's first byte. Lines end at `\n`, `\r\n`, or a lone `\r`, so carriage-return progress updates print as separate lines. History is @@ -39,7 +40,9 @@ before printing anything and suggests running without `--follow`. It prints the history, then streams each instance's new records through the owner from the last record read, so no record is repeated or skipped between history and live output. With `--tail 0` no history is read and the owner starts each follow at -its current end. The owner's log shipping to Analytics reads the persisted logs +its current end. A selected instance the running owner does not serve is named +in a warning and not followed; when it serves none of them, the command fails +with status 1 after printing the history. The owner's log shipping to Analytics reads the persisted logs separately and does not affect this command. ## Output diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.format.ts b/apps/cli/src/commands/experimental/stack/logs/logs.format.ts index 3df0d15ab3..13f3ab1511 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.format.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.format.ts @@ -58,7 +58,7 @@ export interface LogWindow { readonly total: number; } -export const tailWindow = (records: ReadonlyArray, tail: number): LogWindow => { +const tailWindow = (records: ReadonlyArray, tail: number): LogWindow => { const total = records.filter(isLine).length; if (total <= tail) return { records, shown: total, total }; let seen = 0; @@ -85,19 +85,28 @@ const compareRecords = (left: StackLogRecord, right: StackLogRecord) => right.position ?? right.resumeAt ?? origin, ); +interface Launch { + readonly id: number; + readonly timestamp: string; +} + interface InstanceTail { records: Array; head: number; lines: number; total: number; - launched: string | undefined; + launch: Launch | undefined; } +/** Launch ids increase per instance; a gap marker carries none, so its time decides. */ +const fromLaunch = (record: StackLogRecord, launch: Launch) => + record.launchId === undefined + ? record.timestamp >= launch.timestamp + : record.launchId >= launch.id; + /** - * Collects the history window from records streamed per instance in file order, holding each - * instance's newest `tail` lines in display order; `positions` holds each instance's last record - * in file order, where a follow resumes. With `fromLatestLaunch`, an instance's records before its latest - * `launch` record are dropped; an instance whose launch record was not retained keeps all of them. + * Collects each instance's newest `tail` lines and last position, optionally only the records of + * its highest launch. */ export const makeHistoryCollector = (tail: number, fromLatestLaunch: boolean) => { const instances = new Map(); @@ -109,20 +118,23 @@ export const makeHistoryCollector = (tail: number, fromLatestLaunch: boolean) => positions.set(record.instanceId, position); let state = instances.get(record.instanceId); if (state === undefined) { - state = { records: [], head: 0, lines: 0, total: 0, launched: undefined }; + state = { records: [], head: 0, lines: 0, total: 0, launch: undefined }; instances.set(record.instanceId, state); } if (fromLatestLaunch) { - if (record.kind === "launch") { - const launched = record.timestamp; - state.records = state.records - .slice(state.head) - .filter((kept) => kept.timestamp >= launched); + const latest = state.launch; + if ( + record.kind === "launch" && + record.launchId !== undefined && + (latest === undefined || record.launchId > latest.id) + ) { + const launch = { id: record.launchId, timestamp: record.timestamp }; + state.records = state.records.slice(state.head).filter((kept) => fromLaunch(kept, launch)); state.head = 0; state.lines = state.records.filter(isLine).length; state.total = state.lines; - state.launched = launched; - } else if (state.launched !== undefined && record.timestamp < state.launched) return; + state.launch = launch; + } else if (latest !== undefined && !fromLaunch(record, latest)) return; } const line = isLine(record); if (line) state.total += 1; diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.format.unit.test.ts b/apps/cli/src/commands/experimental/stack/logs/logs.format.unit.test.ts index f63a3fa4fc..559f4180c9 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.format.unit.test.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.format.unit.test.ts @@ -1,16 +1,9 @@ import { DateTime } from "effect"; import { describe, expect, it } from "vitest"; import type { StackLogRecord } from "@supabase/stack/effect"; -import { makeHistoryCollector, parseSince, tailWindow } from "./logs.format.ts"; +import { makeHistoryCollector, parseSince } from "./logs.format.ts"; const now = Date.parse("2026-09-29T12:00:00.000Z"); -const record = (kind: StackLogRecord["kind"], text: string): StackLogRecord => ({ - kind, - timestamp: "2026-09-29T10:00:00.000Z", - text, - service: "auth", - instanceId: "auth-a", -}); describe("parseSince", () => { it("subtracts compound durations from now", () => { @@ -41,14 +34,18 @@ describe("makeHistoryCollector", () => { offset: number, kind: StackLogRecord["kind"], text = "", + launchId = 1, ): StackLogRecord => ({ kind, timestamp: DateTime.formatIso(DateTime.makeUnsafe(now + offset)), + launchId, text, service: "auth", instanceId, position: { generation: 1, byteOffset: offset }, }); + const textsOf = (records: ReadonlyArray) => + records.filter(({ kind }) => kind !== "launch").map(({ text }) => text); it("keeps the newest lines across instances, counts every line, and tracks positions", () => { const collector = makeHistoryCollector(2, false); @@ -82,41 +79,50 @@ describe("makeHistoryCollector", () => { expect(positions.get("a")).toEqual({ generation: 1, byteOffset: 40 }); }); - it("starts each instance at its latest launch and keeps instances without one", () => { + it("starts each instance at its highest launch and keeps instances without one", () => { const collector = makeHistoryCollector(10, true); - collector.push(at("a", 0, "launch")); - collector.push(at("a", 1, "stdout", "a old")); - collector.push(at("a", 5, "launch")); - collector.push(at("a", 3, "stderr", "a late old")); - collector.push(at("a", 6, "stdout", "a new")); + collector.push(at("a", 0, "launch", "", 1)); + collector.push(at("a", 1, "stdout", "a old", 1)); + collector.push(at("a", 5, "launch", "", 2)); + collector.push(at("a", 3, "stderr", "a late old", 1)); + collector.push(at("a", 6, "stdout", "a new", 2)); collector.push(at("b", 2, "stdout", "b retained")); const { window } = collector.finish(); - expect(window.records.filter(({ kind }) => kind !== "launch").map(({ text }) => text)).toEqual([ - "b retained", - "a new", - ]); + expect(textsOf(window.records)).toEqual(["b retained", "a new"]); expect(window.total).toBe(2); }); -}); -describe("tailWindow", () => { - it("counts only output lines and keeps the markers between the shown lines", () => { - const records = [ - record("launch", ""), - record("stdout", "one"), - record("launch", ""), - record("stderr", "two"), - record("lost", ""), - record("stdout", "three"), - ]; - - const window = tailWindow(records, 2); - - expect(window.records.map(({ kind }) => kind)).toEqual(["stderr", "lost", "stdout"]); - expect(window).toMatchObject({ shown: 2, total: 3 }); - expect(tailWindow(records, 0)).toEqual({ records: [], shown: 0, total: 3 }); - expect(tailWindow(records, 5).records).toBe(records); + it("keeps the highest launch when an older launch's first output arrives after it", () => { + const collector = makeHistoryCollector(10, true); + collector.push(at("a", 10, "launch", "", 2)); + collector.push(at("a", 11, "stdout", "second launch", 2)); + collector.push(at("a", 20, "launch", "", 1)); + collector.push(at("a", 21, "stdout", "delayed first launch", 1)); + collector.push(at("a", 30, "stdout", "second launch again", 2)); + + const { window } = collector.finish(); + + expect(textsOf(window.records)).toEqual(["second launch", "second launch again"]); + expect(window.records.filter(({ kind }) => kind === "launch")).toHaveLength(1); + }); + + it("starts at the launch of the latest owner run, whose ids continue the earlier run's", () => { + const collector = makeHistoryCollector(10, true); + collector.push({ ...at("a", 0, "launch", "", 3), position: { generation: 1, byteOffset: 0 } }); + collector.push({ + ...at("a", 1, "stdout", "earlier run", 3), + position: { generation: 1, byteOffset: 40 }, + }); + collector.push({ ...at("a", 50, "launch", "", 4), position: { generation: 2, byteOffset: 0 } }); + collector.push({ + ...at("a", 51, "stdout", "restarted run", 4), + position: { generation: 2, byteOffset: 40 }, + }); + + const { window } = collector.finish(); + + expect(textsOf(window.records)).toEqual(["restarted run"]); }); }); diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.handler.ts b/apps/cli/src/commands/experimental/stack/logs/logs.handler.ts index cc91d681a4..1a93dfa637 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.handler.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.handler.ts @@ -176,6 +176,20 @@ export const stackLogs = Effect.fn("experimental.stack.logs")(function* (flags: handle, ]), ); + const missing = selected + .filter(({ id }) => !handles.has(id)) + .map(({ id, service }) => `${service} (${id})`); + if (missing.length === selected.length) + return yield* new StackCommandLogsError({ + reason: "lifecycle", + message: + "The running stack serves none of the selected services, so there are no new lines to follow.", + suggestion: "Run supabase stack logs without --follow to read retained logs.", + }); + if (missing.length > 0) + yield* output.warn( + `Not following ${missing.join(", ")}, which the running stack does not serve.`, + ); const streams = selected.flatMap(({ id, service }) => { const handle = handles.get(id); if (handle === undefined) return []; diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts b/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts index baedd7cdbf..27175bc37a 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts @@ -17,7 +17,7 @@ import { Stream, } from "effect"; import { - readStackLogs, + streamStackLogs, type LogRecord, type ReadLogsOptions, type SavedStack, @@ -370,29 +370,24 @@ describe("stack logs", () => { (stack: { readonly stateRoot: string; readonly stackId: string }) => (id: string, text: string) => followingMail(id, (options) => - Stream.unwrap( - readStackLogs({ ...stack, instances: [id] }).pipe( - Effect.orDie, - Effect.provide(BunServices.layer), - Effect.map((records) => - Stream.fromIterable([ - ...records.filter( - ({ position }) => - options?.tail !== 0 && - (options?.from === undefined || - (position !== undefined && - position.byteOffset >= options.from.byteOffset)), - ), - { - kind: "stdout" as const, - timestamp: iso(t0 + 100), - launchId: 1, - text, - position: { generation: 1, byteOffset: 1_000_000 }, - }, - ]), - ), + streamStackLogs({ ...stack, instances: [id] }).pipe( + Stream.orDie, + Stream.filter( + ({ position }) => + options?.tail !== 0 && + (options?.from === undefined || + (position !== undefined && position.byteOffset >= options.from.byteOffset)), ), + Stream.concat( + Stream.make({ + kind: "stdout" as const, + timestamp: iso(t0 + 100), + launchId: 1, + text, + position: { generation: 1, byteOffset: 1_000_000 }, + }), + ), + Stream.provide(BunServices.layer), ), ); const f = yield* fixture({ @@ -435,6 +430,53 @@ describe("stack logs", () => { }).pipe(Effect.scoped, Effect.provide(live)), ); + it.live("follows the services the running stack serves and names the others", () => + Effect.gen(function* () { + const served = (text: string) => + Stream.make({ + kind: "stdout" as const, + timestamp: iso(t0), + launchId: 1, + text, + position: { generation: 1, byteOffset: 0 }, + }); + const f = yield* fixture({ + instances: [mail("mail-a"), mail("mail-b")], + members: ["mail-a", "mail-b"], + running: true, + handles: () => [followingMail("mail-a", () => served("a live"))], + }); + + const { exit, output } = yield* f.run({ follow: true, tail: 0 }, "stream-json"); + + expect(Exit.isSuccess(exit)).toBe(true); + expect(eventLines(output.events)).toEqual(["a live"]); + expect(output.messages).toContainEqual({ + type: "warn", + message: "Not following mail (mail-b), which the running stack does not serve.", + }); + }).pipe(Effect.scoped, Effect.provide(live)), + ); + + it.live("fails --follow when the running stack serves none of the selected services", () => + Effect.gen(function* () { + const f = yield* fixture({ + instances: [mail("mail-a")], + members: ["mail-a"], + running: true, + handles: () => [], + }); + + const { exit } = yield* f.run({ follow: true, tail: 0 }); + + expect(failure(exit)).toMatchObject({ + reason: "lifecycle", + message: + "The running stack serves none of the selected services, so there are no new lines to follow.", + }); + }).pipe(Effect.scoped, Effect.provide(live)), + ); + it.live("interrupts a follow and releases its subscriptions", () => Effect.gen(function* () { const subscribed = yield* Deferred.make(); diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 0d558590ce..ce5e2cd22f 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -643,24 +643,25 @@ The state root is the stack registry root. Each stack keeps one state document a ``` The owner is the only subscriber of each instance's output. It writes one record per line, ended by -`\n`, `\r\n` or a lone `\r` so carriage-return progress updates become separate records, ` [ truncated] | `, into immutable generation-named +`\n`, `\r\n` or a lone `\r` so carriage-return progress updates become separate records, ` + [ truncated] | `, into immutable generation-named segments: each owner start opens a new generation on the instance's first output, and a record that would take a segment past 5 MiB starts the next one. The oldest closed segments except the newest are deleted while an instance holds more than 10 MiB or 64 segments. Records carry the publish time -of their first byte. Line state is kept per launch, process and stream; chunk sequence numbers span -a launch's processes, so a process whose output was all dropped still shows as `lost`. A late +of their first byte. Launch ids keep increasing across owner starts, since each saved instance +records its latest one. Line state is kept per launch, process and stream; chunk sequence numbers +span a launch's processes, so a process whose output was all dropped still shows as `lost`. A late partial line of an ended launch waits for its newline for two seconds of quiet, or until the store closes. Lines are cut at 32 KiB; chunks the in-memory output buffer dropped or that were still queued when a bounded drain at close ran out, and records that failed to write, are recorded as `lost`; a write that fails part-way can count records that did land, so the count is an upper bound. Closing aborts filesystem work still pending after 5 seconds; output not yet written by then is not -recorded. Every -reader, live or offline, reads segments by position, so history and following share one path; a -follow can resume at a record position, and a reader that finds its segment deleted reports a -`lost` gap marker, which carries `resumeAt` instead of a record position. Destroying an instance deletes its segments, and an owner start removes directories of -instances no longer saved, so a failed deletion is retried; destroying the stack removes `logs/`; -resetting database data keeps them. +recorded. Every reader, live or offline, reads segments by position, so history and following share +one path; a follow can resume at a record position, and a reader that finds its segment deleted +reports a `lost` gap marker, which carries `resumeAt` instead of a record position. Destroying an +instance deletes its segments, and an owner start removes directories of instances no longer saved, +so a failed deletion is retried; destroying the stack removes `logs/`; resetting database data keeps +them. While the composed Analytics instance is running and healthy, the owner ships the persisted stdout/stderr records of Auth, REST, Realtime, Storage, Functions and database instances to its diff --git a/packages/stack/README.md b/packages/stack/README.md index 5d1d618c47..8110f2d2d7 100644 --- a/packages/stack/README.md +++ b/packages/stack/README.md @@ -157,7 +157,7 @@ Bindings supply ordinary configuration values; a URL alone never creates a depen Exit confirmation is bounded. If cleanup is acknowledged but owner exit cannot be confirmed, the operation fails with `operation: "shutdown-exit"` and the owner PID in the message. A failed or cancelled call does not guarantee that teardown has completed. Do not start or restart the same stack concurrently with whole-stack shutdown; separate stacks remain independent. -Each service exposes `status`, `followStatus`, `readLogs`, and `credentials`. The owner persists service output under `//logs`; `readLogs({ from?, since?, tail?, follow? })` streams an instance's records through its live owner (`from` and `tail` are mutually exclusive), `readStackLogs({ stateRoot, stackId, instances?, since?, tail? })` reads it without an owner, ordered by timestamp, service, instance and position, and `streamStackLogs` from `@supabase/stack/effect` streams the same records per instance in file order. Observations include the currently bound public endpoints, including listeners for sleeping services. Credentials default to host addressing. Use `from: "runtime"` for a URL passed to a service or command container. +Each service exposes `status`, `followStatus`, `readLogs`, and `credentials`. The owner persists service output under `//logs`; `readLogs({ from?, since?, tail?, follow? })` streams an instance's records through its live owner (`from` and `tail` are mutually exclusive), and `streamStackLogs({ stateRoot, stackId, instances?, since? })` from `@supabase/stack/effect` reads them without an owner, one instance after another in file order. Observations include the currently bound public endpoints, including listeners for sleeping services. Credentials default to host addressing. Use `from: "runtime"` for a URL passed to a service or command container. ## Testing diff --git a/packages/stack/src/Owner.logs.integration.test.ts b/packages/stack/src/Owner.logs.integration.test.ts index 684eebfbc6..81a5880c30 100644 --- a/packages/stack/src/Owner.logs.integration.test.ts +++ b/packages/stack/src/Owner.logs.integration.test.ts @@ -16,7 +16,7 @@ import { import { tmpdir } from "node:os"; import { ownerFor } from "../tests/owner-rpc.ts"; import type { LogRecord } from "./host/LogRecord.ts"; -import { readStackLogs, StackError } from "./index.ts"; +import { StackError, streamStackLogs } from "./effect.ts"; import * as LogStore from "./host/LogStore.ts"; import * as State from "./State.ts"; import type { SavedStack } from "./State.ts"; @@ -94,7 +94,9 @@ describe("owner persisted logs", () => { expect(history).toContainEqual(followed); const directory = path.join(owner.logsRoot, "mail", mail.id); expect(yield* fs.readDirectory(directory)).toEqual(["0000000001.log"]); - const offline = yield* LogStore.readStackLogs({ root: owner.logsRoot }); + const offline = yield* LogStore.streamStackLogs({ root: owner.logsRoot }).pipe( + Stream.runCollect, + ); expect(offline.slice(0, history.length).map(({ position }) => position)).toEqual( history.map(({ position }) => position), ); @@ -105,7 +107,7 @@ describe("owner persisted logs", () => { ).pipe(Effect.provide(services)), ); - it.live("reads history through the public API after the owner stops", () => + it.live("streams history through the public API after the owner stops", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -125,24 +127,56 @@ describe("owner persisted logs", () => { yield* owner.rpc.stopService({ id: mail.id }); yield* Scope.close(ownerScope, Exit.void); - const records = yield* Effect.promise(() => - readStackLogs({ stateRoot: owner.stateRoot, stackId: owner.stack.id }), - ); - const rejected = (options: { readonly since?: string; readonly tail?: number }) => - Effect.promise(() => - readStackLogs({ stateRoot: owner.stateRoot, stackId: owner.stack.id, ...options }).then( - () => undefined, - (error: unknown) => error, - ), - ); - const invalid = yield* Effect.forEach( - [{ since: "soon" }, { tail: -1 }, { tail: 1.5 }, { tail: Number.POSITIVE_INFINITY }], - rejected, + const selection = { stateRoot: owner.stateRoot, stackId: owner.stack.id }; + const records = yield* streamStackLogs(selection).pipe(Stream.runCollect); + const invalid = yield* streamStackLogs({ ...selection, since: "soon" }).pipe( + Stream.runDrain, + Effect.flip, ); expect(records[0]).toMatchObject({ kind: "launch", service: "mail", instanceId: mail.id }); expect(records.map(({ text }) => text)).toContain(followed?.text); - for (const error of invalid) expect(error).toBeInstanceOf(StackError); + expect(invalid).toBeInstanceOf(StackError); + }), + ).pipe(Effect.provide(services)), + ); + + it.live("continues an instance's launch ids after the owner restarts", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "owner-logs-relaunch-" }); + const firstRun = yield* Scope.make(); + const owner = yield* openOwner("owner-logs-relaunch-", "native", { root }).pipe( + Scope.provide(firstRun), + ); + const mail = yield* owner.rpc.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + yield* owner.rpc.startService({ id: mail.id }); + yield* owner.rpc.readyService({ id: mail.id }); + yield* owner.rpc.stopService({ id: mail.id }); + yield* Scope.close(firstRun, Exit.void); + const state = Context.get( + yield* Layer.build(State.layer({ root: owner.stateRoot })), + State.Service, + ); + const saved = yield* state.read(owner.stack.id); + if (saved === undefined) return yield* Effect.die("stack state missing"); + const restarted = yield* ownerFor({ saved, state, root: `${root}/data`, cacheRoot }); + yield* Effect.addFinalizer(() => restarted.namespace.destroy.pipe(Effect.ignore)); + + yield* restarted.rpc.startService({ id: mail.id }); + yield* restarted.rpc.readyService({ id: mail.id }); + const records = yield* restarted.rpc + .readLogs({ id: mail.id, follow: false }) + .pipe(Stream.runCollect); + + expect( + records.filter(({ kind }) => kind === "launch").map(({ launchId }) => launchId), + ).toEqual([1, 2]); }), ).pipe(Effect.provide(services)), ); diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index 4ffb5dbe11..a9f335d4e6 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -308,15 +308,25 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { ...(options.hostGateway === undefined ? {} : { hostGateway: options.hostGateway }), }).pipe(Effect.provideContext(services)); - const persistCreation = (entry: Pick, creation: ServiceCreation) => + const persistCreation = ( + entry: Pick, + creation: ServiceCreation, + launchId?: number, + ) => updateState((current) => ({ ...current, instances: current.instances.map((instance) => - instance.id === entry.id ? { ...instance, creation } : instance, + instance.id === entry.id + ? { ...instance, creation, ...(launchId === undefined ? {} : { launchId }) } + : instance, ), })).pipe(Effect.andThen(Ref.set(entry.creation, creation))); - const register = Effect.fn("Owner.register")(function* (id: string, recipe: CatalogRecipe) { + const register = Effect.fn("Owner.register")(function* ( + id: string, + recipe: CatalogRecipe, + lastLaunchId?: number, + ) { if (Option.isSome(yield* orchestrator.get(id).pipe(Effect.option))) return yield* new Orchestrator.OrchestratorError({ operation: "register", @@ -329,7 +339,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { { ...recipe.definition, launch: (context) => - persistCreation({ id, creation }, context.config).pipe( + persistCreation({ id, creation }, context.config, context.launchId).pipe( Effect.mapError(serviceError("state")), Effect.andThen(recipe.definition.launch(context)), ), @@ -346,6 +356,8 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { Effect.mapError(serviceError("state")), ), ), + // Shipping writes its cursor into the instance's logs, so it stops before they go. + Effect.andThen(forwarder.detach(id)), Effect.andThen( logStore .remove({ service: initial.service, instanceId: id }) @@ -363,6 +375,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { { id, config: initial, + ...(lastLaunchId === undefined ? {} : { lastLaunchId }), coordinate: (operation, transition) => (drainingBlocks.includes(operation) ? rejectWhileDraining : Effect.void).pipe( Effect.andThen(orchestrator.admissionFor(id)(operation, transition)), @@ -453,7 +466,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { }); for (const saved of options.saved.instances) - yield* register(saved.id, yield* recipeFor(saved.creation, saved.id)); + yield* register(saved.id, yield* recipeFor(saved.creation, saved.id), saved.launchId); yield* logStore.removeOrphans.pipe( Effect.catch((cause) => Effect.logWarning("Orphaned instance logs were not removed", cause)), ); diff --git a/packages/stack/src/Rpc.ts b/packages/stack/src/Rpc.ts index b9e7af8a94..873f1b7b70 100644 --- a/packages/stack/src/Rpc.ts +++ b/packages/stack/src/Rpc.ts @@ -81,7 +81,7 @@ export interface Definition extends Schema.Schema.Type {} const Instance = { id: Schema.String }; /** How many of the latest log records a read starts with. */ -export const LogTail = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)); +const LogTail = Schema.Int.check(Schema.isGreaterThanOrEqualTo(0)); const SnapshotScope = Schema.Literals(snapshotScopes); export const CommandEvent = Schema.TaggedUnion({ diff --git a/packages/stack/src/Service.ts b/packages/stack/src/Service.ts index 462cb0c74b..f15dbb92f5 100644 --- a/packages/stack/src/Service.ts +++ b/packages/stack/src/Service.ts @@ -184,6 +184,8 @@ export const makeService = ( options: { readonly id: string; readonly config: Config; + /** Launch ids continue after this one. */ + readonly lastLaunchId?: number; readonly coordinate?: ( operation: ServiceAdmission, transition: Effect.Effect, @@ -194,7 +196,7 @@ export const makeService = ( const owner = yield* Scope.Scope; const gate = yield* Semaphore.make(1); const revision = yield* Ref.make(0); - const launchCounter = yield* Ref.make(0); + const launchCounter = yield* Ref.make(options.lastLaunchId ?? 0); const current = yield* Ref.make(undefined); // stopNow clears observation.launchId while keeping the error, so readiness must not read that field. const launchError = yield* Ref.make< diff --git a/packages/stack/src/State.integration.test.ts b/packages/stack/src/State.integration.test.ts index d4f8fa5e70..5a121d4554 100644 --- a/packages/stack/src/State.integration.test.ts +++ b/packages/stack/src/State.integration.test.ts @@ -646,7 +646,6 @@ describe("durable stack state", () => { yield* store.remove(initial.id); - expect(yield* Ref.get(held)).toBeLessThan(0); expect(yield* fs.exists(path.join(root, initial.id))).toBe(false); }), ), diff --git a/packages/stack/src/State.ts b/packages/stack/src/State.ts index e035d5519d..07d7eb2ff0 100644 --- a/packages/stack/src/State.ts +++ b/packages/stack/src/State.ts @@ -33,6 +33,8 @@ const SafeId = Schema.String.pipe( const SavedInstance = Schema.Struct({ id: SafeId, creation: Schema.toCodecJson(ServiceCreation), + /** The latest launch id; the next owner continues after it, so launch ids keep increasing. */ + launchId: Schema.optionalKey(Schema.Int), }); interface SavedInstance extends Schema.Schema.Type {} @@ -177,15 +179,12 @@ const checkId = (id: string): Effect.Effect => ? Effect.void : Effect.fail(stateError("identity", `Invalid state id: ${id}`)); -const isRecord = (value: unknown): value is Readonly> => - typeof value === "object" && value !== null && !Array.isArray(value); - const retainedEntries = ( entries: unknown, retired: (entry: Readonly>) => boolean, ): unknown => Array.isArray(entries) - ? entries.filter((entry: unknown) => !(isRecord(entry) && retired(entry))) + ? entries.filter((entry: unknown) => !(Predicate.isReadonlyObject(entry) && retired(entry))) : entries; /** @@ -195,11 +194,12 @@ const retainedEntries = ( const withoutVectorInstances = ( document: unknown, ): { readonly document: unknown; readonly removed: ReadonlyArray } => { - if (!isRecord(document) || !Array.isArray(document.instances)) return { document, removed: [] }; + if (!Predicate.isReadonlyObject(document) || !Array.isArray(document.instances)) + return { document, removed: [] }; const removed = document.instances.flatMap((instance: unknown) => - isRecord(instance) && + Predicate.isReadonlyObject(instance) && typeof instance.id === "string" && - isRecord(instance.creation) && + Predicate.isReadonlyObject(instance.creation) && instance.creation.service === "vector" ? [instance.id] : [], @@ -212,7 +212,7 @@ const withoutVectorInstances = ( document: { ...document, instances: retainedEntries(document.instances, (instance) => ids.has(instance.id)), - composition: isRecord(composition) + composition: Predicate.isReadonlyObject(composition) ? { ...composition, members: retainedEntries(composition.members, (member) => ids.has(member.id)), @@ -671,6 +671,7 @@ const makeState = ( if ( entry === "vector.yaml" || entry === "vector-api.yaml" || + entry === "vector.rendered.yaml" || entry.startsWith(".vector-write-") ) yield* fs.remove(path.join(configRoot, entry), { recursive: true, force: true }); @@ -681,20 +682,29 @@ const makeState = ( Effect.logWarning(`Unable to remove the files of Vector instance ${instanceId}`, cause), ), ); + /** The saved document and its Vector instance ids, when it still holds one. */ + const legacyVector = (target: string) => + Effect.gen(function* () { + if (!(yield* fs.exists(target).pipe(retryTransientRead))) return undefined; + const text = yield* fs.readFileString(target).pipe(retryTransientRead); + const { removed } = yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))( + text, + ).pipe( + Effect.map(withoutVectorInstances), + Effect.orElseSucceed(() => ({ removed: [] })), + ); + return removed.length === 0 ? undefined : { text, removed }; + }); const migrate = Effect.fn("State.migrate")(function* (id: string) { yield* checkId(id); + const target = statePath(id); + // Only a document that still holds Vector takes the registry lock, to migrate it. + if ((yield* legacyVector(target)) === undefined) return; const removed = yield* withLock( Effect.gen(function* () { - const target = statePath(id); - if (!(yield* fs.exists(target).pipe(retryTransientRead))) return []; - const text = yield* fs.readFileString(target).pipe(retryTransientRead); - const { removed } = yield* Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown))( - text, - ).pipe( - Effect.map(withoutVectorInstances), - Effect.orElseSucceed(() => ({ removed: [] })), - ); - if (removed.length === 0) return removed; + const legacy = yield* legacyVector(target); + if (legacy === undefined) return []; + const { text, removed } = legacy; const state = yield* decodeState(text, id, target); if (state.id !== id) return yield* stateError("identity", "State document identity does not match its path"); diff --git a/packages/stack/src/State.vector-migration.integration.test.ts b/packages/stack/src/State.vector-migration.integration.test.ts index 6c6b4fc9d3..1903bff63a 100644 --- a/packages/stack/src/State.vector-migration.integration.test.ts +++ b/packages/stack/src/State.vector-migration.integration.test.ts @@ -1,6 +1,6 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; -import { Context, Effect, FileSystem, Layer, Path, Schema } from "effect"; +import { Context, Deferred, Effect, Exit, Fiber, FileSystem, Layer, Path, Schema } from "effect"; import { create, open } from "./effect.ts"; import * as State from "./State.ts"; import { destroyTestStack } from "../tests/stack-cleanup.ts"; @@ -75,6 +75,10 @@ const writeLegacyState = Effect.fn("test.writeLegacyState")(function* ( yield* fs.makeDirectory(path.join(data, "vector", "runtime", "vector"), { recursive: true }); yield* fs.writeFileString(path.join(data, "vector", "runtime", "vector", "vector.yaml"), ""); yield* fs.writeFileString(path.join(data, "vector", "runtime", "vector", "vector-api.yaml"), ""); + yield* fs.writeFileString( + path.join(data, "vector", "runtime", "vector", "vector.rendered.yaml"), + "rendered", + ); yield* fs.writeFileString(path.join(data, "vector", "pipeline.yaml"), "caller"); const owned = path.join(data, "vector-owned", "runtime", "vector"); yield* fs.makeDirectory(path.join(owned, ".vector-write-1"), { recursive: true }); @@ -137,6 +141,28 @@ describe("saved Vector instance migration", () => { }).pipe(Effect.scoped, Effect.provide(layer)), ); + it.live("checks a state without Vector while another operation holds the registry lock", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-vector-unlocked-" }); + const state = yield* stateFor(root); + yield* writeLegacyState(root, "legacy", root); + yield* state.migrate("legacy"); + const held = yield* Deferred.make(); + const release = yield* Deferred.make(); + const holder = yield* state + .withLock(Deferred.succeed(held, undefined).pipe(Effect.andThen(Deferred.await(release)))) + .pipe(Effect.forkChild); + yield* Deferred.await(held); + + const checked = yield* state.migrate("legacy").pipe(Effect.exit); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(holder); + + expect(Exit.isSuccess(checked)).toBe(true); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + it.live("refuses to migrate a state document copied from another stack", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/packages/stack/src/effect.ts b/packages/stack/src/effect.ts index b8ab3d857d..958f569abf 100644 --- a/packages/stack/src/effect.ts +++ b/packages/stack/src/effect.ts @@ -44,12 +44,8 @@ import { deriveStackId, resolveStackIdentity } from "./identity/Identity.ts"; import { failureMessage } from "./internal/failure-message.ts"; import * as State from "./State.ts"; import type { SavedStack, StackCredentials, StackKeysInput } from "./State.ts"; -import { LogTail, StackError, type Definition, type Observation } from "./Rpc.ts"; -import { - readStackLogs as readPersistedLogs, - sinceMillis, - streamStackLogs as streamPersistedLogs, -} from "./host/LogStore.ts"; +import { StackError, type Definition, type Observation } from "./Rpc.ts"; +import { sinceMillis, streamStackLogs as streamPersistedLogs } from "./host/LogStore.ts"; import type { LogPosition, LogRecord, StackLogRecord } from "./host/LogRecord.ts"; import { reclaimStack } from "./Sweep.ts"; import { @@ -1039,36 +1035,17 @@ export const find = Effect.fn("Stack.find")( ); /** Selects the persisted logs of a stack; its owner does not need to run. */ -export interface ReadStackLogsOptions extends Pick { +export interface StreamStackLogsOptions extends Pick { readonly stackId: string; /** Instance ids to read; every instance with persisted logs by default. */ readonly instances?: ReadonlyArray; /** An ISO-8601 timestamp; older records are skipped. */ readonly since?: string; - /** Returns only this many of the latest records across the selected instances. */ - readonly tail?: number; } -/** Reads a stack's persisted records ordered by timestamp, service, instance and position. */ -export const readStackLogs = Effect.fn("Stack.readStackLogs")( - function* (options: ReadStackLogsOptions) { - const path = yield* Path.Path; - const root = yield* State.stackLogsRoot(path, options.stateRoot, options.stackId); - return yield* readPersistedLogs({ - root, - ...(options.instances === undefined ? {} : { instances: options.instances }), - ...(options.since === undefined ? {} : { since: yield* sinceMillis(options.since) }), - ...(options.tail === undefined - ? {} - : { tail: yield* Schema.decodeEffect(LogTail)(options.tail) }), - }); - }, - Effect.mapError((cause) => failure("readStackLogs", cause)), -); - /** Streams a stack's persisted records one instance after another, each in file order. */ export const streamStackLogs = ( - options: Omit, + options: StreamStackLogsOptions, ): Stream.Stream => Stream.unwrap( Effect.gen(function* () { diff --git a/packages/stack/src/host/LogForwarder.integration.test.ts b/packages/stack/src/host/LogForwarder.integration.test.ts index 6b6ae7e775..1999276079 100644 --- a/packages/stack/src/host/LogForwarder.integration.test.ts +++ b/packages/stack/src/host/LogForwarder.integration.test.ts @@ -3,13 +3,17 @@ import { describe, expect, it } from "@effect/vitest"; import { createServer } from "node:http"; // oxlint-disable-line effecttsgo/node-builtin-import -- real socket fixture. import { Clock, + Deferred, Effect, Exit, + Fiber, FileSystem, Layer, + Logger, Path, PubSub, Queue, + References, Schema, Scope, Stream, @@ -194,6 +198,8 @@ const startForwarder = ( ) => Effect.gen(function* () { const scope = yield* Scope.make(); + // Stops before the test's store and temp directory close, so no cursor write races removal. + yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); const forwarder = yield* LogForwarder.make({ composition, logs: store }).pipe( Scope.provide(scope), ); @@ -205,7 +211,7 @@ const startForwarder = ( Stream.take(1), Stream.runDrain, ); - return { awaitShipping, stop: Scope.close(scope, Exit.void) }; + return { awaitShipping, detach: forwarder.detach, stop: Scope.close(scope, Exit.void) }; }); const fixture = (options: Omit = {}) => @@ -242,9 +248,9 @@ describe("LogForwarder", () => { expect(caughtUp.apiKey).toBe("test-key"); expect(messages(caughtUp)).toEqual(["while asleep 1", "while asleep 2"]); expect(messages(live)).toEqual(["while awake"]); - const persisted = (yield* LogStore.readStackLogs({ root })).filter( - (record) => record.kind === "stdout", - ); + const persisted = (yield* LogStore.streamStackLogs({ root }).pipe( + Stream.runCollect, + )).filter((record) => record.kind === "stdout"); expect([...caughtUp.events, ...live.events].map((event) => event.timestamp)).toEqual( persisted.map((record) => record.timestamp), ); @@ -256,6 +262,76 @@ describe("LogForwarder", () => { }).pipe(Effect.scoped, Effect.provide(layer)), ); + it.live("stops shipping quietly when the log store closes", () => + Effect.gen(function* () { + const { store, sink, database, analytics } = yield* fixture(); + const warnings: Array = []; + const stopped = yield* Deferred.make(); + const captured = Logger.layer([ + Logger.make(({ logLevel, message }) => { + if (logLevel === "Warn") warnings.push(message); + if (logLevel === "Debug" && String(message).includes("stopped")) + Deferred.doneUnsafe(stopped, Effect.void); + }), + ]); + const forwarder = yield* startForwarder(store, [analytics.instance, database.instance]).pipe( + Effect.provide(captured), + Effect.provideService(References.MinimumLogLevel, "Debug"), + ); + yield* analytics.set(true); + yield* forwarder.awaitShipping(true); + yield* database.log("before removal"); + const shipped = yield* sink.next; + + yield* store.close; + yield* Deferred.await(stopped); + yield* forwarder.stop; + + expect(messages(shipped)).toEqual(["before removal"]); + expect(warnings).toEqual([]); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("detaches an instance only once its cursor write landed, so its logs can go", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { root, store, sink, database, analytics } = yield* fixture(); + const renaming = yield* Deferred.make(); + const release = yield* Deferred.make(); + const injected: FileSystem.FileSystem = { + ...fs, + rename: (from, to) => + to.endsWith("cursor.json") + ? Deferred.succeed(renaming, undefined).pipe( + Effect.andThen(Deferred.await(release)), + Effect.andThen(fs.rename(from, to)), + ) + : fs.rename(from, to), + }; + const forwarder = yield* startForwarder(store, [analytics.instance, database.instance]).pipe( + Effect.provideService(FileSystem.FileSystem, injected), + ); + yield* analytics.set(true); + yield* forwarder.awaitShipping(true); + yield* database.log("before removal"); + yield* sink.next; + yield* Deferred.await(renaming); + + const directory = path.join(root, "database", "database"); + const detaching = yield* Effect.forkChild(forwarder.detach("database"), { + startImmediately: true, + }); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(detaching); + const cursorLanded = yield* fs.exists(path.join(directory, "cursor.json")); + yield* store.remove({ service: "database", instanceId: "database" }); + + expect(cursorLanded).toBe(true); + expect(yield* fs.exists(directory)).toBe(false); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + it.live("resumes from its persisted cursor after the forwarder restarts", () => Effect.gen(function* () { const { store, sink, database, analytics } = yield* fixture(); @@ -374,7 +450,7 @@ describe("LogForwarder", () => { yield* analytics.set(false); yield* forwarder.awaitShipping(false); for (let index = 0; index < 30; index++) yield* database.log(`asleep ${index}`); - const retained = (yield* LogStore.readStackLogs({ root })).filter( + const retained = (yield* LogStore.streamStackLogs({ root }).pipe(Stream.runCollect)).filter( (record) => record.kind === "stdout", ); diff --git a/packages/stack/src/host/LogForwarder.ts b/packages/stack/src/host/LogForwarder.ts index 9f81111cca..5f12f6116a 100644 --- a/packages/stack/src/host/LogForwarder.ts +++ b/packages/stack/src/host/LogForwarder.ts @@ -3,13 +3,13 @@ import { Data, Duration, Effect, + FiberMap, FileSystem, Option, Path, Ref, Schedule, Schema, - Scope, Semaphore, Stream, SubscriptionRef, @@ -42,6 +42,8 @@ export interface ForwardedInstance { interface Interface { /** Ships a shipped service's persisted logs, or tracks an Analytics instance as the target. */ readonly attach: (instance: ForwardedInstance) => Effect.Effect; + /** Stops following an instance; returns once no cursor write of it can still land. */ + readonly detach: (instanceId: string) => Effect.Effect; /** Re-selects the shipping target after the composition changes. */ readonly rebind: Effect.Effect; /** Emits whether records are currently shipped; the current value first. */ @@ -51,7 +53,6 @@ interface Interface { export interface LogForwarderOptions { readonly composition: Effect.Effect; readonly logs: LogStore.Interface; - readonly platform?: NodeJS.Platform; } interface Candidate { @@ -145,7 +146,6 @@ const isBodyRejection = (error: unknown): boolean => { * Analytics sleeps are shipped after it wakes; shipping never wakes Analytics. */ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwarderOptions) { - const scope = yield* Scope.Scope; const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; @@ -236,7 +236,6 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar directory, target: path.join(directory, cursorFile), content, - ...(options.platform === undefined ? {} : { platform: options.platform }), }), ), ); @@ -353,10 +352,12 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar for (const { body, last } of bodies(pending)) { yield* deliver(current, source, body); yield* Ref.set(cursor, last.position); + // Detaching waits for a started write instead of leaving files in the directory. yield* writeCursor(directory, last.position).pipe( Effect.catch((error) => Effect.logDebug(`Log cursor of ${instanceId} was not saved`, error), ), + Effect.uninterruptible, ); } }), @@ -372,29 +373,38 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar Stream.runDrain, ); + /** Ships until the instance unregisters or the store detaches its logs, which ends a session. */ const forward = (instance: ForwardedInstance, service: ShippedService) => Effect.gen(function* () { - const current = yield* serving; - // A stopped target pauses shipping; the cursor keeps the position to resume from. - yield* session(instance.id, service, current).pipe( - Effect.catch((error) => - (error._tag === "StaleTarget" - ? Effect.void - : Effect.logWarning(`Log shipping of ${instance.id} paused`, error) - ).pipe(Effect.andThen(retargeted(current))), - ), - Effect.raceFirst(retargeted(current)), - ); - }).pipe(Effect.forever, Effect.raceFirst(unregistered(instance))); + while (true) { + const current = yield* serving; + // A stopped target pauses shipping; the cursor keeps the position to resume from. + const detached = yield* session(instance.id, service, current).pipe( + Effect.as(true), + Effect.catch((error) => + (error._tag === "StaleTarget" + ? Effect.void + : Effect.logWarning(`Log shipping of ${instance.id} paused`, error) + ).pipe(Effect.andThen(retargeted(current)), Effect.as(false)), + ), + Effect.raceFirst(retargeted(current).pipe(Effect.as(false))), + ); + if (detached) + return yield* Effect.logDebug(`Log shipping of ${instance.id} stopped with its logs`); + } + }).pipe(Effect.raceFirst(unregistered(instance))); + const followers = yield* FiberMap.make(); const attach = Effect.fn("LogForwarder.attach")(function* (instance: ForwardedInstance) { - if (instance.service === "analytics") yield* Effect.forkIn(trackTarget(instance), scope); + if (instance.service === "analytics") + yield* FiberMap.run(followers, instance.id, trackTarget(instance)); else if (isShippedService(instance.service)) - yield* Effect.forkIn(forward(instance, instance.service), scope); + yield* FiberMap.run(followers, instance.id, forward(instance, instance.service)); }); return { attach, + detach: (instanceId) => FiberMap.remove(followers, instanceId), rebind, shipping: SubscriptionRef.changes(target).pipe( Stream.map((current) => current !== undefined), diff --git a/packages/stack/src/host/LogRecord.ts b/packages/stack/src/host/LogRecord.ts index 1481c932e2..162e850e6b 100644 --- a/packages/stack/src/host/LogRecord.ts +++ b/packages/stack/src/host/LogRecord.ts @@ -12,10 +12,8 @@ export const LogPosition = Schema.Struct({ export interface LogPosition extends Schema.Schema.Type {} /** - * One log record: an output line, the first output of a launch, or chunks lost before they were - * written, each identified by its `position`. A reader reports segments that retention deleted - * before it reached them as a `lost` gap marker, which is not persisted and has `resumeAt`, the - * position reading continues from, instead of a position. + * One output line, launch, or `lost` marker; a gap over deleted segments has `resumeAt` instead + * of `position`. */ export const LogRecord = Schema.Struct({ kind: Schema.Literals(["stdout", "stderr", "launch", "lost"]), @@ -55,7 +53,7 @@ export type LogEntry = }; /** Lines are cut at this many UTF-8 bytes. */ -export const maxLineBytes = 32 * 1024; +const maxLineBytes = 32 * 1024; const encoder = new TextEncoder(); diff --git a/packages/stack/src/host/LogRecord.unit.test.ts b/packages/stack/src/host/LogRecord.unit.test.ts index 7552efe710..65e3e7afcc 100644 --- a/packages/stack/src/host/LogRecord.unit.test.ts +++ b/packages/stack/src/host/LogRecord.unit.test.ts @@ -4,7 +4,6 @@ import { encodeEntry, makeSplitter, endedLineGraceMillis, - maxLineBytes, parseRecord, segmentGeneration, segmentName, @@ -284,15 +283,6 @@ describe("splitter", () => { ]); }); - it("caps lines at 32 KiB by default", () => { - const splitter = makeSplitter(); - - const [, cut] = push(splitter, chunk(1, 0, `${"x".repeat(maxLineBytes + 10)}\n`), 0); - - expect(cut).toMatchObject({ kind: "stdout", truncated: true }); - expect(cut?.kind === "stdout" ? cut.text.length : 0).toBe(maxLineBytes); - }); - it("keeps stdout and stderr lines apart", () => { const splitter = makeSplitter(); push(splitter, chunk(1, 0, "out "), 0); diff --git a/packages/stack/src/host/LogStore.integration.test.ts b/packages/stack/src/host/LogStore.integration.test.ts index 69fb38494d..b90e3c83c9 100644 --- a/packages/stack/src/host/LogStore.integration.test.ts +++ b/packages/stack/src/host/LogStore.integration.test.ts @@ -105,6 +105,10 @@ const replaceWrites = ( ), }); +/** Reads persisted records offline, one instance after another in file order. */ +const persisted = (options: Parameters[0]) => + LogStore.streamStackLogs(options).pipe(Stream.runCollect); + const lostCounts = (records: Iterable) => Array.from(records).flatMap(({ kind, launchId, stream, count }) => kind === "lost" ? [{ launchId, stream, count }] : [], @@ -182,7 +186,7 @@ describe("LogStore", () => { yield* Fiber.join(follower); expect(texts(seen)).toEqual(Array.from({ length: 1_000 }, (_, index) => `line ${index}`)); - const offline = yield* LogStore.readStackLogs({ root }); + const offline = yield* persisted({ root }); expect(offline.map(({ position }) => position)).toEqual(seen.map(({ position }) => position)); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); @@ -381,7 +385,7 @@ describe("LogStore", () => { yield* TestClock.adjust(200); yield* instance.publish({ ...instance.chunk(2, "after\n"), seq: 1 }); const followed = yield* untilLast(reader, "after"); - const offline = yield* LogStore.readStackLogs({ root }); + const offline = yield* persisted({ root }); const expectedLost = [ { launchId: 1, stream: "stdout", count: 1 }, @@ -440,8 +444,7 @@ describe("LogStore", () => { const followed = yield* untilLast(reader, "after"); expect(texts(followed)).toEqual(["whole", "after"]); - expect(texts(yield* LogStore.readStackLogs({ root }))).toEqual(["whole", "after"]); - expect(texts(yield* LogStore.readStackLogs({ root, tail: 3 }))).toEqual(["whole", "after"]); + expect(texts(yield* persisted({ root }))).toEqual(["whole", "after"]); expect(yield* fs.readFileString(torn)).toBe(content); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); @@ -465,7 +468,7 @@ describe("LogStore", () => { const segments = yield* fs.readDirectory(path.join(root, "auth", "rotate")); expect(segments.length).toBeGreaterThan(3); expect(texts(rest).at(-1)).toBe("rotating line 39"); - const all = yield* LogStore.readStackLogs({ root }); + const all = yield* persisted({ root }); expect(texts(all)).toEqual( Array.from({ length: 40 }, (_, index) => `rotating line ${index}`), ); @@ -496,8 +499,10 @@ describe("LogStore", () => { yield* instance.publish(instance.chunk(1, `retained line ${index}\n`)); yield* untilLast(progress, `retained line ${index}`); } - // Opening an end reader takes the writer lock, so the last rotation's retention finished. - yield* store.read("retention", { from: "end", follow: false }).pipe(Effect.asVoid); + // An empty tailed read takes the writer lock, so the last rotation's retention finished. + yield* store + .read("retention", { from: "oldest", tail: 0, follow: false }) + .pipe(Effect.asVoid); const directory = path.join(root, "auth", "retention"); const sizes = yield* Effect.forEach(yield* fs.readDirectory(directory), (name) => @@ -529,12 +534,12 @@ describe("LogStore", () => { yield* instance.publish(instance.chunk(1, `counted line ${index}\n`)); yield* untilLast(progress, `counted line ${index}`); } - // Opening an end reader takes the writer lock, so the last rotation's retention finished. - yield* store.read("count", { from: "end", follow: false }).pipe(Effect.asVoid); + // An empty tailed read takes the writer lock, so the last rotation's retention finished. + yield* store.read("count", { from: "oldest", tail: 0, follow: false }).pipe(Effect.asVoid); const segments = yield* fs.readDirectory(path.join(root, "auth", "count")); expect(segments.length).toBeLessThanOrEqual(3); - expect(texts(yield* LogStore.readStackLogs({ root })).at(-1)).toBe("counted line 9"); + expect(texts(yield* persisted({ root })).at(-1)).toBe("counted line 9"); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); @@ -570,10 +575,6 @@ describe("LogStore", () => { expect(texts(history)).toEqual(["tailed line 17", "tailed line 18", "tailed line 19"]); expect(followed?.text).toBe("after tail"); - expect(texts(yield* LogStore.readStackLogs({ root, tail: 2 }))).toEqual([ - "tailed line 19", - "after tail", - ]); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); @@ -592,7 +593,7 @@ describe("LogStore", () => { yield* fs.writeFileString(path.join(root, ".DS_Store"), ""); yield* fs.writeFileString(path.join(root, "auth", ".DS_Store"), ""); - const records = yield* LogStore.readStackLogs({ root }); + const records = yield* persisted({ root }); expect(texts(records)).toEqual(["kept line"]); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), @@ -612,15 +613,15 @@ describe("LogStore", () => { ...Array.from({ length: 60 }, (_, index) => instance.chunk(1, `batched line ${index}\n`)), ); yield* untilLast(progress, "batched line 59"); - // Opening an end reader takes the writer lock, so the last rotation's retention finished. - yield* store.read("big", { from: "end", follow: false }).pipe(Effect.asVoid); + // An empty tailed read takes the writer lock, so the last rotation's retention finished. + yield* store.read("big", { from: "oldest", tail: 0, follow: false }).pipe(Effect.asVoid); const directory = path.join(root, "auth", "big"); const sizes = yield* Effect.forEach(yield* fs.readDirectory(directory), (name) => fs.stat(path.join(directory, name)).pipe(Effect.map((info) => Number(info.size))), ); expect(Math.max(...sizes)).toBeLessThanOrEqual(200); - const kept = texts(yield* LogStore.readStackLogs({ root })); + const kept = texts(yield* persisted({ root })); expect(kept.length).toBeGreaterThan(0); expect(kept.at(-1)).toBe("batched line 59"); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), @@ -638,7 +639,7 @@ describe("LogStore", () => { ); yield* opened.close; - const records = yield* LogStore.readStackLogs({ root }); + const records = yield* persisted({ root }); expect(texts(records)).toEqual( Array.from({ length: 1_000 }, (_, index) => `queued line ${index}`), ); @@ -663,15 +664,13 @@ describe("LogStore", () => { yield* instance.setLaunch(undefined); yield* untilLast(progress, "early partial"); - const since = { since: 1_500, tail: 10 }; - const offline = yield* LogStore.readStackLogs({ root, ...since }); const live = yield* (yield* store.read("flushed", { from: "oldest", follow: false, - ...since, + since: 1_500, + tail: 10, })).pipe(Stream.runCollect); - expect(texts(offline)).toEqual(["later line"]); expect(texts(live)).toEqual(["later line"]); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); @@ -698,14 +697,12 @@ describe("LogStore", () => { yield* instance.setLaunch(undefined); yield* untilLast(progress, "early stderr"); - const offline = yield* LogStore.readStackLogs({ root, tail: 1 }); const live = yield* (yield* store.read("flushes", { from: "oldest", follow: false, tail: 1, })).pipe(Stream.runCollect); - expect(texts(offline)).toEqual(["latest"]); expect(texts(live)).toEqual(["latest"]); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); @@ -741,8 +738,8 @@ describe("LogStore", () => { yield* instance.publish(instance.chunk(1, `stuck line ${index}\n`)); yield* untilLast(progress, `stuck line ${index}`); } - // Opening an end reader takes the writer lock, so the last rotation's retention finished. - yield* store.read("stuck", { from: "end", follow: false }).pipe(Effect.asVoid); + // An empty tailed read takes the writer lock, so the last rotation's retention finished. + yield* store.read("stuck", { from: "oldest", tail: 0, follow: false }).pipe(Effect.asVoid); const segments = yield* fs.readDirectory(path.join(root, "auth", "stuck")); const resumed = yield* (yield* store.read("stuck", { @@ -815,7 +812,7 @@ describe("LogStore", () => { segmentName(1), segmentName(2), ]); - expect(texts(yield* LogStore.readStackLogs({ root }))).toEqual(["before", "after"]); + expect(texts(yield* persisted({ root }))).toEqual(["before", "after"]); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); @@ -841,7 +838,7 @@ describe("LogStore", () => { }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); - it.effect("reads persisted records offline after the store closes, merged and tailed", () => + it.effect("reads persisted records offline after the store closes, per instance", () => Effect.gen(function* () { const root = yield* tempRoot("log-store-offline-"); const opened = yield* openStore(root); @@ -867,23 +864,16 @@ describe("LogStore", () => { yield* restReader.take(1); yield* opened.close; - const all = yield* LogStore.readStackLogs({ root }); - const tailed = yield* LogStore.readStackLogs({ root, tail: 2 }); - const recent = yield* LogStore.readStackLogs({ root, since: 2_000, instances: ["rest-1"] }); - const streamed = yield* LogStore.streamStackLogs({ root, instances: ["rest-1"] }).pipe( - Stream.runCollect, - ); + const all = yield* persisted({ root }); + const restOnly = yield* persisted({ root, instances: ["rest-1"] }); + const recent = yield* persisted({ root, since: 2_000, instances: ["rest-1"] }); - expect(Array.from(streamed)).toEqual(all.filter(({ service }) => service === "rest")); - expect(all.map(({ service, kind }) => `${service}:${kind}`)).toEqual([ - "rest:launch", - "rest:stdout", - "auth:launch", - "auth:stdout", - "rest:stdout", + expect(texts(all).toSorted()).toEqual(["auth second", "rest first", "rest third"]); + expect(restOnly.map(({ kind, text }) => [kind, text])).toEqual([ + ["launch", undefined], + ["stdout", "rest first"], + ["stdout", "rest third"], ]); - expect(texts(all)).toEqual(["rest first", "auth second", "rest third"]); - expect(texts(tailed)).toEqual(["auth second", "rest third"]); expect(texts(recent)).toEqual(["rest third"]); expect(recent[0]).toMatchObject({ service: "rest", instanceId: "rest-1" }); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), @@ -898,7 +888,9 @@ describe("LogStore", () => { const opened = yield* openStore(root); const instance = yield* fakeInstance("restart"); yield* opened.store.attach(instance); - const reader = yield* collect(opened.store.read("restart", { from: "end", follow: true })); + const reader = yield* collect( + opened.store.read("restart", { from: "oldest", tail: 0, follow: true }), + ); yield* instance.publish(instance.chunk(1, `${text}\n`)); yield* reader.take(2); yield* opened.close; @@ -909,13 +901,13 @@ describe("LogStore", () => { "0000000001.log", "0000000002.log", ]); - const records = yield* LogStore.readStackLogs({ root }); + const records = yield* persisted({ root }); expect(texts(records)).toEqual(["before restart", "after restart"]); expect(records.map(({ position }) => position?.generation)).toEqual([1, 1, 2, 2]); }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); - it.effect("tails only records written after an end reader subscribes", () => + it.effect("follows only records written after an empty tailed read subscribes", () => Effect.gen(function* () { const root = yield* tempRoot("log-store-end-"); const { store } = yield* openStore(root); @@ -925,7 +917,7 @@ describe("LogStore", () => { yield* instance.publish(instance.chunk(1, "old\n")); yield* history.take(2); - const reader = yield* collect(store.read("end", { from: "end", follow: true })); + const reader = yield* collect(store.read("end", { from: "oldest", tail: 0, follow: true })); yield* instance.publish(instance.chunk(1, "new\n")); const [record] = yield* reader.take(1); diff --git a/packages/stack/src/host/LogStore.ts b/packages/stack/src/host/LogStore.ts index fd6c083ecb..b4ff40ad3d 100644 --- a/packages/stack/src/host/LogStore.ts +++ b/packages/stack/src/host/LogStore.ts @@ -59,13 +59,12 @@ export interface LogStoreOptions { readonly retainBytes?: number; /** Oldest closed segments are deleted while an instance has more segments than this. */ readonly retainSegments?: number; - readonly platform?: NodeJS.Platform; } /** Selects the records a read returns. */ export interface ReadOptions { - /** `end` skips every record written before the read starts; a position resumes there. */ - readonly from: "oldest" | "end" | LogPosition; + /** The oldest retained record, or a record position to resume at. */ + readonly from: "oldest" | LogPosition; /** Epoch milliseconds; older records are skipped. */ readonly since?: number; /** Returns only the last records of the history before following; requires `from: "oldest"`. */ @@ -469,14 +468,14 @@ const makeReader = (fs: FileSystem.FileSystem, path: Path.Path) => { }); const wake = options.follow && live !== undefined ? yield* PubSub.subscribe(live.wake) : undefined; - const start = - options.from === "oldest" - ? { generation: 0, byteOffset: 0 } - : options.from === "end" - ? live === undefined - ? { generation: 0, byteOffset: 0 } - : yield* live.lock.withPermits(1)(Ref.get(live.end)) - : options.from; + const start = options.from === "oldest" ? { generation: 0, byteOffset: 0 } : options.from; + // An empty tail starts at the writer's end now, before the stream runs. + if (options.tail === 0 && live !== undefined) { + const end = yield* live.lock.withPermits(1)(Ref.get(live.end)); + return options.follow + ? follow(directory, { cursor: end, listed: undefined }, options, live, wake) + : Stream.empty; + } if (options.tail === undefined) return follow(directory, { cursor: start, listed: undefined }, options, live, wake); return Stream.unwrap( @@ -522,7 +521,7 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio const rotateBytes = options.rotateBytes ?? defaultRotateBytes; const retainBytes = options.retainBytes ?? defaultRetainBytes; const retainSegments = options.retainSegments ?? defaultRetainSegments; - const retryShared = retrySharingViolation(options.platform); + const retryShared = retrySharingViolation(); interface Attached extends Live { readonly service: string; @@ -990,17 +989,16 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio /** Deletes log directories whose instance is no longer attached, left by a failed removal. */ const removeOrphans = Effect.fn("LogStore.removeOrphans")(function* () { const current = yield* Ref.get(instances); - for (const service of yield* listDirectories(fs, path, options.root)) - for (const instanceId of yield* listDirectories(fs, path, path.join(options.root, service))) - if (!current.has(instanceId)) { - const directory = path.join(options.root, service, instanceId); - yield* fs.remove(directory, { recursive: true, force: true }).pipe( - retryShared, - Effect.catch((error) => - Effect.logWarning(`Unable to remove orphaned logs ${directory}`, error), - ), - ); - } + for (const { service, instanceId } of yield* persistedInstances(fs, path, options.root)) + if (!current.has(instanceId)) { + const directory = path.join(options.root, service, instanceId); + yield* fs.remove(directory, { recursive: true, force: true }).pipe( + retryShared, + Effect.catch((error) => + Effect.logWarning(`Unable to remove orphaned logs ${directory}`, error), + ), + ); + } }); const close = Effect.gen(function* () { @@ -1026,7 +1024,7 @@ const persistedInstances = Effect.fnUntraced(function* ( fs: FileSystem.FileSystem, path: Path.Path, root: string, - instances: ReadonlyArray | undefined, + instances?: ReadonlyArray, ) { const selected: Array<{ readonly service: string; readonly instanceId: string }> = []; for (const service of yield* listDirectories(fs, path, root)) @@ -1057,44 +1055,3 @@ export const streamStackLogs = (options: { ); }), ); - -/** Reads persisted records of a stack's instances, merged by time, service, instance and position. */ -export const readStackLogs = Effect.fn("LogStore.readStackLogs")(function* (options: { - readonly root: string; - readonly instances?: ReadonlyArray; - readonly since?: number; - readonly tail?: number; -}) { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; - const reader = makeReader(fs, path); - const selected = yield* persistedInstances(fs, path, options.root, options.instances); - const perInstance = yield* Effect.forEach( - selected, - ({ service, instanceId }) => - reader - .history( - path.join(options.root, service, instanceId), - { generation: 0, byteOffset: 0 }, - options, - undefined, - ) - .pipe( - Effect.map(({ records }) => - records.map((record): StackLogRecord => ({ ...record, service, instanceId })), - ), - ), - { concurrency: 4 }, - ); - const merged = perInstance.flat().toSorted(compareStackRecords); - return options.tail === undefined - ? merged - : merged.slice(Math.max(0, merged.length - options.tail)); -}); - -/** Orders records by timestamp, service, instance and position. */ -const compareStackRecords = (left: StackLogRecord, right: StackLogRecord) => - compareText(left.timestamp, right.timestamp) || - compareText(left.service, right.service) || - compareText(left.instanceId, right.instanceId) || - compareRecords(left, right); diff --git a/packages/stack/src/host/LogflareEvents.ts b/packages/stack/src/host/LogflareEvents.ts index 432200106e..b591bbc00d 100644 --- a/packages/stack/src/host/LogflareEvents.ts +++ b/packages/stack/src/host/LogflareEvents.ts @@ -1,4 +1,4 @@ -import { DateTime, Option } from "effect"; +import { DateTime, Option, Predicate } from "effect"; import type { ServiceCreation } from "../services/Catalog.ts"; /** Logflare source per shipped service kind; Studio's Logs pages query these names. */ @@ -25,14 +25,11 @@ export interface LogflareEvent { readonly metadata: Record; } -const isRecord = (value: unknown): value is Record => - typeof value === "object" && value !== null && !Array.isArray(value); - const parseJsonObject = (text: string): Record | undefined => { if (!text.startsWith("{")) return undefined; try { const value: unknown = JSON.parse(text); - return isRecord(value) ? value : undefined; + return Predicate.isObject(value) ? value : undefined; } catch { return undefined; } diff --git a/packages/stack/src/index.ts b/packages/stack/src/index.ts index cf415fa053..d504bb5fa0 100644 --- a/packages/stack/src/index.ts +++ b/packages/stack/src/index.ts @@ -30,7 +30,6 @@ export type { PgProveOptions, PlannedInstance, ReadLogsOptions, - ReadStackLogsOptions, SavedStack, StackLogRecord, StackLocations, @@ -67,8 +66,3 @@ export const discover = ( /** Reads one saved stack by id or by project identity; resolves `undefined` when none is saved. */ export const find = (options: StackEffect.FindOptions, callOptions?: CallOptions) => runOnce(StackEffect.find(options).pipe(Effect.map(Option.getOrUndefined)), callOptions); -/** Reads a stack's persisted log records without its owner. */ -export const readStackLogs = ( - options: StackEffect.ReadStackLogsOptions, - callOptions?: CallOptions, -) => runOnce(StackEffect.readStackLogs(options), callOptions); diff --git a/packages/stack/src/services/Database.ts b/packages/stack/src/services/Database.ts index c1fc5eab12..641afc1044 100644 --- a/packages/stack/src/services/Database.ts +++ b/packages/stack/src/services/Database.ts @@ -44,6 +44,7 @@ import { publishProcessLogs, runtimeSessionFromContainer, launchOutputPublisher, + type LaunchOutput, } from "../runtime/Session.ts"; import { ServiceError, @@ -65,7 +66,7 @@ import { resolveNativePostgresUser, type PasswdEntry, } from "../runtime/postgres-user.ts"; -import { EndpointIntent, serviceCreation, type CatalogLog, type CatalogLogs } from "./Recipe.ts"; +import { EndpointIntent, serviceCreation, type CatalogLogs } from "./Recipe.ts"; import { containerInstancePath, ensureOwnedInstanceRoot, @@ -472,7 +473,7 @@ export const makeDatabase = ( const crypto = yield* Crypto.Crypto; const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; const client = yield* HttpClient.HttpClient; - const logs = yield* PubSub.sliding(256); + const logs = yield* PubSub.sliding(256); const endpoint = yield* Ref.make(undefined); const prepared = yield* Ref.make>(new Map()); if (!/^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$/u.test(String(options.stackId))) diff --git a/packages/stack/src/services/ProcessRecipe.ts b/packages/stack/src/services/ProcessRecipe.ts index 8761929c6e..51eb58007b 100644 --- a/packages/stack/src/services/ProcessRecipe.ts +++ b/packages/stack/src/services/ProcessRecipe.ts @@ -40,6 +40,7 @@ import { processExit as sharedProcessExit, publishProcessLogs, runtimeSessionFromContainer, + type LaunchOutput, type PublishOutput, } from "../runtime/Session.ts"; import { @@ -54,7 +55,6 @@ import { removeOwnedInstanceRoot, } from "./InstanceRoot.ts"; import { - type CatalogLog, CatalogError, type CatalogOptions, type ProcessRecipeResult, @@ -247,7 +247,7 @@ const outputDrainGrace = Duration.seconds(2); const startupOutputTailLines = 20; const startupOutputLineChars = 1_000; -type StartupOutput = Readonly>>; +type StartupOutput = Readonly>>; const clipLine = (line: string) => line.length > startupOutputLineChars @@ -335,7 +335,7 @@ const collectNativeOutput = Effect.fn("ProcessRecipe.collectNativeOutput")(funct const consume = Effect.fnUntraced(function* ( stream: Stream.Stream, - name: CatalogLog["stream"], + name: LaunchOutput["stream"], tail: Ref.Ref>, ) { const partial = yield* Ref.make(""); @@ -423,7 +423,7 @@ export const makeProcessRecipe = const prepared = yield* Ref.make(undefined); const preparedRoot = yield* Ref.make(undefined); const endpoints = yield* Ref.make>(new Map()); - const logs = yield* PubSub.sliding(256); + const logs = yield* PubSub.sliding(256); const instanceRoot = deps.path.join(options.root, options.instanceId); const ownedInstanceRoot = { fs: deps.fs, diff --git a/packages/stack/src/services/Recipe.ts b/packages/stack/src/services/Recipe.ts index 4e0c0224df..e6e9aa3a00 100644 --- a/packages/stack/src/services/Recipe.ts +++ b/packages/stack/src/services/Recipe.ts @@ -58,10 +58,8 @@ export interface CatalogOptions { readonly hostGateway?: HostGateway; } -export type CatalogLog = LaunchOutput; - /** Subscribes to a recipe's launch output; chunks published before the subscription are missed. */ -export type CatalogLogs = Effect.Effect, never, Scope.Scope>; +export type CatalogLogs = Effect.Effect, never, Scope.Scope>; export interface CatalogRecipe { readonly creation: C; From 3990b4f9d3ce97d4fec88ea3f8962c11c107b333 Mon Sep 17 00:00:00 2001 From: avallete Date: Thu, 1 Oct 2026 19:37:54 +0200 Subject: [PATCH 16/35] fix(stack): continue launch ids from logs for instances saved without one An instance saved before launch ids were persisted resumes after the highest launch id at the end of its newest log segment, so its next launch never reuses an id its logs already hold. Co-Authored-By: Claude Opus 5.5 --- packages/stack/ARCHITECTURE.md | 26 ++++----- .../stack/src/Owner.logs.integration.test.ts | 45 +++++++++++++++ packages/stack/src/Owner.ts | 8 ++- ...State.vector-migration.integration.test.ts | 2 +- packages/stack/src/host/LogStore.ts | 55 ++++++++++++++++--- 5 files changed, 114 insertions(+), 22 deletions(-) diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index ce5e2cd22f..1850f6136a 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -649,19 +649,19 @@ segments: each owner start opens a new generation on the instance's first output would take a segment past 5 MiB starts the next one. The oldest closed segments except the newest are deleted while an instance holds more than 10 MiB or 64 segments. Records carry the publish time of their first byte. Launch ids keep increasing across owner starts, since each saved instance -records its latest one. Line state is kept per launch, process and stream; chunk sequence numbers -span a launch's processes, so a process whose output was all dropped still shows as `lost`. A late -partial line of an ended launch waits for its newline for two seconds of quiet, or until the store -closes. Lines are cut at 32 KiB; chunks the in-memory output buffer dropped or that were still -queued when a bounded drain at close ran out, and records that failed to write, are recorded as -`lost`; a write that fails part-way can count records that did land, so the count is an upper bound. -Closing aborts filesystem work still pending after 5 seconds; output not yet written by then is not -recorded. Every reader, live or offline, reads segments by position, so history and following share -one path; a follow can resume at a record position, and a reader that finds its segment deleted -reports a `lost` gap marker, which carries `resumeAt` instead of a record position. Destroying an -instance deletes its segments, and an owner start removes directories of instances no longer saved, -so a failed deletion is retried; destroying the stack removes `logs/`; resetting database data keeps -them. +records its latest one; an instance saved without one continues after the launch ids at the end of +its newest segment. Line state is kept per launch, process and stream; chunk sequence numbers span a +launch's processes, so a process whose output was all dropped still shows as `lost`. A late partial +line of an ended launch waits for its newline for two seconds of quiet, or until the store closes. +Lines are cut at 32 KiB; chunks the in-memory output buffer dropped or that were still queued when a +bounded drain at close ran out, and records that failed to write, are recorded as `lost`; a write +that fails part-way can count records that did land, so the count is an upper bound. Closing aborts +filesystem work still pending after 5 seconds; output not yet written by then is not recorded. Every +reader, live or offline, reads segments by position, so history and following share one path; a +follow can resume at a record position, and a reader that finds its segment deleted reports a `lost` +gap marker, which carries `resumeAt` instead of a record position. Destroying an instance deletes +its segments, and an owner start removes directories of instances no longer saved, so a failed +deletion is retried; destroying the stack removes `logs/`; resetting database data keeps them. While the composed Analytics instance is running and healthy, the owner ships the persisted stdout/stderr records of Auth, REST, Realtime, Storage, Functions and database instances to its diff --git a/packages/stack/src/Owner.logs.integration.test.ts b/packages/stack/src/Owner.logs.integration.test.ts index 81a5880c30..9fa450ad41 100644 --- a/packages/stack/src/Owner.logs.integration.test.ts +++ b/packages/stack/src/Owner.logs.integration.test.ts @@ -181,6 +181,51 @@ describe("owner persisted logs", () => { ).pipe(Effect.provide(services)), ); + it.live("continues after the launch ids in its logs when its saved state has none", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "owner-logs-unsaved-launch-" }); + const firstRun = yield* Scope.make(); + const owner = yield* openOwner("owner-logs-unsaved-launch-", "native", { root }).pipe( + Scope.provide(firstRun), + ); + const mail = yield* owner.rpc.createService({ + service: "mail", + config: {}, + endpoints: { http: { port: "auto" } }, + }); + yield* Scope.close(firstRun, Exit.void); + const state = Context.get( + yield* Layer.build(State.layer({ root: owner.stateRoot })), + State.Service, + ); + const saved = yield* state.read(owner.stack.id); + if (saved === undefined) return yield* Effect.die("stack state missing"); + // Logs a state saved before launch ids were persisted left behind at launch 3. + const directory = `${owner.logsRoot}/mail/${mail.id}`; + yield* fs.makeDirectory(directory, { recursive: true }); + yield* fs.writeFileString( + `${directory}/0000000001.log`, + "2026-01-01T00:00:00.000Z launch 3 | \n2026-01-01T00:00:00.001Z stdout 3 | earlier\n", + ); + const restarted = yield* ownerFor({ saved, state, root: `${root}/data`, cacheRoot }); + yield* Effect.addFinalizer(() => restarted.namespace.destroy.pipe(Effect.ignore)); + + yield* restarted.rpc.startService({ id: mail.id }); + yield* restarted.rpc.readyService({ id: mail.id }); + const records = yield* restarted.rpc + .readLogs({ id: mail.id, follow: false }) + .pipe(Stream.runCollect); + + expect(saved.instances.find(({ id }) => id === mail.id)?.launchId).toBeUndefined(); + expect( + records.filter(({ kind }) => kind === "launch").map(({ launchId }) => launchId), + ).toEqual([3, 4]); + }), + ).pipe(Effect.provide(services)), + ); + it.live("resumes a follow at a record position without replaying earlier records", () => Effect.scoped( Effect.gen(function* () { diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index a9f335d4e6..fe47cc36fa 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -333,6 +333,12 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { message: `Duplicate instance ${id}`, }); const initial = recipe.creation; + // A state saved before launch ids were persisted continues after the ids its logs hold. + const resumeAfter = + lastLaunchId ?? + Option.getOrUndefined( + yield* logStore.latestLaunchId({ service: initial.service, instanceId: id }), + ); const creation = yield* Ref.make(initial); const namespaceRef = yield* Ref.make(undefined); const core = yield* makeService( @@ -375,7 +381,7 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { { id, config: initial, - ...(lastLaunchId === undefined ? {} : { lastLaunchId }), + ...(resumeAfter === undefined ? {} : { lastLaunchId: resumeAfter }), coordinate: (operation, transition) => (drainingBlocks.includes(operation) ? rejectWhileDraining : Effect.void).pipe( Effect.andThen(orchestrator.admissionFor(id)(operation, transition)), diff --git a/packages/stack/src/State.vector-migration.integration.test.ts b/packages/stack/src/State.vector-migration.integration.test.ts index 1903bff63a..5fd2e39233 100644 --- a/packages/stack/src/State.vector-migration.integration.test.ts +++ b/packages/stack/src/State.vector-migration.integration.test.ts @@ -155,7 +155,7 @@ describe("saved Vector instance migration", () => { .pipe(Effect.forkChild); yield* Deferred.await(held); - const checked = yield* state.migrate("legacy").pipe(Effect.exit); + const checked = yield* state.migrate("legacy").pipe(Effect.timeout("1 second"), Effect.exit); yield* Deferred.succeed(release, undefined); yield* Fiber.join(holder); diff --git a/packages/stack/src/host/LogStore.ts b/packages/stack/src/host/LogStore.ts index b4ff40ad3d..20cd33991b 100644 --- a/packages/stack/src/host/LogStore.ts +++ b/packages/stack/src/host/LogStore.ts @@ -97,6 +97,11 @@ export interface Interface { }) => Effect.Effect; /** Deletes log directories of instances that are not attached; failures are logged. */ readonly removeOrphans: Effect.Effect; + /** The highest launch id at the end of an instance's newest segment, if one is readable. */ + readonly latestLaunchId: (instance: { + readonly service: string; + readonly instanceId: string; + }) => Effect.Effect>; /** Stops every writer and reader; later reads fail. */ readonly close: Effect.Effect; } @@ -254,6 +259,43 @@ const makeReader = (fs: FileSystem.FileSystem, path: Path.Path) => { Effect.mapError(storeError("read")), ); + /** Reads only the last chunk of the newest segment, skipping its first, possibly cut, line. */ + const latestLaunchId = (directory: string) => + generations(directory).pipe( + Effect.flatMap((listed) => { + const newest = listed.at(-1); + if (newest === undefined) return Effect.succeed(Option.none()); + const file = path.join(directory, segmentName(newest)); + return fs.stat(file).pipe( + Effect.mapError(storeError("read")), + Effect.flatMap((info) => { + const offset = Math.max(0, Number(info.size) - readChunkBytes); + return readChunk(file, offset).pipe( + Effect.map( + Option.match({ + onNone: () => Option.none(), + onSome: (bytes) => { + let latest: number | undefined; + for (const line of decoder + .decode(bytes) + .split("\n") + .slice(offset > 0 ? 1 : 0, -1)) { + const launchId = parseRecord(line, { + generation: newest, + byteOffset: 0, + })?.launchId; + if (launchId !== undefined) latest = Math.max(latest ?? 0, launchId); + } + return Option.fromNullishOr(latest); + }, + }), + ), + ); + }), + ); + }), + ); + const guarded = (live: Live | undefined, effect: Effect.Effect) => live === undefined ? effect : live.passes.withPermits(1)(effect); @@ -402,12 +444,7 @@ const makeReader = (fs: FileSystem.FileSystem, path: Path.Path) => { const listed = yield* generations(directory); const tail = options.tail; if (tail !== undefined) { - if (tail === 0) - return { - records: [], - cursor: live === undefined ? cursor : yield* live.lock.withPermits(1)(Ref.get(live.end)), - listed, - }; + if (tail === 0) return { records: [], cursor, listed }; // A partial line is written when it ends, so any segment can hold the newest records. let records: ReadonlyArray = []; let end: LogPosition = cursor; @@ -509,7 +546,7 @@ const makeReader = (fs: FileSystem.FileSystem, path: Path.Path) => { undefined, ); - return { read, history, records, segments, generations }; + return { read, history, records, segments, generations, latestLaunchId }; }; /** Creates the owner's log store; closing its scope flushes and closes every writer. */ @@ -1015,6 +1052,10 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio directory: (instanceId) => Effect.map(attached(instanceId), (instance) => instance.directory), remove, removeOrphans: removeOrphans(), + latestLaunchId: (instance) => + reader + .latestLaunchId(path.join(options.root, instance.service, instance.instanceId)) + .pipe(Effect.orElseSucceed(() => Option.none())), close, } satisfies Interface; }); From a267a43fa134d2a5837456471059fb558524b59a Mon Sep 17 00:00:00 2001 From: avallete Date: Thu, 1 Oct 2026 19:45:58 +0200 Subject: [PATCH 17/35] fix(stack): redact S3 presigned URL signatures in gateway access lines Co-Authored-By: Claude Opus 5.5 --- .../src/commands/experimental/stack/start/SIDE_EFFECTS.md | 7 ++++--- packages/stack/src/HttpProxy.ts | 6 +++++- packages/stack/src/HttpProxy.unit.test.ts | 6 ++++++ 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index 5c3c5c40de..4f40624488 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -44,9 +44,10 @@ PostgREST runs with `PGRST_LOG_LEVEL=info`, so every request line, query string persisted and shipped to Analytics. The shared API port also records one nginx combined line plus duration (`... "GET /rest/v1/todos?select=* HTTP/1.1" 200 126 "-" "curl/8.7.1" 12ms`) per request and WebSocket upgrade under `logs/gateway/gateway/`, with the same retention; credential query and -fragment values (`apikey`, `token`, `token_hash`, `code`, and access, refresh, ID, and provider -tokens), in the request target and the Referer, are written as `redacted`, as are values that -themselves carry such a pair (a `redirect_to` URL with a token) and URL userinfo. +fragment values (`apikey`, `token`, `token_hash`, `code`, access, refresh, ID, and provider +tokens, and the `X-Amz-Signature`, `X-Amz-Credential`, and `X-Amz-Security-Token` of S3 +presigned URLs), in the request target and the Referer, are written as `redacted`, as are values +that themselves carry such a pair (a `redirect_to` URL with a token) and URL userinfo. When an owner starts a stack saved with a Vector instance, it removes that instance, its composition members, dependencies and port claims from `state.json`, and its stack-owned Vector config files under `data//runtime/vector/`; its containers go with the stack's container sweep. diff --git a/packages/stack/src/HttpProxy.ts b/packages/stack/src/HttpProxy.ts index 3c6ec89959..a1132aca94 100644 --- a/packages/stack/src/HttpProxy.ts +++ b/packages/stack/src/HttpProxy.ts @@ -163,7 +163,8 @@ const decodeNested = (value: string) => { return current; }; -// Auth puts PKCE codes, OTP token hashes and OAuth tokens in redirect and verify URLs. +// Auth puts PKCE codes, OTP token hashes and OAuth tokens in redirect and verify URLs; S3 +// presigned URLs carry their SigV4 signature, credential scope and session token. const credentialParameters = [ "apikey", "access_token", @@ -174,6 +175,9 @@ const credentialParameters = [ "id_token", "provider_token", "provider_refresh_token", + "x-amz-signature", + "x-amz-credential", + "x-amz-security-token", ]; const scheme = String.raw`[a-z][a-z\d+.-]*`; diff --git a/packages/stack/src/HttpProxy.unit.test.ts b/packages/stack/src/HttpProxy.unit.test.ts index 414cfc6025..f05992d06f 100644 --- a/packages/stack/src/HttpProxy.unit.test.ts +++ b/packages/stack/src/HttpProxy.unit.test.ts @@ -13,6 +13,12 @@ describe("redactCredentials", () => { url: "/auth/v1/verify?code=pkce&token_hash=h&type=signup", redacted: "/auth/v1/verify?code=redacted&token_hash=redacted&type=signup", }, + { + case: "redacts S3 presigned URL signatures and session tokens", + url: "/storage/v1/s3/b/o.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AK%2F20261001%2Flocal%2Fs3%2Faws4_request&X-Amz-Expires=60&X-Amz-Security-Token=st&X-Amz-Signature=abc", + redacted: + "/storage/v1/s3/b/o.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=redacted&X-Amz-Expires=60&X-Amz-Security-Token=redacted&X-Amz-Signature=redacted", + }, { case: "redacts fragment credentials of an absolute URL", url: "http://127.0.0.1:54321/x?token=t1&select=*#access_token=frag&type=bearer", From 245e78e5aa17dec403d13d512da6a3092f75552e Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 2 Oct 2026 07:56:21 +0200 Subject: [PATCH 18/35] fix(stack): retry log reads, bound stack logs by launch, and harden migration - Retry failed log reads from the saved cursor with capped backoff instead of pausing shipping until Analytics restarts. - Bound stack logs --since start, history and follow, by each instance's saved launch id. - Keep the PostgREST time when a message contains ": ". - Delete Vector files before committing the migration so a failure is retried, keep reclaiming a session stack when migration fails, and reap stale cursor-write directories. Co-Authored-By: Claude Opus 5.5 --- .../experimental/stack/logs/SIDE_EFFECTS.md | 6 +- .../experimental/stack/logs/logs.format.ts | 46 +++++++++---- .../stack/logs/logs.format.unit.test.ts | 10 +-- .../experimental/stack/logs/logs.handler.ts | 22 +++++-- .../stack/logs/logs.integration.test.ts | 65 +++++++++++++++++++ .../experimental/stack/start/SIDE_EFFECTS.md | 4 +- packages/stack/ARCHITECTURE.md | 4 +- packages/stack/src/State.ts | 22 ++++--- ...State.vector-migration.integration.test.ts | 57 +++++++++++++++- packages/stack/src/Sweep.integration.test.ts | 38 +++++++++++ packages/stack/src/Sweep.ts | 9 ++- .../src/host/LogForwarder.integration.test.ts | 63 ++++++++++++++++++ packages/stack/src/host/LogForwarder.ts | 53 +++++++++++---- packages/stack/src/host/LogflareEvents.ts | 2 +- .../src/host/LogflareEvents.unit.test.ts | 10 +++ 15 files changed, 362 insertions(+), 49 deletions(-) diff --git a/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md index 0f5d9d6fc0..7684224b65 100644 --- a/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/logs/SIDE_EFFECTS.md @@ -26,8 +26,10 @@ service configuration, artifacts, logs, or data are changed. `--tail N` (default 200, 0 prints none) keeps the newest N output lines across the selected services, with the `launch` and `lost` markers between them. `--since` accepts a duration before now (`30s`, `10m`, `1h30m`, `2d`), an -ISO-8601 time, or `start`, which keeps the records of each instance's highest -launch id, which keeps increasing across owner restarts (all of them when +ISO-8601 time, or `start`, which keeps the records of each instance's current +launch and later ones, in history and while following. The current launch is the +launch id saved in the stack definition, which keeps increasing across owner +restarts, or else the highest launch record in history (all records when retention removed that launch record). Records are ordered by timestamp, service, instance, and file position. Timestamps are the owner's clock at each line's first byte. Lines end at `\n`, `\r\n`, or a lone diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.format.ts b/apps/cli/src/commands/experimental/stack/logs/logs.format.ts index 13f3ab1511..087f4a4100 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.format.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.format.ts @@ -87,7 +87,8 @@ const compareRecords = (left: StackLogRecord, right: StackLogRecord) => interface Launch { readonly id: number; - readonly timestamp: string; + /** Unknown until its launch record is read; a saved launch may not have written one. */ + readonly timestamp: string | undefined; } interface InstanceTail { @@ -98,19 +99,28 @@ interface InstanceTail { launch: Launch | undefined; } +/** Whether a record belongs to `launchId` or a later launch; a gap marker carries no launch id. */ +export const isFromLaunch = (record: LogRecord, launchId: number | undefined) => + launchId === undefined || record.launchId === undefined || record.launchId >= launchId; + /** Launch ids increase per instance; a gap marker carries none, so its time decides. */ const fromLaunch = (record: StackLogRecord, launch: Launch) => record.launchId === undefined - ? record.timestamp >= launch.timestamp - : record.launchId >= launch.id; + ? launch.timestamp === undefined || record.timestamp >= launch.timestamp + : isFromLaunch(record, launch.id); /** - * Collects each instance's newest `tail` lines and last position, optionally only the records of - * its highest launch. + * Collects each instance's newest `tail` lines and last position. With `startLaunches`, it keeps + * only the records of each instance's current launch: its saved launch id, or else its highest + * launch record, which `launches` reports for following. */ -export const makeHistoryCollector = (tail: number, fromLatestLaunch: boolean) => { +export const makeHistoryCollector = ( + tail: number, + startLaunches: ReadonlyMap | undefined, +) => { const instances = new Map(); const positions = new Map(); + const launches = new Map(startLaunches); const push = (record: StackLogRecord) => { const position = record.position; const last = positions.get(record.instanceId); @@ -118,15 +128,24 @@ export const makeHistoryCollector = (tail: number, fromLatestLaunch: boolean) => positions.set(record.instanceId, position); let state = instances.get(record.instanceId); if (state === undefined) { - state = { records: [], head: 0, lines: 0, total: 0, launch: undefined }; + const saved = startLaunches?.get(record.instanceId); + state = { + records: [], + head: 0, + lines: 0, + total: 0, + launch: saved === undefined ? undefined : { id: saved, timestamp: undefined }, + }; instances.set(record.instanceId, state); } - if (fromLatestLaunch) { + if (startLaunches !== undefined) { const latest = state.launch; if ( record.kind === "launch" && record.launchId !== undefined && - (latest === undefined || record.launchId > latest.id) + (latest === undefined || + record.launchId > latest.id || + (record.launchId === latest.id && latest.timestamp === undefined)) ) { const launch = { id: record.launchId, timestamp: record.timestamp }; state.records = state.records.slice(state.head).filter((kept) => fromLaunch(kept, launch)); @@ -134,6 +153,7 @@ export const makeHistoryCollector = (tail: number, fromLatestLaunch: boolean) => state.lines = state.records.filter(isLine).length; state.total = state.lines; state.launch = launch; + launches.set(record.instanceId, launch.id); } else if (latest !== undefined && !fromLaunch(record, latest)) return; } const line = isLine(record); @@ -163,11 +183,15 @@ export const makeHistoryCollector = (tail: number, fromLatestLaunch: boolean) => state.head = 0; } }; - const finish = (): { readonly window: LogWindow; readonly positions: typeof positions } => { + const finish = (): { + readonly window: LogWindow; + readonly positions: typeof positions; + readonly launches: ReadonlyMap; + } => { const kept = [...instances.values()].flatMap((state) => state.records.slice(state.head)); const total = [...instances.values()].reduce((sum, state) => sum + state.total, 0); const window = tailWindow(kept.toSorted(compareRecords), tail); - return { window: { ...window, total }, positions }; + return { window: { ...window, total }, positions, launches }; }; return { push, finish }; }; diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.format.unit.test.ts b/apps/cli/src/commands/experimental/stack/logs/logs.format.unit.test.ts index 559f4180c9..2e2bf2a006 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.format.unit.test.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.format.unit.test.ts @@ -48,7 +48,7 @@ describe("makeHistoryCollector", () => { records.filter(({ kind }) => kind !== "launch").map(({ text }) => text); it("keeps the newest lines across instances, counts every line, and tracks positions", () => { - const collector = makeHistoryCollector(2, false); + const collector = makeHistoryCollector(2, undefined); for (let offset = 0; offset < 1_000; offset += 2) { collector.push(at("a", offset, "stdout", `a ${offset}`)); collector.push(at("b", offset + 1, "stdout", `b ${offset + 1}`)); @@ -62,7 +62,7 @@ describe("makeHistoryCollector", () => { }); it("keeps the newest line by time when an older partial line is flushed after it", () => { - const collector = makeHistoryCollector(1, false); + const collector = makeHistoryCollector(1, undefined); collector.push({ ...at("a", 10, "stdout", "newer"), position: { generation: 1, byteOffset: 0 }, @@ -80,7 +80,7 @@ describe("makeHistoryCollector", () => { }); it("starts each instance at its highest launch and keeps instances without one", () => { - const collector = makeHistoryCollector(10, true); + const collector = makeHistoryCollector(10, new Map()); collector.push(at("a", 0, "launch", "", 1)); collector.push(at("a", 1, "stdout", "a old", 1)); collector.push(at("a", 5, "launch", "", 2)); @@ -95,7 +95,7 @@ describe("makeHistoryCollector", () => { }); it("keeps the highest launch when an older launch's first output arrives after it", () => { - const collector = makeHistoryCollector(10, true); + const collector = makeHistoryCollector(10, new Map()); collector.push(at("a", 10, "launch", "", 2)); collector.push(at("a", 11, "stdout", "second launch", 2)); collector.push(at("a", 20, "launch", "", 1)); @@ -109,7 +109,7 @@ describe("makeHistoryCollector", () => { }); it("starts at the launch of the latest owner run, whose ids continue the earlier run's", () => { - const collector = makeHistoryCollector(10, true); + const collector = makeHistoryCollector(10, new Map()); collector.push({ ...at("a", 0, "launch", "", 3), position: { generation: 1, byteOffset: 0 } }); collector.push({ ...at("a", 1, "stdout", "earlier run", 3), diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.handler.ts b/apps/cli/src/commands/experimental/stack/logs/logs.handler.ts index 1a93dfa637..54ca3d1e6a 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.handler.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.handler.ts @@ -16,6 +16,7 @@ import { StackCommandLogsError } from "./logs.errors.ts"; import type { StackLogsFlags } from "./logs.command.ts"; import { isAfter, + isFromLaunch, logEvent, makeHistoryCollector, makeTextFormatter, @@ -38,15 +39,17 @@ type SavedInstance = SavedStack["instances"][number]; interface Selected { readonly id: string; readonly service: string; + readonly launchId: number | undefined; } const select = ( definition: SavedStack, requested: ReadonlyArray, ): Effect.Effect, StackCommandLogsError> => { - const subject = ({ id, creation }: SavedInstance): Selected => ({ + const subject = ({ id, creation, launchId }: SavedInstance): Selected => ({ id, service: creation.service, + launchId, }); if (requested.length === 0) { const members = new Set(definition.composition.members.map(({ id }) => id)); @@ -133,7 +136,16 @@ export const stackLogs = Effect.fn("experimental.stack.logs")(function* (flags: cacheRoot: path.join(settings.supabaseHome, "cache", "stack"), }; const sinceTime = since?.kind === "time" ? { since: since.iso } : {}; - const collector = makeHistoryCollector(flags.tail, since?.kind === "start"); + const collector = makeHistoryCollector( + flags.tail, + since?.kind === "start" + ? new Map( + selected.flatMap(({ id, launchId }) => + launchId === undefined ? [] : [[id, launchId] as const], + ), + ) + : undefined, + ); if (flags.tail > 0) yield* streamStackLogs({ stateRoot: locations.stateRoot, @@ -144,7 +156,7 @@ export const stackLogs = Effect.fn("experimental.stack.logs")(function* (flags: Stream.mapError(logsError), Stream.runForEach((record) => Effect.sync(() => collector.push(record))), ); - const { window, positions: printed } = collector.finish(); + const { window, positions: printed, launches } = collector.finish(); if (output.format === "json") return yield* output.result(window.records.map((record) => logEvent(record, "history"))); @@ -198,7 +210,9 @@ export const stackLogs = Effect.fn("experimental.stack.logs")(function* (flags: const start = flags.tail === 0 ? { tail: 0 } : from === undefined ? {} : { from }; return [ handle.readLogs({ follow: true, ...start, ...sinceTime }).pipe( - Stream.filter((record) => isAfter(record, from)), + Stream.filter( + (record) => isAfter(record, from) && isFromLaunch(record, launches.get(id)), + ), Stream.map((record): StackLogRecord => ({ ...record, service, instanceId: id })), ), ]; diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts b/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts index 27175bc37a..672d9adfa0 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts @@ -275,6 +275,37 @@ describe("stack logs", () => { }).pipe(Effect.scoped, Effect.provide(live)), ); + it.live("starts --since start at each instance's saved launch, even before it logged", () => + Effect.gen(function* () { + const f = yield* fixture({ + instances: [ + { ...mail("mail-a"), launchId: 3 }, + { ...mail("mail-b"), launchId: 2 }, + ], + members: ["mail-a", "mail-b"], + }); + yield* f.writeSegment("mail", "mail-a", [ + launch(t0, 2), + line(t0 + 1, "stdout", "a previous launch", 2), + ]); + yield* f.writeSegment("mail", "mail-b", [ + launch(t0 + 2, 1), + line(t0 + 3, "stdout", "b previous launch", 1), + launch(t0 + 4, 2), + line(t0 + 5, "stdout", "b current launch", 2), + ]); + + const { output } = yield* f.run({ since: Option.some("start") }, "stream-json"); + + expect( + output.events.map(({ instance_id, kind, line: text }) => [instance_id, kind ?? text]), + ).toEqual([ + ["mail-b", "launch"], + ["mail-b", "b current launch"], + ]); + }).pipe(Effect.scoped, Effect.provide(live)), + ); + it.live("tails by record time when an older partial line was flushed last", () => Effect.gen(function* () { const f = yield* fixture({ instances: [mail("mail-a")], members: ["mail-a"] }); @@ -430,6 +461,40 @@ describe("stack logs", () => { }).pipe(Effect.scoped, Effect.provide(live)), ); + it.live("follows --since start without delayed records of an earlier launch", () => + Effect.gen(function* () { + const record = (offset: number, launchId: number, text: string): LogRecord => ({ + kind: "stdout", + timestamp: iso(t0 + offset), + launchId, + text, + position: { generation: 1, byteOffset: offset }, + }); + const f = yield* fixture({ + instances: [{ ...mail("mail-a"), launchId: 3 }], + members: ["mail-a"], + running: true, + handles: () => [ + followingMail("mail-a", () => + Stream.make( + record(0, 3, "current launch"), + record(1, 2, "delayed earlier launch"), + record(2, 4, "newer launch"), + ), + ), + ], + }); + + const { exit, output } = yield* f.run( + { follow: true, since: Option.some("start") }, + "stream-json", + ); + + expect(Exit.isSuccess(exit)).toBe(true); + expect(eventLines(output.events)).toEqual(["current launch", "newer launch"]); + }).pipe(Effect.scoped, Effect.provide(live)), + ); + it.live("follows the services the running stack serves and names the others", () => Effect.gen(function* () { const served = (text: string) => diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index 0636d9e740..06139e4f8d 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -99,7 +99,9 @@ each instance keeps its position in `logs///cursor.json`, while Analytics is stopped, starting, or unhealthy are shipped with their original timestamps once it is healthy again, including after an owner restart. A missing or unreadable position starts from the oldest retained line. Lines already deleted by log retention are skipped, and Analytics refusing -the API key pauses shipping until Analytics restarts or the composition changes. Each event carries an id derived from its instance and position, so a request that fails +the API key pauses shipping until Analytics stops or turns unhealthy and is healthy again, or the +composition selects a different Analytics instance. A failed log read is retried from the saved +position with a backoff. Each event carries an id derived from its instance and position, so a request that fails after it may have been ingested is repeated without duplicating rows. Shipping never wakes Analytics and does not count as idle activity, so a lazy Analytics still stops on its idle timer while other services log. Service log streams and `supabase stack logs` are never blocked by shipping. diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 1850f6136a..29674f8bfb 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -669,7 +669,9 @@ direct backend (never the proxy, so shipping neither wakes it nor counts as acti at most 256 events and 1 MiB. Each instance reads from `cursor.json` in its logs directory, the position of its last shipped record, written atomically after each body settles: a missing cursor or unreadable cursor starts at the oldest retained segment, and a cursor in a deleted segment -resumes at the next retained one. Bodies of an instance are sequential. Event ids +resumes at the next retained one. A failed log read restarts the instance from its cursor after a +capped backoff, and attaching an instance removes temporary cursor writes a dead owner left behind. +Bodies of an instance are sequential. Event ids derive from the instance and record position, and Logflare keeps the first row per id, so a failed body is posted again until it settles or the target changes. A 401, 403 or 404 response pauses the instance with its cursor until the target changes; any other 4xx except 408 and 429 skips the body. The target is re-selected when the composition or Analytics' health changes. diff --git a/packages/stack/src/State.ts b/packages/stack/src/State.ts index 07d7eb2ff0..ed4d4773ce 100644 --- a/packages/stack/src/State.ts +++ b/packages/stack/src/State.ts @@ -272,7 +272,7 @@ const stateWritePrefix = ".state-write-"; const staleWriteAgeMillis = 24 * 60 * 60 * 1000; /** Best-effort removal of temporary write directories a killed writer left behind. */ -const reapStaleWrites = (fs: FileSystem.FileSystem, path: Path.Path, root: string) => +export const reapStaleWrites = (fs: FileSystem.FileSystem, path: Path.Path, root: string) => Effect.gen(function* () { const now = yield* Clock.currentTimeMillis; const entries = yield* fs.readDirectory(root); @@ -678,8 +678,12 @@ const makeState = ( for (const directory of [configRoot, path.dirname(configRoot), instanceRoot]) yield* removeEmptyDirectory(directory); }).pipe( + Effect.as(true), Effect.catchCause((cause) => - Effect.logWarning(`Unable to remove the files of Vector instance ${instanceId}`, cause), + Effect.logWarning( + `Unable to remove the files of Vector instance ${instanceId}; the next owner start retries`, + cause, + ).pipe(Effect.as(false)), ), ); /** The saved document and its Vector instance ids, when it still holds one. */ @@ -700,21 +704,21 @@ const makeState = ( const target = statePath(id); // Only a document that still holds Vector takes the registry lock, to migrate it. if ((yield* legacyVector(target)) === undefined) return; - const removed = yield* withLock( + yield* withLock( Effect.gen(function* () { const legacy = yield* legacyVector(target); - if (legacy === undefined) return []; + if (legacy === undefined) return; const { text, removed } = legacy; const state = yield* decodeState(text, id, target); if (state.id !== id) return yield* stateError("identity", "State document identity does not match its path"); - yield* save(state); - return removed; + // The saved ids keep a failed file removal retried by the next migration. + const cleaned = yield* Effect.forEach(removed, (instanceId) => + removeVectorData(id, instanceId), + ); + if (cleaned.every(Boolean)) yield* save(state); }), ); - yield* Effect.forEach(removed, (instanceId) => removeVectorData(id, instanceId), { - discard: true, - }); }); return { read, diff --git a/packages/stack/src/State.vector-migration.integration.test.ts b/packages/stack/src/State.vector-migration.integration.test.ts index 5fd2e39233..ecc9f79891 100644 --- a/packages/stack/src/State.vector-migration.integration.test.ts +++ b/packages/stack/src/State.vector-migration.integration.test.ts @@ -1,6 +1,18 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; -import { Context, Deferred, Effect, Exit, Fiber, FileSystem, Layer, Path, Schema } from "effect"; +import { + Context, + Deferred, + Effect, + Exit, + Fiber, + FileSystem, + Layer, + Path, + PlatformError, + Ref, + Schema, +} from "effect"; import { create, open } from "./effect.ts"; import * as State from "./State.ts"; import { destroyTestStack } from "../tests/stack-cleanup.ts"; @@ -141,6 +153,49 @@ describe("saved Vector instance migration", () => { }).pipe(Effect.scoped, Effect.provide(layer)), ); + it.live( + "keeps Vector saved when removing its files fails, and retries at the next migration", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-vector-retry-" }); + const failNext = yield* Ref.make(true); + const injected: FileSystem.FileSystem = { + ...fs, + remove: (target, options) => + Effect.gen(function* () { + if ( + target.endsWith("vector.rendered.yaml") && + (yield* Ref.getAndSet(failNext, false)) + ) + return yield* PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "remove", + pathOrDescriptor: target, + }); + return yield* fs.remove(target, options); + }), + }; + const state = yield* stateFor(root).pipe( + Effect.provideService(FileSystem.FileSystem, injected), + ); + const file = yield* writeLegacyState(root, "legacy", root); + const runtime = path.join(root, "legacy", "data", "vector", "runtime"); + + yield* state.migrate("legacy"); + const afterFailure = yield* savedServices(yield* fs.readFileString(file)); + const keptFiles = yield* fs.exists(runtime); + yield* state.migrate("legacy"); + + expect(afterFailure).toEqual(["analytics", "mail", "vector", "vector"]); + expect(keptFiles).toBe(true); + expect(yield* savedServices(yield* fs.readFileString(file))).toEqual(["analytics", "mail"]); + expect(yield* fs.exists(runtime)).toBe(false); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + it.live("checks a state without Vector while another operation holds the registry lock", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/packages/stack/src/Sweep.integration.test.ts b/packages/stack/src/Sweep.integration.test.ts index 70c1754af0..e57462b0f1 100644 --- a/packages/stack/src/Sweep.integration.test.ts +++ b/packages/stack/src/Sweep.integration.test.ts @@ -11,12 +11,14 @@ import { Layer, Option, Path, + Schema, Stream, } from "effect"; import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { launchHost } from "./HostProcess.ts"; import { makeContainerRuntime } from "./runtime/Container.ts"; import * as State from "./State.ts"; +import { reclaimStack } from "./Sweep.ts"; import { makeDockerDatabaseRoot } from "../tests/docker-fixture.ts"; import { shutdownOwner, watchLeaseRelease } from "../tests/owner.ts"; @@ -208,3 +210,39 @@ it.live.skipIf(process.platform === "win32")( ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), { timeout: 180_000 }, ); + +it.live("destroys a dead session stack whose saved Vector instance fails to migrate", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const stateRoot = yield* fs.makeTempDirectoryScoped({ prefix: "stack-sweep-migrate-" }); + const real = Context.get(yield* Layer.build(State.layer({ root: stateRoot })), State.Service); + const state: State.Interface = { + ...real, + migrate: () => + Effect.fail(new State.StateError({ operation: "lock", message: "registry is busy" })), + }; + const id = "session-vector"; + yield* fs.makeDirectory(path.join(stateRoot, id), { recursive: true }); + yield* fs.writeFileString( + path.join(stateRoot, id, "state.json"), + yield* Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown))({ + ...saved(id, path.join(stateRoot, "project"), "native", "session"), + instances: [{ id: "vector", creation: { service: "vector", config: {} } }], + composition: { members: [{ id: "vector", activation: "eager" }], dependencies: [] }, + }), + ); + + const reclaimed = yield* reclaimStack({ + state, + stateRoot, + cacheRoot: path.join(stateRoot, "cache"), + id, + }); + + expect(reclaimed).toBe(true); + expect(yield* real.read(id)).toBeUndefined(); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), +); diff --git a/packages/stack/src/Sweep.ts b/packages/stack/src/Sweep.ts index 413b422d0c..e6376d7b25 100644 --- a/packages/stack/src/Sweep.ts +++ b/packages/stack/src/Sweep.ts @@ -48,7 +48,14 @@ export const reclaimStack = Effect.fn("Sweep.reclaimStack")(function* (options: if (saved === undefined) return true; const dataRoot = path.join(yield* fs.realPath(options.stateRoot), id, "data"); yield* Owner.sweepContainers(saved, dataRoot); - yield* state.migrate(id); + // Reading drops saved Vector instances, so destroying does not depend on the migration. + yield* state + .migrate(id) + .pipe( + Effect.catch((error) => + Effect.logWarning(`Unable to migrate the saved state of stack ${id}`, error), + ), + ); if (saved.lifetime === "session") yield* destroyStack(state, saved, dataRoot, options.cacheRoot); return true; diff --git a/packages/stack/src/host/LogForwarder.integration.test.ts b/packages/stack/src/host/LogForwarder.integration.test.ts index 1999276079..650fbcb2fa 100644 --- a/packages/stack/src/host/LogForwarder.integration.test.ts +++ b/packages/stack/src/host/LogForwarder.integration.test.ts @@ -13,6 +13,7 @@ import { Path, PubSub, Queue, + Ref, References, Schema, Scope, @@ -292,6 +293,68 @@ describe("LogForwarder", () => { }).pipe(Effect.scoped, Effect.provide(layer)), ); + it.live("ships again after a failed log read while Analytics stays healthy", () => + Effect.gen(function* () { + const { store, sink, database, analytics } = yield* fixture(); + const reads = yield* Ref.make(0); + const flaky: LogStore.Interface = { + ...store, + read: (instanceId, options) => + Ref.getAndUpdate(reads, (count) => count + 1).pipe( + Effect.flatMap((count) => + count === 0 + ? Effect.succeed( + Stream.fail( + new LogStore.LogStoreError({ operation: "read", message: "injected" }), + ), + ) + : store.read(instanceId, options), + ), + ), + }; + const warnings: Array = []; + const captured = Logger.layer([ + Logger.make(({ logLevel, message }) => { + if (logLevel === "Warn") warnings.push(message); + }), + ]); + yield* database.log("after a failed read"); + yield* analytics.set(true); + + yield* startForwarder(flaky, [analytics.instance, database.instance]).pipe( + Effect.provide(captured), + ); + const shipped = yield* sink.next; + + expect(messages(shipped)).toEqual(["after a failed read"]); + expect(yield* Ref.get(reads)).toBe(2); + expect(warnings).toEqual([ + ["Reading database logs to ship failed; retrying", expect.anything()], + ]); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("removes an instance's stale cursor write directories when it attaches", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { root, store, database, analytics } = yield* fixture(); + const directory = path.join(root, "database", "database"); + const stale = path.join(directory, ".state-write-stale"); + const recent = path.join(directory, ".state-write-recent"); + yield* fs.makeDirectory(stale, { recursive: true }); + yield* fs.makeDirectory(recent, { recursive: true }); + // A numeric file time is in seconds. + const twoDaysAgo = (yield* Clock.currentTimeMillis) / 1000 - 2 * 24 * 60 * 60; + yield* fs.utimes(stale, twoDaysAgo, twoDaysAgo); + + yield* startForwarder(store, [analytics.instance, database.instance]); + + expect(yield* fs.exists(stale)).toBe(false); + expect(yield* fs.exists(recent)).toBe(true); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + it.live("detaches an instance only once its cursor write landed, so its logs can go", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/packages/stack/src/host/LogForwarder.ts b/packages/stack/src/host/LogForwarder.ts index 5f12f6116a..d9f7e93d0a 100644 --- a/packages/stack/src/host/LogForwarder.ts +++ b/packages/stack/src/host/LogForwarder.ts @@ -19,7 +19,7 @@ import type { CompositionConfig } from "../Orchestrator.ts"; import type { ServiceObservation } from "../Service.ts"; import type { ServiceCreation } from "../services/Catalog.ts"; import type { CatalogError, ServiceEndpoint } from "../services/Recipe.ts"; -import { writeFileAtomically } from "../State.ts"; +import { reapStaleWrites, writeFileAtomically } from "../State.ts"; import { isShippedService, logflareEvent, @@ -80,8 +80,8 @@ const cursorFile = "cursor.json"; const encoder = new TextEncoder(); const decodeCursor = Schema.decodeUnknownEffect(Schema.fromJsonString(LogPosition)); const encodeCursor = Schema.encodeEffect(Schema.fromJsonString(LogPosition)); -/** Delivery is idempotent per event id, so a body that did not settle is posted again. */ -const redeliverSchedule = Schedule.exponential("250 millis", 2).pipe( +/** Delivery is idempotent per event id, so an unsettled body or a failed log read is retried. */ +const retrySchedule = Schedule.exponential("250 millis", 2).pipe( Schedule.modifyDelay(({ duration }) => Effect.succeed(Duration.min(duration, Duration.seconds(10))), ), @@ -302,7 +302,7 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar ), ), Effect.retry({ - schedule: redeliverSchedule, + schedule: retrySchedule, while: (error) => error._tag !== "StaleTarget" && !isBodyRejection(error), }), Effect.catch((error) => @@ -314,7 +314,12 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar }); /** Ships one instance's records from its cursor until the target changes. */ - const session = (instanceId: string, service: ShippedService, current: Target) => + const session = ( + instanceId: string, + service: ShippedService, + current: Target, + failing: Ref.Ref, + ) => Effect.scoped( Effect.gen(function* () { const directory = yield* options.logs.directory(instanceId); @@ -352,6 +357,7 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar for (const { body, last } of bodies(pending)) { yield* deliver(current, source, body); yield* Ref.set(cursor, last.position); + yield* Ref.set(failing, false); // Detaching waits for a started write instead of leaving files in the directory. yield* writeCursor(directory, last.position).pipe( Effect.catch((error) => @@ -378,15 +384,30 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar Effect.gen(function* () { while (true) { const current = yield* serving; - // A stopped target pauses shipping; the cursor keeps the position to resume from. - const detached = yield* session(instance.id, service, current).pipe( - Effect.as(true), - Effect.catch((error) => - (error._tag === "StaleTarget" + const failing = yield* Ref.make(false); + // A stopped or refused target pauses shipping until it changes; a failed log read resumes + // from the cursor after a backoff. + const detached = yield* session(instance.id, service, current, failing).pipe( + Effect.tapError((error) => + error._tag === "StaleTarget" ? Effect.void - : Effect.logWarning(`Log shipping of ${instance.id} paused`, error) - ).pipe(Effect.andThen(retargeted(current)), Effect.as(false)), + : Ref.getAndSet(failing, true).pipe( + Effect.flatMap((already) => + already + ? Effect.void + : Effect.logWarning( + `Reading ${instance.id} logs to ship failed; retrying`, + error, + ), + ), + ), ), + Effect.retry({ + schedule: retrySchedule, + while: (error) => error._tag !== "StaleTarget", + }), + Effect.as(true), + Effect.catch(() => retargeted(current).pipe(Effect.as(false))), Effect.raceFirst(retargeted(current).pipe(Effect.as(false))), ); if (detached) @@ -398,8 +419,14 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar const attach = Effect.fn("LogForwarder.attach")(function* (instance: ForwardedInstance) { if (instance.service === "analytics") yield* FiberMap.run(followers, instance.id, trackTarget(instance)); - else if (isShippedService(instance.service)) + else if (isShippedService(instance.service)) { + // A cursor write cut short by a crash leaves its temporary directory behind. + yield* options.logs.directory(instance.id).pipe( + Effect.flatMap((directory) => reapStaleWrites(fs, path, directory)), + Effect.ignore, + ); yield* FiberMap.run(followers, instance.id, forward(instance, instance.service)); + } }); return { diff --git a/packages/stack/src/host/LogflareEvents.ts b/packages/stack/src/host/LogflareEvents.ts index b591bbc00d..5cc8c68d32 100644 --- a/packages/stack/src/host/LogflareEvents.ts +++ b/packages/stack/src/host/LogflareEvents.ts @@ -100,7 +100,7 @@ const remaps: Record LogflareEvent> = status: Number(request[5]), }, }; - const match = /^(.*): (.*)$/u.exec(event.event_message); + const match = /^(.*?): (.*)$/u.exec(event.event_message); const timestamp = match === null ? undefined : parsePostgrestTime(match[1] ?? ""); return match === null || timestamp === undefined ? event diff --git a/packages/stack/src/host/LogflareEvents.unit.test.ts b/packages/stack/src/host/LogflareEvents.unit.test.ts index ebbf816ad6..158e51def6 100644 --- a/packages/stack/src/host/LogflareEvents.unit.test.ts +++ b/packages/stack/src/host/LogflareEvents.unit.test.ts @@ -33,6 +33,16 @@ it("splits PostgREST's timestamp prefix from the message and converts its zone o }); }); +it("splits PostgREST's timestamp prefix from a message that contains a colon", () => { + const event = logflareEvent( + "rest", + received, + "28/Sep/2026:12:30:15 +0200: Failed to load schema cache: connection refused", + ); + expect(event.event_message).toBe("Failed to load schema cache: connection refused"); + expect(event.timestamp).toBe("2026-09-28T10:30:15.000Z"); +}); + it("parses a PostgREST request line into request metadata stamped with the request time", () => { const line = '172.18.0.1 - anon [28/Sep/2026:12:30:15 +0200] "GET /items?select=id HTTP/1.1" 200 2 "" "curl/8.7.1"'; From 9903cc50c08cc880b92a81eb3ef260f914c31035 Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 2 Oct 2026 12:50:21 +0200 Subject: [PATCH 19/35] fix(stack): confirm log delivery in Analytics before advancing the cursor Analytics acknowledges a post before its batched insert, and one already stored id makes that insert drop the whole batch. Shipping now records the pending ids before each post, checks which are stored in Analytics' tables, posts only the missing ones, resends after the flush window, and advances the cursor only once every id is stored. Co-Authored-By: Claude Opus 5.5 --- .../experimental/stack/start/SIDE_EFFECTS.md | 8 +- packages/stack/ARCHITECTURE.md | 30 +- .../src/Owner.analytics.integration.test.ts | 35 +- packages/stack/src/Owner.ts | 28 + .../src/host/LogForwarder.integration.test.ts | 642 +++++++++++++----- packages/stack/src/host/LogForwarder.ts | 404 ++++++++--- packages/stack/src/host/LogflareStorage.ts | 110 +++ packages/stack/src/services/Analytics.ts | 7 +- packages/stack/tests/fake-logflare.ts | 159 +++++ 9 files changed, 1162 insertions(+), 261 deletions(-) create mode 100644 packages/stack/src/host/LogflareStorage.ts create mode 100644 packages/stack/tests/fake-logflare.ts diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index 06139e4f8d..cc196c8b1b 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -101,8 +101,12 @@ it is healthy again, including after an owner restart. A missing or unreadable p the oldest retained line. Lines already deleted by log retention are skipped, and Analytics refusing the API key pauses shipping until Analytics stops or turns unhealthy and is healthy again, or the composition selects a different Analytics instance. A failed log read is retried from the saved -position with a backoff. Each event carries an id derived from its instance and position, so a request that fails -after it may have been ingested is repeated without duplicating rows. Shipping never wakes Analytics +position with a backoff. Each event carries an id derived from its instance and position. Before +posting a request, the owner records it as pending in `cursor.json`; it then reads Analytics' +Postgres tables (`_analytics.sources` and `_analytics.log_events_` in the stack database) +to confirm which events are stored, posts only the missing ones, and advances the position once +all are stored. Events still missing 5 seconds after a request are posted again, and a pending +request is confirmed the same way after an owner restart or an Analytics restart. Shipping never wakes Analytics and does not count as idle activity, so a lazy Analytics still stops on its idle timer while other services log. Service log streams and `supabase stack logs` are never blocked by shipping. diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 00a0831773..16cd5e427b 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -667,14 +667,28 @@ While the composed Analytics instance is running and healthy, the owner ships th stdout/stderr records of Auth, REST, Realtime, Storage, Functions and database instances to its direct backend (never the proxy, so shipping neither wakes it nor counts as activity), in bodies of at most 256 events and 1 MiB. Each instance reads from `cursor.json` in its logs directory, the -position of its last shipped record, written atomically after each body settles: a missing cursor -or unreadable cursor starts at the oldest retained segment, and a cursor in a deleted segment -resumes at the next retained one. A failed log read restarts the instance from its cursor after a -capped backoff, and attaching an instance removes temporary cursor writes a dead owner left behind. -Bodies of an instance are sequential. Event ids -derive from the instance and record position, and Logflare keeps the first row per id, so a failed -body is posted again until it settles or the target changes. A 401, 403 or 404 response pauses the -instance with its cursor until the target changes; any other 4xx except 408 and 429 skips the body. The target is re-selected when the composition or Analytics' health changes. +position of its last confirmed record plus an optional pending body (its end position, source, +send time and event ids): a missing cursor or unreadable cursor starts at the oldest retained +segment, and a cursor in a deleted segment resumes at the next retained one. A failed log read +restarts the instance from its cursor after a capped backoff, and attaching an instance removes +temporary cursor writes a dead owner left behind. + +Bodies of an instance are sequential, and event ids derive from the instance and record position. +Logflare answers a post once it queued the events and stores them later in per-source batches; a +batch holding an id already stored is dropped whole. So delivery is confirmed in Analytics' +Postgres database: the owner reads which ids `_analytics.log_events_` holds, resolving the +token from `_analytics.sources`, through the database Analytics is composed with. The pending body +is written atomically before it is posted, and a failed write blocks posting. Only ids not yet +stored are posted. After a success, timeout, 5xx or interruption the owner polls the stored ids; +once all are stored the cursor advances and the pending body is cleared, and ids still missing 5 +seconds after a post are posted again. A start, restart or retarget reconciles a pending body +before later records; pending records already deleted by retention are skipped with a warning. A +401, 403 or 404 response clears the pending body and pauses the instance with its cursor until the +target changes; any other 4xx except 408 and 429 skips the body. A retarget waits for a post in +flight. The target is re-selected when the composition or Analytics' health changes. A batch that +Analytics commits more than 5 seconds after its post can still conflict with a repost, and the +check depends on Logflare's private table layout, which the owner's Analytics integration test +pins. Registry updates use an OS-backed lock through a private `node:sqlite` connection to `/.registry-lock.sqlite`. Each `withLock` call opens its own connection, disables diff --git a/packages/stack/src/Owner.analytics.integration.test.ts b/packages/stack/src/Owner.analytics.integration.test.ts index 61dfe7c031..c8ee96f37e 100644 --- a/packages/stack/src/Owner.analytics.integration.test.ts +++ b/packages/stack/src/Owner.analytics.integration.test.ts @@ -7,11 +7,13 @@ import { DateTime, Effect, Fiber, + FileSystem, Layer, Option, Path, Redacted, Schedule, + Schema, Stream, } from "effect"; import { HttpClient } from "effect/unstable/http"; @@ -41,6 +43,9 @@ const query = (url: string, statement: string) => }), ); +const Confirmed = Schema.Struct({ generation: Schema.Int, byteOffset: Schema.Int }); +const decodeConfirmed = Schema.decodeUnknownEffect(Schema.fromJsonString(Confirmed)); + interface StoredEvent { readonly message: string; readonly severity: string | null; @@ -53,6 +58,7 @@ it.live( () => Effect.scoped( Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; const client = yield* HttpClient.HttpClient; @@ -189,6 +195,27 @@ it.live( timestamp: expect.any(String), }, ]); + // The cursor passes a body only once shipping read its events back from Analytics' tables. + const awakeRecord = Option.getOrUndefined( + yield* owner.rpc.readLogs({ id: databaseId, follow: false }).pipe( + Stream.filter((record) => record.text?.includes(`LOG: ${awakeMarker}`) === true), + Stream.runHead, + ), + ); + const passedAwake = ({ generation, byteOffset }: typeof Confirmed.Type) => + awakeRecord?.position !== undefined && + (generation > awakeRecord.position.generation || + (generation === awakeRecord.position.generation && + byteOffset >= awakeRecord.position.byteOffset)); + const confirmed = yield* fs + .readFileString(path.join(state.logsRoot(stackId), "database", databaseId, "cursor.json")) + .pipe( + Effect.flatMap(decodeConfirmed), + Effect.filterOrFail(passedAwake), + Effect.retry(Schedule.spaced("250 millis")), + Effect.timeout("60 seconds"), + ); + expect(passedAwake(confirmed)).toBe(true); const restPath = `/${stackId}-probe`; yield* client.get(`${rest.url}${restPath}`); expect( @@ -216,9 +243,10 @@ it.live( Stream.runHead, Effect.forkScoped, ); + const raisedAt = DateTime.toEpochMillis(yield* DateTime.now); yield* raise(asleepMarker); const asleepRecord = Option.getOrUndefined(yield* Fiber.join(persistedAsleep)); - const wokeAt = DateTime.formatIso(yield* DateTime.now); + const wakeRequestedAt = DateTime.toEpochMillis(yield* DateTime.now); yield* wakeAnalytics; yield* Effect.forkScoped(keepAwake); @@ -227,7 +255,10 @@ it.live( `body->>'event_message' LIKE '%LOG: ${asleepMarker}'`, ); expect(shippedAsleep?.timestamp).toBe(asleepRecord?.timestamp); - expect((shippedAsleep?.timestamp ?? "") < wokeAt).toBe(true); + // The owner stamps a line with host time on arrival, possibly in the millisecond it is read. + const shippedAt = Date.parse(shippedAsleep?.timestamp ?? ""); + expect(shippedAt).toBeGreaterThanOrEqual(raisedAt); + expect(shippedAt).toBeLessThanOrEqual(wakeRequestedAt); }), ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), { timeout: 600_000 }, diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index fe47cc36fa..70a793e3d2 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -61,6 +61,7 @@ import { type ServiceCreationInput, } from "./services/Catalog.ts"; import type { CatalogError } from "./services/Recipe.ts"; +import { databaseConnection } from "./services/ServiceConfig.ts"; import * as Container from "./runtime/Container.ts"; import { projectSegmentFor } from "./identity/Identity.ts"; import { stackError, type OwnerRpc } from "./Rpc.ts"; @@ -68,6 +69,7 @@ import * as State from "./State.ts"; import type { SavedStack, StackCredentials, StackKeysInput } from "./State.ts"; import { makeDockerHelperRegistry } from "./storage/DockerHelperRegistry.ts"; import * as LogForwarder from "./host/LogForwarder.ts"; +import * as LogflareStorage from "./host/LogflareStorage.ts"; import * as LogStore from "./host/LogStore.ts"; export interface OwnerOptions { @@ -189,9 +191,35 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { const logStore = yield* LogStore.make({ root: options.state.logsRoot(options.saved.id) }).pipe( Effect.provideContext(services), ); + /** The database Analytics stores events in, on its direct backend, with Analytics' credentials. */ + const analyticsDatabase = (analyticsId: string) => + Effect.gen(function* () { + const creation = yield* Ref.get((yield* orchestrator.get(analyticsId)).creation); + const databaseUrl = + creation.service === "analytics" ? creation.config.databaseUrl : undefined; + const dependency = (yield* orchestrator.composition).dependencies.find( + ({ to, bindings }) => + to === analyticsId && bindings?.some(({ input }) => input === "databaseUrl") === true, + ); + if (databaseUrl === undefined || dependency === undefined) + return yield* new ServiceError({ + operation: "analytics", + message: "Analytics has no composed database", + }); + const connection = yield* databaseConnection(databaseUrl); + const endpoint = yield* (yield* orchestrator.get(dependency.from)).recipe.endpoint("sql"); + return { + host: (endpoint.kind === "unix" ? endpoint.path : endpoint.host) ?? "127.0.0.1", + port: endpoint.port, + database: connection.database, + username: connection.username, + password: connection.password, + } satisfies LogflareStorage.AnalyticsDatabase; + }); const forwarder = yield* LogForwarder.make({ composition: orchestrator.composition, logs: logStore, + storedEvents: (analytics) => LogflareStorage.make(analyticsDatabase(analytics.id)), }).pipe(Effect.provideContext(services)); const definitionGate = yield* Semaphore.make(1); const draining = yield* Ref.make(false); diff --git a/packages/stack/src/host/LogForwarder.integration.test.ts b/packages/stack/src/host/LogForwarder.integration.test.ts index 650fbcb2fa..4e1ff18c33 100644 --- a/packages/stack/src/host/LogForwarder.integration.test.ts +++ b/packages/stack/src/host/LogForwarder.integration.test.ts @@ -1,9 +1,9 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { describe, expect, it } from "@effect/vitest"; -import { createServer } from "node:http"; // oxlint-disable-line effecttsgo/node-builtin-import -- real socket fixture. import { Clock, Deferred, + Duration, Effect, Exit, Fiber, @@ -11,15 +11,21 @@ import { Layer, Logger, Path, + PlatformError, PubSub, Queue, Ref, References, - Schema, Scope, Stream, SubscriptionRef, } from "effect"; +import { + makeFakeLogflare, + type FakeLogflare, + type FakeLogflareOptions, + type Received, +} from "../../tests/fake-logflare.ts"; import type { ServiceObservation } from "../Service.ts"; import type { LaunchOutput } from "../runtime/Session.ts"; import { CatalogError } from "../services/Recipe.ts"; @@ -27,73 +33,77 @@ import type { LogRecord } from "./LogRecord.ts"; import * as LogForwarder from "./LogForwarder.ts"; import * as LogStore from "./LogStore.ts"; -const Body = Schema.Struct({ - batch: Schema.Array( - Schema.Struct({ - id: Schema.String, - event_message: Schema.String, - appname: Schema.String, - timestamp: Schema.String, - metadata: Schema.Record(Schema.String, Schema.Unknown), - }), - ), -}); -type Event = Schema.Schema.Type["batch"][number]; - -interface Received { - readonly url: string; - readonly apiKey: string | undefined; - readonly events: ReadonlyArray; -} - -const decodeBody = Schema.decodeUnknownSync(Body); - -/** A Logflare stand-in that records posts and answers the next `failures` of them with `status`. */ -const makeSink = Effect.gen(function* () { - const received = yield* Queue.unbounded(); - const control = { failures: 0, status: 500 }; - const server = yield* Effect.acquireRelease( - Effect.callback<{ readonly port: number; readonly server: ReturnType }>( - (resume) => { - const server = createServer((request, response) => { - const chunks: Array = []; - request.on("data", (chunk: Uint8Array) => chunks.push(chunk)); - request.on("end", () => { - const apiKey = request.headers["x-api-key"]; - Queue.offerUnsafe(received, { - url: request.url ?? "", - apiKey: typeof apiKey === "string" ? apiKey : undefined, - events: decodeBody(JSON.parse(new TextDecoder().decode(Buffer.concat(chunks)))).batch, - }); - response.statusCode = control.failures > 0 ? control.status : 200; - control.failures = Math.max(0, control.failures - 1); - response.end(); - }); - }); - server.listen(0, "127.0.0.1", () => { - const address = server.address(); - if (address === null || typeof address === "string") - return resume(Effect.die("sink has no address")); - resume(Effect.succeed({ port: address.port, server })); - }); - }, - ), - ({ server }) => - Effect.callback((resume) => { - server.closeAllConnections(); - server.close(() => resume(Effect.void)); +const { flushWindowMillis, pollMillis } = LogForwarder; + +/** A clock that moves only when the test advances it; `sleeping` waits for a pending sleep. */ +const makeManualClock = Effect.gen(function* () { + let now = yield* Clock.currentTimeMillis; + const sleeps = new Set<{ + readonly millis: number; + readonly until: number; + readonly woken: Deferred.Deferred; + }>(); + const waiters = new Set<{ readonly millis: number; readonly ready: Deferred.Deferred }>(); + const nanos = () => BigInt(now) * 1_000_000n; + const clock: Clock.Clock = { + currentTimeMillisUnsafe: () => now, + currentTimeMillis: Effect.sync(() => now), + currentTimeNanosUnsafe: nanos, + currentTimeNanos: Effect.sync(nanos), + monotonicTimeNanosUnsafe: nanos, + monotonicTimeNanos: Effect.sync(nanos), + sleep: (duration) => + Effect.suspend(() => { + const millis = Duration.toMillis(duration); + if (millis <= 0) return Effect.void; + const entry = { millis, until: now + millis, woken: Deferred.makeUnsafe() }; + sleeps.add(entry); + for (const waiter of waiters) + if (waiter.millis === millis) { + waiters.delete(waiter); + Deferred.doneUnsafe(waiter.ready, Effect.void); + } + return Deferred.await(entry.woken).pipe( + Effect.ensuring(Effect.sync(() => sleeps.delete(entry))), + ); }), - ); + }; return { - port: server.port, - next: Queue.take(received), - failNext: (count: number, status = 500) => + clock, + sleeping: (millis: number) => + Effect.suspend(() => { + if ([...sleeps].some((entry) => entry.millis === millis)) return Effect.void; + const waiter = { millis, ready: Deferred.makeUnsafe() }; + waiters.add(waiter); + return Deferred.await(waiter.ready).pipe( + Effect.ensuring(Effect.sync(() => waiters.delete(waiter))), + ); + }), + advance: (millis: number) => Effect.sync(() => { - control.failures = count; - control.status = status; + now += millis; + for (const entry of sleeps) + if (entry.until <= now) { + sleeps.delete(entry); + Deferred.doneUnsafe(entry.woken, Effect.void); + } }), }; }); +type ManualClock = Effect.Success; + +/** Takes the next post, running each stored-id poll the forwarder waits on until one arrives. */ +const nextPost = (logflare: FakeLogflare, manual: ManualClock): Effect.Effect => + logflare.next.pipe( + Effect.raceFirst( + manual + .sleeping(pollMillis) + .pipe( + Effect.andThen(manual.advance(pollMillis)), + Effect.andThen(Effect.suspend(() => nextPost(logflare, manual))), + ), + ), + ); const observationOf = ( id: string, @@ -165,10 +175,11 @@ const databaseSource = (store: LogStore.Interface) => }; }); -const analyticsTarget = (port: number) => +const analyticsTarget = (port: Effect.Effect) => Effect.gen(function* () { const observation = yield* SubscriptionRef.make(observationOf("analytics", {})); return { + observation, set: (awake: boolean) => SubscriptionRef.set( observation, @@ -177,7 +188,14 @@ const analyticsTarget = (port: number) => instance: { id: "analytics", service: "analytics" as const, - endpoint: () => Effect.succeed({ kind: "tcp" as const, host: "127.0.0.1" as const, port }), + endpoint: () => + port.pipe( + Effect.map((current) => ({ + kind: "tcp" as const, + host: "127.0.0.1" as const, + port: current, + })), + ), creation: Effect.succeed({ service: "analytics" as const, config: { apiKey: "test-key" } }), observation: SubscriptionRef.changes(observation), } satisfies LogForwarder.ForwardedInstance, @@ -195,15 +213,19 @@ const composition = Effect.succeed({ /** Starts a forwarder in its own scope so a test can stop it like an owner. */ const startForwarder = ( store: LogStore.Interface, + logflare: FakeLogflare, instances: ReadonlyArray, + clock?: Clock.Clock, ) => Effect.gen(function* () { const scope = yield* Scope.make(); // Stops before the test's store and temp directory close, so no cursor write races removal. yield* Effect.addFinalizer(() => Scope.close(scope, Exit.void)); - const forwarder = yield* LogForwarder.make({ composition, logs: store }).pipe( - Scope.provide(scope), - ); + const forwarder = yield* LogForwarder.make({ + composition, + logs: store, + storedEvents: () => logflare.storedEvents, + }).pipe(Scope.provide(scope)); for (const instance of instances) yield* forwarder.attach(instance); yield* forwarder.rebind; const awaitShipping = (value: boolean) => @@ -213,20 +235,29 @@ const startForwarder = ( Stream.runDrain, ); return { awaitShipping, detach: forwarder.detach, stop: Scope.close(scope, Exit.void) }; - }); - -const fixture = (options: Omit = {}) => + }).pipe(clock === undefined ? (effect) => effect : Effect.provideService(Clock.Clock, clock)); + +/** Without `logflare` options, the fake stores each accepted post right away. */ +const fixture = ( + options: { + readonly store?: Omit; + readonly logflare?: FakeLogflareOptions; + } = {}, +) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const root = yield* fs.makeTempDirectoryScoped({ prefix: "log-forwarder-" }); - const store = yield* LogStore.make({ root, ...options }); - const sink = yield* makeSink; + const store = yield* LogStore.make({ root, ...options.store }); + const logflare = yield* makeFakeLogflare(options.logflare ?? { flushMillis: 0 }); const database = yield* databaseSource(store); - const analytics = yield* analyticsTarget(sink.port); - return { root, store, sink, database, analytics }; + const analytics = yield* analyticsTarget(Effect.succeed(logflare.port)); + return { root, store, logflare, database, analytics }; }); const messages = (post: Received) => post.events.map((event) => event.event_message); +const ids = (post: Received) => post.events.map((event) => event.id); +const storedMessages = (logflare: FakeLogflare) => + logflare.stored.pipe(Effect.map((events) => events.map((event) => event.event_message))); const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); @@ -235,15 +266,18 @@ describe("LogForwarder", () => { "ships records written while Analytics slept with their timestamps, without markers", () => Effect.gen(function* () { - const { root, store, sink, database, analytics } = yield* fixture(); - const forwarder = yield* startForwarder(store, [analytics.instance, database.instance]); + const { root, store, logflare, database, analytics } = yield* fixture(); + const forwarder = yield* startForwarder(store, logflare, [ + analytics.instance, + database.instance, + ]); yield* database.log("while asleep 1", "while asleep 2"); yield* analytics.set(true); yield* forwarder.awaitShipping(true); - const caughtUp = yield* sink.next; + const caughtUp = yield* logflare.next; yield* database.log("while awake"); - const live = yield* sink.next; + const live = yield* logflare.next; expect(caughtUp.url).toBe("/api/logs?source_name=postgres.logs"); expect(caughtUp.apiKey).toBe("test-key"); @@ -255,17 +289,76 @@ describe("LogForwarder", () => { expect([...caughtUp.events, ...live.events].map((event) => event.timestamp)).toEqual( persisted.map((record) => record.timestamp), ); - const ids = [...caughtUp.events, ...live.events].map((event) => event.id); - expect(new Set(ids).size).toBe(3); - expect(ids.every((id) => /^[0-9a-f]{8}-[0-9a-f]{4}-8[0-9a-f]{3}-[89ab]/u.test(id))).toBe( - true, + const shipped = [...ids(caughtUp), ...ids(live)]; + expect(new Set(shipped).size).toBe(3); + expect( + shipped.every((id) => /^[0-9a-f]{8}-[0-9a-f]{4}-8[0-9a-f]{3}-[89ab]/u.test(id)), + ).toBe(true); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live( + "ships a record persisted while Analytics slept once it wakes on a new port, without repeating the post it let finish", + () => + Effect.gen(function* () { + const { store, logflare, database } = yield* fixture({ logflare: {} }); + const manual = yield* makeManualClock; + const firstLaunch = yield* Scope.make(); + const asleepPort = yield* logflare.listen(firstLaunch); + const port = yield* Ref.make(asleepPort); + const analytics = yield* analyticsTarget(Ref.get(port)); + yield* analytics.set(true); + const forwarder = yield* startForwarder( + store, + logflare, + [analytics.instance, database.instance], + manual.clock, + ); + const release = yield* logflare.hold; + yield* database.log("while awake"); + const beforeSleep = yield* logflare.next; + + yield* SubscriptionRef.set( + analytics.observation, + observationOf("analytics", { lifecycle: "stopping", launchId: 1 }), ); + yield* forwarder.awaitShipping(false); + yield* release; + yield* logflare.apply(); + yield* Scope.close(firstLaunch, Exit.void); + yield* SubscriptionRef.set(analytics.observation, observationOf("analytics", {})); + yield* database.log("while asleep"); + yield* Ref.set(port, logflare.port); + yield* logflare.respond(503); + yield* SubscriptionRef.set( + analytics.observation, + observationOf("analytics", { lifecycle: "starting", health: "starting", launchId: 2 }), + ); + yield* SubscriptionRef.set( + analytics.observation, + observationOf("analytics", { lifecycle: "running", health: "starting", launchId: 2 }), + ); + yield* SubscriptionRef.set(analytics.observation, { ...serving("analytics"), launchId: 2 }); + const unavailable = yield* logflare.next; + yield* manual.sleeping(pollMillis); + yield* manual.advance(flushWindowMillis); + const caughtUp = yield* logflare.next; + yield* logflare.apply(); + + expect(beforeSleep.port).toBe(asleepPort); + expect(messages(beforeSleep)).toEqual(["while awake"]); + expect(yield* logflare.aborted).toBe(0); + expect(unavailable.port).toBe(logflare.port); + expect(messages(unavailable)).toEqual(["while asleep"]); + expect(ids(caughtUp)).toEqual(ids(unavailable)); + expect(yield* storedMessages(logflare)).toEqual(["while awake", "while asleep"]); + expect(yield* logflare.dropped).toBe(0); }).pipe(Effect.scoped, Effect.provide(layer)), ); it.live("stops shipping quietly when the log store closes", () => Effect.gen(function* () { - const { store, sink, database, analytics } = yield* fixture(); + const { store, logflare, database, analytics } = yield* fixture(); const warnings: Array = []; const stopped = yield* Deferred.make(); const captured = Logger.layer([ @@ -275,14 +368,14 @@ describe("LogForwarder", () => { Deferred.doneUnsafe(stopped, Effect.void); }), ]); - const forwarder = yield* startForwarder(store, [analytics.instance, database.instance]).pipe( - Effect.provide(captured), - Effect.provideService(References.MinimumLogLevel, "Debug"), - ); + const forwarder = yield* startForwarder(store, logflare, [ + analytics.instance, + database.instance, + ]).pipe(Effect.provide(captured), Effect.provideService(References.MinimumLogLevel, "Debug")); yield* analytics.set(true); yield* forwarder.awaitShipping(true); yield* database.log("before removal"); - const shipped = yield* sink.next; + const shipped = yield* logflare.next; yield* store.close; yield* Deferred.await(stopped); @@ -295,7 +388,7 @@ describe("LogForwarder", () => { it.live("ships again after a failed log read while Analytics stays healthy", () => Effect.gen(function* () { - const { store, sink, database, analytics } = yield* fixture(); + const { store, logflare, database, analytics } = yield* fixture(); const reads = yield* Ref.make(0); const flaky: LogStore.Interface = { ...store, @@ -321,10 +414,10 @@ describe("LogForwarder", () => { yield* database.log("after a failed read"); yield* analytics.set(true); - yield* startForwarder(flaky, [analytics.instance, database.instance]).pipe( + yield* startForwarder(flaky, logflare, [analytics.instance, database.instance]).pipe( Effect.provide(captured), ); - const shipped = yield* sink.next; + const shipped = yield* logflare.next; expect(messages(shipped)).toEqual(["after a failed read"]); expect(yield* Ref.get(reads)).toBe(2); @@ -338,7 +431,7 @@ describe("LogForwarder", () => { Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const { root, store, database, analytics } = yield* fixture(); + const { root, store, logflare, database, analytics } = yield* fixture(); const directory = path.join(root, "database", "database"); const stale = path.join(directory, ".state-write-stale"); const recent = path.join(directory, ".state-write-recent"); @@ -348,7 +441,7 @@ describe("LogForwarder", () => { const twoDaysAgo = (yield* Clock.currentTimeMillis) / 1000 - 2 * 24 * 60 * 60; yield* fs.utimes(stale, twoDaysAgo, twoDaysAgo); - yield* startForwarder(store, [analytics.instance, database.instance]); + yield* startForwarder(store, logflare, [analytics.instance, database.instance]); expect(yield* fs.exists(stale)).toBe(false); expect(yield* fs.exists(recent)).toBe(true); @@ -359,7 +452,7 @@ describe("LogForwarder", () => { Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const { root, store, sink, database, analytics } = yield* fixture(); + const { root, store, logflare, database, analytics } = yield* fixture(); const renaming = yield* Deferred.make(); const release = yield* Deferred.make(); const injected: FileSystem.FileSystem = { @@ -372,13 +465,13 @@ describe("LogForwarder", () => { ) : fs.rename(from, to), }; - const forwarder = yield* startForwarder(store, [analytics.instance, database.instance]).pipe( - Effect.provideService(FileSystem.FileSystem, injected), - ); + const forwarder = yield* startForwarder(store, logflare, [ + analytics.instance, + database.instance, + ]).pipe(Effect.provideService(FileSystem.FileSystem, injected)); yield* analytics.set(true); yield* forwarder.awaitShipping(true); yield* database.log("before removal"); - yield* sink.next; yield* Deferred.await(renaming); const directory = path.join(root, "database", "database"); @@ -397,85 +490,331 @@ describe("LogForwarder", () => { it.live("resumes from its persisted cursor after the forwarder restarts", () => Effect.gen(function* () { - const { store, sink, database, analytics } = yield* fixture(); + const { store, logflare, database, analytics } = yield* fixture(); yield* analytics.set(true); - const first = yield* startForwarder(store, [analytics.instance, database.instance]); + const first = yield* startForwarder(store, logflare, [analytics.instance, database.instance]); yield* database.log("line 1", "line 2"); - const shipped = yield* sink.next; - // Shipping line 3 implies the cursor passed lines 1 and 2 before it. + const shipped = yield* logflare.next; yield* database.log("line 3"); - const third = yield* sink.next; + const third = yield* logflare.next; yield* first.stop; yield* database.log("line 4"); - yield* startForwarder(store, [analytics.instance, database.instance]); - const resumed: Array = []; - while (resumed.at(-1)?.event_message !== "line 4") resumed.push(...(yield* sink.next).events); + yield* startForwarder(store, logflare, [analytics.instance, database.instance]); + const resumed = yield* logflare.next; expect(messages(shipped)).toEqual(["line 1", "line 2"]); expect(messages(third)).toEqual(["line 3"]); - expect(["line 3", "line 4"]).toEqual( - expect.arrayContaining(resumed.map((e) => e.event_message)), - ); - const repeated = resumed.find((event) => event.event_message === "line 3"); - if (repeated !== undefined) expect(repeated.id).toBe(third.events[0]?.id); + expect(messages(resumed)).toEqual(["line 4"]); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live( + "does not post an accepted body again after an interruption before it was confirmed", + () => + Effect.gen(function* () { + const { store, logflare, database, analytics } = yield* fixture({ logflare: {} }); + const manual = yield* makeManualClock; + const instances = [analytics.instance, database.instance]; + yield* analytics.set(true); + const first = yield* startForwarder(store, logflare, instances, manual.clock); + const release = yield* logflare.hold; + yield* database.log("line 1"); + const accepted = yield* logflare.next; + + const stopping = yield* Effect.forkChild(first.stop, { startImmediately: true }); + yield* release; + yield* Fiber.join(stopping); + yield* logflare.apply(); + yield* startForwarder(store, logflare, instances, manual.clock); + yield* database.log("line 2"); + const resumed = yield* logflare.next; + yield* logflare.apply(); + + expect(messages(accepted)).toEqual(["line 1"]); + expect(messages(resumed)).toEqual(["line 2"]); + expect(yield* storedMessages(logflare)).toEqual(["line 1", "line 2"]); + expect(yield* logflare.dropped).toBe(0); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("does not post stored events again when its cursor is lost", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const { root, store, logflare, database, analytics } = yield* fixture({ logflare: {} }); + const instances = [analytics.instance, database.instance]; + yield* analytics.set(true); + const first = yield* startForwarder(store, logflare, instances); + yield* database.log("line 1"); + const shipped = yield* logflare.next; + yield* logflare.apply(); + yield* first.stop; + yield* fs.remove(path.join(root, "database", "database", "cursor.json"), { force: true }); + + yield* startForwarder(store, logflare, instances); + yield* database.log("line 2"); + const resumed = yield* logflare.next; + + expect(messages(shipped)).toEqual(["line 1"]); + expect(messages(resumed)).toEqual(["line 2"]); }).pipe(Effect.scoped, Effect.provide(layer)), ); - it.live("posts an unsettled body again before any later body", () => + it.live("posts a body Analytics failed again before any later body", () => Effect.gen(function* () { - const { store, sink, database, analytics } = yield* fixture(); + const { store, logflare, database, analytics } = yield* fixture({ logflare: {} }); + const manual = yield* makeManualClock; yield* analytics.set(true); - yield* startForwarder(store, [analytics.instance, database.instance]); - yield* sink.failNext(1); + yield* startForwarder(store, logflare, [analytics.instance, database.instance], manual.clock); + yield* logflare.respond(500); const lines = Array.from({ length: 300 }, (_, index) => `burst ${index}`); yield* database.log(...lines); - const failed = yield* sink.next; - const retried = yield* sink.next; - const shipped = [...retried.events]; - while (shipped.length < lines.length) shipped.push(...(yield* sink.next).events); + const failed = yield* logflare.next; + yield* manual.sleeping(pollMillis); + yield* manual.advance(flushWindowMillis); + const retried = yield* logflare.next; + yield* logflare.apply(); + const rest = yield* nextPost(logflare, manual); + + expect(ids(retried)).toEqual(ids(failed)); + expect([...messages(retried), ...messages(rest)]).toEqual(lines); + expect(failed.events.length).toBeLessThanOrEqual(256); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); - expect(retried.events.map((event) => event.id)).toEqual( - failed.events.map((event) => event.id), + it.live("posts the events of a batch Analytics dropped again once the flush window passes", () => + Effect.gen(function* () { + const { store, logflare, database, analytics } = yield* fixture({ logflare: {} }); + const manual = yield* makeManualClock; + yield* analytics.set(true); + yield* startForwarder(store, logflare, [analytics.instance, database.instance], manual.clock); + yield* database.log("line 1", "line 2"); + const posted = yield* logflare.next; + yield* logflare.discard; + + yield* manual.sleeping(pollMillis); + yield* manual.advance(flushWindowMillis); + const resent = yield* logflare.next; + yield* logflare.apply(); + yield* database.log("line 3"); + const later = yield* nextPost(logflare, manual); + + expect(ids(resent)).toEqual(ids(posted)); + expect(messages(later)).toEqual(["line 3"]); + expect(yield* storedMessages(logflare)).toEqual(["line 1", "line 2"]); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("posts again only the events Analytics did not store", () => + Effect.gen(function* () { + const { store, logflare, database, analytics } = yield* fixture({ + logflare: { batchEvents: 2 }, + }); + const manual = yield* makeManualClock; + yield* analytics.set(true); + yield* startForwarder(store, logflare, [analytics.instance, database.instance], manual.clock); + yield* database.log("line 1", "line 2", "line 3", "line 4"); + const posted = yield* logflare.next; + yield* logflare.apply(1); + yield* logflare.discard; + + yield* manual.sleeping(pollMillis); + yield* manual.advance(flushWindowMillis); + const resent = yield* logflare.next; + yield* logflare.apply(); + + expect(messages(posted)).toEqual(["line 1", "line 2", "line 3", "line 4"]); + expect(messages(resent)).toEqual(["line 3", "line 4"]); + expect(yield* storedMessages(logflare)).toEqual(["line 1", "line 2", "line 3", "line 4"]); + expect(yield* logflare.dropped).toBe(0); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live( + "waits for a post that timed out after Analytics queued it instead of posting it again", + () => + Effect.gen(function* () { + const { store, logflare, database, analytics } = yield* fixture({ logflare: {} }); + const manual = yield* makeManualClock; + yield* analytics.set(true); + yield* startForwarder( + store, + logflare, + [analytics.instance, database.instance], + manual.clock, + ); + const release = yield* logflare.hold; + yield* database.log("line 1"); + const queued = yield* logflare.next; + + yield* manual.sleeping(5_000); + yield* manual.advance(5_000); + yield* manual.sleeping(pollMillis); + yield* logflare.apply(); + yield* database.log("line 2"); + const later = yield* nextPost(logflare, manual); + yield* release; + + expect(messages(queued)).toEqual(["line 1"]); + expect(messages(later)).toEqual(["line 2"]); + expect(yield* logflare.dropped).toBe(0); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("posts nothing while it cannot save its cursor", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const { store, logflare, database, analytics } = yield* fixture({ logflare: {} }); + const manual = yield* makeManualClock; + const broken = yield* Ref.make(true); + const injected: FileSystem.FileSystem = { + ...fs, + rename: (from, to) => + Ref.get(broken).pipe( + Effect.flatMap((failing) => + failing && to.endsWith("cursor.json") + ? Effect.fail( + PlatformError.systemError({ + _tag: "Unknown", + module: "FileSystem", + method: "rename", + pathOrDescriptor: to, + description: "injected cursor write failure", + }), + ) + : fs.rename(from, to), + ), + ), + }; + yield* analytics.set(true); + yield* startForwarder( + store, + logflare, + [analytics.instance, database.instance], + manual.clock, + ).pipe(Effect.provideService(FileSystem.FileSystem, injected)); + + yield* database.log("line 1"); + const first = yield* logflare.next.pipe( + Effect.as("posted"), + Effect.raceFirst(manual.sleeping(250).pipe(Effect.as("retrying"))), + ); + yield* Ref.set(broken, false); + yield* manual.advance(250); + const posted = yield* logflare.next; + + expect(first).toBe("retrying"); + expect(messages(posted)).toEqual(["line 1"]); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("posts a pending body again after a restart once the flush window passes", () => + Effect.gen(function* () { + const { store, logflare, database, analytics } = yield* fixture({ logflare: {} }); + const manual = yield* makeManualClock; + const instances = [analytics.instance, database.instance]; + yield* analytics.set(true); + const first = yield* startForwarder(store, logflare, instances, manual.clock); + yield* database.log("line 1"); + const posted = yield* logflare.next; + yield* logflare.discard; + yield* manual.sleeping(pollMillis); + yield* first.stop; + + yield* startForwarder(store, logflare, instances, manual.clock); + yield* manual.sleeping(pollMillis); + yield* manual.advance(flushWindowMillis); + const resent = yield* logflare.next; + + expect(ids(resent)).toEqual(ids(posted)); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("warns about pending events retention deleted and ships the retained records", () => + Effect.gen(function* () { + const { root, store, logflare, database, analytics } = yield* fixture({ + store: { rotateBytes: 200, retainBytes: 400 }, + logflare: {}, + }); + const manual = yield* makeManualClock; + const instances = [analytics.instance, database.instance]; + const warnings: Array = []; + const captured = Logger.layer([ + Logger.make(({ logLevel, message }) => { + if (logLevel === "Warn") warnings.push(message); + }), + ]); + yield* analytics.set(true); + const first = yield* startForwarder(store, logflare, instances, manual.clock); + yield* database.log("first"); + yield* logflare.next; + yield* logflare.discard; + yield* manual.sleeping(pollMillis); + yield* first.stop; + for (let index = 0; index < 30; index++) yield* database.log(`later ${index}`); + const retained = (yield* LogStore.streamStackLogs({ root }).pipe(Stream.runCollect)).filter( + (record) => record.kind === "stdout", ); - expect(shipped.map((event) => event.event_message)).toEqual(lines); - expect(Math.max(failed.events.length, retried.events.length)).toBeLessThanOrEqual(256); + + yield* startForwarder(store, logflare, instances, manual.clock).pipe( + Effect.provide(captured), + ); + yield* manual.sleeping(pollMillis); + yield* manual.advance(flushWindowMillis); + const shipped: Array = []; + while (shipped.at(-1) !== "later 29") { + shipped.push(...messages(yield* nextPost(logflare, manual))); + yield* logflare.apply(); + } + + expect(retained.map((record) => record.text)).not.toContain("first"); + expect(shipped).toEqual(retained.map((record) => record.text)); + expect(warnings).toEqual([ + [ + "Logs of database instance database were deleted before Analytics stored them; skipping 1 lines", + ], + [ + "Logs of database instance database were deleted before shipping; resuming from the next retained record", + ], + ]); }).pipe(Effect.scoped, Effect.provide(layer)), ); it.live("pauses without advancing when Analytics refuses its credentials", () => Effect.gen(function* () { - const { store, sink, database, analytics } = yield* fixture(); + const { store, logflare, database, analytics } = yield* fixture(); yield* analytics.set(true); - const forwarder = yield* startForwarder(store, [analytics.instance, database.instance]); - yield* sink.failNext(1, 401); + const forwarder = yield* startForwarder(store, logflare, [ + analytics.instance, + database.instance, + ]); + yield* logflare.respond(401); yield* database.log("refused line"); - const refused = yield* sink.next; + const refused = yield* logflare.next; yield* analytics.set(false); yield* forwarder.awaitShipping(false); yield* analytics.set(true); - const resumed = yield* sink.next; + const resumed = yield* logflare.next; expect(messages(refused)).toEqual(["refused line"]); - expect(resumed.events.map((event) => event.id)).toEqual( - refused.events.map((event) => event.id), - ); + expect(ids(resumed)).toEqual(ids(refused)); }).pipe(Effect.scoped, Effect.provide(layer)), ); it.live("skips a body Analytics rejects and ships the next one", () => Effect.gen(function* () { - const { store, sink, database, analytics } = yield* fixture(); + const { store, logflare, database, analytics } = yield* fixture(); yield* analytics.set(true); - yield* startForwarder(store, [analytics.instance, database.instance]); - yield* sink.failNext(1, 400); + yield* startForwarder(store, logflare, [analytics.instance, database.instance]); + yield* logflare.respond(400); yield* database.log("rejected line"); - const rejected = yield* sink.next; + const rejected = yield* logflare.next; yield* database.log("accepted line"); - const accepted = yield* sink.next; + const accepted = yield* logflare.next; expect(messages(rejected)).toEqual(["rejected line"]); expect(messages(accepted)).toEqual(["accepted line"]); @@ -486,13 +825,13 @@ describe("LogForwarder", () => { Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const { root, store, sink, database, analytics } = yield* fixture(); + const { root, store, logflare, database, analytics } = yield* fixture(); yield* database.log("before a corrupt cursor"); yield* fs.writeFileString(path.join(root, "database", "database", "cursor.json"), "{oops"); yield* analytics.set(true); - yield* startForwarder(store, [analytics.instance, database.instance]); - const shipped = yield* sink.next; + yield* startForwarder(store, logflare, [analytics.instance, database.instance]); + const shipped = yield* logflare.next; expect(messages(shipped)).toEqual(["before a corrupt cursor"]); }).pipe(Effect.scoped, Effect.provide(layer)), @@ -500,16 +839,18 @@ describe("LogForwarder", () => { it.live("restarts from the oldest retained segment when its cursor's segment was deleted", () => Effect.gen(function* () { - const { root, store, sink, database, analytics } = yield* fixture({ - rotateBytes: 200, - retainBytes: 400, + const { root, store, logflare, database, analytics } = yield* fixture({ + store: { rotateBytes: 200, retainBytes: 400 }, }); yield* analytics.set(true); - const forwarder = yield* startForwarder(store, [analytics.instance, database.instance]); + const forwarder = yield* startForwarder(store, logflare, [ + analytics.instance, + database.instance, + ]); yield* database.log("first"); - yield* sink.next; + yield* logflare.next; yield* database.log("second"); - yield* sink.next; + yield* logflare.next; yield* analytics.set(false); yield* forwarder.awaitShipping(false); for (let index = 0; index < 30; index++) yield* database.log(`asleep ${index}`); @@ -518,14 +859,11 @@ describe("LogForwarder", () => { ); yield* analytics.set(true); - const shipped: Array = []; - while (shipped.at(-1)?.event_message !== "asleep 29") - shipped.push(...(yield* sink.next).events); + const shipped: Array = []; + while (shipped.at(-1) !== "asleep 29") shipped.push(...messages(yield* logflare.next)); expect(retained.map((record) => record.text)).not.toContain("second"); - expect(shipped.map((event) => event.event_message)).toEqual( - retained.map((record) => record.text), - ); + expect(shipped).toEqual(retained.map((record) => record.text)); }).pipe(Effect.scoped, Effect.provide(layer)), ); }); diff --git a/packages/stack/src/host/LogForwarder.ts b/packages/stack/src/host/LogForwarder.ts index d9f7e93d0a..63cc0762b6 100644 --- a/packages/stack/src/host/LogForwarder.ts +++ b/packages/stack/src/host/LogForwarder.ts @@ -1,8 +1,10 @@ import { + Clock, Crypto, Data, Duration, Effect, + Exit, FiberMap, FileSystem, Option, @@ -10,6 +12,7 @@ import { Ref, Schedule, Schema, + Scope, Semaphore, Stream, SubscriptionRef, @@ -39,6 +42,20 @@ export interface ForwardedInstance { readonly observation: Stream.Stream>; } +export class StoredEventsError extends Schema.TaggedError()( + "StoredEventsError", + { message: Schema.String, cause: Schema.optionalKey(Schema.Defect()) }, +) {} + +/** Which shipped events an Analytics instance has stored. */ +export interface StoredEvents { + /** The subset of `ids` stored for `source`. */ + readonly storedIds: ( + source: string, + ids: ReadonlyArray, + ) => Effect.Effect, StoredEventsError>; +} + interface Interface { /** Ships a shipped service's persisted logs, or tracks an Analytics instance as the target. */ readonly attach: (instance: ForwardedInstance) => Effect.Effect; @@ -53,6 +70,10 @@ interface Interface { export interface LogForwarderOptions { readonly composition: Effect.Effect; readonly logs: LogStore.Interface; + /** Opens the stored-event view of an Analytics target; its scope closes when the target changes. */ + readonly storedEvents: ( + analytics: ForwardedInstance, + ) => Effect.Effect; } interface Candidate { @@ -63,48 +84,110 @@ interface Candidate { interface Target { readonly instance: ForwardedInstance; readonly epoch: number; + readonly stored: StoredEvents; } -/** A shipped event; its id makes a repeated post of the same record a no-op in Logflare. */ +/** A shipped event; its id derives from the instance and record position. */ interface ShippedEvent extends LogflareEvent { readonly id: string; } class StaleTarget extends Data.TaggedError("StaleTarget")<{}> {} +/** A body handed to Analytics whose events are not all confirmed stored yet. */ +const Pending = Schema.Struct({ + ...LogPosition.fields, + source: Schema.String, + sentAt: Schema.Finite, + ids: Schema.Array(Schema.String), +}); +interface Pending extends Schema.Schema.Type {} + +/** The last confirmed position, if any, and the pending body after it. */ +const Cursor = Schema.Union([ + Schema.Struct({ ...LogPosition.fields, pending: Schema.optionalKey(Pending) }), + Schema.Struct({ pending: Pending }), +]); +type Cursor = Schema.Schema.Type; + +interface Progress { + readonly confirmed: LogPosition | undefined; + readonly pending: Pending | undefined; +} + +/** One instance's shipping against one target. */ +interface Session { + readonly instanceId: string; + readonly service: ShippedService; + readonly directory: string; + readonly current: Target; + readonly progress: Ref.Ref; + /** Held while a post is in flight. */ + readonly posting: Semaphore.Semaphore; +} + +type Delivery = "sent" | "unsettled" | "refused" | "rejected"; + const batchEvents = 256; export const batchBytes = 1024 * 1024; +/** + * How long after a post a missing event counts as dropped and is resent; Logflare flushes its + * per-source batches within about a second. + */ +export const flushWindowMillis = 5_000; +/** How often stored ids are checked while a body is unconfirmed. */ +export const pollMillis = 250; const envelopeBytes = '{"batch":[]}'.length; const postTimeout = "5 seconds"; const cursorFile = "cursor.json"; const encoder = new TextEncoder(); -const decodeCursor = Schema.decodeUnknownEffect(Schema.fromJsonString(LogPosition)); -const encodeCursor = Schema.encodeEffect(Schema.fromJsonString(LogPosition)); -/** Delivery is idempotent per event id, so an unsettled body or a failed log read is retried. */ +const decodeCursor = Schema.decodeUnknownEffect(Schema.fromJsonString(Cursor)); +const encodeCursor = Schema.encodeEffect(Schema.fromJsonString(Cursor)); const retrySchedule = Schedule.exponential("250 millis", 2).pipe( Schedule.modifyDelay(({ duration }) => Effect.succeed(Duration.min(duration, Duration.seconds(10))), ), ); -const after = (record: LogRecord, cursor: LogPosition | undefined) => +const after = (position: LogPosition, cursor: LogPosition | undefined) => cursor === undefined || - record.position === undefined || - record.position.generation > cursor.generation || - (record.position.generation === cursor.generation && - record.position.byteOffset > cursor.byteOffset); + position.generation > cursor.generation || + (position.generation === cursor.generation && position.byteOffset > cursor.byteOffset); + +const positionOf = ({ generation, byteOffset }: LogPosition): LogPosition => ({ + generation, + byteOffset, +}); + +const progressOf = (cursor: Cursor): Progress => ({ + confirmed: "generation" in cursor ? positionOf(cursor) : undefined, + pending: cursor.pending, +}); + +const cursorOf = ({ confirmed, pending }: Progress): Cursor | undefined => { + if (confirmed !== undefined) return pending === undefined ? confirmed : { ...confirmed, pending }; + return pending === undefined ? undefined : { pending }; +}; + +interface Body { + readonly body: string; + readonly items: ReadonlyArray; + readonly last: A; +} /** Groups serialized events into request bodies of at most 256 events and `batchBytes` bytes. */ export const bodies = ( events: ReadonlyArray, -): ReadonlyArray<{ readonly body: string; readonly last: A }> => { - const result: Array<{ readonly body: string; readonly last: A }> = []; +): ReadonlyArray> => { + const result: Array> = []; let current: Array = []; + let items: Array = []; let size = envelopeBytes; - let last: A | undefined; const close = () => { - if (last !== undefined) result.push({ body: `{"batch":[${current.join(",")}]}`, last }); + const last = items.at(-1); + if (last !== undefined) result.push({ body: `{"batch":[${current.join(",")}]}`, items, last }); current = []; + items = []; size = envelopeBytes; }; for (const item of events) { @@ -114,9 +197,9 @@ export const bodies = ( close(); size += (current.length > 0 ? 1 : 0) + bytes; current.push(serialized); - last = item; + items.push(item); } - if (current.length > 0) close(); + close(); return result; }; @@ -140,18 +223,40 @@ const isBodyRejection = (error: unknown): boolean => { ); }; +const warnOnce = (warned: Ref.Ref, message: string, error: unknown) => + Ref.getAndSet(warned, true).pipe( + Effect.flatMap((already) => (already ? Effect.void : Effect.logWarning(message, error))), + ); + +/** Retries with the shared backoff, warning on the first failure only. */ +const retryWarningOnce = + (message: string) => + (effect: Effect.Effect) => + Ref.make(false).pipe( + Effect.flatMap((warned) => + effect.pipe( + Effect.tapError((error) => warnOnce(warned, message, error)), + Effect.retry(retrySchedule), + ), + ), + ); + /** * Ships persisted service logs to Analytics' direct backend while the composed Analytics instance * is running and healthy. Each instance resumes from its cursor, so records written while - * Analytics sleeps are shipped after it wakes; shipping never wakes Analytics. + * Analytics sleeps are shipped after it wakes; shipping never wakes Analytics. Logflare accepts a + * post before storing it and drops a whole batch holding an already stored id, so the cursor + * passes a body only once `storedEvents` reports all of it, and only missing events are posted. */ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwarderOptions) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const crypto = yield* Crypto.Crypto; const client = (yield* HttpClient.HttpClient).pipe(HttpClient.filterStatusOk); + const forwarderScope = yield* Effect.scope; const candidates = yield* Ref.make>(new Map()); const target = yield* SubscriptionRef.make(undefined); + const targetScope = yield* Ref.make(undefined); const epochs = yield* Ref.make(0); const rebinding = yield* Semaphore.make(1); @@ -163,10 +268,18 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar const previous = yield* SubscriptionRef.get(target); if (serving?.instance === previous?.instance) return; const epoch = yield* Ref.updateAndGet(epochs, (value) => value + 1); - yield* SubscriptionRef.set( - target, - serving === undefined ? undefined : { instance: serving.instance, epoch }, - ); + const scope = serving === undefined ? undefined : yield* Scope.fork(forwarderScope); + const next = + serving === undefined || scope === undefined + ? undefined + : { + instance: serving.instance, + epoch, + stored: yield* options.storedEvents(serving.instance).pipe(Scope.provide(scope)), + }; + const retired = yield* Ref.getAndSet(targetScope, scope); + yield* SubscriptionRef.set(target, next); + if (retired !== undefined) yield* Scope.close(retired, Exit.void); }).pipe(Semaphore.withPermits(rebinding, 1)); const trackTarget = (instance: ForwardedInstance) => @@ -219,25 +332,31 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar ), Effect.flatMap( Option.match({ - onNone: () => Effect.succeed("oldest"), - onSome: (text) => decodeCursor(text), + onNone: () => Effect.succeed({ confirmed: undefined, pending: undefined }), + onSome: (text) => decodeCursor(text).pipe(Effect.map(progressOf)), }), ), Effect.catch(() => Effect.logWarning( `Unreadable log cursor in ${directory}; shipping from the oldest retained record`, - ).pipe(Effect.as("oldest")), + ).pipe(Effect.as({ confirmed: undefined, pending: undefined })), ), ); - const writeCursor = (directory: string, position: LogPosition) => - encodeCursor(position).pipe( - Effect.flatMap((content) => - writeFileAtomically(fs, path, { - directory, - target: path.join(directory, cursorFile), - content, - }), - ), + const writeCursor = Effect.fnUntraced(function* (directory: string, progress: Progress) { + const cursor = cursorOf(progress); + const target = path.join(directory, cursorFile); + if (cursor === undefined) return yield* fs.remove(target, { force: true }); + const content = yield* encodeCursor(cursor); + yield* writeFileAtomically(fs, path, { directory, target, content }); + }); + + /** Saves progress before acting on it, so the in-memory position never passes the file. */ + const persist = (session: Session, next: Progress) => + writeCursor(session.directory, next).pipe( + // Detaching waits for a started write instead of leaving files in the directory. + Effect.uninterruptible, + retryWarningOnce(`Saving the log cursor of ${session.instanceId} failed; retrying`), + Effect.andThen(Ref.set(session.progress, next)), ); const eventId = Effect.fnUntraced(function* (instanceId: string, position: LogPosition) { @@ -252,6 +371,20 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar return `${text.slice(0, 8)}-${text.slice(8, 12)}-${text.slice(12, 16)}-${text.slice(16, 20)}-${text.slice(20)}`; }); + /** The event of an output record, or `undefined` for markers and deletion gaps. */ + const shippedEvent = Effect.fnUntraced(function* (session: Session, record: LogRecord) { + if (record.position === undefined || (record.kind !== "stdout" && record.kind !== "stderr")) + return undefined; + const shipped: { readonly position: LogPosition; readonly event: ShippedEvent } = { + position: record.position, + event: { + id: yield* eventId(session.instanceId, record.position), + ...logflareEvent(session.service, record.timestamp, record.text ?? ""), + }, + }; + return shipped; + }); + const post = Effect.fn("LogForwarder.post")(function* ( current: Target, source: string, @@ -278,39 +411,122 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar ); }); - /** - * Posts one body until it settles; the cursor passes it only afterwards. A rejected target pauses - * shipping until the next retarget, and a rejected body is skipped. - */ - const deliver = Effect.fnUntraced(function* (current: Target, source: string, body: string) { - const warned = yield* Ref.make(false); - yield* post(current, source, body).pipe( + /** Posts one body once and classifies the answer. */ + const deliver = (session: Session, source: string, body: string, warned: Ref.Ref) => + post(session.current, source, body).pipe( + session.posting.withPermits(1), + Effect.as("sent"), Effect.catchIf(isTargetRejection, (error) => Effect.logWarning(`Analytics refused ${source} logs; pausing until it changes`, error).pipe( - Effect.andThen(Effect.fail(new StaleTarget())), + Effect.as("refused"), ), ), - Effect.tapError((error) => - error._tag === "StaleTarget" || isBodyRejection(error) - ? Effect.void - : Ref.getAndSet(warned, true).pipe( - Effect.flatMap((already) => - already - ? Effect.void - : Effect.logWarning(`Posting ${source} logs failed; retrying`, error), + Effect.catch((error) => { + if (error._tag === "StaleTarget") return Effect.fail(error); + if (isBodyRejection(error)) + return Effect.logWarning( + `Analytics rejected a ${source} log body; skipping it`, + error, + ).pipe(Effect.as("rejected")); + return warnOnce(warned, `Posting ${source} logs failed; retrying`, error).pipe( + Effect.as("unsettled"), + ); + }), + ); + + /** + * Advances past a pending body once Analytics stored all its events. Missing events are posted; + * after a post, those still missing once `flushWindowMillis` passes are posted again. A refused + * target clears the body and pauses; a rejected body is skipped. + */ + const settle = Effect.fnUntraced(function* ( + session: Session, + initial: Pending, + events: ReadonlyMap, + alreadyPosted: boolean, + ) { + const warned = yield* Ref.make(false); + let pending = initial; + let posted = alreadyPosted; + let due = posted ? pending.sentAt + flushWindowMillis : 0; + settling: while (true) { + const stored = yield* session.current.stored + .storedIds(pending.source, pending.ids) + .pipe( + retryWarningOnce( + `Checking which ${pending.source} logs Analytics stored failed; retrying`, + ), + ); + const missing = pending.ids.filter((id) => !stored.has(id)); + if (missing.length === 0) break; + const now = yield* Clock.currentTimeMillis; + if (now < due) { + yield* Effect.sleep(pollMillis); + continue; + } + const resend = missing.flatMap((id) => { + const event = events.get(id); + return event === undefined ? [] : [{ event }]; + }); + if (resend.length < missing.length) { + yield* Effect.logWarning( + `Logs of ${session.service} instance ${session.instanceId} were deleted before Analytics stored them; skipping ${missing.length - resend.length} lines`, + ); + pending = { ...pending, ids: pending.ids.filter((id) => stored.has(id) || events.has(id)) }; + } + if (resend.length === 0) break; + if (posted) { + pending = { ...pending, sentAt: now }; + yield* persist(session, { + confirmed: (yield* Ref.get(session.progress)).confirmed, + pending, + }); + } + for (const { body } of bodies(resend)) { + const delivery = yield* deliver(session, pending.source, body, warned); + if (delivery === "refused") { + yield* persist(session, { + confirmed: (yield* Ref.get(session.progress)).confirmed, + pending: undefined, + }); + return yield* new StaleTarget(); + } + if (delivery === "rejected") break settling; + } + posted = true; + due = (yield* Clock.currentTimeMillis) + flushWindowMillis; + } + yield* persist(session, { confirmed: positionOf(pending), pending: undefined }); + }); + + /** Rebuilds a pending body from its retained records and settles it. */ + const reconcile = Effect.fnUntraced(function* (session: Session, saved: Progress) { + const { confirmed, pending } = saved; + if (pending === undefined) return; + const end = positionOf(pending); + const wanted = new Set(pending.ids); + const events = new Map(); + yield* Effect.scoped( + options.logs.read(session.instanceId, { from: confirmed ?? "oldest", follow: false }).pipe( + Effect.flatMap((records) => + records.pipe( + Stream.takeWhile((record) => { + const at = record.position ?? record.resumeAt; + return at === undefined || !after(at, end); + }), + Stream.runForEach((record) => + shippedEvent(session, record).pipe( + Effect.map((shipped) => { + if (shipped !== undefined && wanted.has(shipped.event.id)) + events.set(shipped.event.id, shipped.event); + }), ), ), - ), - Effect.retry({ - schedule: retrySchedule, - while: (error) => error._tag !== "StaleTarget" && !isBodyRejection(error), - }), - Effect.catch((error) => - error._tag === "StaleTarget" - ? Effect.fail(error) - : Effect.logWarning(`Analytics rejected a ${source} log body; skipping it`, error), + ), + ), ), ); + yield* settle(session, pending, events, true); }); /** Ships one instance's records from its cursor until the target changes. */ @@ -319,23 +535,25 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar service: ShippedService, current: Target, failing: Ref.Ref, + posting: Semaphore.Semaphore, ) => Effect.scoped( Effect.gen(function* () { const directory = yield* options.logs.directory(instanceId); - const from = yield* readCursor(directory); - const cursor = yield* Ref.make( - typeof from === "string" ? undefined : from, - ); + const saved = yield* readCursor(directory); + const progress = yield* Ref.make(saved); + const shipping: Session = { instanceId, service, directory, current, progress, posting }; + yield* reconcile(shipping, saved); + const from = (yield* Ref.get(progress)).confirmed ?? "oldest"; const records = yield* options.logs.read(instanceId, { from, follow: true }); const source = logflareSources[service]; yield* records.pipe( Stream.runForEachArray((page) => Effect.gen(function* () { - const shipped = yield* Ref.get(cursor); - const pending: Array<{ - readonly event: ShippedEvent; + const { confirmed } = yield* Ref.get(progress); + const unshipped: Array<{ readonly position: LogPosition; + readonly event: ShippedEvent; }> = []; for (const record of page) { if (record.position === undefined) { @@ -344,27 +562,28 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar ); continue; } - if (!after(record, shipped)) continue; - if (record.kind !== "stdout" && record.kind !== "stderr") continue; - pending.push({ - position: record.position, - event: { - id: yield* eventId(instanceId, record.position), - ...logflareEvent(service, record.timestamp, record.text ?? ""), - }, - }); + if (!after(record.position, confirmed)) continue; + const shipped = yield* shippedEvent(shipping, record); + if (shipped !== undefined) unshipped.push(shipped); } - for (const { body, last } of bodies(pending)) { - yield* deliver(current, source, body); - yield* Ref.set(cursor, last.position); - yield* Ref.set(failing, false); - // Detaching waits for a started write instead of leaving files in the directory. - yield* writeCursor(directory, last.position).pipe( - Effect.catch((error) => - Effect.logDebug(`Log cursor of ${instanceId} was not saved`, error), - ), - Effect.uninterruptible, + for (const { items, last } of bodies(unshipped)) { + const pending: Pending = { + ...last.position, + source, + sentAt: yield* Clock.currentTimeMillis, + ids: items.map(({ event }) => event.id), + }; + yield* persist(shipping, { + confirmed: (yield* Ref.get(progress)).confirmed, + pending, + }); + yield* settle( + shipping, + pending, + new Map(items.map(({ event }) => [event.id, event])), + false, ); + yield* Ref.set(failing, false); } }), ), @@ -385,22 +604,14 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar while (true) { const current = yield* serving; const failing = yield* Ref.make(false); + const posting = yield* Semaphore.make(1); // A stopped or refused target pauses shipping until it changes; a failed log read resumes // from the cursor after a backoff. - const detached = yield* session(instance.id, service, current, failing).pipe( + const detached = yield* session(instance.id, service, current, failing, posting).pipe( Effect.tapError((error) => error._tag === "StaleTarget" ? Effect.void - : Ref.getAndSet(failing, true).pipe( - Effect.flatMap((already) => - already - ? Effect.void - : Effect.logWarning( - `Reading ${instance.id} logs to ship failed; retrying`, - error, - ), - ), - ), + : warnOnce(failing, `Reading ${instance.id} logs to ship failed; retrying`, error), ), Effect.retry({ schedule: retrySchedule, @@ -408,7 +619,10 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar }), Effect.as(true), Effect.catch(() => retargeted(current).pipe(Effect.as(false))), - Effect.raceFirst(retargeted(current).pipe(Effect.as(false))), + // A retarget lets a started post finish, so its answer decides the pending body. + Effect.raceFirst( + retargeted(current).pipe(Effect.andThen(posting.take(1)), Effect.as(false)), + ), ); if (detached) return yield* Effect.logDebug(`Log shipping of ${instance.id} stopped with its logs`); diff --git a/packages/stack/src/host/LogflareStorage.ts b/packages/stack/src/host/LogflareStorage.ts new file mode 100644 index 0000000000..8b35a87c9f --- /dev/null +++ b/packages/stack/src/host/LogflareStorage.ts @@ -0,0 +1,110 @@ +import { PgClient } from "@effect/sql-pg"; +import { + Cache, + Context, + Duration, + Effect, + Exit, + Layer, + Option, + RcRef, + Redacted, + Schema, +} from "effect"; +import { failureMessage } from "../internal/failure-message.ts"; +import { schema } from "../services/Analytics.ts"; +import { StoredEventsError, type StoredEvents } from "./LogForwarder.ts"; + +/** The Postgres database Analytics' Postgres backend writes to, reachable from the host. */ +export interface AnalyticsDatabase { + readonly host: string; + readonly port: number; + readonly database: string; + readonly username: string | undefined; + readonly password: string | undefined; +} + +/** A source token with `-` replaced by `_`, as Logflare names its event tables. */ +const tablePattern = /^log_events_[0-9a-f]{8}(?:_[0-9a-f]{4}){3}_[0-9a-f]{12}$/u; + +/** + * Reads stored event ids from the `log_events_` table Logflare's Postgres backend keeps per + * source. This couples to Logflare's private table layout, which the owner's Analytics + * integration test pins. + */ +export const make = Effect.fn("LogflareStorage.make")(function* ( + database: Effect.Effect, +) { + const client = yield* RcRef.make({ + acquire: database.pipe( + Effect.flatMap((connection) => + Layer.build( + PgClient.layer({ + host: connection.host, + port: connection.port, + database: connection.database, + username: connection.username, + password: + connection.password === undefined ? undefined : Redacted.make(connection.password), + connectTimeout: "2 seconds", + }), + ), + ), + Effect.map((context) => Context.get(context, PgClient.PgClient)), + ), + idleTimeToLive: Duration.infinity, + }); + const withClient = (run: (sql: PgClient.PgClient) => Effect.Effect) => + Effect.scoped(RcRef.get(client).pipe(Effect.flatMap(run))); + + // A source or table Analytics has not created yet holds no events and is looked up again. + const tables = yield* Cache.makeWith( + (source: string) => + withClient( + (sql) => + sql<{ readonly name: string }>` + SELECT 'log_events_' || replace(token::text, '-', '_') AS name + FROM ${sql(schema)}.sources + WHERE name = ${source} + AND to_regclass(format('%I.%I', ${schema}::text, 'log_events_' || replace(token::text, '-', '_'))) IS NOT NULL`, + ).pipe( + Effect.flatMap(([row]) => { + if (row === undefined) return Effect.succeed(Option.none()); + return tablePattern.test(row.name) + ? Effect.succeed(Option.some(row.name)) + : Effect.fail( + new StoredEventsError({ message: `Unexpected Analytics table name ${row.name}` }), + ); + }), + ), + { + capacity: 64, + timeToLive: (exit) => + Exit.isSuccess(exit) && Option.isSome(exit.value) ? Duration.infinity : Duration.zero, + }, + ); + + return { + storedIds: (source, ids) => + Effect.gen(function* () { + if (ids.length === 0) return new Set(); + const table = yield* Cache.get(tables, source); + if (Option.isNone(table)) return new Set(); + const rows = yield* withClient( + (sql) => + sql<{ readonly id: string }>` + SELECT id::text AS id FROM ${sql(schema)}.${sql(table.value)} WHERE id IN ${sql.in(ids)}`, + ); + return new Set(rows.map(({ id }) => id)); + }).pipe( + Effect.mapError((cause) => + Schema.is(StoredEventsError)(cause) + ? cause + : new StoredEventsError({ + message: `Unable to read stored ${source} events: ${failureMessage(cause)}`, + cause, + }), + ), + ), + } satisfies StoredEvents; +}); diff --git a/packages/stack/src/services/Analytics.ts b/packages/stack/src/services/Analytics.ts index a8d25fd4b2..fda10aaec4 100644 --- a/packages/stack/src/services/Analytics.ts +++ b/packages/stack/src/services/Analytics.ts @@ -15,6 +15,9 @@ export const Endpoints = Schema.Struct({ http: Schema.optionalKey(EndpointIntent export interface Endpoints extends Schema.Schema.Type {} export const Creation = serviceCreation("analytics", Config, Endpoints); +/** The schema holding Analytics' sources and the event tables of its Postgres backend. */ +export const schema = "_analytics"; + export interface Creation extends Schema.Schema.Type {} export const makeSpec = (): ProcessRecipeSpec => ({ @@ -37,7 +40,7 @@ export const makeSpec = (): ProcessRecipeSpec => ({ ? {} : { PORT: String(http.port), PHX_HTTP_PORT: String(http.port) }), DB_DATABASE: "_supabase", - DB_SCHEMA: "_analytics", + DB_SCHEMA: schema, DB_HOSTNAME: db.host, DB_PORT: db.port, DB_USERNAME: db.username ?? "supabase_admin", @@ -49,7 +52,7 @@ export const makeSpec = (): ProcessRecipeSpec => ({ ? {} : { LOGFLARE_PRIVATE_ACCESS_TOKEN: creation.config.apiKey }), POSTGRES_BACKEND_URL: databaseUrl, - POSTGRES_BACKEND_SCHEMA: "_analytics", + POSTGRES_BACKEND_SCHEMA: schema, }; }), args: () => Effect.succeed(["start"]), diff --git a/packages/stack/tests/fake-logflare.ts b/packages/stack/tests/fake-logflare.ts new file mode 100644 index 0000000000..ec152b420e --- /dev/null +++ b/packages/stack/tests/fake-logflare.ts @@ -0,0 +1,159 @@ +import { createServer } from "node:http"; // oxlint-disable-line effecttsgo/node-builtin-import -- real socket fixture. +import { Effect, Queue, Schema, Scope } from "effect"; +import type { StoredEvents } from "../src/host/LogForwarder.ts"; + +const Body = Schema.Struct({ + batch: Schema.Array( + Schema.Struct({ + id: Schema.String, + event_message: Schema.String, + appname: Schema.String, + timestamp: Schema.String, + metadata: Schema.Record(Schema.String, Schema.Unknown), + }), + ), +}); +export type FakeEvent = Schema.Schema.Type["batch"][number]; + +const decodeBody = Schema.decodeUnknownSync(Body); + +/** One ingest request as the fake received it. */ +export interface Received { + readonly port: number; + readonly url: string; + readonly apiKey: string | undefined; + readonly events: ReadonlyArray; +} + +export interface FakeLogflareOptions { + /** Events per stored batch; Logflare's Postgres pipeline uses 350. */ + readonly batchEvents?: number; + /** Stores queued events this long after they arrive; without it only `apply` stores them. */ + readonly flushMillis?: number; +} + +/** + * A Logflare stand-in with its asynchronous Postgres pipeline: an accepted post only queues its + * events, and each per-source batch is stored atomically, or dropped whole when one of its ids is + * already stored. Listeners share the storage, like an Analytics restarted on a new port. + */ +export const makeFakeLogflare = (options: FakeLogflareOptions = {}) => + Effect.gen(function* () { + const batchEvents = options.batchEvents ?? 350; + const received = yield* Queue.unbounded(); + const stored = new Map>(); + const queued = new Map>(); + const statuses: Array = []; + const holds: Array<{ readonly released: Promise; readonly closed: () => void }> = []; + const counts = { dropped: 0, aborted: 0 }; + const flushes = yield* Queue.unbounded(); + + const applyBatches = (limit = Number.POSITIVE_INFINITY) => { + let applied = 0; + for (const [source, events] of queued) { + const table = stored.get(source) ?? new Map(); + stored.set(source, table); + while (events.length > 0 && applied < limit) { + const batch = events.splice(0, batchEvents); + applied++; + const ids = new Set(batch.map(({ id }) => id)); + if (ids.size < batch.length || batch.some(({ id }) => table.has(id))) + counts.dropped += batch.length; + else for (const event of batch) table.set(event.id, event); + } + } + }; + const { flushMillis } = options; + if (flushMillis !== undefined) + yield* Queue.take(flushes).pipe( + Effect.andThen(Effect.sleep(flushMillis)), + Effect.andThen(Queue.clear(flushes)), + Effect.andThen(Effect.sync(() => applyBatches())), + Effect.forever, + Effect.forkScoped, + ); + + const listen = Effect.acquireRelease( + Effect.callback<{ readonly port: number; readonly server: ReturnType }>( + (resume) => { + const server = createServer((request, response) => { + const chunks: Array = []; + request.on("data", (chunk: Uint8Array) => chunks.push(chunk)); + request.on("end", () => { + const url = new URL(request.url ?? "", "http://fake"); + const apiKey = request.headers["x-api-key"]; + const { batch } = decodeBody( + JSON.parse(new TextDecoder().decode(Buffer.concat(chunks))), + ); + const status = statuses.shift() ?? 200; + if (status >= 200 && status < 300) { + const source = url.searchParams.get("source_name") ?? ""; + queued.set(source, [...(queued.get(source) ?? []), ...batch]); + Queue.offerUnsafe(flushes, undefined); + } + const hold = holds.shift(); + response.on("finish", () => hold?.closed()); + response.on("close", () => { + if (!response.writableFinished) counts.aborted++; + hold?.closed(); + }); + Queue.offerUnsafe(received, { + port: Number(request.socket.localPort), + url: request.url ?? "", + apiKey: typeof apiKey === "string" ? apiKey : undefined, + events: batch, + }); + void (hold?.released ?? Promise.resolve()).then(() => { + response.statusCode = status; + response.end(); + }); + }); + }); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (address === null || typeof address === "string") + return resume(Effect.die("fake Logflare has no address")); + resume(Effect.succeed({ port: address.port, server })); + }); + }, + ), + ({ server }) => + Effect.callback((resume) => { + server.closeAllConnections(); + server.close(() => resume(Effect.void)); + }), + ).pipe(Effect.map(({ port }) => port)); + + const port = yield* listen; + const storedIds: StoredEvents["storedIds"] = (source, ids) => + Effect.sync(() => new Set(ids.filter((id) => stored.get(source)?.has(id) === true))); + + return { + port, + /** Opens another ingest port sharing this fake's storage. */ + listen: (scope: Scope.Scope) => listen.pipe(Scope.provide(scope)), + next: Queue.take(received), + /** The next posts answer with these statuses; only a 2xx queues its events. */ + respond: (...next: ReadonlyArray) => Effect.sync(() => statuses.push(...next)), + /** Holds the next post's response until the returned effect runs; it ends once the response does. */ + hold: Effect.sync(() => { + const released = Promise.withResolvers(); + const closed = Promise.withResolvers(); + holds.push({ released: released.promise, closed: closed.resolve }); + return Effect.sync(released.resolve).pipe( + Effect.andThen(Effect.promise(() => closed.promise)), + ); + }), + /** Stores queued batches now, at most `limit` of them. */ + apply: (limit?: number) => Effect.sync(() => applyBatches(limit)), + /** Loses every queued event, like Analytics stopping before its pipeline flushed. */ + discard: Effect.sync(() => queued.clear()), + storedIds, + storedEvents: Effect.succeed({ storedIds } satisfies StoredEvents), + stored: Effect.sync(() => [...stored.values()].flatMap((table) => [...table.values()])), + dropped: Effect.sync(() => counts.dropped), + aborted: Effect.sync(() => counts.aborted), + }; + }); + +export type FakeLogflare = Effect.Success>; From 715c5f01693dc7a6363a83e547a558e5a923933b Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 2 Oct 2026 14:29:21 +0200 Subject: [PATCH 20/35] fix(stack): never re-post queued log events and harden confirmed delivery - Do not re-post events to the Analytics launch that may still have them queued (its ingest lag exceeds any fixed window); re-post after that launch ends, with a bounded per-event backstop that skips what can never be stored. - Replace NUL and unpaired surrogates that Analytics' jsonb rejects. - Derive Analytics' backend connection once for Analytics and the stored-id check, treat unknown stored state as an error, and drop a failed connection. - Keep a retired target's storage open while its sessions finish, trace the stored-id query, and remove logs before the state document on destroy. Co-Authored-By: Claude Opus 5.5 --- .../experimental/stack/start/SIDE_EFFECTS.md | 13 +- packages/stack/ARCHITECTURE.md | 38 ++- packages/stack/src/Owner.ts | 27 +- packages/stack/src/State.integration.test.ts | 44 +++ packages/stack/src/State.ts | 9 +- .../src/host/LogForwarder.integration.test.ts | 248 ++++++++++++---- packages/stack/src/host/LogForwarder.ts | 266 ++++++++++++------ packages/stack/src/host/LogflareEvents.ts | 27 +- .../src/host/LogflareEvents.unit.test.ts | 13 + .../host/LogflareStorage.integration.test.ts | 149 ++++++++++ packages/stack/src/host/LogflareStorage.ts | 130 ++++++--- .../src/host/LogflareStorage.unit.test.ts | 35 +++ packages/stack/src/services/Analytics.ts | 52 +++- .../stack/src/services/Analytics.unit.test.ts | 44 +++ packages/stack/tests/fake-logflare.ts | 40 ++- 15 files changed, 915 insertions(+), 220 deletions(-) create mode 100644 packages/stack/src/host/LogflareStorage.integration.test.ts create mode 100644 packages/stack/src/host/LogflareStorage.unit.test.ts create mode 100644 packages/stack/src/services/Analytics.unit.test.ts diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index cc196c8b1b..473353c332 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -103,10 +103,15 @@ the API key pauses shipping until Analytics stops or turns unhealthy and is heal composition selects a different Analytics instance. A failed log read is retried from the saved position with a backoff. Each event carries an id derived from its instance and position. Before posting a request, the owner records it as pending in `cursor.json`; it then reads Analytics' -Postgres tables (`_analytics.sources` and `_analytics.log_events_` in the stack database) -to confirm which events are stored, posts only the missing ones, and advances the position once -all are stored. Events still missing 5 seconds after a request are posted again, and a pending -request is confirmed the same way after an owner restart or an Analytics restart. Shipping never wakes Analytics +Postgres tables (`_analytics.sources` and `_analytics.log_events_` in the stack's +`_supabase` database) to confirm which events are stored, posts only the missing ones, and +advances the position once all are stored. While those tables cannot be read, shipping waits and +logs one warning. Events an Analytics process accepted are not posted to it again while it may +still store them; events still missing are posted again after Analytics restarts or the owner +restarts, once 5 seconds passed since the request ended, and a pending request is confirmed the +same way. Events still missing 60 seconds after Analytics accepted them are posted one at a time, +and one that is still not stored is skipped with a warning. NUL characters and unpaired +surrogates in shipped lines are replaced with U+FFFD. Shipping never wakes Analytics and does not count as idle activity, so a lazy Analytics still stops on its idle timer while other services log. Service log streams and `supabase stack logs` are never blocked by shipping. diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 16cd5e427b..1d87bca10e 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -668,7 +668,8 @@ stdout/stderr records of Auth, REST, Realtime, Storage, Functions and database i direct backend (never the proxy, so shipping neither wakes it nor counts as activity), in bodies of at most 256 events and 1 MiB. Each instance reads from `cursor.json` in its logs directory, the position of its last confirmed record plus an optional pending body (its end position, source, -send time and event ids): a missing cursor or unreadable cursor starts at the oldest retained +event ids, the Analytics launch its latest post went to, and when that post ends at the latest): +a missing cursor or unreadable cursor starts at the oldest retained segment, and a cursor in a deleted segment resumes at the next retained one. A failed log read restarts the instance from its cursor after a capped backoff, and attaching an instance removes temporary cursor writes a dead owner left behind. @@ -677,18 +678,29 @@ Bodies of an instance are sequential, and event ids derive from the instance and Logflare answers a post once it queued the events and stores them later in per-source batches; a batch holding an id already stored is dropped whole. So delivery is confirmed in Analytics' Postgres database: the owner reads which ids `_analytics.log_events_` holds, resolving the -token from `_analytics.sources`, through the database Analytics is composed with. The pending body -is written atomically before it is posted, and a failed write blocks posting. Only ids not yet -stored are posted. After a success, timeout, 5xx or interruption the owner polls the stored ids; -once all are stored the cursor advances and the pending body is cleared, and ids still missing 5 -seconds after a post are posted again. A start, restart or retarget reconciles a pending body -before later records; pending records already deleted by retention are skipped with a warning. A -401, 403 or 404 response clears the pending body and pauses the instance with its cursor until the -target changes; any other 4xx except 408 and 429 skips the body. A retarget waits for a post in -flight. The target is re-selected when the composition or Analytics' health changes. A batch that -Analytics commits more than 5 seconds after its post can still conflict with a repost, and the -check depends on Logflare's private table layout, which the owner's Analytics integration test -pins. +token from `_analytics.sources`. That database is the one Logflare keeps its sources and events in: +`_supabase` on the host and port of Analytics' database URL (reached at the bound database's +endpoint when the composition binds it), with `supabase_admin`/`postgres` for credentials the URL +omits. A source Analytics does not know, or a failed query, leaves the stored state unknown: the +check is retried and shipping waits, with one warning per outage. A known source without a table +holds no events. The pending body is written atomically before each post, and a failed write +blocks posting. Only ids not yet stored are posted. A post ends by its 5 second timeout, which the +pending body records. Logflare can take many seconds to store what it accepted, so a post that +succeeded, timed out or was interrupted is not repeated to the Analytics launch that took it +while it may still store it: the owner polls the stored ids until all are stored, then advances +the cursor and clears the pending body. Ids still missing are posted again once that launch ended +(a new launch, or another owner) and 5 seconds passed since the post ended; a 5xx answer is posted +again 5 seconds after it. Ids the accepting launch leaves unstored for 60 seconds count as a lost +batch: each is posted on its own, and one that still is not stored alone is skipped with a +warning, so one unstorable event cannot block an instance. Event text and metadata strings have +NUL and unpaired surrogates replaced with U+FFFD before posting, because Postgres `jsonb` rejects +them and drops the whole batch. A start, restart or retarget reconciles a pending body before +later records; pending records already deleted by retention are skipped with a warning. A 401, 403 +or 404 response clears the pending body and pauses the instance with its cursor until the target +changes; any other 4xx except 408 and 429 skips the body. A retarget waits for a post in flight, +and a retired target's stored-event view stays open until its sessions end. The target is +re-selected when the composition, Analytics' health or its launch changes. The check depends on +Logflare's private table layout, which the owner's Analytics integration test pins. Registry updates use an OS-backed lock through a private `node:sqlite` connection to `/.registry-lock.sqlite`. Each `withLock` call opens its own connection, disables diff --git a/packages/stack/src/Owner.ts b/packages/stack/src/Owner.ts index 70a793e3d2..14d08d377c 100644 --- a/packages/stack/src/Owner.ts +++ b/packages/stack/src/Owner.ts @@ -61,7 +61,6 @@ import { type ServiceCreationInput, } from "./services/Catalog.ts"; import type { CatalogError } from "./services/Recipe.ts"; -import { databaseConnection } from "./services/ServiceConfig.ts"; import * as Container from "./runtime/Container.ts"; import { projectSegmentFor } from "./identity/Identity.ts"; import { stackError, type OwnerRpc } from "./Rpc.ts"; @@ -191,30 +190,26 @@ const makeOwner = Effect.fn("Owner.make")(function* (options: OwnerOptions) { const logStore = yield* LogStore.make({ root: options.state.logsRoot(options.saved.id) }).pipe( Effect.provideContext(services), ); - /** The database Analytics stores events in, on its direct backend, with Analytics' credentials. */ + /** The database Analytics stores events in, as the host reaches it. */ const analyticsDatabase = (analyticsId: string) => Effect.gen(function* () { const creation = yield* Ref.get((yield* orchestrator.get(analyticsId)).creation); const databaseUrl = creation.service === "analytics" ? creation.config.databaseUrl : undefined; + if (databaseUrl === undefined) + return yield* new ServiceError({ + operation: "analytics", + message: "Analytics has no database URL", + }); const dependency = (yield* orchestrator.composition).dependencies.find( ({ to, bindings }) => to === analyticsId && bindings?.some(({ input }) => input === "databaseUrl") === true, ); - if (databaseUrl === undefined || dependency === undefined) - return yield* new ServiceError({ - operation: "analytics", - message: "Analytics has no composed database", - }); - const connection = yield* databaseConnection(databaseUrl); - const endpoint = yield* (yield* orchestrator.get(dependency.from)).recipe.endpoint("sql"); - return { - host: (endpoint.kind === "unix" ? endpoint.path : endpoint.host) ?? "127.0.0.1", - port: endpoint.port, - database: connection.database, - username: connection.username, - password: connection.password, - } satisfies LogflareStorage.AnalyticsDatabase; + const bound = + dependency === undefined + ? undefined + : yield* (yield* orchestrator.get(dependency.from)).recipe.endpoint("sql"); + return yield* LogflareStorage.analyticsDatabase(databaseUrl, bound); }); const forwarder = yield* LogForwarder.make({ composition: orchestrator.composition, diff --git a/packages/stack/src/State.integration.test.ts b/packages/stack/src/State.integration.test.ts index 5a121d4554..cc1f9c3b63 100644 --- a/packages/stack/src/State.integration.test.ts +++ b/packages/stack/src/State.integration.test.ts @@ -651,6 +651,50 @@ describe("durable stack state", () => { ), ); + it.live("keeps a stack listed when its logs cannot be removed, so a second removal retries", () => + run( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-state-remove-failed-" }); + const denied = yield* Ref.make(true); + const injectedFs = Layer.succeed(FileSystem.FileSystem, { + ...fs, + remove: (target: string, options?: Parameters[1]) => + Effect.gen(function* () { + if (target.endsWith(`${path.sep}logs`) && (yield* Ref.get(denied))) + return yield* PlatformError.systemError({ + _tag: "PermissionDenied", + module: "FileSystem", + method: "remove", + pathOrDescriptor: target, + }); + return yield* fs.remove(target, options); + }), + }); + const store = yield* Layer.build( + State.layer({ root }).pipe(Layer.provide(injectedFs)), + ).pipe(Effect.map((context) => Context.get(context, State.Service))); + yield* store.save(initial); + const segment = path.join(store.logsRoot(initial.id), "auth", "instance", "0000000001.log"); + yield* fs.makeDirectory(path.dirname(segment), { recursive: true }); + yield* fs.writeFileString(segment, "record\n"); + + const failed = yield* store.remove(initial.id).pipe(Effect.exit); + const listed = (yield* store.list).map(({ id }) => id); + const logsKept = yield* fs.exists(segment); + yield* Ref.set(denied, false); + yield* store.remove(initial.id); + + expect(Exit.isFailure(failed)).toBe(true); + expect(listed).toEqual([initial.id]); + expect(logsKept).toBe(true); + expect(yield* store.read(initial.id)).toBeUndefined(); + expect(yield* fs.exists(path.join(root, initial.id))).toBe(false); + }), + ), + ); + it.live.skipIf(process.platform === "win32")( "restricts the state root to its owner while keeping a traverse-only grant", () => diff --git a/packages/stack/src/State.ts b/packages/stack/src/State.ts index ed4d4773ce..00b810dec9 100644 --- a/packages/stack/src/State.ts +++ b/packages/stack/src/State.ts @@ -445,14 +445,15 @@ const makeState = ( }); const remove = Effect.fn("State.remove")(function* (id: string) { yield* checkId(id); - for (const file of [statePath(id), ownerPath(id), ownerLog(id)]) - yield* fs - .remove(file, { force: true }) - .pipe(Effect.mapError((cause) => stateError("remove", cause))); + // Logs go before the state file, so a stack whose logs remain stays listed for another removal. yield* fs.remove(logsRoot(id), { recursive: true, force: true }).pipe( retryShared, Effect.mapError((cause) => stateError("remove", cause)), ); + for (const file of [statePath(id), ownerPath(id), ownerLog(id)]) + yield* fs + .remove(file, { force: true }) + .pipe(Effect.mapError((cause) => stateError("remove", cause))); yield* removeEmptyDirectory(path.join(stackRoot(id), "data")); yield* removeEmptyDirectory(stackRoot(id)); }); diff --git a/packages/stack/src/host/LogForwarder.integration.test.ts b/packages/stack/src/host/LogForwarder.integration.test.ts index 4e1ff18c33..cfcfbb3470 100644 --- a/packages/stack/src/host/LogForwarder.integration.test.ts +++ b/packages/stack/src/host/LogForwarder.integration.test.ts @@ -33,9 +33,13 @@ import type { LogRecord } from "./LogRecord.ts"; import * as LogForwarder from "./LogForwarder.ts"; import * as LogStore from "./LogStore.ts"; -const { flushWindowMillis, pollMillis } = LogForwarder; +const { flushWindowMillis, pollMillis, postTimeoutMillis, retryMillis, stuckMillis } = LogForwarder; -/** A clock that moves only when the test advances it; `sleeping` waits for a pending sleep. */ +/** + * A clock that moves only when the test advances it; `sleeping` waits for a pending sleep of that + * duration. Wake-ups run on a later macrotask, since completing a deferred resumes its fiber + * inline and a sleeper and the test would otherwise drive each other without yielding to I/O. + */ const makeManualClock = Effect.gen(function* () { let now = yield* Clock.currentTimeMillis; const sleeps = new Set<{ @@ -44,6 +48,8 @@ const makeManualClock = Effect.gen(function* () { readonly woken: Deferred.Deferred; }>(); const waiters = new Set<{ readonly millis: number; readonly ready: Deferred.Deferred }>(); + const wake = (deferred: Deferred.Deferred) => + setImmediate(() => Deferred.doneUnsafe(deferred, Effect.void)); const nanos = () => BigInt(now) * 1_000_000n; const clock: Clock.Clock = { currentTimeMillisUnsafe: () => now, @@ -61,7 +67,7 @@ const makeManualClock = Effect.gen(function* () { for (const waiter of waiters) if (waiter.millis === millis) { waiters.delete(waiter); - Deferred.doneUnsafe(waiter.ready, Effect.void); + wake(waiter.ready); } return Deferred.await(entry.woken).pipe( Effect.ensuring(Effect.sync(() => sleeps.delete(entry))), @@ -85,7 +91,7 @@ const makeManualClock = Effect.gen(function* () { for (const entry of sleeps) if (entry.until <= now) { sleeps.delete(entry); - Deferred.doneUnsafe(entry.woken, Effect.void); + wake(entry.woken); } }), }; @@ -185,6 +191,11 @@ const analyticsTarget = (port: Effect.Effect) => observation, awake ? serving("analytics") : observationOf("analytics", { lifecycle: "stopped" }), ), + /** Restarts Analytics as a new launch, which loses whatever the previous one had queued. */ + relaunch: (launchId: number) => + SubscriptionRef.set(observation, observationOf("analytics", { lifecycle: "stopped" })).pipe( + Effect.andThen(SubscriptionRef.set(observation, { ...serving("analytics"), launchId })), + ), instance: { id: "analytics", service: "analytics" as const, @@ -216,6 +227,7 @@ const startForwarder = ( logflare: FakeLogflare, instances: ReadonlyArray, clock?: Clock.Clock, + stored?: LogForwarder.StoredEvents, ) => Effect.gen(function* () { const scope = yield* Scope.make(); @@ -224,7 +236,7 @@ const startForwarder = ( const forwarder = yield* LogForwarder.make({ composition, logs: store, - storedEvents: () => logflare.storedEvents, + storedEvents: () => (stored === undefined ? logflare.storedEvents : Effect.succeed(stored)), }).pipe(Scope.provide(scope)); for (const instance of instances) yield* forwarder.attach(instance); yield* forwarder.rebind; @@ -584,27 +596,38 @@ describe("LogForwarder", () => { }).pipe(Effect.scoped, Effect.provide(layer)), ); - it.live("posts the events of a batch Analytics dropped again once the flush window passes", () => - Effect.gen(function* () { - const { store, logflare, database, analytics } = yield* fixture({ logflare: {} }); - const manual = yield* makeManualClock; - yield* analytics.set(true); - yield* startForwarder(store, logflare, [analytics.instance, database.instance], manual.clock); - yield* database.log("line 1", "line 2"); - const posted = yield* logflare.next; - yield* logflare.discard; + it.live( + "posts accepted events Analytics has not stored again to its next launch, not to the launch that may still store them", + () => + Effect.gen(function* () { + const { store, logflare, database, analytics } = yield* fixture({ logflare: {} }); + const manual = yield* makeManualClock; + yield* analytics.set(true); + yield* startForwarder( + store, + logflare, + [analytics.instance, database.instance], + manual.clock, + ); + yield* database.log("line 1", "line 2"); + const posted = yield* logflare.next; - yield* manual.sleeping(pollMillis); - yield* manual.advance(flushWindowMillis); - const resent = yield* logflare.next; - yield* logflare.apply(); - yield* database.log("line 3"); - const later = yield* nextPost(logflare, manual); + yield* manual.sleeping(pollMillis); + yield* manual.advance(stuckMillis - pollMillis); + yield* manual.sleeping(pollMillis); + const resentToSameLaunch = yield* logflare.unread; + yield* logflare.discard; + yield* analytics.relaunch(2); + const resent = yield* logflare.next; + yield* logflare.apply(); + yield* database.log("line 3"); + const later = yield* nextPost(logflare, manual); - expect(ids(resent)).toEqual(ids(posted)); - expect(messages(later)).toEqual(["line 3"]); - expect(yield* storedMessages(logflare)).toEqual(["line 1", "line 2"]); - }).pipe(Effect.scoped, Effect.provide(layer)), + expect(resentToSameLaunch).toBe(0); + expect(ids(resent)).toEqual(ids(posted)); + expect(messages(later)).toEqual(["line 3"]); + expect(yield* storedMessages(logflare)).toEqual(["line 1", "line 2"]); + }).pipe(Effect.scoped, Effect.provide(layer)), ); it.live("posts again only the events Analytics did not store", () => @@ -621,7 +644,8 @@ describe("LogForwarder", () => { yield* logflare.discard; yield* manual.sleeping(pollMillis); - yield* manual.advance(flushWindowMillis); + yield* manual.advance(postTimeoutMillis + flushWindowMillis); + yield* analytics.relaunch(2); const resent = yield* logflare.next; yield* logflare.apply(); @@ -649,8 +673,8 @@ describe("LogForwarder", () => { yield* database.log("line 1"); const queued = yield* logflare.next; - yield* manual.sleeping(5_000); - yield* manual.advance(5_000); + yield* manual.sleeping(postTimeoutMillis); + yield* manual.advance(postTimeoutMillis); yield* manual.sleeping(pollMillis); yield* logflare.apply(); yield* database.log("line 2"); @@ -699,10 +723,10 @@ describe("LogForwarder", () => { yield* database.log("line 1"); const first = yield* logflare.next.pipe( Effect.as("posted"), - Effect.raceFirst(manual.sleeping(250).pipe(Effect.as("retrying"))), + Effect.raceFirst(manual.sleeping(retryMillis).pipe(Effect.as("retrying"))), ); yield* Ref.set(broken, false); - yield* manual.advance(250); + yield* manual.advance(retryMillis); const posted = yield* logflare.next; expect(first).toBe("retrying"); @@ -710,26 +734,47 @@ describe("LogForwarder", () => { }).pipe(Effect.scoped, Effect.provide(layer)), ); - it.live("posts a pending body again after a restart once the flush window passes", () => - Effect.gen(function* () { - const { store, logflare, database, analytics } = yield* fixture({ logflare: {} }); - const manual = yield* makeManualClock; - const instances = [analytics.instance, database.instance]; - yield* analytics.set(true); - const first = yield* startForwarder(store, logflare, instances, manual.clock); - yield* database.log("line 1"); - const posted = yield* logflare.next; - yield* logflare.discard; - yield* manual.sleeping(pollMillis); - yield* first.stop; + it.live( + "posts a pending body again after a restart only once the flush window passed since its slow post ended", + () => + Effect.gen(function* () { + const { store, logflare, database, analytics } = yield* fixture({ logflare: {} }); + const manual = yield* makeManualClock; + const instances = [analytics.instance, database.instance]; + const checking = yield* Queue.unbounded(); + const checked = yield* Deferred.make(); + const slowCheck: LogForwarder.StoredEvents = { + storedIds: (source, eventIds) => + Queue.offer(checking, undefined).pipe( + Effect.andThen(Deferred.await(checked)), + Effect.andThen(logflare.storedIds(source, eventIds)), + ), + }; + yield* analytics.set(true); + const first = yield* startForwarder(store, logflare, instances, manual.clock, slowCheck); + yield* database.log("line 1"); + yield* Queue.take(checking); + yield* manual.advance(2_000); + const release = yield* logflare.hold; + yield* Deferred.succeed(checked, undefined); + const posted = yield* logflare.next; + yield* manual.advance(2_000); + yield* release; + yield* manual.sleeping(pollMillis); + yield* logflare.discard; + yield* first.stop; - yield* startForwarder(store, logflare, instances, manual.clock); - yield* manual.sleeping(pollMillis); - yield* manual.advance(flushWindowMillis); - const resent = yield* logflare.next; + yield* startForwarder(store, logflare, instances, manual.clock); + yield* manual.sleeping(pollMillis); + yield* manual.advance(flushWindowMillis - 1); + yield* manual.sleeping(pollMillis); + const resentInWindow = yield* logflare.unread; + yield* manual.advance(postTimeoutMillis); + const resent = yield* logflare.next; - expect(ids(resent)).toEqual(ids(posted)); - }).pipe(Effect.scoped, Effect.provide(layer)), + expect(resentInWindow).toBe(0); + expect(ids(resent)).toEqual(ids(posted)); + }).pipe(Effect.scoped, Effect.provide(layer)), ); it.live("warns about pending events retention deleted and ships the retained records", () => @@ -762,7 +807,7 @@ describe("LogForwarder", () => { Effect.provide(captured), ); yield* manual.sleeping(pollMillis); - yield* manual.advance(flushWindowMillis); + yield* manual.advance(postTimeoutMillis + flushWindowMillis); const shipped: Array = []; while (shipped.at(-1) !== "later 29") { shipped.push(...messages(yield* nextPost(logflare, manual))); @@ -821,6 +866,113 @@ describe("LogForwarder", () => { }).pipe(Effect.scoped, Effect.provide(layer)), ); + it.live("stores a line holding NUL, replaced, instead of losing its batch", () => + Effect.gen(function* () { + const { store, logflare, database, analytics } = yield* fixture(); + yield* analytics.set(true); + yield* startForwarder(store, logflare, [analytics.instance, database.instance]); + + yield* database.log("before \u0000 after", "next line"); + const posted = yield* logflare.next; + yield* database.log("later"); + yield* logflare.next; + + expect(messages(posted)).toEqual(["before � after", "next line"]); + expect((yield* storedMessages(logflare)).slice(0, 2)).toEqual(messages(posted)); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live( + "posts events a batch lost one at a time and skips the one that never stores, then ships later lines", + () => + Effect.gen(function* () { + const { store, logflare, database, analytics } = yield* fixture({ + logflare: { + unstorable: (event) => event.event_message === "never stored", + }, + }); + const manual = yield* makeManualClock; + const warnings: Array = []; + const captured = Logger.layer([ + Logger.make(({ logLevel, message }) => { + if (logLevel === "Warn") warnings.push(message); + }), + ]); + yield* analytics.set(true); + yield* startForwarder( + store, + logflare, + [analytics.instance, database.instance], + manual.clock, + ).pipe(Effect.provide(captured)); + + yield* database.log("good 1", "never stored", "good 2"); + const batch = yield* logflare.next; + yield* logflare.apply(); + yield* manual.sleeping(pollMillis); + yield* manual.advance(stuckMillis); + const first = yield* logflare.next; + yield* logflare.apply(); + const poison = yield* nextPost(logflare, manual); + yield* logflare.apply(); + yield* manual.sleeping(pollMillis); + yield* manual.advance(stuckMillis); + const second = yield* logflare.next; + yield* logflare.apply(); + yield* database.log("later"); + const later = yield* nextPost(logflare, manual); + yield* logflare.apply(); + + expect(messages(batch)).toEqual(["good 1", "never stored", "good 2"]); + expect([first, poison, second, later].map(messages)).toEqual([ + ["good 1"], + ["never stored"], + ["good 2"], + ["later"], + ]); + expect((yield* storedMessages(logflare)).slice(0, 2)).toEqual(["good 1", "good 2"]); + expect(warnings).toEqual([ + [ + "Analytics has not stored 3 postgres.logs lines of database it accepted; posting them one at a time", + ], + [ + "Analytics did not store a postgres.logs line of database posted on its own; skipping it", + ], + ]); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("keeps shipping to the new target after a retarget while a post is in flight", () => + Effect.gen(function* () { + const { store, logflare, database } = yield* fixture(); + const firstLaunch = yield* Scope.make(); + const firstPort = yield* logflare.listen(firstLaunch); + const port = yield* Ref.make(firstPort); + const analytics = yield* analyticsTarget(Ref.get(port)); + yield* analytics.set(true); + const forwarder = yield* startForwarder(store, logflare, [ + analytics.instance, + database.instance, + ]); + const release = yield* logflare.hold; + yield* database.log("in flight"); + const inFlight = yield* logflare.next; + + yield* analytics.set(false); + yield* forwarder.awaitShipping(false); + yield* Ref.set(port, logflare.port); + yield* SubscriptionRef.set(analytics.observation, { ...serving("analytics"), launchId: 2 }); + yield* forwarder.awaitShipping(true); + yield* release; + yield* database.log("after the retarget"); + const after = yield* logflare.next; + + expect(inFlight.port).toBe(firstPort); + expect(after.port).toBe(logflare.port); + expect(messages(after)).toEqual(["after the retarget"]); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + it.live("ships from the oldest retained record when its cursor is unreadable", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/packages/stack/src/host/LogForwarder.ts b/packages/stack/src/host/LogForwarder.ts index 63cc0762b6..0ac763de4d 100644 --- a/packages/stack/src/host/LogForwarder.ts +++ b/packages/stack/src/host/LogForwarder.ts @@ -49,7 +49,7 @@ export class StoredEventsError extends Schema.TaggedError()( /** Which shipped events an Analytics instance has stored. */ export interface StoredEvents { - /** The subset of `ids` stored for `source`. */ + /** The subset of `ids` stored for `source`; failures are retried and reported by the view. */ readonly storedIds: ( source: string, ids: ReadonlyArray, @@ -79,12 +79,17 @@ export interface LogForwarderOptions { interface Candidate { readonly instance: ForwardedInstance; readonly serving: boolean; + readonly launchId: number | undefined; } interface Target { readonly instance: ForwardedInstance; readonly epoch: number; + /** Names the Analytics launch; launches seen by different forwarders never share a name. */ + readonly launch: string; readonly stored: StoredEvents; + /** One permit per session using `stored`; the view closes once a retired target has none. */ + readonly users: Semaphore.Semaphore; } /** A shipped event; its id derives from the instance and record position. */ @@ -94,13 +99,22 @@ interface ShippedEvent extends LogflareEvent { class StaleTarget extends Data.TaggedError("StaleTarget")<{}> {} -/** A body handed to Analytics whose events are not all confirmed stored yet. */ -const Pending = Schema.Struct({ +/** A body's end position, source and event ids. */ +const Shipment = Schema.Struct({ ...LogPosition.fields, source: Schema.String, - sentAt: Schema.Finite, ids: Schema.Array(Schema.String), }); +interface Shipment extends Schema.Schema.Type {} + +/** A shipment posted to Analytics whose events are not all confirmed stored yet. */ +const Pending = Schema.Struct({ + ...Shipment.fields, + /** The Analytics launch the latest post went to. */ + launch: Schema.String, + /** When the latest post ends at the latest, by its timeout. */ + postEndsAt: Schema.Finite, +}); interface Pending extends Schema.Schema.Type {} /** The last confirmed position, if any, and the pending body after it. */ @@ -126,24 +140,29 @@ interface Session { readonly posting: Semaphore.Semaphore; } -type Delivery = "sent" | "unsettled" | "refused" | "rejected"; +/** `unsettled` posts got no answer, so Analytics may have queued them; `failed` ones got a 5xx. */ +type Delivery = "sent" | "unsettled" | "failed" | "refused" | "rejected"; const batchEvents = 256; export const batchBytes = 1024 * 1024; -/** - * How long after a post a missing event counts as dropped and is resent; Logflare flushes its - * per-source batches within about a second. - */ +/** How long after a post ended its events may still be stored by a launch that stopped. */ export const flushWindowMillis = 5_000; /** How often stored ids are checked while a body is unconfirmed. */ export const pollMillis = 250; +/** The first delay of the shared retry backoff. */ +export const retryMillis = 500; +/** The longest a post may take, so a pending body knows when its post ended at the latest. */ +export const postTimeoutMillis = 5_000; +/** How long events may stay unstored in the launch that accepted them before they count as dropped. */ +export const stuckMillis = 60_000; +/** More sessions than can share one target. */ +const targetUsers = 2 ** 30; const envelopeBytes = '{"batch":[]}'.length; -const postTimeout = "5 seconds"; const cursorFile = "cursor.json"; const encoder = new TextEncoder(); const decodeCursor = Schema.decodeUnknownEffect(Schema.fromJsonString(Cursor)); const encodeCursor = Schema.encodeEffect(Schema.fromJsonString(Cursor)); -const retrySchedule = Schedule.exponential("250 millis", 2).pipe( +const retrySchedule = Schedule.exponential(Duration.millis(retryMillis), 2).pipe( Schedule.modifyDelay(({ duration }) => Effect.succeed(Duration.min(duration, Duration.seconds(10))), ), @@ -256,9 +275,13 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar const forwarderScope = yield* Effect.scope; const candidates = yield* Ref.make>(new Map()); const target = yield* SubscriptionRef.make(undefined); - const targetScope = yield* Ref.make(undefined); + const targetScope = yield* Ref.make< + { readonly scope: Scope.Closeable; readonly users: Semaphore.Semaphore } | undefined + >(undefined); const epochs = yield* Ref.make(0); const rebinding = yield* Semaphore.make(1); + const run = yield* crypto.randomUUIDv4; + const launchOf = (candidate: Candidate) => `${run}:${candidate.launchId ?? "unknown"}`; const rebind = Effect.gen(function* () { const members = new Set((yield* options.composition).members.map(({ id }) => id)); @@ -266,7 +289,11 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar (candidate) => candidate.serving && members.has(candidate.instance.id), ); const previous = yield* SubscriptionRef.get(target); - if (serving?.instance === previous?.instance) return; + if ( + serving?.instance === previous?.instance && + (serving === undefined || launchOf(serving) === previous?.launch) + ) + return; const epoch = yield* Ref.updateAndGet(epochs, (value) => value + 1); const scope = serving === undefined ? undefined : yield* Scope.fork(forwarderScope); const next = @@ -275,11 +302,20 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar : { instance: serving.instance, epoch, + launch: launchOf(serving), stored: yield* options.storedEvents(serving.instance).pipe(Scope.provide(scope)), + users: yield* Semaphore.make(targetUsers), }; - const retired = yield* Ref.getAndSet(targetScope, scope); + const retired = yield* Ref.getAndSet( + targetScope, + scope === undefined || next === undefined ? undefined : { scope, users: next.users }, + ); yield* SubscriptionRef.set(target, next); - if (retired !== undefined) yield* Scope.close(retired, Exit.void); + if (retired !== undefined) + yield* Effect.forkIn( + retired.users.take(targetUsers).pipe(Effect.andThen(Scope.close(retired.scope, Exit.void))), + forwarderScope, + ); }).pipe(Semaphore.withPermits(rebinding, 1)); const trackTarget = (instance: ForwardedInstance) => @@ -290,6 +326,7 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar new Map(current).set(instance.id, { instance, serving: observation.lifecycle === "running" && observation.health === "healthy", + launchId: observation.launchId, }), ).pipe(Effect.andThen(rebind)), ), @@ -389,12 +426,14 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar current: Target, source: string, body: string, + deadline: number, ) { yield* requireCurrent(current); const creation = yield* current.instance.creation; const apiKey = creation.service === "analytics" ? creation.config.apiKey : undefined; if (apiKey === undefined) return yield* new StaleTarget(); const endpoint = yield* current.instance.endpoint("http"); + const remaining = deadline - (yield* Clock.currentTimeMillis); yield* client .execute( HttpClientRequest.post( @@ -407,13 +446,19 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar ) .pipe( Effect.flatMap((response) => response.text), - Effect.timeout(postTimeout), + Effect.timeout(Duration.millis(Math.max(remaining, 0))), ); }); - /** Posts one body once and classifies the answer. */ - const deliver = (session: Session, source: string, body: string, warned: Ref.Ref) => - post(session.current, source, body).pipe( + /** Posts one body once, ending by `deadline`, and classifies the answer. */ + const deliver = ( + session: Session, + source: string, + body: string, + deadline: number, + warned: Ref.Ref, + ) => + post(session.current, source, body, deadline).pipe( session.posting.withPermits(1), Effect.as("sent"), Effect.catchIf(isTargetRejection, (error) => @@ -428,75 +473,132 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar `Analytics rejected a ${source} log body; skipping it`, error, ).pipe(Effect.as("rejected")); - return warnOnce(warned, `Posting ${source} logs failed; retrying`, error).pipe( - Effect.as("unsettled"), - ); + if ((rejectedStatus(error) ?? 0) >= 500) + return warnOnce(warned, `Posting ${source} logs failed; retrying`, error).pipe( + Effect.as("failed"), + ); + return warnOnce( + warned, + `Posting ${source} logs got no answer; waiting until Analytics stores them`, + error, + ).pipe(Effect.as("unsettled")); }), ); /** - * Advances past a pending body once Analytics stored all its events. Missing events are posted; - * after a post, those still missing once `flushWindowMillis` passes are posted again. A refused - * target clears the body and pauses; a rejected body is skipped. + * Advances past a shipment once Analytics stored all its events, posting only missing events. + * A post that got no 5xx may be queued in the launch that took it, so its missing events are + * posted again only once that launch ended and `flushWindowMillis` passed since the post ended. + * Events that launch leaves unstored for `stuckMillis` are posted one at a time, and one that + * still is not stored on its own is skipped. A refused target clears the pending body and + * pauses; a rejected body is skipped. */ - const settle = Effect.fnUntraced(function* ( + const settle = Effect.fn("LogForwarder.settle")(function* ( session: Session, - initial: Pending, + shipment: Shipment, events: ReadonlyMap, - alreadyPosted: boolean, + posted: Pending | undefined, ) { - const warned = yield* Ref.make(false); - let pending = initial; - let posted = alreadyPosted; - let due = posted ? pending.sentAt + flushWindowMillis : 0; - settling: while (true) { - const stored = yield* session.current.stored - .storedIds(pending.source, pending.ids) - .pipe( - retryWarningOnce( - `Checking which ${pending.source} logs Analytics stored failed; retrying`, - ), - ); - const missing = pending.ids.filter((id) => !stored.has(id)); - if (missing.length === 0) break; - const now = yield* Clock.currentTimeMillis; - if (now < due) { - yield* Effect.sleep(pollMillis); - continue; - } - const resend = missing.flatMap((id) => { - const event = events.get(id); - return event === undefined ? [] : [{ event }]; - }); - if (resend.length < missing.length) { - yield* Effect.logWarning( - `Logs of ${session.service} instance ${session.instanceId} were deleted before Analytics stored them; skipping ${missing.length - resend.length} lines`, - ); - pending = { ...pending, ids: pending.ids.filter((id) => stored.has(id) || events.has(id)) }; - } - if (resend.length === 0) break; - if (posted) { - pending = { ...pending, sentAt: now }; - yield* persist(session, { - confirmed: (yield* Ref.get(session.progress)).confirmed, - pending, + const { source } = shipment; + yield* Effect.annotateCurrentSpan({ source, event_count: shipment.ids.length }); + const postWarned = yield* Ref.make(false); + const counts = { checks: 0, posts: 0, skipped: 0 }; + let remaining = shipment.ids; + let latest: { readonly queuedIn: string | undefined; readonly endedAt: number } | undefined = + posted === undefined ? undefined : { queuedIn: posted.launch, endedAt: posted.postEndsAt }; + + /** Posts `initial` until Analytics stored it, and returns the ids left unstored for `stuckMillis`. */ + const deliverAll = Effect.fnUntraced(function* (initial: ReadonlyArray) { + let wanted = initial; + delivering: while (true) { + // Only the first check is traced; the rest are counted on this span. + const stored = yield* session.current.stored + .storedIds(source, wanted) + .pipe( + Effect.retry(retrySchedule), + counts.checks++ === 0 ? (check) => check : Effect.withTracerEnabled(false), + ); + const missing = wanted.filter((id) => !stored.has(id)); + if (missing.length === 0) return []; + const now = yield* Clock.currentTimeMillis; + if (latest !== undefined) { + if (latest.queuedIn === session.current.launch) { + if (now >= latest.endedAt + stuckMillis) return missing; + yield* Effect.sleep(pollMillis); + continue; + } + if (now < latest.endedAt + flushWindowMillis) { + yield* Effect.sleep(pollMillis); + continue; + } + } + const deleted = new Set(missing.filter((id) => !events.has(id))); + if (deleted.size > 0) { + yield* Effect.logWarning( + `Logs of ${session.service} instance ${session.instanceId} were deleted before Analytics stored them; skipping ${deleted.size} lines`, + ); + wanted = wanted.filter((id) => !deleted.has(id)); + remaining = remaining.filter((id) => !deleted.has(id)); + } + const resend = missing.flatMap((id) => { + const event = events.get(id); + return event === undefined ? [] : [{ event }]; }); - } - for (const { body } of bodies(resend)) { - const delivery = yield* deliver(session, pending.source, body, warned); - if (delivery === "refused") { + if (resend.length === 0) return []; + let queuedIn: string | undefined; + for (const { body } of bodies(resend)) { + const deadline = (yield* Clock.currentTimeMillis) + postTimeoutMillis; yield* persist(session, { confirmed: (yield* Ref.get(session.progress)).confirmed, - pending: undefined, + pending: { + ...positionOf(shipment), + source, + ids: remaining, + launch: session.current.launch, + postEndsAt: deadline, + }, }); - return yield* new StaleTarget(); + // A slow cursor write leaves no time to post by the saved deadline. + if ((yield* Clock.currentTimeMillis) >= deadline) continue delivering; + counts.posts++; + const delivery = yield* deliver(session, source, body, deadline, postWarned); + if (delivery === "refused") { + yield* persist(session, { + confirmed: (yield* Ref.get(session.progress)).confirmed, + pending: undefined, + }); + return yield* new StaleTarget(); + } + if (delivery === "rejected") return "rejected"; + if (delivery !== "failed") queuedIn = session.current.launch; } - if (delivery === "rejected") break settling; + latest = { queuedIn, endedAt: yield* Clock.currentTimeMillis }; + } + }); + + const stuck = yield* deliverAll(shipment.ids); + if (stuck !== "rejected" && stuck.length > 0) { + yield* Effect.logWarning( + `Analytics has not stored ${stuck.length} ${source} lines of ${session.instanceId} it accepted; posting them one at a time`, + ); + for (const id of stuck) { + latest = undefined; + const alone = yield* deliverAll([id]); + if (alone !== "rejected" && alone.length === 0) continue; + if (alone !== "rejected") + yield* Effect.logWarning( + `Analytics did not store a ${source} line of ${session.instanceId} posted on its own; skipping it`, + ); + remaining = remaining.filter((other) => other !== id); + counts.skipped++; } - posted = true; - due = (yield* Clock.currentTimeMillis) + flushWindowMillis; } - yield* persist(session, { confirmed: positionOf(pending), pending: undefined }); + yield* Effect.annotateCurrentSpan({ + check_count: counts.checks, + post_count: counts.posts, + skipped_count: counts.skipped, + }); + yield* persist(session, { confirmed: positionOf(shipment), pending: undefined }); }); /** Rebuilds a pending body from its retained records and settles it. */ @@ -526,7 +628,7 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar ), ), ); - yield* settle(session, pending, events, true); + yield* settle(session, pending, events, pending); }); /** Ships one instance's records from its cursor until the target changes. */ @@ -567,21 +669,11 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar if (shipped !== undefined) unshipped.push(shipped); } for (const { items, last } of bodies(unshipped)) { - const pending: Pending = { - ...last.position, - source, - sentAt: yield* Clock.currentTimeMillis, - ids: items.map(({ event }) => event.id), - }; - yield* persist(shipping, { - confirmed: (yield* Ref.get(progress)).confirmed, - pending, - }); yield* settle( shipping, - pending, + { ...last.position, source, ids: items.map(({ event }) => event.id) }, new Map(items.map(({ event }) => [event.id, event])), - false, + undefined, ); yield* Ref.set(failing, false); } @@ -589,7 +681,7 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar ), ); }), - ); + ).pipe(current.users.withPermits(1)); const unregistered = (instance: ForwardedInstance) => instance.observation.pipe( diff --git a/packages/stack/src/host/LogflareEvents.ts b/packages/stack/src/host/LogflareEvents.ts index 5cc8c68d32..1b507f1df7 100644 --- a/packages/stack/src/host/LogflareEvents.ts +++ b/packages/stack/src/host/LogflareEvents.ts @@ -153,16 +153,39 @@ const remaps: Record LogflareEvent> = }, }; +/** + * Replaces NUL and unpaired surrogates with U+FFFD: Logflare stores events as Postgres `jsonb`, + * which rejects both, and one rejected event drops its whole batch. + */ +const storableText = (text: string): string => text.replaceAll("\u0000", "�").toWellFormed(); + +const storableValue = (value: unknown): unknown => { + if (typeof value === "string") return storableText(value); + if (Array.isArray(value)) return value.map(storableValue); + return typeof value === "object" && value !== null ? storableRecord(value) : value; +}; + +const storableRecord = (record: object): Record => + Object.fromEntries( + Object.entries(record).map(([key, value]) => [storableText(key), storableValue(value)]), + ); + /** Builds the Logflare event for one service log line received at `timestamp`. */ export const logflareEvent = ( service: ShippedService, timestamp: string, message: string, -): LogflareEvent => - remaps[service]({ +): LogflareEvent => { + const event = remaps[service]({ project: "default", event_message: message, appname: service, timestamp, metadata: {}, }); + return { + ...event, + event_message: storableText(event.event_message), + metadata: storableRecord(event.metadata), + }; +}; diff --git a/packages/stack/src/host/LogflareEvents.unit.test.ts b/packages/stack/src/host/LogflareEvents.unit.test.ts index 158e51def6..ab6e8b9c2f 100644 --- a/packages/stack/src/host/LogflareEvents.unit.test.ts +++ b/packages/stack/src/host/LogflareEvents.unit.test.ts @@ -122,3 +122,16 @@ it("derives the Postgres severity from the last level marker and defaults to LOG error_severity: "LOG", }); }); + +it("replaces NUL and unpaired surrogates, which Postgres jsonb rejects, in the message and metadata", () => { + const line = String.raw`{"msg":"a\u0000b","detail":["\ud800"],"k\u0000":"\udc00 x"}`; + const event = logflareEvent("auth", received, line); + + expect(logflareEvent("database", received, "a\u0000b").event_message).toBe("a�b"); + expect(event.metadata).toMatchObject({ + msg: "a�b", + detail: ["�"], + "k�": "� x", + }); + expect(event.event_message).toBe(line); +}); diff --git a/packages/stack/src/host/LogflareStorage.integration.test.ts b/packages/stack/src/host/LogflareStorage.integration.test.ts new file mode 100644 index 0000000000..a4aa18d1fa --- /dev/null +++ b/packages/stack/src/host/LogflareStorage.integration.test.ts @@ -0,0 +1,149 @@ +import { NodeHttpClient, NodeServices } from "@effect/platform-node"; +import { PgClient } from "@effect/sql-pg"; +import { describe, expect, it } from "@effect/vitest"; +import { Context, Effect, Exit, FileSystem, Layer, Redacted, Ref } from "effect"; +import { connect, createServer, type Socket } from "node:net"; // oxlint-disable-line effecttsgo/node-builtin-import -- real socket fixture. +import { tmpdir } from "node:os"; +import { makeService } from "../Service.ts"; +import { makeDatabase, type DatabaseConfig } from "../services/Database.ts"; +import * as LogflareStorage from "./LogflareStorage.ts"; + +const config: DatabaseConfig = { + version: "17", + databasePassword: Redacted.make("logflare-storage-password"), + jwtSecret: Redacted.make("logflare-storage-jwt-secret-with-32-chars"), + jwtExpiry: 3600, + healthTimeoutMs: 120_000, +}; + +const sourceToken = "0b6d7c8e-1f2a-4b3c-8d4e-5f6a7b8c9d0e"; +const uncreatedToken = "1c7e8d9f-2a3b-4c5d-9e6f-7a8b9c0d1e2f"; +const storedId = "8f0e7a9b-6c5d-8e4f-9a3b-2c1d0e9f8a7b"; +const missingId = "9a1b2c3d-4e5f-8a6b-9c7d-8e9f0a1b2c3d"; + +/** A native stack database whose `_supabase` database holds a Logflare-shaped source table. */ +const analyticsDatabase = Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const root = yield* fs.makeTempDirectoryScoped({ prefix: "stack-logflare-storage-" }); + const recipe = yield* makeDatabase({ + stackId: "logflare-storage", + instanceId: "database", + root, + cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, + runtime: "native", + }); + const service = yield* makeService(recipe.definition, { id: "database", config }); + yield* service.start; + yield* service.ready; + const endpoint = yield* recipe.endpoint; + if (endpoint.kind !== "unix") return yield* Effect.die("native database has no socket"); + const database: LogflareStorage.AnalyticsDatabase = { + host: endpoint.path, + port: endpoint.port, + database: "_supabase", + username: "supabase_admin", + password: Redacted.value(config.databasePassword), + }; + yield* Effect.scoped( + Effect.gen(function* () { + const services = yield* Layer.build( + PgClient.layer({ ...database, password: config.databasePassword }), + ); + yield* Context.get(services, PgClient.PgClient).unsafe(` + CREATE SCHEMA IF NOT EXISTS _analytics; + CREATE TABLE _analytics.sources (name text NOT NULL, token uuid NOT NULL); + INSERT INTO _analytics.sources VALUES + ('postgres.logs', '${sourceToken}'), ('auth.logs', '${uncreatedToken}'); + CREATE TABLE _analytics."log_events_${sourceToken.replaceAll("-", "_")}" (id uuid PRIMARY KEY); + INSERT INTO _analytics."log_events_${sourceToken.replaceAll("-", "_")}" VALUES ('${storedId}'); + `); + }), + ); + return { database, destroy: service.destroy }; +}); + +/** Relays TCP connections to a Unix socket until `cut` drops them all and refuses new ones. */ +const socketRelay = (path: string) => + Effect.acquireRelease( + Effect.callback<{ + readonly port: number; + readonly cut: () => void; + readonly close: () => void; + }>((resume) => { + const sockets = new Set(); + let cut = false; + const server = createServer((client) => { + if (cut) return client.destroy(); + const upstream = connect(path); + for (const socket of [client, upstream]) { + sockets.add(socket); + socket.on("close", () => sockets.delete(socket)); + socket.on("error", () => socket.destroy()); + } + client.pipe(upstream); + upstream.pipe(client); + }); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + if (address === null || typeof address === "string") + return resume(Effect.die("relay has no address")); + resume( + Effect.succeed({ + port: address.port, + cut: () => { + cut = true; + for (const socket of sockets) socket.destroy(); + }, + close: () => server.close(), + }), + ); + }); + }), + (relay) => Effect.sync(relay.close), + ); + +const layer = Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp); + +describe("LogflareStorage", { timeout: 180_000 }, () => { + it.live( + "reads stored ids from a source's table, none before Analytics creates it, and fails for an unknown source", + () => + Effect.gen(function* () { + const { database, destroy } = yield* analyticsDatabase; + const storage = yield* LogflareStorage.make(Effect.succeed(database)); + + const stored = yield* storage.storedIds("postgres.logs", [storedId, missingId]); + const uncreated = yield* storage.storedIds("auth.logs", [storedId]); + const unknown = yield* storage.storedIds("storage.logs", [storedId]).pipe(Effect.flip); + + expect([...stored]).toEqual([storedId]); + expect([...uncreated]).toEqual([]); + expect(unknown.message).toBe("Analytics has no storage.logs sources"); + yield* destroy; + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("connects again after a failed query, so it follows a database that moved", () => + Effect.gen(function* () { + const { database, destroy } = yield* analyticsDatabase; + const relay = yield* socketRelay(`${database.host}/.s.PGSQL.${database.port}`); + const location = yield* Ref.make({ + ...database, + host: "127.0.0.1", + port: relay.port, + }); + const storage = yield* LogflareStorage.make(Ref.get(location)); + const before = yield* storage.storedIds("postgres.logs", [storedId]); + + relay.cut(); + yield* Ref.set(location, database); + const failed = yield* storage.storedIds("postgres.logs", [storedId]).pipe(Effect.exit); + const after = yield* storage.storedIds("postgres.logs", [storedId]); + + expect([...before]).toEqual([storedId]); + expect(Exit.isFailure(failed)).toBe(true); + expect([...after]).toEqual([storedId]); + yield* destroy; + }).pipe(Effect.scoped, Effect.provide(layer)), + ); +}); diff --git a/packages/stack/src/host/LogflareStorage.ts b/packages/stack/src/host/LogflareStorage.ts index 8b35a87c9f..bdc8ff95e1 100644 --- a/packages/stack/src/host/LogflareStorage.ts +++ b/packages/stack/src/host/LogflareStorage.ts @@ -9,28 +9,51 @@ import { Option, RcRef, Redacted, + Ref, Schema, } from "effect"; import { failureMessage } from "../internal/failure-message.ts"; -import { schema } from "../services/Analytics.ts"; +import { backendConnection, schema } from "../services/Analytics.ts"; +import type { ServiceEndpoint } from "../services/Recipe.ts"; import { StoredEventsError, type StoredEvents } from "./LogForwarder.ts"; -/** The Postgres database Analytics' Postgres backend writes to, reachable from the host. */ +/** The database Analytics' Postgres backend writes to, as the host reaches it. */ export interface AnalyticsDatabase { readonly host: string; readonly port: number; readonly database: string; - readonly username: string | undefined; - readonly password: string | undefined; + readonly username: string; + readonly password: string; } +/** + * Locates Logflare's backend database from Analytics' database URL. A bound database is reached at + * its endpoint, because the bound URL addresses it from Analytics' runtime. + */ +export const analyticsDatabase = Effect.fn("LogflareStorage.analyticsDatabase")(function* ( + databaseUrl: string, + bound: ServiceEndpoint | undefined, +) { + const backend = yield* backendConnection(databaseUrl); + return { + host: + bound === undefined + ? backend.host + : ((bound.kind === "unix" ? bound.path : bound.host) ?? "127.0.0.1"), + port: bound === undefined ? Number(backend.port) : bound.port, + database: backend.database, + username: backend.username, + password: backend.password, + } satisfies AnalyticsDatabase; +}); + /** A source token with `-` replaced by `_`, as Logflare names its event tables. */ const tablePattern = /^log_events_[0-9a-f]{8}(?:_[0-9a-f]{4}){3}_[0-9a-f]{12}$/u; /** * Reads stored event ids from the `log_events_` table Logflare's Postgres backend keeps per * source. This couples to Logflare's private table layout, which the owner's Analytics - * integration test pins. + * integration test pins. Failures are retried by the caller and warned about once per outage. */ export const make = Effect.fn("LogflareStorage.make")(function* ( database: Effect.Effect, @@ -44,37 +67,53 @@ export const make = Effect.fn("LogflareStorage.make")(function* ( port: connection.port, database: connection.database, username: connection.username, - password: - connection.password === undefined ? undefined : Redacted.make(connection.password), + password: Redacted.make(connection.password), connectTimeout: "2 seconds", }), ), ), Effect.map((context) => Context.get(context, PgClient.PgClient)), + Effect.mapError( + (cause) => + new StoredEventsError({ + message: `Unable to reach the database Analytics stores events in: ${failureMessage(cause)}`, + cause, + }), + ), ), idleTimeToLive: Duration.infinity, }); + // A failed query can mean the database moved, so the next one connects again. const withClient = (run: (sql: PgClient.PgClient) => Effect.Effect) => - Effect.scoped(RcRef.get(client).pipe(Effect.flatMap(run))); + Effect.scoped(RcRef.get(client).pipe(Effect.flatMap(run))).pipe( + Effect.tapError(() => RcRef.invalidate(client)), + ); - // A source or table Analytics has not created yet holds no events and is looked up again. + // Only a source Analytics knows answers for its events; its table appears once Analytics uses it. const tables = yield* Cache.makeWith( (source: string) => withClient( (sql) => - sql<{ readonly name: string }>` - SELECT 'log_events_' || replace(token::text, '-', '_') AS name + sql<{ readonly name: string; readonly created: boolean }>` + SELECT 'log_events_' || replace(token::text, '-', '_') AS name, + to_regclass(format('%I.%I', ${schema}::text, 'log_events_' || replace(token::text, '-', '_'))) IS NOT NULL AS created FROM ${sql(schema)}.sources WHERE name = ${source} - AND to_regclass(format('%I.%I', ${schema}::text, 'log_events_' || replace(token::text, '-', '_'))) IS NOT NULL`, + LIMIT 2`, ).pipe( - Effect.flatMap(([row]) => { - if (row === undefined) return Effect.succeed(Option.none()); - return tablePattern.test(row.name) - ? Effect.succeed(Option.some(row.name)) - : Effect.fail( - new StoredEventsError({ message: `Unexpected Analytics table name ${row.name}` }), - ); + Effect.flatMap((rows) => { + const [row] = rows; + if (row === undefined || rows.length > 1) + return Effect.fail( + new StoredEventsError({ + message: `Analytics has ${row === undefined ? "no" : "several"} ${source} sources`, + }), + ); + if (!tablePattern.test(row.name)) + return Effect.fail( + new StoredEventsError({ message: `Unexpected Analytics table name ${row.name}` }), + ); + return Effect.succeed(row.created ? Option.some(row.name) : Option.none()); }), ), { @@ -83,28 +122,47 @@ export const make = Effect.fn("LogflareStorage.make")(function* ( Exit.isSuccess(exit) && Option.isSome(exit.value) ? Duration.infinity : Duration.zero, }, ); + const warned = yield* Ref.make(false); return { - storedIds: (source, ids) => - Effect.gen(function* () { + storedIds: Effect.fn("LogflareStorage.storedIds")( + function* (source: string, ids: ReadonlyArray) { + yield* Effect.annotateCurrentSpan({ source, requested_count: ids.length }); if (ids.length === 0) return new Set(); const table = yield* Cache.get(tables, source); - if (Option.isNone(table)) return new Set(); - const rows = yield* withClient( - (sql) => - sql<{ readonly id: string }>` - SELECT id::text AS id FROM ${sql(schema)}.${sql(table.value)} WHERE id IN ${sql.in(ids)}`, - ); + const rows = Option.isNone(table) + ? [] + : yield* withClient( + (sql) => + sql<{ readonly id: string }>` + SELECT id::text AS id FROM ${sql(schema)}.${sql(table.value)} WHERE id IN ${sql.in(ids)}`, + ); + yield* Effect.annotateCurrentSpan({ stored_count: rows.length }); return new Set(rows.map(({ id }) => id)); - }).pipe( - Effect.mapError((cause) => - Schema.is(StoredEventsError)(cause) - ? cause - : new StoredEventsError({ - message: `Unable to read stored ${source} events: ${failureMessage(cause)}`, - cause, - }), + }, + (effect, source) => + effect.pipe( + Effect.mapError((cause) => + Schema.is(StoredEventsError)(cause) + ? cause + : new StoredEventsError({ + message: `Unable to read stored ${source} events: ${failureMessage(cause)}`, + cause, + }), + ), + Effect.tapError((error) => + Ref.getAndSet(warned, true).pipe( + Effect.flatMap((already) => + already + ? Effect.void + : Effect.logWarning( + `Log shipping waits until Analytics' stored events can be read: ${error.message}`, + ), + ), + ), + ), + Effect.tap(() => Ref.set(warned, false)), ), - ), + ), } satisfies StoredEvents; }); diff --git a/packages/stack/src/host/LogflareStorage.unit.test.ts b/packages/stack/src/host/LogflareStorage.unit.test.ts new file mode 100644 index 0000000000..c6c1d9ecff --- /dev/null +++ b/packages/stack/src/host/LogflareStorage.unit.test.ts @@ -0,0 +1,35 @@ +import { expect, it } from "@effect/vitest"; +import { Effect } from "effect"; +import { analyticsDatabase } from "./LogflareStorage.ts"; + +it.effect("reaches a configured Analytics database at its URL's address", () => + Effect.gen(function* () { + expect( + yield* analyticsDatabase("postgresql://db.example.test:6543/postgres", undefined), + ).toEqual({ + host: "db.example.test", + port: 6543, + database: "_supabase", + username: "supabase_admin", + password: "postgres", + }); + }), +); + +it.effect("reaches a bound Analytics database at its endpoint with the URL's credentials", () => + Effect.gen(function* () { + expect( + yield* analyticsDatabase("postgresql://supabase_admin:secret@db-runtime:5432/_supabase", { + kind: "unix", + path: "/tmp/stack-socket", + port: 5432, + }), + ).toEqual({ + host: "/tmp/stack-socket", + port: 5432, + database: "_supabase", + username: "supabase_admin", + password: "secret", + }); + }), +); diff --git a/packages/stack/src/services/Analytics.ts b/packages/stack/src/services/Analytics.ts index fda10aaec4..d8da75c2b2 100644 --- a/packages/stack/src/services/Analytics.ts +++ b/packages/stack/src/services/Analytics.ts @@ -1,4 +1,5 @@ import { Effect, Schema } from "effect"; +import { ServiceError } from "../Service.ts"; import { EndpointIntent, serviceCreation } from "./Recipe.ts"; import { databaseConnection, requiredInput } from "./ServiceConfig.ts"; import { type ProcessRecipeSpec } from "./ProcessRecipe.ts"; @@ -20,6 +21,43 @@ export const schema = "_analytics"; export interface Creation extends Schema.Schema.Type {} +/** The database Logflare keeps its sources and its Postgres backend's event tables in. */ +interface BackendConnection { + readonly host: string; + readonly port: string; + readonly database: string; + readonly username: string; + readonly password: string; + /** The connection as a URL, with the query parameters of Analytics' database URL. */ + readonly url: string; +} + +/** Derives Logflare's backend database from Analytics' database URL and its credential defaults. */ +export const backendConnection = Effect.fn("Analytics.backendConnection")(function* ( + databaseUrl: string, +) { + const db = yield* databaseConnection(databaseUrl); + const connection = { + host: db.host, + port: db.port, + database: "_supabase", + username: db.username ?? "supabase_admin", + password: db.password ?? "postgres", + }; + const url = yield* Effect.try({ + try: () => { + const parsed = new URL(databaseUrl); + parsed.username = encodeURIComponent(connection.username); + parsed.password = encodeURIComponent(connection.password); + parsed.pathname = `/${connection.database}`; + return parsed.toString(); + }, + catch: (cause) => + new ServiceError({ operation: "launch", message: "Invalid Analytics database URL", cause }), + }); + return { ...connection, url } satisfies BackendConnection; +}); + export const makeSpec = (): ProcessRecipeSpec => ({ service: "analytics", executable: "bin/logflare", @@ -33,25 +71,25 @@ export const makeSpec = (): ProcessRecipeSpec => ({ "databaseUrl", creation.config.databaseUrl, ); - const db = yield* databaseConnection(databaseUrl); + const backend = yield* backendConnection(databaseUrl); return { DATABASE_URL: databaseUrl, ...(http === undefined ? {} : { PORT: String(http.port), PHX_HTTP_PORT: String(http.port) }), - DB_DATABASE: "_supabase", + DB_DATABASE: backend.database, DB_SCHEMA: schema, - DB_HOSTNAME: db.host, - DB_PORT: db.port, - DB_USERNAME: db.username ?? "supabase_admin", - DB_PASSWORD: db.password ?? "postgres", + DB_HOSTNAME: backend.host, + DB_PORT: backend.port, + DB_USERNAME: backend.username, + DB_PASSWORD: backend.password, LOGFLARE_SUPABASE_MODE: "true", LOGFLARE_SINGLE_TENANT: "true", ...(container ? {} : { LOGFLARE_GRPC_PORT: "0" }), ...(creation.config.apiKey === undefined ? {} : { LOGFLARE_PRIVATE_ACCESS_TOKEN: creation.config.apiKey }), - POSTGRES_BACKEND_URL: databaseUrl, + POSTGRES_BACKEND_URL: backend.url, POSTGRES_BACKEND_SCHEMA: schema, }; }), diff --git a/packages/stack/src/services/Analytics.unit.test.ts b/packages/stack/src/services/Analytics.unit.test.ts new file mode 100644 index 0000000000..c267474ab9 --- /dev/null +++ b/packages/stack/src/services/Analytics.unit.test.ts @@ -0,0 +1,44 @@ +import { expect, it } from "@effect/vitest"; +import { Effect } from "effect"; +import { makeSpec, type Creation } from "./Analytics.ts"; + +const creation = (databaseUrl: string): Creation => ({ + service: "analytics", + config: { databaseUrl, apiKey: "analytics-key" }, +}); + +it.effect("keeps Logflare's events in the internal database it keeps its sources in", () => + Effect.gen(function* () { + const env = yield* makeSpec().env( + creation("postgresql://reader:se%40cret@db.internal:6543/postgres?sslmode=disable"), + new Map(), + true, + ); + + expect(env).toMatchObject({ + DB_HOSTNAME: "db.internal", + DB_PORT: "6543", + DB_DATABASE: "_supabase", + DB_USERNAME: "reader", + DB_PASSWORD: "se@cret", + POSTGRES_BACKEND_URL: + "postgresql://reader:se%40cret@db.internal:6543/_supabase?sslmode=disable", + }); + }), +); + +it.effect("gives Logflare's sources and events the default credentials a URL leaves out", () => + Effect.gen(function* () { + const env = yield* makeSpec().env( + creation("postgresql://db.internal/_supabase"), + new Map(), + true, + ); + + expect(env).toMatchObject({ + DB_USERNAME: "supabase_admin", + DB_PASSWORD: "postgres", + POSTGRES_BACKEND_URL: "postgresql://supabase_admin:postgres@db.internal/_supabase", + }); + }), +); diff --git a/packages/stack/tests/fake-logflare.ts b/packages/stack/tests/fake-logflare.ts index ec152b420e..280827af72 100644 --- a/packages/stack/tests/fake-logflare.ts +++ b/packages/stack/tests/fake-logflare.ts @@ -30,12 +30,24 @@ export interface FakeLogflareOptions { readonly batchEvents?: number; /** Stores queued events this long after they arrive; without it only `apply` stores them. */ readonly flushMillis?: number; + /** Events whose batch fails to store, besides those holding text Postgres jsonb rejects. */ + readonly unstorable?: (event: FakeEvent) => boolean; } +/** Postgres jsonb rejects NUL and unpaired surrogates anywhere in a document. */ +const jsonbStorable = (value: unknown): boolean => { + if (typeof value === "string") return !value.includes("\u0000") && value.isWellFormed(); + if (Array.isArray(value)) return value.every(jsonbStorable); + return typeof value === "object" && value !== null + ? Object.entries(value).every(([key, item]) => jsonbStorable(key) && jsonbStorable(item)) + : true; +}; + /** * A Logflare stand-in with its asynchronous Postgres pipeline: an accepted post only queues its * events, and each per-source batch is stored atomically, or dropped whole when one of its ids is - * already stored. Listeners share the storage, like an Analytics restarted on a new port. + * already stored or one of its events cannot be stored. Listeners share the storage, like an + * Analytics restarted on a new port. */ export const makeFakeLogflare = (options: FakeLogflareOptions = {}) => Effect.gen(function* () { @@ -57,7 +69,15 @@ export const makeFakeLogflare = (options: FakeLogflareOptions = {}) => const batch = events.splice(0, batchEvents); applied++; const ids = new Set(batch.map(({ id }) => id)); - if (ids.size < batch.length || batch.some(({ id }) => table.has(id))) + if ( + ids.size < batch.length || + batch.some( + (event) => + table.has(event.id) || + !jsonbStorable(event) || + options.unstorable?.(event) === true, + ) + ) counts.dropped += batch.length; else for (const event of batch) table.set(event.id, event); } @@ -133,6 +153,8 @@ export const makeFakeLogflare = (options: FakeLogflareOptions = {}) => /** Opens another ingest port sharing this fake's storage. */ listen: (scope: Scope.Scope) => listen.pipe(Scope.provide(scope)), next: Queue.take(received), + /** How many received posts `next` has not taken yet. */ + unread: Queue.size(received), /** The next posts answer with these statuses; only a 2xx queues its events. */ respond: (...next: ReadonlyArray) => Effect.sync(() => statuses.push(...next)), /** Holds the next post's response until the returned effect runs; it ends once the response does. */ @@ -149,7 +171,19 @@ export const makeFakeLogflare = (options: FakeLogflareOptions = {}) => /** Loses every queued event, like Analytics stopping before its pipeline flushed. */ discard: Effect.sync(() => queued.clear()), storedIds, - storedEvents: Effect.succeed({ storedIds } satisfies StoredEvents), + /** A stored-event view that interrupts its callers once its scope closed, like Logflare's. */ + storedEvents: Effect.gen(function* () { + let closed = false; + yield* Effect.addFinalizer(() => + Effect.sync(() => { + closed = true; + }), + ); + return { + storedIds: (source, ids) => + Effect.suspend(() => (closed ? Effect.interrupt : storedIds(source, ids))), + } satisfies StoredEvents; + }), stored: Effect.sync(() => [...stored.values()].flatMap((table) => [...table.values()])), dropped: Effect.sync(() => counts.dropped), aborted: Effect.sync(() => counts.aborted), From 46d7c09f1c4049b77bdd3732397485502761c4a8 Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 2 Oct 2026 15:16:52 +0200 Subject: [PATCH 21/35] fix(stack): retry refused log posts and skip events only when Analytics stores - Count any HTTP status and pre-send failure as a failed post that retries; only timeouts and post-send transport errors leave a body possibly queued. - Re-post a previous owner's possibly-queued events only after a grace that exceeds Analytics' shutdown. - Isolate unstored events by halves and skip one only after Analytics stored a later post on the same launch, so an Analytics that stores nothing keeps every event pending. - Start the backstop when shipping resumes, document the ignored database name, and keep pure helpers untraced. Co-Authored-By: Claude Opus 5.5 --- .../experimental/stack/start/SIDE_EFFECTS.md | 9 +- packages/stack/ARCHITECTURE.md | 18 +- .../src/host/LogForwarder.integration.test.ts | 193 +++++++++++++++--- packages/stack/src/host/LogForwarder.ts | 162 ++++++++++++--- .../host/LogflareStorage.integration.test.ts | 19 +- packages/stack/src/host/LogflareStorage.ts | 2 +- .../src/host/LogflareStorage.unit.test.ts | 14 -- packages/stack/src/services/Analytics.ts | 5 +- 8 files changed, 330 insertions(+), 92 deletions(-) diff --git a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md index 473353c332..a3b73dd9ac 100644 --- a/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/experimental/stack/start/SIDE_EFFECTS.md @@ -107,10 +107,11 @@ Postgres tables (`_analytics.sources` and `_analytics.log_events_` in the `_supabase` database) to confirm which events are stored, posts only the missing ones, and advances the position once all are stored. While those tables cannot be read, shipping waits and logs one warning. Events an Analytics process accepted are not posted to it again while it may -still store them; events still missing are posted again after Analytics restarts or the owner -restarts, once 5 seconds passed since the request ended, and a pending request is confirmed the -same way. Events still missing 60 seconds after Analytics accepted them are posted one at a time, -and one that is still not stored is skipped with a warning. NUL characters and unpaired +still store them; events still missing are posted again 5 seconds after the request ended once +Analytics restarted, or 60 seconds after it once the owner restarted, and a pending request is +confirmed the same way. Events still missing after 60 seconds of Analytics serving are posted +again in halves; one that is still not stored on its own is skipped with a warning once Analytics +stored a later request, and kept otherwise. NUL characters and unpaired surrogates in shipped lines are replaced with U+FFFD. Shipping never wakes Analytics and does not count as idle activity, so a lazy Analytics still stops on its idle timer while other services log. Service log streams and `supabase stack logs` are never blocked by shipping. diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 1d87bca10e..11960a9d8c 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -686,13 +686,17 @@ check is retried and shipping waits, with one warning per outage. A known source holds no events. The pending body is written atomically before each post, and a failed write blocks posting. Only ids not yet stored are posted. A post ends by its 5 second timeout, which the pending body records. Logflare can take many seconds to store what it accepted, so a post that -succeeded, timed out or was interrupted is not repeated to the Analytics launch that took it -while it may still store it: the owner polls the stored ids until all are stored, then advances -the cursor and clears the pending body. Ids still missing are posted again once that launch ended -(a new launch, or another owner) and 5 seconds passed since the post ended; a 5xx answer is posted -again 5 seconds after it. Ids the accepting launch leaves unstored for 60 seconds count as a lost -batch: each is posted on its own, and one that still is not stored alone is skipped with a -warning, so one unstorable event cannot block an instance. Event text and metadata strings have +succeeded, timed out or broke after it was sent is not repeated to the Analytics launch that took +it while it may still store it: the owner polls the stored ids until all are stored, then advances +the cursor and clears the pending body. Ids still missing are posted again once that launch can no +longer store them: 5 seconds after the post ended when it was a launch of this owner, which starts +a launch only after the previous one exited, and 60 seconds after it when it was another owner's, +whose process may still be draining. Any other HTTP status and a request that was never sent are +posted again 5 seconds later. Ids the serving launch leaves unstored for 60 seconds of serving +count as a lost batch: they are posted again in halves until each unstored one was posted alone. +A lone unstored id is skipped with a warning once a later post to the launch was stored, which +shows Analytics is storing; otherwise it stays pending and is posted alone again, with one warning, +so an Analytics that stores nothing loses nothing. Event text and metadata strings have NUL and unpaired surrogates replaced with U+FFFD before posting, because Postgres `jsonb` rejects them and drops the whole batch. A start, restart or retarget reconciles a pending body before later records; pending records already deleted by retention are skipped with a warning. A 401, 403 diff --git a/packages/stack/src/host/LogForwarder.integration.test.ts b/packages/stack/src/host/LogForwarder.integration.test.ts index cfcfbb3470..90aa46963c 100644 --- a/packages/stack/src/host/LogForwarder.integration.test.ts +++ b/packages/stack/src/host/LogForwarder.integration.test.ts @@ -111,6 +111,30 @@ const nextPost = (logflare: FakeLogflare, manual: ManualClock): Effect.Effect) => Effect.Effect, +) => + Effect.gen(function* () { + const posts: Array = []; + for (let step = 0; !(yield* done(posts)); step++) { + if (step > 5_000) return yield* Effect.die("the forwarder stopped making progress"); + if ((yield* logflare.unread) > 0) { + posts.push(yield* logflare.next); + yield* logflare.apply(); + continue; + } + yield* manual.sleeping(pollMillis); + if ((yield* logflare.unread) === 0) yield* manual.advance(stuckMillis); + } + return posts; + }); + const observationOf = ( id: string, overrides: Partial>, @@ -735,7 +759,7 @@ describe("LogForwarder", () => { ); it.live( - "posts a pending body again after a restart only once the flush window passed since its slow post ended", + "posts a pending body again after a restart only once the previous owner's launch had time to store it since its slow post ended", () => Effect.gen(function* () { const { store, logflare, database, analytics } = yield* fixture({ logflare: {} }); @@ -766,7 +790,7 @@ describe("LogForwarder", () => { yield* startForwarder(store, logflare, instances, manual.clock); yield* manual.sleeping(pollMillis); - yield* manual.advance(flushWindowMillis - 1); + yield* manual.advance(stuckMillis - 1); yield* manual.sleeping(pollMillis); const resentInWindow = yield* logflare.unread; yield* manual.advance(postTimeoutMillis); @@ -777,6 +801,77 @@ describe("LogForwarder", () => { }).pipe(Effect.scoped, Effect.provide(layer)), ); + it.live( + "does not post a body again after a restart while the previous owner's launch still stores it late", + () => + Effect.gen(function* () { + const { store, logflare, database, analytics } = yield* fixture({ logflare: {} }); + const manual = yield* makeManualClock; + const instances = [analytics.instance, database.instance]; + yield* analytics.set(true); + const first = yield* startForwarder(store, logflare, instances, manual.clock); + yield* database.log("line 1"); + yield* logflare.next; + yield* manual.sleeping(pollMillis); + yield* first.stop; + + yield* startForwarder(store, logflare, instances, manual.clock); + yield* manual.sleeping(pollMillis); + yield* manual.advance(postTimeoutMillis + 2 * flushWindowMillis); + yield* manual.sleeping(pollMillis); + const resentBeforeDrain = yield* logflare.unread; + yield* logflare.apply(); + yield* database.log("line 2"); + const later = yield* nextPost(logflare, manual); + yield* logflare.apply(); + + expect(resentBeforeDrain).toBe(0); + expect(messages(later)).toEqual(["line 2"]); + expect(yield* storedMessages(logflare)).toEqual(["line 1", "line 2"]); + expect(yield* logflare.dropped).toBe(0); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + + it.live("posts a body again after 429s without skipping any of it", () => + Effect.gen(function* () { + const { store, logflare, database, analytics } = yield* fixture({ logflare: {} }); + const manual = yield* makeManualClock; + const warnings: Array = []; + const captured = Logger.layer([ + Logger.make(({ logLevel, message }) => { + if (logLevel === "Warn") warnings.push(message); + }), + ]); + yield* analytics.set(true); + yield* startForwarder( + store, + logflare, + [analytics.instance, database.instance], + manual.clock, + ).pipe(Effect.provide(captured)); + yield* logflare.respond(429, 429, 429); + + yield* database.log("line 1", "line 2"); + const attempts = [yield* logflare.next]; + while (attempts.length < 4) { + yield* manual.sleeping(pollMillis); + yield* manual.advance(flushWindowMillis); + attempts.push(yield* logflare.next); + } + yield* logflare.apply(); + yield* database.log("line 3"); + const later = yield* nextPost(logflare, manual); + yield* logflare.apply(); + + expect(attempts.map(messages)).toEqual(Array.from({ length: 4 }, () => ["line 1", "line 2"])); + expect(messages(later)).toEqual(["line 3"]); + expect(yield* storedMessages(logflare)).toEqual(["line 1", "line 2", "line 3"]); + expect(warnings).toEqual([ + ["Posting postgres.logs logs failed; retrying", expect.anything()], + ]); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + it.live("warns about pending events retention deleted and ships the retained records", () => Effect.gen(function* () { const { root, store, logflare, database, analytics } = yield* fixture({ @@ -807,7 +902,7 @@ describe("LogForwarder", () => { Effect.provide(captured), ); yield* manual.sleeping(pollMillis); - yield* manual.advance(postTimeoutMillis + flushWindowMillis); + yield* manual.advance(postTimeoutMillis + stuckMillis); const shipped: Array = []; while (shipped.at(-1) !== "later 29") { shipped.push(...messages(yield* nextPost(logflare, manual))); @@ -883,13 +978,11 @@ describe("LogForwarder", () => { ); it.live( - "posts events a batch lost one at a time and skips the one that never stores, then ships later lines", + "isolates one unstorable event among 256 by halving, skips it alone, and ships the rest", () => Effect.gen(function* () { const { store, logflare, database, analytics } = yield* fixture({ - logflare: { - unstorable: (event) => event.event_message === "never stored", - }, + logflare: { unstorable: (event) => event.event_message === "line 100" }, }); const manual = yield* makeManualClock; const warnings: Array = []; @@ -905,43 +998,81 @@ describe("LogForwarder", () => { [analytics.instance, database.instance], manual.clock, ).pipe(Effect.provide(captured)); + const lines = Array.from({ length: 256 }, (_, index) => `line ${index}`); + const storedCount = storedMessages(logflare).pipe(Effect.map((stored) => stored.length)); - yield* database.log("good 1", "never stored", "good 2"); - const batch = yield* logflare.next; - yield* logflare.apply(); - yield* manual.sleeping(pollMillis); - yield* manual.advance(stuckMillis); - const first = yield* logflare.next; - yield* logflare.apply(); - const poison = yield* nextPost(logflare, manual); - yield* logflare.apply(); - yield* manual.sleeping(pollMillis); - yield* manual.advance(stuckMillis); - const second = yield* logflare.next; - yield* logflare.apply(); + yield* database.log(...lines); + const posts = yield* drive(logflare, manual, () => + Effect.map(storedCount, (n) => n >= 255), + ); yield* database.log("later"); - const later = yield* nextPost(logflare, manual); - yield* logflare.apply(); + yield* drive(logflare, manual, () => Effect.map(storedCount, (n) => n >= 256)); - expect(messages(batch)).toEqual(["good 1", "never stored", "good 2"]); - expect([first, poison, second, later].map(messages)).toEqual([ - ["good 1"], - ["never stored"], - ["good 2"], - ["later"], + expect(yield* storedMessages(logflare)).toEqual([ + ...lines.filter((line) => line !== "line 100"), + "later", ]); - expect((yield* storedMessages(logflare)).slice(0, 2)).toEqual(["good 1", "good 2"]); + expect(posts.filter((post) => post.events.length === 1).map(messages)).toContainEqual([ + "line 100", + ]); + expect(posts.length).toBeLessThan(40); expect(warnings).toEqual([ [ - "Analytics has not stored 3 postgres.logs lines of database it accepted; posting them one at a time", + "Analytics has not stored 256 postgres.logs lines of database it accepted; posting them in halves", ], [ - "Analytics did not store a postgres.logs line of database posted on its own; skipping it", + "Analytics did not store a postgres.logs line of database posted on its own while it stored later posts; skipping it", ], ]); }).pipe(Effect.scoped, Effect.provide(layer)), ); + it.live("keeps every event while Analytics stores nothing and ships them once it recovers", () => + Effect.gen(function* () { + let storing = false; + const { store, logflare, database, analytics } = yield* fixture({ + logflare: { unstorable: () => !storing }, + }); + const manual = yield* makeManualClock; + const warnings: Array = []; + const captured = Logger.layer([ + Logger.make(({ logLevel, message }) => { + if (logLevel === "Warn") warnings.push(message); + }), + ]); + yield* analytics.set(true); + yield* startForwarder( + store, + logflare, + [analytics.instance, database.instance], + manual.clock, + ).pipe(Effect.provide(captured)); + const lines = ["line 1", "line 2", "line 3", "line 4"]; + const storedCount = storedMessages(logflare).pipe(Effect.map((stored) => stored.length)); + + yield* database.log(...lines); + yield* drive(logflare, manual, () => Effect.sync(() => warnings.length >= 2)); + const retried = yield* drive(logflare, manual, (posts) => Effect.succeed(posts.length >= 8)); + const warnedWhileBroken = [...warnings]; + storing = true; + yield* drive(logflare, manual, () => Effect.map(storedCount, (n) => n >= 4)); + yield* database.log("later"); + yield* drive(logflare, manual, () => Effect.map(storedCount, (n) => n >= 5)); + + expect(yield* storedMessages(logflare)).toEqual([...lines, "later"]); + expect(warnedWhileBroken).toEqual([ + [ + "Analytics has not stored 4 postgres.logs lines of database it accepted; posting them in halves", + ], + [ + "Analytics stored none of 4 postgres.logs lines of database posted on their own; waiting until it stores them", + ], + ]); + expect(retried.map(messages)).toEqual([...lines, ...lines].map((line) => [line])); + expect(warnings).toEqual(warnedWhileBroken); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + it.live("keeps shipping to the new target after a retarget while a post is in flight", () => Effect.gen(function* () { const { store, logflare, database } = yield* fixture(); diff --git a/packages/stack/src/host/LogForwarder.ts b/packages/stack/src/host/LogForwarder.ts index 0ac763de4d..ce08c77c04 100644 --- a/packages/stack/src/host/LogForwarder.ts +++ b/packages/stack/src/host/LogForwarder.ts @@ -1,4 +1,5 @@ import { + Cause, Clock, Crypto, Data, @@ -9,6 +10,7 @@ import { FileSystem, Option, Path, + Predicate, Ref, Schedule, Schema, @@ -90,6 +92,8 @@ interface Target { readonly stored: StoredEvents; /** One permit per session using `stored`; the view closes once a retired target has none. */ readonly users: Semaphore.Semaphore; + /** The latest post to this launch, by sequence number, that Analytics stored events of. */ + readonly storedPost: Ref.Ref; } /** A shipped event; its id derives from the instance and record position. */ @@ -138,9 +142,11 @@ interface Session { readonly progress: Ref.Ref; /** Held while a post is in flight. */ readonly posting: Semaphore.Semaphore; + /** When this session started serving the target. */ + readonly resumedAt: number; } -/** `unsettled` posts got no answer, so Analytics may have queued them; `failed` ones got a 5xx. */ +/** `unsettled` posts may have reached Analytics without an answer; `failed` ones were not queued. */ type Delivery = "sent" | "unsettled" | "failed" | "refused" | "rejected"; const batchEvents = 256; @@ -153,7 +159,10 @@ export const pollMillis = 250; export const retryMillis = 500; /** The longest a post may take, so a pending body knows when its post ended at the latest. */ export const postTimeoutMillis = 5_000; -/** How long events may stay unstored in the launch that accepted them before they count as dropped. */ +/** + * How long a live launch may take to store what it accepted: past it, the serving launch's + * unstored events count as dropped, and a previous owner's launch has stopped or drained. + */ export const stuckMillis = 60_000; /** More sessions than can share one target. */ const targetUsers = 2 ** 30; @@ -222,6 +231,12 @@ export const bodies = ( return result; }; +/** Splits ids into two halves, the first one longer by at most one. */ +const halves = (ids: ReadonlyArray): Array> => { + const middle = Math.ceil(ids.length / 2); + return [ids.slice(0, middle), ids.slice(middle)].filter((half) => half.length > 0); +}; + const rejectedStatus = (error: unknown): number | undefined => error instanceof HttpClientError.HttpClientError && error.reason._tag === "StatusCodeError" ? error.reason.response.status @@ -242,6 +257,32 @@ const isBodyRejection = (error: unknown): boolean => { ); }; +/** Connection failures that mean a request never reached Analytics. */ +const unsentCodes = new Set([ + "ECONNREFUSED", + "ENOTFOUND", + "EHOSTUNREACH", + "ENETUNREACH", + "EADDRNOTAVAIL", + "EAI_AGAIN", +]); + +const errorCode = (cause: unknown): unknown => + Predicate.hasProperty(cause, "code") + ? cause.code + : Predicate.hasProperty(cause, "cause") + ? errorCode(cause.cause) + : undefined; + +/** A post that timed out or broke after it was sent, so Analytics may have queued its events. */ +const mayHaveQueued = (error: unknown): boolean => { + if (Cause.isTimeoutError(error)) return true; + if (!(error instanceof HttpClientError.HttpClientError)) return false; + const { reason } = error; + if (reason._tag === "TransportError") return !unsentCodes.has(String(errorCode(reason.cause))); + return reason._tag === "DecodeError" || reason._tag === "EmptyBodyError"; +}; + const warnOnce = (warned: Ref.Ref, message: string, error: unknown) => Ref.getAndSet(warned, true).pipe( Effect.flatMap((already) => (already ? Effect.void : Effect.logWarning(message, error))), @@ -281,7 +322,9 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar const epochs = yield* Ref.make(0); const rebinding = yield* Semaphore.make(1); const run = yield* crypto.randomUUIDv4; - const launchOf = (candidate: Candidate) => `${run}:${candidate.launchId ?? "unknown"}`; + const launchPrefix = `${run}:`; + const launchOf = (candidate: Candidate) => `${launchPrefix}${candidate.launchId ?? "unknown"}`; + let postSequence = 0; const rebind = Effect.gen(function* () { const members = new Set((yield* options.composition).members.map(({ id }) => id)); @@ -305,6 +348,7 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar launch: launchOf(serving), stored: yield* options.storedEvents(serving.instance).pipe(Scope.provide(scope)), users: yield* Semaphore.make(targetUsers), + storedPost: yield* Ref.make(0), }; const retired = yield* Ref.getAndSet( targetScope, @@ -473,7 +517,7 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar `Analytics rejected a ${source} log body; skipping it`, error, ).pipe(Effect.as("rejected")); - if ((rejectedStatus(error) ?? 0) >= 500) + if (!mayHaveQueued(error)) return warnOnce(warned, `Posting ${source} logs failed; retrying`, error).pipe( Effect.as("failed"), ); @@ -487,11 +531,12 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar /** * Advances past a shipment once Analytics stored all its events, posting only missing events. - * A post that got no 5xx may be queued in the launch that took it, so its missing events are - * posted again only once that launch ended and `flushWindowMillis` passed since the post ended. - * Events that launch leaves unstored for `stuckMillis` are posted one at a time, and one that - * still is not stored on its own is skipped. A refused target clears the pending body and - * pauses; a rejected body is skipped. + * A post that may have been queued is posted again only once the launch that took it can no + * longer store it: after `flushWindowMillis` for an ended launch of this owner, after + * `stuckMillis` for a previous owner's launch. Events the current launch leaves unstored for + * `stuckMillis` are posted again in halves; one that still is not stored on its own is skipped + * once a later post to the launch was stored, and kept pending otherwise. A refused + * target clears the pending body and pauses; a rejected body is skipped. */ const settle = Effect.fn("LogForwarder.settle")(function* ( session: Session, @@ -504,6 +549,9 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar const postWarned = yield* Ref.make(false); const counts = { checks: 0, posts: 0, skipped: 0 }; let remaining = shipment.ids; + /** The sequence number of the latest post, and the ids it carried. */ + let lastPost = 0; + let lastPosted: ReadonlySet = new Set(); let latest: { readonly queuedIn: string | undefined; readonly endedAt: number } | undefined = posted === undefined ? undefined : { queuedIn: posted.launch, endedAt: posted.postEndsAt }; @@ -518,16 +566,26 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar Effect.retry(retrySchedule), counts.checks++ === 0 ? (check) => check : Effect.withTracerEnabled(false), ); + if (lastPost > 0 && [...lastPosted].some((id) => stored.has(id))) + yield* Ref.update(session.current.storedPost, (post) => Math.max(post, lastPost)); const missing = wanted.filter((id) => !stored.has(id)); if (missing.length === 0) return []; const now = yield* Clock.currentTimeMillis; if (latest !== undefined) { - if (latest.queuedIn === session.current.launch) { - if (now >= latest.endedAt + stuckMillis) return missing; + const { queuedIn, endedAt } = latest; + if (queuedIn === session.current.launch) { + // Only time this session served the launch counts, not time Analytics was unhealthy. + if (now >= Math.max(endedAt, session.resumedAt) + stuckMillis) return missing; yield* Effect.sleep(pollMillis); continue; } - if (now < latest.endedAt + flushWindowMillis) { + // This owner starts a launch only after the previous one exited; another owner's launch + // may still be draining its queue. + const drained = + queuedIn === undefined || queuedIn.startsWith(launchPrefix) + ? flushWindowMillis + : stuckMillis; + if (now < endedAt + drained) { yield* Effect.sleep(pollMillis); continue; } @@ -561,6 +619,8 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar // A slow cursor write leaves no time to post by the saved deadline. if ((yield* Clock.currentTimeMillis) >= deadline) continue delivering; counts.posts++; + lastPost = ++postSequence; + lastPosted = new Set(resend.map(({ event }) => event.id)); const delivery = yield* deliver(session, source, body, deadline, postWarned); if (delivery === "refused") { yield* persist(session, { @@ -576,22 +636,66 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar } }); + const skip = (id: string, reason: string) => + Effect.logWarning( + `Analytics did not store a ${source} line of ${session.instanceId} posted on its own ${reason}; skipping it`, + ).pipe( + Effect.andThen( + Effect.sync(() => { + remaining = remaining.filter((other) => other !== id); + counts.skipped++; + }), + ), + ); + + /** + * Posts the ids Analytics left unstored in halves until each unstored one was posted alone. + * A lone unstored id is skipped only once a later post to this launch got stored, which shows + * Analytics is storing; otherwise it is posted alone again in the next round. + */ + const isolate = Effect.fnUntraced(function* (initial: ReadonlyArray) { + const idleWarned = yield* Ref.make(false); + let groups = halves(initial); + while (groups.length > 0) { + const suspects: Array<{ readonly id: string; readonly post: number }> = []; + for (let group = groups.shift(); group !== undefined; group = groups.shift()) { + latest = undefined; + const left = yield* deliverAll(group); + const [lone] = group; + if (left === "rejected") { + if (group.length === 1 && lone !== undefined) + yield* skip(lone, "as Analytics rejected it"); + else groups.unshift(...halves(group)); + } else if (left.length > 0) { + if (group.length === 1 && lone !== undefined) + suspects.push({ id: lone, post: lastPost }); + else groups.unshift(...halves(left)); + } + } + const proof = yield* Ref.get(session.current.storedPost); + for (const suspect of suspects) + if (proof > suspect.post) yield* skip(suspect.id, "while it stored later posts"); + const kept = new Set(remaining); + groups = suspects.flatMap(({ id }) => (kept.has(id) ? [[id]] : [])); + if (groups.length > 0) + yield* Ref.getAndSet(idleWarned, true).pipe( + Effect.flatMap((already) => + already + ? Effect.void + : Effect.logWarning( + `Analytics stored none of ${groups.length} ${source} lines of ${session.instanceId} posted on their own; waiting until it stores them`, + ), + ), + ); + } + }); + const stuck = yield* deliverAll(shipment.ids); if (stuck !== "rejected" && stuck.length > 0) { yield* Effect.logWarning( - `Analytics has not stored ${stuck.length} ${source} lines of ${session.instanceId} it accepted; posting them one at a time`, + `Analytics has not stored ${stuck.length} ${source} lines of ${session.instanceId} it accepted; posting them in halves`, ); - for (const id of stuck) { - latest = undefined; - const alone = yield* deliverAll([id]); - if (alone !== "rejected" && alone.length === 0) continue; - if (alone !== "rejected") - yield* Effect.logWarning( - `Analytics did not store a ${source} line of ${session.instanceId} posted on its own; skipping it`, - ); - remaining = remaining.filter((other) => other !== id); - counts.skipped++; - } + yield* isolate(stuck); } yield* Effect.annotateCurrentSpan({ check_count: counts.checks, @@ -644,7 +748,15 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar const directory = yield* options.logs.directory(instanceId); const saved = yield* readCursor(directory); const progress = yield* Ref.make(saved); - const shipping: Session = { instanceId, service, directory, current, progress, posting }; + const shipping: Session = { + instanceId, + service, + directory, + current, + progress, + posting, + resumedAt: yield* Clock.currentTimeMillis, + }; yield* reconcile(shipping, saved); const from = (yield* Ref.get(progress)).confirmed ?? "oldest"; const records = yield* options.logs.read(instanceId, { from, follow: true }); diff --git a/packages/stack/src/host/LogflareStorage.integration.test.ts b/packages/stack/src/host/LogflareStorage.integration.test.ts index a4aa18d1fa..9c13287d3e 100644 --- a/packages/stack/src/host/LogflareStorage.integration.test.ts +++ b/packages/stack/src/host/LogflareStorage.integration.test.ts @@ -1,7 +1,7 @@ import { NodeHttpClient, NodeServices } from "@effect/platform-node"; import { PgClient } from "@effect/sql-pg"; import { describe, expect, it } from "@effect/vitest"; -import { Context, Effect, Exit, FileSystem, Layer, Redacted, Ref } from "effect"; +import { Context, Effect, Exit, FileSystem, Layer, Redacted, Ref, Scope } from "effect"; import { connect, createServer, type Socket } from "node:net"; // oxlint-disable-line effecttsgo/node-builtin-import -- real socket fixture. import { tmpdir } from "node:os"; import { makeService } from "../Service.ts"; @@ -32,7 +32,14 @@ const analyticsDatabase = Effect.gen(function* () { cacheRoot: `${tmpdir()}/supabase-stack-artifacts`, runtime: "native", }); - const service = yield* makeService(recipe.definition, { id: "database", config }); + // The service runs its operations in its own scope, which must stay open while it is destroyed. + const serviceScope = yield* Scope.make(); + const service = yield* makeService(recipe.definition, { id: "database", config }).pipe( + Scope.provide(serviceScope), + ); + yield* Effect.addFinalizer(() => + service.destroy.pipe(Effect.ignore, Effect.andThen(Scope.close(serviceScope, Exit.void))), + ); yield* service.start; yield* service.ready; const endpoint = yield* recipe.endpoint; @@ -59,7 +66,7 @@ const analyticsDatabase = Effect.gen(function* () { `); }), ); - return { database, destroy: service.destroy }; + return database; }); /** Relays TCP connections to a Unix socket until `cut` drops them all and refuses new ones. */ @@ -109,7 +116,7 @@ describe("LogflareStorage", { timeout: 180_000 }, () => { "reads stored ids from a source's table, none before Analytics creates it, and fails for an unknown source", () => Effect.gen(function* () { - const { database, destroy } = yield* analyticsDatabase; + const database = yield* analyticsDatabase; const storage = yield* LogflareStorage.make(Effect.succeed(database)); const stored = yield* storage.storedIds("postgres.logs", [storedId, missingId]); @@ -119,13 +126,12 @@ describe("LogflareStorage", { timeout: 180_000 }, () => { expect([...stored]).toEqual([storedId]); expect([...uncreated]).toEqual([]); expect(unknown.message).toBe("Analytics has no storage.logs sources"); - yield* destroy; }).pipe(Effect.scoped, Effect.provide(layer)), ); it.live("connects again after a failed query, so it follows a database that moved", () => Effect.gen(function* () { - const { database, destroy } = yield* analyticsDatabase; + const database = yield* analyticsDatabase; const relay = yield* socketRelay(`${database.host}/.s.PGSQL.${database.port}`); const location = yield* Ref.make({ ...database, @@ -143,7 +149,6 @@ describe("LogflareStorage", { timeout: 180_000 }, () => { expect([...before]).toEqual([storedId]); expect(Exit.isFailure(failed)).toBe(true); expect([...after]).toEqual([storedId]); - yield* destroy; }).pipe(Effect.scoped, Effect.provide(layer)), ); }); diff --git a/packages/stack/src/host/LogflareStorage.ts b/packages/stack/src/host/LogflareStorage.ts index bdc8ff95e1..f6e6157ac7 100644 --- a/packages/stack/src/host/LogflareStorage.ts +++ b/packages/stack/src/host/LogflareStorage.ts @@ -30,7 +30,7 @@ export interface AnalyticsDatabase { * Locates Logflare's backend database from Analytics' database URL. A bound database is reached at * its endpoint, because the bound URL addresses it from Analytics' runtime. */ -export const analyticsDatabase = Effect.fn("LogflareStorage.analyticsDatabase")(function* ( +export const analyticsDatabase = Effect.fnUntraced(function* ( databaseUrl: string, bound: ServiceEndpoint | undefined, ) { diff --git a/packages/stack/src/host/LogflareStorage.unit.test.ts b/packages/stack/src/host/LogflareStorage.unit.test.ts index c6c1d9ecff..2a67444654 100644 --- a/packages/stack/src/host/LogflareStorage.unit.test.ts +++ b/packages/stack/src/host/LogflareStorage.unit.test.ts @@ -2,20 +2,6 @@ import { expect, it } from "@effect/vitest"; import { Effect } from "effect"; import { analyticsDatabase } from "./LogflareStorage.ts"; -it.effect("reaches a configured Analytics database at its URL's address", () => - Effect.gen(function* () { - expect( - yield* analyticsDatabase("postgresql://db.example.test:6543/postgres", undefined), - ).toEqual({ - host: "db.example.test", - port: 6543, - database: "_supabase", - username: "supabase_admin", - password: "postgres", - }); - }), -); - it.effect("reaches a bound Analytics database at its endpoint with the URL's credentials", () => Effect.gen(function* () { expect( diff --git a/packages/stack/src/services/Analytics.ts b/packages/stack/src/services/Analytics.ts index d8da75c2b2..ccb807e3ec 100644 --- a/packages/stack/src/services/Analytics.ts +++ b/packages/stack/src/services/Analytics.ts @@ -5,6 +5,7 @@ import { databaseConnection, requiredInput } from "./ServiceConfig.ts"; import { type ProcessRecipeSpec } from "./ProcessRecipe.ts"; export const Config = Schema.Struct({ + /** The Postgres server Logflare uses; its path is ignored, as Logflare always uses `_supabase`. */ databaseUrl: Schema.optionalKey(Schema.String), backend: Schema.optionalKey(Schema.Literal("postgres")), apiKey: Schema.optionalKey(Schema.String), @@ -33,9 +34,7 @@ interface BackendConnection { } /** Derives Logflare's backend database from Analytics' database URL and its credential defaults. */ -export const backendConnection = Effect.fn("Analytics.backendConnection")(function* ( - databaseUrl: string, -) { +export const backendConnection = Effect.fnUntraced(function* (databaseUrl: string) { const db = yield* databaseConnection(databaseUrl); const connection = { host: db.host, From 33e4f00b0eb3018538971873a4ddc6b79aa708e6 Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 2 Oct 2026 19:31:42 +0200 Subject: [PATCH 22/35] fix(stack): prove log storage per source and name launches by instance Co-Authored-By: Claude Opus 5.5 --- packages/stack/src/host/LogForwarder.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/stack/src/host/LogForwarder.ts b/packages/stack/src/host/LogForwarder.ts index ce08c77c04..95ba25064b 100644 --- a/packages/stack/src/host/LogForwarder.ts +++ b/packages/stack/src/host/LogForwarder.ts @@ -92,8 +92,8 @@ interface Target { readonly stored: StoredEvents; /** One permit per session using `stored`; the view closes once a retired target has none. */ readonly users: Semaphore.Semaphore; - /** The latest post to this launch, by sequence number, that Analytics stored events of. */ - readonly storedPost: Ref.Ref; + /** Per source, the latest post to this launch, by sequence number, that Analytics stored events of. */ + readonly storedPost: Ref.Ref>; } /** A shipped event; its id derives from the instance and record position. */ @@ -323,7 +323,8 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar const rebinding = yield* Semaphore.make(1); const run = yield* crypto.randomUUIDv4; const launchPrefix = `${run}:`; - const launchOf = (candidate: Candidate) => `${launchPrefix}${candidate.launchId ?? "unknown"}`; + const launchOf = (candidate: Candidate) => + `${launchPrefix}${candidate.instance.id}:${candidate.launchId ?? "unknown"}`; let postSequence = 0; const rebind = Effect.gen(function* () { @@ -348,7 +349,7 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar launch: launchOf(serving), stored: yield* options.storedEvents(serving.instance).pipe(Scope.provide(scope)), users: yield* Semaphore.make(targetUsers), - storedPost: yield* Ref.make(0), + storedPost: yield* Ref.make>(new Map()), }; const retired = yield* Ref.getAndSet( targetScope, @@ -567,7 +568,9 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar counts.checks++ === 0 ? (check) => check : Effect.withTracerEnabled(false), ); if (lastPost > 0 && [...lastPosted].some((id) => stored.has(id))) - yield* Ref.update(session.current.storedPost, (post) => Math.max(post, lastPost)); + yield* Ref.update(session.current.storedPost, (posts) => + new Map(posts).set(source, Math.max(posts.get(source) ?? 0, lastPost)), + ); const missing = wanted.filter((id) => !stored.has(id)); if (missing.length === 0) return []; const now = yield* Clock.currentTimeMillis; @@ -672,7 +675,7 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar else groups.unshift(...halves(left)); } } - const proof = yield* Ref.get(session.current.storedPost); + const proof = (yield* Ref.get(session.current.storedPost)).get(source) ?? 0; for (const suspect of suspects) if (proof > suspect.post) yield* skip(suspect.id, "while it stored later posts"); const kept = new Set(remaining); From 91458781880e6525660b665874b9fbc07366cb2c Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 2 Oct 2026 19:40:59 +0200 Subject: [PATCH 23/35] fix(stack): redact Edge Function websocket JWTs in gateway logs Co-Authored-By: Claude Opus 5.5 --- packages/stack/src/HttpProxy.ts | 1 + packages/stack/src/HttpProxy.unit.test.ts | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/packages/stack/src/HttpProxy.ts b/packages/stack/src/HttpProxy.ts index a1132aca94..d1f3f6cdbd 100644 --- a/packages/stack/src/HttpProxy.ts +++ b/packages/stack/src/HttpProxy.ts @@ -167,6 +167,7 @@ const decodeNested = (value: string) => { // presigned URLs carry their SigV4 signature, credential scope and session token. const credentialParameters = [ "apikey", + "jwt", "access_token", "token", "token_hash", diff --git a/packages/stack/src/HttpProxy.unit.test.ts b/packages/stack/src/HttpProxy.unit.test.ts index f05992d06f..8b3c2c84b3 100644 --- a/packages/stack/src/HttpProxy.unit.test.ts +++ b/packages/stack/src/HttpProxy.unit.test.ts @@ -8,6 +8,11 @@ describe("redactCredentials", () => { url: "/api/ok?select=*&apikey=sb_secret_x&Access_Token=jwt", redacted: "/api/ok?select=*&apikey=redacted&Access_Token=redacted", }, + { + case: "redacts an Edge Function websocket JWT", + url: "/functions/v1/realtime-chat?jwt=eyJ.p.s", + redacted: "/functions/v1/realtime-chat?jwt=redacted", + }, { case: "redacts Auth codes and OTP token hashes", url: "/auth/v1/verify?code=pkce&token_hash=h&type=signup", From 2cc21137383dcd31fa41b3b25b464a5aff5a1580 Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 2 Oct 2026 20:22:30 +0200 Subject: [PATCH 24/35] fix(stack): make log shipping and persistence easier to debug Settle spans record their counts on every exit, with the instance and launch; stored-id retries no longer emit a span each. The owner logs Analytics target changes, warns once per launch that dropped output and once per failing retention, and summarizes skipped lines per round. functions serve reports lost output. Tests pin the duplicate-id batch drop against real Logflare and the per-source storage proof. Co-Authored-By: Claude Opus 5.5 --- .../stack/logs/logs.integration.test.ts | 15 - .../commands/functions/serve/SIDE_EFFECTS.md | 4 +- .../functions/serve/serve.stack.handler.ts | 18 +- .../serve/serve.stack.integration.test.ts | 5 +- packages/stack/ARCHITECTURE.md | 138 ++++---- .../src/Owner.analytics.integration.test.ts | 297 ++++++++++++------ .../stack/src/Owner.logs.integration.test.ts | 13 +- .../src/host/LogForwarder.integration.test.ts | 248 +++++++-------- packages/stack/src/host/LogForwarder.ts | 181 ++++++----- .../src/host/LogStore.integration.test.ts | 26 -- packages/stack/src/host/LogStore.ts | 34 +- packages/stack/src/host/LogflareStorage.ts | 1 + .../stack/src/internal/sharing-violation.ts | 7 +- packages/stack/tests/manual-clock.ts | 65 ++++ 14 files changed, 632 insertions(+), 420 deletions(-) create mode 100644 packages/stack/tests/manual-clock.ts diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts b/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts index 672d9adfa0..35d5e069ef 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.integration.test.ts @@ -306,21 +306,6 @@ describe("stack logs", () => { }).pipe(Effect.scoped, Effect.provide(live)), ); - it.live("tails by record time when an older partial line was flushed last", () => - Effect.gen(function* () { - const f = yield* fixture({ instances: [mail("mail-a")], members: ["mail-a"] }); - yield* f.writeSegment("mail", "mail-a", [ - launch(t0), - line(t0 + 10, "stdout", "newer"), - line(t0 + 5, "stderr", "late older"), - ]); - - const { output } = yield* f.run({ tail: 1 }, "stream-json"); - - expect(eventLines(output.events)).toEqual(["newer"]); - }).pipe(Effect.scoped, Effect.provide(live)), - ); - it.live("keeps the log-entry fields and reports markers in every output mode", () => Effect.gen(function* () { const f = yield* fixture({ instances: [mail("mail-a")], members: ["mail-a"] }); diff --git a/apps/cli/src/commands/functions/serve/SIDE_EFFECTS.md b/apps/cli/src/commands/functions/serve/SIDE_EFFECTS.md index d7f840fead..8b5cef3bac 100644 --- a/apps/cli/src/commands/functions/serve/SIDE_EFFECTS.md +++ b/apps/cli/src/commands/functions/serve/SIDE_EFFECTS.md @@ -92,6 +92,7 @@ Writes lifecycle text to stderr / stdout while the command is running: - `Skipped serving Function: ` for disabled functions - `File change detected: ()` when a watched file triggers a restart - live `docker logs -f --timestamps` output from the edge-runtime container +- in stack mode, `--- chunks lost ---` on stderr when the stack could not persist Functions output - `Stopped serving supabase/functions` on a user-initiated shutdown (`SIGINT`/`SIGTERM`) - `Edge Runtime exited (code 0). Stopped serving supabase/functions` when the container stops on its own with exit code `0` - `Edge Runtime container stopped (exit ). Stopped serving supabase/functions` when an external supervisor tears the container down (exit `129`/`130`/`131`/`143`) @@ -105,7 +106,8 @@ Long-running raw log / error output only; there is no final success payload obje The legacy backend emits long-running raw log / error events with no terminal `result` event. Stack mode emits a readiness `result` containing the instance ID and URL, then live log events -for Functions output persisted since the command started, stamped with the owner's record time. +for Functions output persisted since the command started, stamped with the owner's record time, +and a `log-marker` event of kind `lost` for output the stack could not persist. ## Notes diff --git a/apps/cli/src/commands/functions/serve/serve.stack.handler.ts b/apps/cli/src/commands/functions/serve/serve.stack.handler.ts index 2df29b6e98..f36ae22bae 100644 --- a/apps/cli/src/commands/functions/serve/serve.stack.handler.ts +++ b/apps/cli/src/commands/functions/serve/serve.stack.handler.ts @@ -48,7 +48,23 @@ const follow = Effect.fn("functions.serve.follow")(function* ( // keeps its first output. const since = DateTime.formatIso(yield* DateTime.now); const logs = yield* instance.readLogs({ follow: true, since }).pipe( - Stream.runForEach(({ kind, timestamp, text }) => { + Stream.runForEach(({ kind, timestamp, text, stream, count }) => { + if (kind === "lost" && count !== undefined) + return output.format === "stream-json" + ? output.event({ + type: "log-marker", + timestamp, + source: "live", + service: "functions", + instance_id: instance.id, + kind, + ...(stream === undefined ? {} : { stream }), + count, + }) + : output.raw( + `--- ${count} ${stream ?? "output"} ${count === 1 ? "chunk" : "chunks"} lost ---\n`, + "stderr", + ); if (kind !== "stdout" && kind !== "stderr") return Effect.void; const line = text ?? ""; return output.format === "stream-json" diff --git a/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts b/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts index 16c710780e..10cc300593 100644 --- a/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts +++ b/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts @@ -510,7 +510,7 @@ describe("experimental Stack Functions serve", () => { }).pipe(Effect.provide(BunServices.layer)), ); - it.live("forwards Functions output written since serve started and drops markers", () => + it.live("forwards Functions output written since serve started and reports lost output", () => Effect.gen(function* () { const drained = yield* Deferred.make(); const records: ReadonlyArray = [ @@ -518,7 +518,7 @@ describe("experimental Stack Functions serve", () => { { kind: "launch", timestamp: "2999-01-01T00:00:00.000Z", launchId: 2 }, { kind: "stdout", timestamp: "2999-01-01T00:00:00.001Z", launchId: 2, text: "hello" }, { kind: "stderr", timestamp: "2999-01-01T00:00:00.002Z", launchId: 2, text: "oops" }, - { kind: "lost", timestamp: "2999-01-01T00:00:00.003Z", launchId: 2, count: 1 }, + { kind: "lost", timestamp: "2999-01-01T00:00:00.003Z", stream: "stdout", count: 2 }, ]; // Honors `since` like the owner, so replaying earlier launches would surface "old". const state = yield* fixture({ @@ -546,6 +546,7 @@ describe("experimental Stack Functions serve", () => { expect(forwarded).toEqual([ { text: "hello\n", stream: "stdout" }, { text: "oops\n", stream: "stderr" }, + { text: "--- 2 stdout chunks lost ---\n", stream: "stderr" }, ]); }), ); diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 11960a9d8c..8bb498a67b 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -642,69 +642,85 @@ The state root is the stack registry root. Each stack keeps one state document a //logs///.log persisted service output ``` -The owner is the only subscriber of each instance's output. It writes one record per line, ended by -`\n`, `\r\n` or a lone `\r` so carriage-return progress updates become separate records, ` - [ truncated] | `, into immutable generation-named -segments: each owner start opens a new generation on the instance's first output, and a record that -would take a segment past 5 MiB starts the next one. The oldest closed segments except the newest -are deleted while an instance holds more than 10 MiB or 64 segments. Records carry the publish time -of their first byte. Launch ids keep increasing across owner starts, since each saved instance -records its latest one; an instance saved without one continues after the launch ids at the end of -its newest segment. Line state is kept per launch, process and stream; chunk sequence numbers span a -launch's processes, so a process whose output was all dropped still shows as `lost`. A late partial -line of an ended launch waits for its newline for two seconds of quiet, or until the store closes. -Lines are cut at 32 KiB; chunks the in-memory output buffer dropped or that were still queued when a -bounded drain at close ran out, and records that failed to write, are recorded as `lost`; a write -that fails part-way can count records that did land, so the count is an upper bound. Closing aborts -filesystem work still pending after 5 seconds; output not yet written by then is not recorded. Every -reader, live or offline, reads segments by position, so history and following share one path; a -follow can resume at a record position, and a reader that finds its segment deleted reports a `lost` -gap marker, which carries `resumeAt` instead of a record position. Destroying an instance deletes -its segments, and an owner start removes directories of instances no longer saved, so a failed -deletion is retried; destroying the stack removes `logs/`; resetting database data keeps them. +#### Persisted service logs + +The owner is the only subscriber of each instance's output and persists it as records: + +- **Format:** one record per line, ended by `\n`, `\r\n` or a lone `\r` so carriage-return progress + updates become separate records: ` [ truncated] | `. + Records carry the publish time of their first byte; lines are cut at 32 KiB. +- **Segments:** immutable and generation-named. Each owner start opens a new generation on the + instance's first output, and a record that would take a segment past 5 MiB starts the next one. +- **Retention:** the oldest closed segments except the newest are deleted while an instance holds + more than 10 MiB or 64 segments. A failed deletion is retried at the next rotation, with one + warning until a pass succeeds. +- **Launch ids:** keep increasing across owner starts, since each saved instance records its latest + one; an instance saved without one continues after the launch ids at the end of its newest segment. +- **Lines:** state is kept per launch, process and stream. A late partial line of an ended launch + waits for its newline for two seconds of quiet, or until the store closes. +- **Loss:** chunk sequence numbers span a launch's processes, so a process whose output was all + dropped still shows as `lost`, and the owner log warns once per launch that dropped output. Chunks the in-memory output buffer dropped, chunks still queued when + a bounded drain at close ran out, and records that failed to write are recorded as `lost`; a write + that fails part-way can count records that did land, so the count is an upper bound. Closing + aborts filesystem work still pending after 5 seconds; output not yet written by then is not recorded. +- **Readers:** every reader, live or offline, reads segments by position, so history and following + share one path. A follow can resume at a record position, and a reader that finds its segment + deleted reports a `lost` gap marker, which carries `resumeAt` instead of a record position. +- **Deletion:** destroying an instance deletes its segments, and an owner start removes directories + of instances no longer saved, so a failed deletion is retried. Destroying the stack removes + `logs/`; resetting database data keeps them. + +#### Shipping logs to Analytics While the composed Analytics instance is running and healthy, the owner ships the persisted stdout/stderr records of Auth, REST, Realtime, Storage, Functions and database instances to its -direct backend (never the proxy, so shipping neither wakes it nor counts as activity), in bodies of -at most 256 events and 1 MiB. Each instance reads from `cursor.json` in its logs directory, the -position of its last confirmed record plus an optional pending body (its end position, source, -event ids, the Analytics launch its latest post went to, and when that post ends at the latest): -a missing cursor or unreadable cursor starts at the oldest retained -segment, and a cursor in a deleted segment resumes at the next retained one. A failed log read -restarts the instance from its cursor after a capped backoff, and attaching an instance removes -temporary cursor writes a dead owner left behind. - -Bodies of an instance are sequential, and event ids derive from the instance and record position. -Logflare answers a post once it queued the events and stores them later in per-source batches; a -batch holding an id already stored is dropped whole. So delivery is confirmed in Analytics' -Postgres database: the owner reads which ids `_analytics.log_events_` holds, resolving the -token from `_analytics.sources`. That database is the one Logflare keeps its sources and events in: -`_supabase` on the host and port of Analytics' database URL (reached at the bound database's -endpoint when the composition binds it), with `supabase_admin`/`postgres` for credentials the URL -omits. A source Analytics does not know, or a failed query, leaves the stored state unknown: the -check is retried and shipping waits, with one warning per outage. A known source without a table -holds no events. The pending body is written atomically before each post, and a failed write -blocks posting. Only ids not yet stored are posted. A post ends by its 5 second timeout, which the -pending body records. Logflare can take many seconds to store what it accepted, so a post that -succeeded, timed out or broke after it was sent is not repeated to the Analytics launch that took -it while it may still store it: the owner polls the stored ids until all are stored, then advances -the cursor and clears the pending body. Ids still missing are posted again once that launch can no -longer store them: 5 seconds after the post ended when it was a launch of this owner, which starts -a launch only after the previous one exited, and 60 seconds after it when it was another owner's, -whose process may still be draining. Any other HTTP status and a request that was never sent are -posted again 5 seconds later. Ids the serving launch leaves unstored for 60 seconds of serving -count as a lost batch: they are posted again in halves until each unstored one was posted alone. -A lone unstored id is skipped with a warning once a later post to the launch was stored, which -shows Analytics is storing; otherwise it stays pending and is posted alone again, with one warning, -so an Analytics that stores nothing loses nothing. Event text and metadata strings have -NUL and unpaired surrogates replaced with U+FFFD before posting, because Postgres `jsonb` rejects -them and drops the whole batch. A start, restart or retarget reconciles a pending body before -later records; pending records already deleted by retention are skipped with a warning. A 401, 403 -or 404 response clears the pending body and pauses the instance with its cursor until the target -changes; any other 4xx except 408 and 429 skips the body. A retarget waits for a post in flight, -and a retired target's stored-event view stays open until its sessions end. The target is -re-selected when the composition, Analytics' health or its launch changes. The check depends on -Logflare's private table layout, which the owner's Analytics integration test pins. +direct backend, never the proxy, so shipping neither wakes it nor counts as activity. The owner logs +each target change, and the target is re-selected when the composition, Analytics' health or its +launch changes. + +- **Bodies:** at most 256 events and 1 MiB. Bodies of an instance are sequential, and event ids + derive from the instance and record position. +- **Cursor:** each instance reads from `cursor.json` in its logs directory: the position of its last + confirmed record plus an optional pending body (its end position, source, event ids, the Analytics + launch its latest post went to, and when that post ends at the latest). A missing or unreadable + cursor starts at the oldest retained segment, and a cursor in a deleted segment resumes at the + next retained one. A failed log read restarts the instance from its cursor after a capped + backoff, and attaching an instance removes temporary cursor writes a dead owner left behind. +- **Why delivery is confirmed:** Logflare answers a post once it queued the events and stores them + later in per-source batches; a batch holding an id already stored is dropped whole. +- **Stored-id check:** the owner reads which ids `_analytics.log_events_` holds, resolving the + token from `_analytics.sources`, in the database Logflare keeps its sources and events in: + `_supabase` on the host and port of Analytics' database URL (reached at the bound database's + endpoint when the composition binds it), with `supabase_admin`/`postgres` for credentials the URL + omits. A source Analytics does not know, or a failed query, leaves the stored state unknown: the + check is retried and shipping waits, with one warning per outage. A known source without a table + holds no events. The check depends on Logflare's private table layout, which the owner's Analytics + integration test pins. +- **Posting:** the pending body is written atomically before each post, and a failed write blocks + posting. Only ids not yet stored are posted. A post ends by its 5 second timeout, which the + pending body records. The owner polls the stored ids until all are stored, then advances the + cursor and clears the pending body. +- **Reposting:** a post that succeeded, timed out or broke after it was sent is not repeated to the + Analytics launch that took it while that launch may still store it. Ids still missing are posted + again once the launch can no longer store them: 5 seconds after the post ended for a launch of + this owner, which starts a launch only after the previous one exited, and 60 seconds after it + for another owner's launch, whose process may still be draining. Any other HTTP status and a + request that was never sent are posted again 5 seconds later. +- **Lost batches:** ids the serving launch leaves unstored for 60 seconds of serving are posted + again in halves until each unstored one was posted alone. A lone unstored id is skipped once a + later post of its source to the launch was stored, which shows Analytics stores that source; + otherwise it stays pending and is posted alone again, so an Analytics that stores nothing loses + nothing. Each round warns once. +- **Statuses:** a 401, 403 or 404 response clears the pending body and pauses the instance with its + cursor until the target changes; any other 4xx except 408 and 429 skips the body. +- **Sanitizing:** event text and metadata strings have NUL and unpaired surrogates replaced with + U+FFFD before posting, because Postgres `jsonb` rejects them and drops the whole batch. +- **Recovery and retargets:** a start, restart or retarget reconciles a pending body before later + records; pending records already deleted by retention are skipped with a warning. A retarget + waits for a post in flight, and a retired target's stored-event view stays open until its + sessions end. + +#### Registry lock Registry updates use an OS-backed lock through a private `node:sqlite` connection to `/.registry-lock.sqlite`. Each `withLock` call opens its own connection, disables @@ -715,6 +731,8 @@ SQLite. No tables, state records, or WAL are created there. Saved stack data rem the lock does not make multi-file operations transactional or recover interrupted operations. This uses the built-in SQLite API available in the pinned Bun runtime and modern Node.js. +#### Stop and destroy + The artifact cache is independent and shared across stacks. Normal stop preserves the stack directory and service data. Destroy removes the state document, owner files and proven-owned, empty parents; the lease file goes last, while its lock is still held; caller-owned paths such as Storage uploads remain untouched. ### Snapshots belong to the database instance diff --git a/packages/stack/src/Owner.analytics.integration.test.ts b/packages/stack/src/Owner.analytics.integration.test.ts index c8ee96f37e..578c863a83 100644 --- a/packages/stack/src/Owner.analytics.integration.test.ts +++ b/packages/stack/src/Owner.analytics.integration.test.ts @@ -16,8 +16,9 @@ import { Schema, Stream, } from "effect"; -import { HttpClient } from "effect/unstable/http"; +import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"; import { tmpdir } from "node:os"; +import { logflareEvent, logflareSources } from "./host/LogflareEvents.ts"; import * as State from "./State.ts"; import type { SavedStack } from "./State.ts"; import type { ServiceCreationInput } from "./services/Catalog.ts"; @@ -53,6 +54,115 @@ interface StoredEvent { readonly timestamp: string; } +/** The table Logflare's Postgres backend stores a source's events in, once the source exists. */ +const eventsTable = (internalDatabaseUrl: string, source: string) => + query<{ readonly token: string }>( + internalDatabaseUrl, + `SELECT replace(token::text, '-', '_') AS token FROM _analytics.sources WHERE name = '${source}'`, + ).pipe( + Effect.map(([row]) => (row === undefined ? undefined : `_analytics."log_events_${row.token}"`)), + ); + +const database: ServiceCreationInput = { + service: "database", + config: { + version: "17", + databasePassword: Redacted.make("owner-logs-password"), + jwtSecret: Redacted.make("owner-logs-jwt-secret-with-at-least-32-chars"), + jwtExpiry: 3600, + }, + endpoints: { sql: { port: "auto" } }, +}; +const analytics: ServiceCreationInput = { + service: "analytics", + config: { backend: "postgres", apiKey: "owner-logs-key" }, + endpoints: { http: { port: "auto" } }, +}; + +/** + * Starts an owner composing `services` in a fresh Docker stack. Its Analytics wakes on proxy + * traffic, which `keepAwake` keeps sending while events must persist. + */ +const startStack = ( + name: string, + services: ReadonlyArray, + analyticsIdleMillis?: number, +) => + Effect.gen(function* () { + const path = yield* Path.Path; + const crypto = yield* Crypto.Crypto; + const client = yield* HttpClient.HttpClient; + const stackId = `${name}-${(yield* crypto.randomUUIDv4).slice(0, 8)}`; + const root = yield* makeDockerDatabaseRoot(`stack-${name}-`, stackId); + const saved: SavedStack = { + id: stackId, + identity: { projectRoot: "/tmp/project", branchContext: name, stackName: stackId }, + runtime: "docker", + instances: [], + lifetime: "detached", + composition: { members: [], dependencies: [] }, + ports: [], + }; + const state = Context.get( + yield* Layer.build(State.layer({ root: path.dirname(path.dirname(root)) })), + State.Service, + ); + yield* state.save(saved); + const owner = yield* ownerFor({ saved, state, root, cacheRoot }); + yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); + + const composed = yield* owner.rpc.supabaseComposition({ services }); + const idOf = (service: string) => { + const id = composed.find((entry) => entry.creation.service === service)?.id; + return id === undefined ? Effect.die(`${service} was not composed`) : Effect.succeed(id); + }; + const analyticsId = yield* idOf("analytics"); + if (analyticsIdleMillis !== undefined) { + const composition = (yield* state.read(stackId))?.composition; + if (composition === undefined) return yield* Effect.die("composition was not saved"); + yield* owner.rpc.configureComposition({ + ...composition, + members: composition.members.map((member) => + member.id === analyticsId ? { ...member, idleMillis: analyticsIdleMillis } : member, + ), + }); + } + yield* owner.rpc.startComposition(); + + const analyticsUrl = (yield* owner.rpc.credentials({ id: analyticsId, from: "host" })).url; + if (analyticsUrl === undefined) return yield* Effect.die("the Analytics URL is missing"); + const awaitAnalytics = ( + predicate: (observation: { + lifecycle: string; + health?: string; + wakeEnabled: boolean; + }) => boolean, + ) => + owner.rpc + .followStatus({ id: analyticsId }) + .pipe(Stream.filter(predicate), Stream.take(1), Stream.runDrain); + return { + owner, + state, + stackId, + idOf, + analyticsUrl, + awaitAnalytics, + wakeAnalytics: client + .get(`${analyticsUrl}/health`) + .pipe( + Effect.andThen( + awaitAnalytics( + ({ lifecycle, health }) => lifecycle === "running" && health === "healthy", + ), + ), + ), + keepAwake: client + .get(`${analyticsUrl}/health`) + .pipe(Effect.ignore, Effect.repeat(Schedule.spaced("1 second"))), + }; + }); + it.live( "ships persisted service logs to Analytics, catching up after it wakes from idle", () => @@ -60,114 +170,37 @@ it.live( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; - const crypto = yield* Crypto.Crypto; const client = yield* HttpClient.HttpClient; - const stackId = `owner-logs-${(yield* crypto.randomUUIDv4).slice(0, 8)}`; - const root = yield* makeDockerDatabaseRoot("stack-owner-logs-", stackId); - const saved: SavedStack = { - id: stackId, - identity: { - projectRoot: "/tmp/project", - branchContext: "owner-logs", - stackName: stackId, - }, - runtime: "docker", - instances: [], - lifetime: "detached", - composition: { members: [], dependencies: [] }, - ports: [], - }; - const state = Context.get( - yield* Layer.build(State.layer({ root: path.dirname(path.dirname(root)) })), - State.Service, - ); - yield* state.save(saved); - const owner = yield* ownerFor({ saved, state, root, cacheRoot }); - yield* Effect.addFinalizer(() => owner.namespace.destroy.pipe(Effect.ignore)); - - const services: ReadonlyArray = [ - { - service: "database", - config: { - version: "17", - databasePassword: Redacted.make("owner-logs-password"), - jwtSecret: Redacted.make("owner-logs-jwt-secret-with-at-least-32-chars"), - jwtExpiry: 3600, - }, - endpoints: { sql: { port: "auto" } }, - }, - { service: "rest", config: {}, endpoints: { http: { port: "auto" } } }, - { - service: "analytics", - config: { backend: "postgres", apiKey: "owner-logs-key" }, - endpoints: { http: { port: "auto" } }, - }, - ]; - const composed = yield* owner.rpc.supabaseComposition({ services }); - const idOf = (service: string) => { - const id = composed.find((entry) => entry.creation.service === service)?.id; - return id === undefined ? Effect.die(`${service} was not composed`) : Effect.succeed(id); - }; - const [databaseId, restId, analyticsId] = yield* Effect.all([ - idOf("database"), - idOf("rest"), - idOf("analytics"), - ]); // A short idle timer lets the test observe Analytics sleeping while services keep logging. - const composition = (yield* state.read(stackId))?.composition; - if (composition === undefined) return yield* Effect.die("composition was not saved"); - yield* owner.rpc.configureComposition({ - ...composition, - members: composition.members.map((member) => - member.id === analyticsId ? { ...member, idleMillis: 3_000 } : member, - ), + const { owner, state, stackId, idOf, awaitAnalytics, wakeAnalytics, keepAwake } = + yield* startStack( + "owner-logs", + [ + database, + { service: "rest", config: {}, endpoints: { http: { port: "auto" } } }, + analytics, + ], + 3_000, + ); + const [databaseId, restId] = yield* Effect.all([idOf("database"), idOf("rest")]); + const { databaseUrl, internalDatabaseUrl } = yield* owner.rpc.credentials({ + id: databaseId, + from: "host", }); - yield* owner.rpc.startComposition(); - - const database = yield* owner.rpc.credentials({ id: databaseId, from: "host" }); const rest = yield* owner.rpc.credentials({ id: restId, from: "host" }); - const analytics = yield* owner.rpc.credentials({ id: analyticsId, from: "host" }); - const { databaseUrl, internalDatabaseUrl } = database; if (databaseUrl === undefined || internalDatabaseUrl === undefined) return yield* Effect.die("database credentials are missing"); - if (rest.url === undefined || analytics.url === undefined) - return yield* Effect.die("service URLs are missing"); + if (rest.url === undefined) return yield* Effect.die("the REST URL is missing"); - const awaitAnalytics = ( - predicate: (observation: { - lifecycle: string; - health?: string; - wakeEnabled: boolean; - }) => boolean, - ) => - owner.rpc - .followStatus({ id: analyticsId }) - .pipe(Stream.filter(predicate), Stream.take(1), Stream.runDrain); - const wakeAnalytics = client - .get(`${analytics.url}/health`) - .pipe( - Effect.andThen( - awaitAnalytics( - ({ lifecycle, health }) => lifecycle === "running" && health === "healthy", - ), - ), - ); - // Proxy traffic is the only idle activity; this stands in for Studio while events must persist. - const keepAwake = client - .get(`${analytics.url}/health`) - .pipe(Effect.ignore, Effect.repeat(Schedule.spaced("1 second"))); const raise = (marker: string) => query(databaseUrl, `DO $$ BEGIN RAISE LOG '${marker}'; END $$`); const stored = (source: string, filter: string) => Effect.gen(function* () { - const [row] = yield* query<{ readonly token: string }>( - internalDatabaseUrl, - `SELECT replace(token::text, '-', '_') AS token FROM _analytics.sources WHERE name = '${source}'`, - ); - if (row === undefined) return []; + const table = yield* eventsTable(internalDatabaseUrl, source); + if (table === undefined) return []; return yield* query( internalDatabaseUrl, - `SELECT body->>'event_message' AS message, body->'metadata'->'parsed'->>'error_severity' AS severity, body->'metadata'->>'host' AS host, to_char(timestamp, 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') AS timestamp FROM _analytics."log_events_${row.token}" WHERE ${filter}`, + `SELECT body->>'event_message' AS message, body->'metadata'->'parsed'->>'error_severity' AS severity, body->'metadata'->>'host' AS host, to_char(timestamp, 'YYYY-MM-DD"T"HH24:MI:SS.MS"Z"') AS timestamp FROM ${table} WHERE ${filter}`, ); }); // Logflare ingests asynchronously and exposes no completion signal; polling is the guard. @@ -263,3 +296,75 @@ it.live( ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), { timeout: 600_000 }, ); + +it.live( + "Logflare accepts a batch holding an already stored id and then stores none of its events", + () => + Effect.scoped( + Effect.gen(function* () { + const crypto = yield* Crypto.Crypto; + const client = yield* HttpClient.HttpClient; + const { owner, idOf, analyticsUrl, wakeAnalytics, keepAwake } = yield* startStack( + "owner-logflare", + [database, analytics], + ); + const { internalDatabaseUrl } = yield* owner.rpc.credentials({ + id: yield* idOf("database"), + from: "host", + }); + if (internalDatabaseUrl === undefined) + return yield* Effect.die("database credentials are missing"); + // No composed service ships to this source, so only this test's posts reach its batches. + const source = logflareSources.realtime; + const event = Effect.fnUntraced(function* (message: string) { + const id = yield* crypto.randomUUIDv4; + const timestamp = DateTime.formatIso(yield* DateTime.now); + return { id, ...logflareEvent("realtime", timestamp, message) }; + }); + const post = (events: ReadonlyArray<{ readonly id: string }>) => + client + .execute( + HttpClientRequest.post(`${analyticsUrl}/api/logs`).pipe( + HttpClientRequest.setUrlParam("source_name", source), + HttpClientRequest.setHeader("x-api-key", "owner-logs-key"), + HttpClientRequest.bodyText(JSON.stringify({ batch: events }), "application/json"), + ), + ) + .pipe(Effect.flatMap(HttpClientResponse.filterStatusOk)); + const storedIds = (ids: ReadonlyArray) => + Effect.gen(function* () { + const table = yield* eventsTable(internalDatabaseUrl, source); + if (table === undefined) return []; + const rows = yield* query<{ readonly id: string }>( + internalDatabaseUrl, + `SELECT id::text AS id FROM ${table} WHERE id IN (${ids.map((id) => `'${id}'`).join(", ")})`, + ); + return rows.map(({ id }) => id); + }); + // Logflare ingests asynchronously and exposes no completion signal; polling is the guard. + const awaitStored = (id: string, attempts: number) => + storedIds([id]).pipe( + Effect.filterOrFail((rows) => rows.length > 0), + Effect.retry({ schedule: Schedule.spaced("500 millis"), times: attempts }), + ); + + yield* wakeAnalytics; + yield* Effect.forkScoped(keepAwake); + const first = yield* event("stored first"); + yield* post([first]); + yield* awaitStored(first.id, 240); + const added = [yield* event("added 1"), yield* event("added 2")]; + yield* post([{ ...first, event_message: "stored again" }, ...added]); + // A stored sentinel shows Logflare processed the batch before it; a sentinel inserted with + // that batch is dropped too, so fresh ones follow until one is stored. + yield* event("sentinel").pipe( + Effect.tap((sentinel) => post([sentinel])), + Effect.flatMap(({ id }) => awaitStored(id, 10)), + Effect.retry({ times: 24 }), + ); + + expect(yield* storedIds([first.id, ...added.map(({ id }) => id)])).toEqual([first.id]); + }), + ).pipe(Effect.provide(Layer.merge(NodeServices.layer, NodeHttpClient.layerNodeHttp))), + { timeout: 600_000 }, +); diff --git a/packages/stack/src/Owner.logs.integration.test.ts b/packages/stack/src/Owner.logs.integration.test.ts index 9fa450ad41..172f21b136 100644 --- a/packages/stack/src/Owner.logs.integration.test.ts +++ b/packages/stack/src/Owner.logs.integration.test.ts @@ -93,7 +93,7 @@ describe("owner persisted logs", () => { expect(history[0]).toMatchObject({ kind: "launch", launchId: 1 }); expect(history).toContainEqual(followed); const directory = path.join(owner.logsRoot, "mail", mail.id); - expect(yield* fs.readDirectory(directory)).toEqual(["0000000001.log"]); + expect(yield* fs.exists(directory)).toBe(true); const offline = yield* LogStore.streamStackLogs({ root: owner.logsRoot }).pipe( Stream.runCollect, ); @@ -302,8 +302,6 @@ describe("owner persisted logs", () => { () => Effect.scoped( Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const path = yield* Path.Path; const owner = yield* openOwner("owner-logs-docker-", "docker"); const mail = yield* owner.rpc.createService({ service: "mail", @@ -314,11 +312,14 @@ describe("owner persisted logs", () => { yield* owner.rpc.readyService({ id: mail.id }); const [followed] = yield* firstOutput(owner.rpc.readLogs({ id: mail.id, follow: true })); + const persisted = yield* LogStore.streamStackLogs({ root: owner.logsRoot }).pipe( + Stream.runCollect, + ); expect(followed).toMatchObject({ launchId: 1 }); - expect(yield* fs.readDirectory(path.join(owner.logsRoot, "mail", mail.id))).toEqual([ - "0000000001.log", - ]); + expect(Array.from(persisted, ({ position }) => position)).toContainEqual( + followed?.position, + ); }), ).pipe(Effect.provide(services)), { timeout: 120_000 }, diff --git a/packages/stack/src/host/LogForwarder.integration.test.ts b/packages/stack/src/host/LogForwarder.integration.test.ts index 90aa46963c..2700b626a3 100644 --- a/packages/stack/src/host/LogForwarder.integration.test.ts +++ b/packages/stack/src/host/LogForwarder.integration.test.ts @@ -3,7 +3,6 @@ import { describe, expect, it } from "@effect/vitest"; import { Clock, Deferred, - Duration, Effect, Exit, Fiber, @@ -26,6 +25,7 @@ import { type FakeLogflareOptions, type Received, } from "../../tests/fake-logflare.ts"; +import { makeManualClock, type ManualClock } from "../../tests/manual-clock.ts"; import type { ServiceObservation } from "../Service.ts"; import type { LaunchOutput } from "../runtime/Session.ts"; import { CatalogError } from "../services/Recipe.ts"; @@ -35,69 +35,6 @@ import * as LogStore from "./LogStore.ts"; const { flushWindowMillis, pollMillis, postTimeoutMillis, retryMillis, stuckMillis } = LogForwarder; -/** - * A clock that moves only when the test advances it; `sleeping` waits for a pending sleep of that - * duration. Wake-ups run on a later macrotask, since completing a deferred resumes its fiber - * inline and a sleeper and the test would otherwise drive each other without yielding to I/O. - */ -const makeManualClock = Effect.gen(function* () { - let now = yield* Clock.currentTimeMillis; - const sleeps = new Set<{ - readonly millis: number; - readonly until: number; - readonly woken: Deferred.Deferred; - }>(); - const waiters = new Set<{ readonly millis: number; readonly ready: Deferred.Deferred }>(); - const wake = (deferred: Deferred.Deferred) => - setImmediate(() => Deferred.doneUnsafe(deferred, Effect.void)); - const nanos = () => BigInt(now) * 1_000_000n; - const clock: Clock.Clock = { - currentTimeMillisUnsafe: () => now, - currentTimeMillis: Effect.sync(() => now), - currentTimeNanosUnsafe: nanos, - currentTimeNanos: Effect.sync(nanos), - monotonicTimeNanosUnsafe: nanos, - monotonicTimeNanos: Effect.sync(nanos), - sleep: (duration) => - Effect.suspend(() => { - const millis = Duration.toMillis(duration); - if (millis <= 0) return Effect.void; - const entry = { millis, until: now + millis, woken: Deferred.makeUnsafe() }; - sleeps.add(entry); - for (const waiter of waiters) - if (waiter.millis === millis) { - waiters.delete(waiter); - wake(waiter.ready); - } - return Deferred.await(entry.woken).pipe( - Effect.ensuring(Effect.sync(() => sleeps.delete(entry))), - ); - }), - }; - return { - clock, - sleeping: (millis: number) => - Effect.suspend(() => { - if ([...sleeps].some((entry) => entry.millis === millis)) return Effect.void; - const waiter = { millis, ready: Deferred.makeUnsafe() }; - waiters.add(waiter); - return Deferred.await(waiter.ready).pipe( - Effect.ensuring(Effect.sync(() => waiters.delete(waiter))), - ); - }), - advance: (millis: number) => - Effect.sync(() => { - now += millis; - for (const entry of sleeps) - if (entry.until <= now) { - sleeps.delete(entry); - wake(entry.woken); - } - }), - }; -}); -type ManualClock = Effect.Success; - /** Takes the next post, running each stored-id poll the forwarder waits on until one arrives. */ const nextPost = (logflare: FakeLogflare, manual: ManualClock): Effect.Effect => logflare.next.pipe( @@ -159,14 +96,14 @@ const serving = (id: string) => const encoder = new TextEncoder(); -/** A database instance whose output a real log store persists. */ -const databaseSource = (store: LogStore.Interface) => +/** A shipped service instance, named after its service, whose output a real log store persists. */ +const serviceSource = (store: LogStore.Interface, service: "database" | "auth" = "database") => Effect.gen(function* () { const logs = yield* PubSub.unbounded(); - const observation = yield* SubscriptionRef.make(serving("database")); + const observation = yield* SubscriptionRef.make(serving(service)); yield* store.attach({ - service: "database", - instanceId: "database", + service, + instanceId: service, logs: PubSub.subscribe(logs), observation: SubscriptionRef.changes(observation).pipe( Stream.map(({ launchId }) => ({ launchId })), @@ -174,7 +111,7 @@ const databaseSource = (store: LogStore.Interface) => }); let seq = 0; const written = yield* Queue.unbounded(); - yield* (yield* store.read("database", { from: "oldest", follow: true })).pipe( + yield* (yield* store.read(service, { from: "oldest", follow: true })).pipe( Stream.runForEach((record) => Queue.offer(written, record)), Effect.forkScoped, ); @@ -196,8 +133,8 @@ const databaseSource = (store: LogStore.Interface) => return { log, instance: { - id: "database", - service: "database" as const, + id: service, + service, endpoint: () => Effect.fail(new CatalogError({ operation: "endpoint", message: "unused" })), creation: Effect.die("Only the Analytics creation is read"), observation: SubscriptionRef.changes(observation), @@ -285,7 +222,7 @@ const fixture = ( const root = yield* fs.makeTempDirectoryScoped({ prefix: "log-forwarder-" }); const store = yield* LogStore.make({ root, ...options.store }); const logflare = yield* makeFakeLogflare(options.logflare ?? { flushMillis: 0 }); - const database = yield* databaseSource(store); + const database = yield* serviceSource(store); const analytics = yield* analyticsTarget(Effect.succeed(logflare.port)); return { root, store, logflare, database, analytics }; }); @@ -325,16 +262,12 @@ describe("LogForwarder", () => { expect([...caughtUp.events, ...live.events].map((event) => event.timestamp)).toEqual( persisted.map((record) => record.timestamp), ); - const shipped = [...ids(caughtUp), ...ids(live)]; - expect(new Set(shipped).size).toBe(3); - expect( - shipped.every((id) => /^[0-9a-f]{8}-[0-9a-f]{4}-8[0-9a-f]{3}-[89ab]/u.test(id)), - ).toBe(true); + expect(new Set([...ids(caughtUp), ...ids(live)]).size).toBe(3); }).pipe(Effect.scoped, Effect.provide(layer)), ); it.live( - "ships a record persisted while Analytics slept once it wakes on a new port, without repeating the post it let finish", + "ships a record persisted while Analytics slept once it wakes on a new port, letting the post in flight across the retarget finish without repeating it", () => Effect.gen(function* () { const { store, logflare, database } = yield* fixture({ logflare: {} }); @@ -359,9 +292,6 @@ describe("LogForwarder", () => { observationOf("analytics", { lifecycle: "stopping", launchId: 1 }), ); yield* forwarder.awaitShipping(false); - yield* release; - yield* logflare.apply(); - yield* Scope.close(firstLaunch, Exit.void); yield* SubscriptionRef.set(analytics.observation, observationOf("analytics", {})); yield* database.log("while asleep"); yield* Ref.set(port, logflare.port); @@ -375,6 +305,10 @@ describe("LogForwarder", () => { observationOf("analytics", { lifecycle: "running", health: "starting", launchId: 2 }), ); yield* SubscriptionRef.set(analytics.observation, { ...serving("analytics"), launchId: 2 }); + yield* forwarder.awaitShipping(true); + yield* logflare.apply(); + yield* release; + yield* Scope.close(firstLaunch, Exit.void); const unavailable = yield* logflare.next; yield* manual.sleeping(pollMillis); yield* manual.advance(flushWindowMillis); @@ -456,7 +390,6 @@ describe("LogForwarder", () => { const shipped = yield* logflare.next; expect(messages(shipped)).toEqual(["after a failed read"]); - expect(yield* Ref.get(reads)).toBe(2); expect(warnings).toEqual([ ["Reading database logs to ship failed; retrying", expect.anything()], ]); @@ -574,26 +507,34 @@ describe("LogForwarder", () => { }).pipe(Effect.scoped, Effect.provide(layer)), ); - it.live("does not post stored events again when its cursor is lost", () => + it.live("posts again only the events Analytics has not stored when its cursor is lost", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; const { root, store, logflare, database, analytics } = yield* fixture({ logflare: {} }); + const manual = yield* makeManualClock; const instances = [analytics.instance, database.instance]; yield* analytics.set(true); - const first = yield* startForwarder(store, logflare, instances); + const first = yield* startForwarder(store, logflare, instances, manual.clock); yield* database.log("line 1"); - const shipped = yield* logflare.next; + yield* logflare.next; yield* logflare.apply(); + yield* database.log("line 2"); + yield* nextPost(logflare, manual); yield* first.stop; + yield* logflare.discard; yield* fs.remove(path.join(root, "database", "database", "cursor.json"), { force: true }); + yield* database.log("line 3"); - yield* startForwarder(store, logflare, instances); - yield* database.log("line 2"); - const resumed = yield* logflare.next; + yield* startForwarder(store, logflare, instances, manual.clock); + const reposted: Array = []; + while (reposted.at(-1) !== "line 3") { + reposted.push(...messages(yield* nextPost(logflare, manual))); + yield* logflare.apply(); + } - expect(messages(shipped)).toEqual(["line 1"]); - expect(messages(resumed)).toEqual(["line 2"]); + expect(reposted).toEqual(["line 2", "line 3"]); + expect(yield* logflare.dropped).toBe(0); }).pipe(Effect.scoped, Effect.provide(layer)), ); @@ -867,7 +808,7 @@ describe("LogForwarder", () => { expect(messages(later)).toEqual(["line 3"]); expect(yield* storedMessages(logflare)).toEqual(["line 1", "line 2", "line 3"]); expect(warnings).toEqual([ - ["Posting postgres.logs logs failed; retrying", expect.anything()], + ["Posting postgres.logs logs of database failed; retrying", expect.anything()], ]); }).pipe(Effect.scoped, Effect.provide(layer)), ); @@ -924,23 +865,51 @@ describe("LogForwarder", () => { it.live("pauses without advancing when Analytics refuses its credentials", () => Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; const { store, logflare, database, analytics } = yield* fixture(); + const manual = yield* makeManualClock; + const refused = yield* Deferred.make(); + const savedAfterRefusal = yield* Deferred.make(); + const saved = (file: string) => + Deferred.isDone(refused).pipe( + Effect.flatMap((done) => + done && file.endsWith("cursor.json") + ? Deferred.succeed(savedAfterRefusal, undefined) + : Effect.void, + ), + ); + // A cursor with neither a confirmed position nor a pending body is removed, not written. + const injected: FileSystem.FileSystem = { + ...fs, + rename: (from, to) => fs.rename(from, to).pipe(Effect.andThen(saved(to))), + remove: (file, options) => fs.remove(file, options).pipe(Effect.andThen(saved(file))), + }; yield* analytics.set(true); - const forwarder = yield* startForwarder(store, logflare, [ - analytics.instance, - database.instance, - ]); + yield* startForwarder( + store, + logflare, + [analytics.instance, database.instance], + manual.clock, + ).pipe(Effect.provideService(FileSystem.FileSystem, injected)); yield* logflare.respond(401); yield* database.log("refused line"); - const refused = yield* logflare.next; - yield* analytics.set(false); - yield* forwarder.awaitShipping(false); - yield* analytics.set(true); + const refusedPost = yield* logflare.next; + yield* Deferred.succeed(refused, undefined); + // A retried post first waits on the clock, so the cursor write wins only when it pauses. + const outcome = yield* Deferred.await(savedAfterRefusal).pipe( + Effect.as("paused"), + Effect.raceFirst(manual.sleeping(pollMillis).pipe(Effect.as("retrying"))), + ); + yield* manual.advance(stuckMillis); + const postedWhilePaused = yield* logflare.unread; + yield* analytics.relaunch(2); const resumed = yield* logflare.next; - expect(messages(refused)).toEqual(["refused line"]); - expect(ids(resumed)).toEqual(ids(refused)); + expect(outcome).toBe("paused"); + expect(postedWhilePaused).toBe(0); + expect(messages(refusedPost)).toEqual(["refused line"]); + expect(ids(resumed)).toEqual(ids(refusedPost)); }).pipe(Effect.scoped, Effect.provide(layer)), ); @@ -1021,12 +990,54 @@ describe("LogForwarder", () => { "Analytics has not stored 256 postgres.logs lines of database it accepted; posting them in halves", ], [ - "Analytics did not store a postgres.logs line of database posted on its own while it stored later posts; skipping it", + "Analytics did not store 1 postgres.logs lines of database posted on their own (0 rejected, 1 unstored while it stored later posts); skipping them", ], ]); }).pipe(Effect.scoped, Effect.provide(layer)), ); + it.live( + "keeps a lone unstored line pending while Analytics stores only another source's posts", + () => + Effect.gen(function* () { + const { store, logflare, database, analytics } = yield* fixture({ + logflare: { unstorable: (event) => event.event_message.startsWith("db ") }, + }); + const auth = yield* serviceSource(store, "auth"); + const manual = yield* makeManualClock; + const isDatabase = (post: Received) => post.url.includes("source_name=postgres.logs"); + yield* analytics.set(true); + yield* startForwarder( + store, + logflare, + [analytics.instance, database.instance, auth.instance], + manual.clock, + ); + + yield* database.log("db line"); + yield* drive(logflare, manual, (posts) => + Effect.succeed(posts.filter(isDatabase).length >= 2), + ); + yield* database.log("db later"); + // Storing before the answer lets auth confirm without a clock tick that could make the + // database line stuck; its next post shows the stored one was recorded. + const release = yield* logflare.hold; + yield* auth.log("auth line"); + yield* logflare.next; + yield* logflare.apply(); + yield* release; + yield* auth.log("auth later"); + yield* logflare.next; + yield* logflare.apply(); + const posts = yield* drive(logflare, manual, (taken) => + Effect.succeed(taken.some(isDatabase)), + ); + + expect(posts.filter(isDatabase).map(messages)).toEqual([["db line"]]); + expect(yield* storedMessages(logflare)).toEqual(["auth line", "auth later"]); + }).pipe(Effect.scoped, Effect.provide(layer)), + ); + it.live("keeps every event while Analytics stores nothing and ships them once it recovers", () => Effect.gen(function* () { let storing = false; @@ -1073,37 +1084,6 @@ describe("LogForwarder", () => { }).pipe(Effect.scoped, Effect.provide(layer)), ); - it.live("keeps shipping to the new target after a retarget while a post is in flight", () => - Effect.gen(function* () { - const { store, logflare, database } = yield* fixture(); - const firstLaunch = yield* Scope.make(); - const firstPort = yield* logflare.listen(firstLaunch); - const port = yield* Ref.make(firstPort); - const analytics = yield* analyticsTarget(Ref.get(port)); - yield* analytics.set(true); - const forwarder = yield* startForwarder(store, logflare, [ - analytics.instance, - database.instance, - ]); - const release = yield* logflare.hold; - yield* database.log("in flight"); - const inFlight = yield* logflare.next; - - yield* analytics.set(false); - yield* forwarder.awaitShipping(false); - yield* Ref.set(port, logflare.port); - yield* SubscriptionRef.set(analytics.observation, { ...serving("analytics"), launchId: 2 }); - yield* forwarder.awaitShipping(true); - yield* release; - yield* database.log("after the retarget"); - const after = yield* logflare.next; - - expect(inFlight.port).toBe(firstPort); - expect(after.port).toBe(logflare.port); - expect(messages(after)).toEqual(["after the retarget"]); - }).pipe(Effect.scoped, Effect.provide(layer)), - ); - it.live("ships from the oldest retained record when its cursor is unreadable", () => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; diff --git a/packages/stack/src/host/LogForwarder.ts b/packages/stack/src/host/LogForwarder.ts index 95ba25064b..494060fdf0 100644 --- a/packages/stack/src/host/LogForwarder.ts +++ b/packages/stack/src/host/LogForwarder.ts @@ -10,7 +10,6 @@ import { FileSystem, Option, Path, - Predicate, Ref, Schedule, Schema, @@ -24,6 +23,7 @@ import type { CompositionConfig } from "../Orchestrator.ts"; import type { ServiceObservation } from "../Service.ts"; import type { ServiceCreation } from "../services/Catalog.ts"; import type { CatalogError, ServiceEndpoint } from "../services/Recipe.ts"; +import { errorCode } from "../internal/sharing-violation.ts"; import { reapStaleWrites, writeFileAtomically } from "../State.ts"; import { isShippedService, @@ -149,6 +149,12 @@ interface Session { /** `unsettled` posts may have reached Analytics without an answer; `failed` ones were not queued. */ type Delivery = "sent" | "unsettled" | "failed" | "refused" | "rejected"; +/** Whether a settle already warned of failed posts and of rejected bodies. */ +interface Warned { + readonly post: Ref.Ref; + readonly rejection: Ref.Ref; +} + const batchEvents = 256; export const batchBytes = 1024 * 1024; /** How long after a post ended its events may still be stored by a launch that stopped. */ @@ -267,19 +273,12 @@ const unsentCodes = new Set([ "EAI_AGAIN", ]); -const errorCode = (cause: unknown): unknown => - Predicate.hasProperty(cause, "code") - ? cause.code - : Predicate.hasProperty(cause, "cause") - ? errorCode(cause.cause) - : undefined; - /** A post that timed out or broke after it was sent, so Analytics may have queued its events. */ const mayHaveQueued = (error: unknown): boolean => { if (Cause.isTimeoutError(error)) return true; if (!(error instanceof HttpClientError.HttpClientError)) return false; const { reason } = error; - if (reason._tag === "TransportError") return !unsentCodes.has(String(errorCode(reason.cause))); + if (reason._tag === "TransportError") return !unsentCodes.has(errorCode(reason.cause) ?? ""); return reason._tag === "DecodeError" || reason._tag === "EmptyBodyError"; }; @@ -356,6 +355,11 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar scope === undefined || next === undefined ? undefined : { scope, users: next.users }, ); yield* SubscriptionRef.set(target, next); + yield* next === undefined + ? Effect.logInfo("Log shipping paused until Analytics is running and healthy") + : Effect.logInfo( + `Log shipping targets Analytics instance ${next.instance.id}, launch ${serving?.launchId ?? "unknown"}`, + ); if (retired !== undefined) yield* Effect.forkIn( retired.users.take(targetUsers).pipe(Effect.andThen(Scope.close(retired.scope, Exit.void))), @@ -495,36 +499,40 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar ); }); - /** Posts one body once, ending by `deadline`, and classifies the answer. */ + /** Posts one body of `lines` events once, ending by `deadline`, and classifies the answer. */ const deliver = ( session: Session, source: string, - body: string, + body: Body<{ readonly event: ShippedEvent }>, deadline: number, - warned: Ref.Ref, + warned: Warned, ) => - post(session.current, source, body, deadline).pipe( + post(session.current, source, body.body, deadline).pipe( session.posting.withPermits(1), Effect.as("sent"), Effect.catchIf(isTargetRejection, (error) => - Effect.logWarning(`Analytics refused ${source} logs; pausing until it changes`, error).pipe( - Effect.as("refused"), - ), + Effect.logWarning( + `Analytics refused ${source} logs of ${session.instanceId}; pausing until it changes`, + error, + ).pipe(Effect.as("refused")), ), Effect.catch((error) => { if (error._tag === "StaleTarget") return Effect.fail(error); if (isBodyRejection(error)) - return Effect.logWarning( - `Analytics rejected a ${source} log body; skipping it`, + return warnOnce( + warned.rejection, + `Analytics rejected a body of ${body.items.length} ${source} lines of ${session.instanceId}`, error, ).pipe(Effect.as("rejected")); if (!mayHaveQueued(error)) - return warnOnce(warned, `Posting ${source} logs failed; retrying`, error).pipe( - Effect.as("failed"), - ); + return warnOnce( + warned.post, + `Posting ${source} logs of ${session.instanceId} failed; retrying`, + error, + ).pipe(Effect.as("failed")); return warnOnce( - warned, - `Posting ${source} logs got no answer; waiting until Analytics stores them`, + warned.post, + `Posting ${source} logs of ${session.instanceId} got no answer; waiting until Analytics stores them`, error, ).pipe(Effect.as("unsettled")); }), @@ -546,9 +554,14 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar posted: Pending | undefined, ) { const { source } = shipment; - yield* Effect.annotateCurrentSpan({ source, event_count: shipment.ids.length }); - const postWarned = yield* Ref.make(false); - const counts = { checks: 0, posts: 0, skipped: 0 }; + yield* Effect.annotateCurrentSpan({ + source, + instance_id: session.instanceId, + launch: session.current.launch, + event_count: shipment.ids.length, + }); + const warned: Warned = { post: yield* Ref.make(false), rejection: yield* Ref.make(false) }; + const counts = { checks: 0, checkAttempts: 0, posts: 0, deleted: 0, stuck: 0, skipped: 0 }; let remaining = shipment.ids; /** The sequence number of the latest post, and the ids it carried. */ let lastPost = 0; @@ -556,17 +569,19 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar let latest: { readonly queuedIn: string | undefined; readonly endedAt: number } | undefined = posted === undefined ? undefined : { queuedIn: posted.launch, endedAt: posted.postEndsAt }; + /** Only the first attempt of the first check is traced; the rest are counted on this span. */ + const storedIds = (ids: ReadonlyArray) => + Effect.suspend(() => { + const check = session.current.stored.storedIds(source, ids); + return counts.checkAttempts++ === 0 ? check : check.pipe(Effect.withTracerEnabled(false)); + }).pipe(Effect.retry(retrySchedule)); + /** Posts `initial` until Analytics stored it, and returns the ids left unstored for `stuckMillis`. */ const deliverAll = Effect.fnUntraced(function* (initial: ReadonlyArray) { let wanted = initial; delivering: while (true) { - // Only the first check is traced; the rest are counted on this span. - const stored = yield* session.current.stored - .storedIds(source, wanted) - .pipe( - Effect.retry(retrySchedule), - counts.checks++ === 0 ? (check) => check : Effect.withTracerEnabled(false), - ); + counts.checks++; + const stored = yield* storedIds(wanted); if (lastPost > 0 && [...lastPosted].some((id) => stored.has(id))) yield* Ref.update(session.current.storedPost, (posts) => new Map(posts).set(source, Math.max(posts.get(source) ?? 0, lastPost)), @@ -598,6 +613,7 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar yield* Effect.logWarning( `Logs of ${session.service} instance ${session.instanceId} were deleted before Analytics stored them; skipping ${deleted.size} lines`, ); + counts.deleted += deleted.size; wanted = wanted.filter((id) => !deleted.has(id)); remaining = remaining.filter((id) => !deleted.has(id)); } @@ -607,7 +623,7 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar }); if (resend.length === 0) return []; let queuedIn: string | undefined; - for (const { body } of bodies(resend)) { + for (const body of bodies(resend)) { const deadline = (yield* Clock.currentTimeMillis) + postTimeoutMillis; yield* persist(session, { confirmed: (yield* Ref.get(session.progress)).confirmed, @@ -623,8 +639,8 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar if ((yield* Clock.currentTimeMillis) >= deadline) continue delivering; counts.posts++; lastPost = ++postSequence; - lastPosted = new Set(resend.map(({ event }) => event.id)); - const delivery = yield* deliver(session, source, body, deadline, postWarned); + lastPosted = new Set(body.items.map(({ event }) => event.id)); + const delivery = yield* deliver(session, source, body, deadline, warned); if (delivery === "refused") { yield* persist(session, { confirmed: (yield* Ref.get(session.progress)).confirmed, @@ -639,18 +655,6 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar } }); - const skip = (id: string, reason: string) => - Effect.logWarning( - `Analytics did not store a ${source} line of ${session.instanceId} posted on its own ${reason}; skipping it`, - ).pipe( - Effect.andThen( - Effect.sync(() => { - remaining = remaining.filter((other) => other !== id); - counts.skipped++; - }), - ), - ); - /** * Posts the ids Analytics left unstored in halves until each unstored one was posted alone. * A lone unstored id is skipped only once a later post to this launch got stored, which shows @@ -661,14 +665,17 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar let groups = halves(initial); while (groups.length > 0) { const suspects: Array<{ readonly id: string; readonly post: number }> = []; + const skipped = new Set(); + let rejected = 0; for (let group = groups.shift(); group !== undefined; group = groups.shift()) { latest = undefined; const left = yield* deliverAll(group); const [lone] = group; if (left === "rejected") { - if (group.length === 1 && lone !== undefined) - yield* skip(lone, "as Analytics rejected it"); - else groups.unshift(...halves(group)); + if (group.length === 1 && lone !== undefined) { + skipped.add(lone); + rejected++; + } else groups.unshift(...halves(group)); } else if (left.length > 0) { if (group.length === 1 && lone !== undefined) suspects.push({ id: lone, post: lastPost }); @@ -676,10 +683,15 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar } } const proof = (yield* Ref.get(session.current.storedPost)).get(source) ?? 0; - for (const suspect of suspects) - if (proof > suspect.post) yield* skip(suspect.id, "while it stored later posts"); - const kept = new Set(remaining); - groups = suspects.flatMap(({ id }) => (kept.has(id) ? [[id]] : [])); + for (const suspect of suspects) if (proof > suspect.post) skipped.add(suspect.id); + if (skipped.size > 0) { + yield* Effect.logWarning( + `Analytics did not store ${skipped.size} ${source} lines of ${session.instanceId} posted on their own (${rejected} rejected, ${skipped.size - rejected} unstored while it stored later posts); skipping them`, + ); + remaining = remaining.filter((id) => !skipped.has(id)); + counts.skipped += skipped.size; + } + groups = suspects.flatMap(({ id }) => (skipped.has(id) ? [] : [[id]])); if (groups.length > 0) yield* Ref.getAndSet(idleWarned, true).pipe( Effect.flatMap((already) => @@ -693,25 +705,49 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar } }); - const stuck = yield* deliverAll(shipment.ids); - if (stuck !== "rejected" && stuck.length > 0) { - yield* Effect.logWarning( - `Analytics has not stored ${stuck.length} ${source} lines of ${session.instanceId} it accepted; posting them in halves`, - ); - yield* isolate(stuck); - } - yield* Effect.annotateCurrentSpan({ - check_count: counts.checks, - post_count: counts.posts, - skipped_count: counts.skipped, - }); - yield* persist(session, { confirmed: positionOf(shipment), pending: undefined }); + yield* Effect.gen(function* () { + const stuck = yield* deliverAll(shipment.ids); + if (stuck === "rejected") { + yield* Effect.logWarning( + `Skipping ${remaining.length} ${source} lines of ${session.instanceId} that Analytics rejected`, + ); + counts.skipped += remaining.length; + } else if (stuck.length > 0) { + counts.stuck = stuck.length; + yield* Effect.logWarning( + `Analytics has not stored ${stuck.length} ${source} lines of ${session.instanceId} it accepted; posting them in halves`, + ); + yield* isolate(stuck); + } + yield* persist(session, { confirmed: positionOf(shipment), pending: undefined }); + }).pipe( + // Counted on every exit, so a stuck or retargeted settle still shows its progress. + Effect.ensuring( + Effect.suspend(() => + Effect.annotateCurrentSpan({ + check_count: counts.checks, + check_attempt_count: counts.checkAttempts, + post_count: counts.posts, + deleted_count: counts.deleted, + stuck_count: counts.stuck, + skipped_count: counts.skipped, + }), + ), + ), + ); }); /** Rebuilds a pending body from its retained records and settles it. */ - const reconcile = Effect.fnUntraced(function* (session: Session, saved: Progress) { - const { confirmed, pending } = saved; - if (pending === undefined) return; + const reconcile = Effect.fn("LogForwarder.reconcile")(function* ( + session: Session, + confirmed: LogPosition | undefined, + pending: Pending, + ) { + yield* Effect.annotateCurrentSpan({ + instance_id: session.instanceId, + source: pending.source, + pending_count: pending.ids.length, + }); const end = positionOf(pending); const wanted = new Set(pending.ids); const events = new Map(); @@ -735,6 +771,7 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar ), ), ); + yield* Effect.annotateCurrentSpan({ recovered_count: events.size }); yield* settle(session, pending, events, pending); }); @@ -760,7 +797,7 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar posting, resumedAt: yield* Clock.currentTimeMillis, }; - yield* reconcile(shipping, saved); + if (saved.pending !== undefined) yield* reconcile(shipping, saved.confirmed, saved.pending); const from = (yield* Ref.get(progress)).confirmed ?? "oldest"; const records = yield* options.logs.read(instanceId, { from, follow: true }); const source = logflareSources[service]; diff --git a/packages/stack/src/host/LogStore.integration.test.ts b/packages/stack/src/host/LogStore.integration.test.ts index b90e3c83c9..509a92d12f 100644 --- a/packages/stack/src/host/LogStore.integration.test.ts +++ b/packages/stack/src/host/LogStore.integration.test.ts @@ -212,32 +212,6 @@ describe("LogStore", () => { }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); - it.effect("attributes a late chunk of a failed launch to that launch", () => - Effect.gen(function* () { - const root = yield* tempRoot("log-store-late-"); - const { store } = yield* openStore(root); - const instance = yield* fakeInstance("late"); - yield* store.attach(instance); - const reader = yield* collect(store.read("late", { from: "oldest", follow: true })); - - yield* instance.publish(instance.chunk(1, "starting\nbo")); - yield* instance.publish({ ...instance.chunk(2, "second\n"), seq: 0 }); - yield* instance.publish({ ...instance.chunk(1, "om\n"), seq: 1 }); - const records = yield* reader.take(6); - - expect(Array.from(records, ({ kind, launchId, text }) => ({ kind, launchId, text }))).toEqual( - [ - { kind: "launch", launchId: 1, text: undefined }, - { kind: "stdout", launchId: 1, text: "starting" }, - { kind: "stdout", launchId: 1, text: "bo" }, - { kind: "launch", launchId: 2, text: undefined }, - { kind: "stdout", launchId: 2, text: "second" }, - { kind: "stdout", launchId: 1, text: "om" }, - ], - ); - }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), - ); - it.effect("writes a launch's partial line when its observation leaves the launch", () => Effect.gen(function* () { const root = yield* tempRoot("log-store-partial-"); diff --git a/packages/stack/src/host/LogStore.ts b/packages/stack/src/host/LogStore.ts index 20cd33991b..447d7809ca 100644 --- a/packages/stack/src/host/LogStore.ts +++ b/packages/stack/src/host/LogStore.ts @@ -660,6 +660,8 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio /** Output not persisted, reported as `lost` by the next successful append. */ const dropped = new Map>(); let broken: { readonly attempts: number; readonly retryAt: number } | undefined; + /** Launches whose dropped output was already reported in the owner log. */ + const warnedLaunches = new Set(); const aborted = new LogStoreError({ operation: "write", @@ -687,35 +689,52 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio }); }); + let retentionWarned = false; + /** Removes the oldest closed segments past the limits; a failed removal waits for the next rotation. */ const enforceRetention = reader.segments(directory).pipe( Effect.flatMap((listed) => Effect.gen(function* () { let total = listed.reduce((sum, segment) => sum + segment.size, 0); let count = listed.length; + let failure: unknown; // The newest closed segment holds the latest records before the open one. const newestClosed = listed .map(({ generation }) => generation) .filter((generation) => generation !== file?.generation) .at(-1); for (const segment of listed) { - if (total <= retainBytes && count <= retainSegments) return; + if (total <= retainBytes && count <= retainSegments) break; if (segment.generation === file?.generation || segment.generation === newestClosed) - return; + break; // A reader holding the file on Windows defers its deletion to the next rotation. const removed = yield* fs .remove(path.join(directory, segmentName(segment.generation)), { force: true }) .pipe( retryShared, Effect.as(true), - Effect.orElseSucceed(() => false), + Effect.catch((error) => + Effect.sync(() => { + failure ??= error; + return false; + }), + ), ); if (!removed) continue; total -= segment.size; count -= 1; } + if (failure !== undefined) return yield* Effect.fail(failure); + retentionWarned = false; }), ), - Effect.ignore, + Effect.catch((error) => { + if (retentionWarned) return Effect.void; + retentionWarned = true; + return Effect.logWarning( + `Removing old logs of ${instance.service} instance ${instance.instanceId} failed; retrying at the next rotation`, + error, + ); + }), ); const openGeneration = Effect.gen(function* () { @@ -837,6 +856,13 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio dropped.delete(dropKey(marker.launchId, marker.stream)); if (Exit.isSuccess(written)) { broken = undefined; + for (const entry of entries) + if (entry.kind === "lost" && !warnedLaunches.has(entry.launchId)) { + warnedLaunches.add(entry.launchId); + yield* Effect.logWarning( + `${instance.service} instance ${instance.instanceId} dropped ${entry.count} ${entry.stream} chunks of launch ${entry.launchId} before they were persisted; later drops of this launch are recorded only in its logs`, + ); + } return; } dropRecords(entries.slice(entriesWritten.written)); diff --git a/packages/stack/src/host/LogflareStorage.ts b/packages/stack/src/host/LogflareStorage.ts index f6e6157ac7..93da83d79d 100644 --- a/packages/stack/src/host/LogflareStorage.ts +++ b/packages/stack/src/host/LogflareStorage.ts @@ -130,6 +130,7 @@ export const make = Effect.fn("LogflareStorage.make")(function* ( yield* Effect.annotateCurrentSpan({ source, requested_count: ids.length }); if (ids.length === 0) return new Set(); const table = yield* Cache.get(tables, source); + yield* Effect.annotateCurrentSpan({ table_found: Option.isSome(table) }); const rows = Option.isNone(table) ? [] : yield* withClient( diff --git a/packages/stack/src/internal/sharing-violation.ts b/packages/stack/src/internal/sharing-violation.ts index b7dd11fcac..a84ad849c7 100644 --- a/packages/stack/src/internal/sharing-violation.ts +++ b/packages/stack/src/internal/sharing-violation.ts @@ -1,12 +1,13 @@ import { Duration, Effect, Predicate, Schedule } from "effect"; -/** The errno code a platform error carries on its cause, also when another error wraps it. */ +/** The errno code an error carries, itself or on a cause, also when other errors wrap it. */ export const errorCode = (error: unknown): string | undefined => { let current = error; - for (let depth = 0; depth < 4 && Predicate.hasProperty(current, "cause"); depth++) { - current = current.cause; + for (let depth = 0; depth <= 4; depth++) { if (Predicate.hasProperty(current, "code") && typeof current.code === "string") return current.code; + if (!Predicate.hasProperty(current, "cause")) return undefined; + current = current.cause; } return undefined; }; diff --git a/packages/stack/tests/manual-clock.ts b/packages/stack/tests/manual-clock.ts new file mode 100644 index 0000000000..8988fac884 --- /dev/null +++ b/packages/stack/tests/manual-clock.ts @@ -0,0 +1,65 @@ +import { Clock, Deferred, Duration, Effect } from "effect"; + +/** + * A clock that moves only when the test advances it; `sleeping` waits for a pending sleep of that + * duration. Wake-ups run on a later macrotask, since completing a deferred resumes its fiber + * inline and a sleeper and the test would otherwise drive each other without yielding to I/O. + */ +export const makeManualClock = Effect.gen(function* () { + let now = yield* Clock.currentTimeMillis; + const sleeps = new Set<{ + readonly millis: number; + readonly until: number; + readonly woken: Deferred.Deferred; + }>(); + const waiters = new Set<{ readonly millis: number; readonly ready: Deferred.Deferred }>(); + const wake = (deferred: Deferred.Deferred) => + setImmediate(() => Deferred.doneUnsafe(deferred, Effect.void)); + const nanos = () => BigInt(now) * 1_000_000n; + const clock: Clock.Clock = { + currentTimeMillisUnsafe: () => now, + currentTimeMillis: Effect.sync(() => now), + currentTimeNanosUnsafe: nanos, + currentTimeNanos: Effect.sync(nanos), + monotonicTimeNanosUnsafe: nanos, + monotonicTimeNanos: Effect.sync(nanos), + sleep: (duration) => + Effect.suspend(() => { + const millis = Duration.toMillis(duration); + if (millis <= 0) return Effect.void; + const entry = { millis, until: now + millis, woken: Deferred.makeUnsafe() }; + sleeps.add(entry); + for (const waiter of waiters) + if (waiter.millis === millis) { + waiters.delete(waiter); + wake(waiter.ready); + } + return Deferred.await(entry.woken).pipe( + Effect.ensuring(Effect.sync(() => sleeps.delete(entry))), + ); + }), + }; + return { + clock, + sleeping: (millis: number) => + Effect.suspend(() => { + if ([...sleeps].some((entry) => entry.millis === millis)) return Effect.void; + const waiter = { millis, ready: Deferred.makeUnsafe() }; + waiters.add(waiter); + return Deferred.await(waiter.ready).pipe( + Effect.ensuring(Effect.sync(() => waiters.delete(waiter))), + ); + }), + advance: (millis: number) => + Effect.sync(() => { + now += millis; + for (const entry of sleeps) + if (entry.until <= now) { + sleeps.delete(entry); + wake(entry.woken); + } + }), + }; +}); + +export type ManualClock = Effect.Success; From 84125ac6e57861cb4840a69ceb9284b64adf207d Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 2 Oct 2026 20:33:10 +0200 Subject: [PATCH 25/35] refactor(stack): move gateway credential redaction into its own module Proxy request and upgrade spans record their route and response status. CLI tests share one unused gateway fake, and the gateway tests leave the line format to the proxy tests and prove a request is recorded once. Co-Authored-By: Claude Opus 5.5 --- .../db-config.integration.test.ts | 3 +- .../commands/db/dump/dump.integration.test.ts | 3 +- .../db/reset/reset.integration.test.ts | 3 +- .../generate/generate.integration.test.ts | 3 +- .../declarative/sync/sync.integration.test.ts | 3 +- .../db/start/start.integration.test.ts | 3 +- .../stack/prepare/prepare.integration.test.ts | 3 +- .../start-export-pointer.integration.test.ts | 3 +- .../stack/start/start.integration.test.ts | 3 +- .../stack/status/status.integration.test.ts | 3 +- .../serve/serve.stack.integration.test.ts | 3 +- apps/cli/tests/helpers/storage.ts | 3 +- apps/cli/tests/helpers/unused-stack.ts | 6 +- .../stack/src/HttpProxy.integration.test.ts | 6 ++ packages/stack/src/HttpProxy.ts | 90 +++---------------- .../stack/src/Owner.logs.integration.test.ts | 6 +- .../stack/src/internal/redact-credentials.ts | 77 ++++++++++++++++ .../redact-credentials.unit.test.ts} | 2 +- 18 files changed, 127 insertions(+), 96 deletions(-) create mode 100644 packages/stack/src/internal/redact-credentials.ts rename packages/stack/src/{HttpProxy.unit.test.ts => internal/redact-credentials.unit.test.ts} (98%) diff --git a/apps/cli/src/command-internal/db-config.integration.test.ts b/apps/cli/src/command-internal/db-config.integration.test.ts index ad1e3d4390..8ce8f853f6 100644 --- a/apps/cli/src/command-internal/db-config.integration.test.ts +++ b/apps/cli/src/command-internal/db-config.integration.test.ts @@ -33,6 +33,7 @@ import { mockTty, } from "../../tests/helpers/mocks.ts"; import { VALID_TOKEN, mockCommandSettings } from "../../tests/helpers/command-mocks.ts"; +import { unusedGateway } from "../../tests/helpers/unused-stack.ts"; import { DebugFlag, DnsResolverFlag, @@ -489,7 +490,7 @@ describe("dbConfigResolver (db-url under the stack backend)", () => { }, stop: unused, destroy: unused, - gateway: { readLogs: () => Stream.die("unused") }, + gateway: unusedGateway, commands: { run: () => unused }, }; return Layer.succeed(StackApi, { diff --git a/apps/cli/src/commands/db/dump/dump.integration.test.ts b/apps/cli/src/commands/db/dump/dump.integration.test.ts index 642203185b..76a7e9518e 100644 --- a/apps/cli/src/commands/db/dump/dump.integration.test.ts +++ b/apps/cli/src/commands/db/dump/dump.integration.test.ts @@ -16,6 +16,7 @@ import { import { ChildProcess, ChildProcessSpawner } from "effect/unstable/process"; import { mockOutput, mockTty, processEnvLayer } from "../../../../tests/helpers/mocks.ts"; +import { unusedGateway } from "../../../../tests/helpers/unused-stack.ts"; import { VALID_REF, mockCommandSettings, @@ -153,7 +154,7 @@ const managedDumpStackApi = (runtime: "native" | "docker") => { }, stop: Effect.void, destroy: Effect.succeed({ runtimeCleanup: "complete" as const }), - gateway: { readLogs: () => Stream.die("unused") }, + gateway: unusedGateway, commands: { run: runCommand }, } satisfies Stack; return Layer.succeed(StackApi, { diff --git a/apps/cli/src/commands/db/reset/reset.integration.test.ts b/apps/cli/src/commands/db/reset/reset.integration.test.ts index 15f5c5b81f..d3fed41fb4 100644 --- a/apps/cli/src/commands/db/reset/reset.integration.test.ts +++ b/apps/cli/src/commands/db/reset/reset.integration.test.ts @@ -39,6 +39,7 @@ import { sequentialExecBatch, transportFailure, } from "../../../../tests/helpers/command-mocks.ts"; +import { unusedGateway } from "../../../../tests/helpers/unused-stack.ts"; import { CommandPlatformApi } from "../../../auth/command-platform-api.service.ts"; import { CommandPlatformApiFactory } from "../../../auth/command-platform-api-factory.service.ts"; import { ProjectRefNotLinkedError } from "../../../config/project-ref.errors.ts"; @@ -808,7 +809,7 @@ function mockResetStackApi(opts: { }, stop: Effect.die("unused"), destroy: Effect.die("unused"), - gateway: { readLogs: () => Stream.die("unused") }, + gateway: unusedGateway, commands: { run: () => Effect.die("unused") }, }; return { diff --git a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts index cfeca95094..74e8cee717 100644 --- a/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/generate/generate.integration.test.ts @@ -14,6 +14,7 @@ import { } from "effect"; import { StackError, type DatabaseInstance, type Stack } from "@supabase/stack/effect"; import { stripAnsi } from "../../../../../../tests/helpers/ansi.ts"; +import { unusedGateway } from "../../../../../../tests/helpers/unused-stack.ts"; import { alwaysReadyHttpClientLayer, @@ -164,7 +165,7 @@ function generateStackApi(workdir: string) { }, stop: unusedStack, destroy: unusedStack, - gateway: { readLogs: () => Stream.die("unused") }, + gateway: unusedGateway, commands: { run: unusedStackFn }, }; const identity = { projectRoot: workdir, branchContext: "main", stackName: "default" }; diff --git a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts index bb222990b2..4d0321f07d 100644 --- a/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts +++ b/apps/cli/src/commands/db/schema/declarative/sync/sync.integration.test.ts @@ -14,6 +14,7 @@ import { } from "effect"; import { stripAnsi } from "../../../../../../tests/helpers/ansi.ts"; +import { unusedGateway } from "../../../../../../tests/helpers/unused-stack.ts"; import { alwaysReadyHttpClientLayer, defaultLocalResetRoute, @@ -166,7 +167,7 @@ function syncStackApi(workdir: string, port: number) { }, stop: unusedSync, destroy: unusedSync, - gateway: { readLogs: () => Stream.die("unused") }, + gateway: unusedGateway, commands: { run: unusedSyncFn }, }; const identity = { projectRoot: workdir, branchContext: "main", stackName: "default" }; diff --git a/apps/cli/src/commands/db/start/start.integration.test.ts b/apps/cli/src/commands/db/start/start.integration.test.ts index fdeca26575..eeaaefa772 100644 --- a/apps/cli/src/commands/db/start/start.integration.test.ts +++ b/apps/cli/src/commands/db/start/start.integration.test.ts @@ -23,6 +23,7 @@ import { mockProcessControl, mockRuntimeInfo, } from "../../../../tests/helpers/mocks.ts"; +import { unusedGateway } from "../../../../tests/helpers/unused-stack.ts"; import { mockCommandSettings, mockLocalDockerEngineUnavailableLayer, @@ -1733,7 +1734,7 @@ describe("db start stack backend", () => { }, stop: Effect.void, destroy: Effect.succeed({ runtimeCleanup: "complete" as const }), - gateway: { readLogs: () => Stream.die("unused") }, + gateway: unusedGateway, commands: { run: () => Effect.die("unused") }, }; return { stack, state }; diff --git a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts index c377fd3316..bce7ca9e63 100644 --- a/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/prepare/prepare.integration.test.ts @@ -17,6 +17,7 @@ import { containerEngineSpawner, type ContainerEngineState, } from "../../../../../tests/helpers/child-process-spawner.ts"; +import { unusedGateway } from "../../../../../tests/helpers/unused-stack.ts"; import { mockCommandSettings, mockTelemetryStateTracked, @@ -157,7 +158,7 @@ const makeFixture = (root: string, options: FixtureOptions = {}) => { }, stop: Effect.void, destroy: Effect.succeed({ runtimeCleanup: "complete" as const }), - gateway: { readLogs: () => Stream.die("unused") }, + gateway: unusedGateway, commands: { run: () => Effect.die("unused") }, }; const output = mockOutput(); diff --git a/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts index f8ac2d7753..dc425005fb 100644 --- a/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start-export-pointer.integration.test.ts @@ -19,6 +19,7 @@ import { mockTty, } from "../../../../../tests/helpers/mocks.ts"; import { mockTelemetryStateTracked } from "../../../../../tests/helpers/command-mocks.ts"; +import { unusedGateway } from "../../../../../tests/helpers/unused-stack.ts"; import { commandRuntimeLayer } from "../../../../shared/runtime/command-runtime.layer.ts"; import { CliArgs } from "../../../../shared/cli/cli-args.service.ts"; import { textCliOutputFormatter } from "../../../../shared/output/text-formatter.ts"; @@ -112,7 +113,7 @@ function makeDatabaseStack(sqlPort: number, credentials: StackCredentials): Stac }, stop: Effect.die("unused"), destroy: Effect.die("unused"), - gateway: { readLogs: () => Stream.die("unused") }, + gateway: unusedGateway, commands: { run: () => Effect.die("unused") }, } satisfies Stack; } diff --git a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts index 5b0d9379a7..0df657500d 100644 --- a/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/start/start.integration.test.ts @@ -34,6 +34,7 @@ import { withEnvVar, } from "../../../../../tests/helpers/command-mocks.ts"; import { mockOutput, mockRuntimeInfo, mockTty } from "../../../../../tests/helpers/mocks.ts"; +import { unusedGateway } from "../../../../../tests/helpers/unused-stack.ts"; import { containerEngineSpawner } from "../../../../../tests/helpers/child-process-spawner.ts"; import { DbConnection, @@ -352,7 +353,7 @@ const fakeStack = (compositionStart?: Stack["composition"]["start"]) => { hostDestroyed += 1; return { runtimeCleanup: "complete" as const }; }), - gateway: { readLogs: () => Stream.die("unused") }, + gateway: unusedGateway, commands: { run: () => Effect.die("command not used") }, }; return { diff --git a/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts b/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts index 2abbb751c0..c41c585456 100644 --- a/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts +++ b/apps/cli/src/commands/experimental/stack/status/status.integration.test.ts @@ -9,6 +9,7 @@ import { StackError, } from "@supabase/stack/effect"; import { mockOutput } from "../../../../../tests/helpers/mocks.ts"; +import { unusedGateway } from "../../../../../tests/helpers/unused-stack.ts"; import { mockCommandSettings, mockTelemetryStateTracked, @@ -176,7 +177,7 @@ const makeStack = ( }, stop: Effect.die("unused"), destroy: Effect.die("unused"), - gateway: { readLogs: () => Stream.die("unused") }, + gateway: unusedGateway, commands: { run: (_tool, _options) => Effect.die("unused"), }, diff --git a/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts b/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts index 0f9ba4f130..0dc3a08250 100644 --- a/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts +++ b/apps/cli/src/commands/functions/serve/serve.stack.integration.test.ts @@ -26,6 +26,7 @@ import { import { StackApi } from "../../../command-internal/stack-api.ts"; import { OutputFlag } from "../../../command-internal/global-flags.ts"; import { mockCommandSettings } from "../../../../tests/helpers/command-mocks.ts"; +import { unusedGateway } from "../../../../tests/helpers/unused-stack.ts"; import { mockOutput, mockProcessControl, @@ -287,7 +288,7 @@ const fixture = ( }, stop: Effect.die("unused"), destroy: Effect.die("unused"), - gateway: { readLogs: () => Stream.die("unused") }, + gateway: unusedGateway, commands: { run: () => Effect.die("unused") }, } satisfies Stack; const identity = { projectRoot: "/project", branchContext: "main", stackName: "default" }; diff --git a/apps/cli/tests/helpers/storage.ts b/apps/cli/tests/helpers/storage.ts index 3e5b651316..33430edc67 100644 --- a/apps/cli/tests/helpers/storage.ts +++ b/apps/cli/tests/helpers/storage.ts @@ -18,6 +18,7 @@ import { CliArgs } from "../../src/shared/cli/cli-args.service.ts"; import { CommandPlatformApi } from "../../src/auth/command-platform-api.service.ts"; import { CommandPlatformApiFactory } from "../../src/auth/command-platform-api-factory.service.ts"; import { ProjectRefNotLinkedError } from "../../src/config/project-ref.errors.ts"; +import { unusedGateway } from "./unused-stack.ts"; import { ProjectRefResolver } from "../../src/config/project-ref.service.ts"; import { generateGoJwt } from "../../src/command-internal/go-jwt.ts"; import { YesFlag } from "../../src/command-internal/global-flags.ts"; @@ -263,7 +264,7 @@ export function buildStorageStackApi( }, stop: Effect.die("unused"), destroy: Effect.die("unused"), - gateway: { readLogs: () => Stream.die("unused") }, + gateway: unusedGateway, commands: { run: () => Effect.die("unused") }, }; const definition = { diff --git a/apps/cli/tests/helpers/unused-stack.ts b/apps/cli/tests/helpers/unused-stack.ts index ea1039f398..381dc84a8a 100644 --- a/apps/cli/tests/helpers/unused-stack.ts +++ b/apps/cli/tests/helpers/unused-stack.ts @@ -1,4 +1,5 @@ -import { Effect, Layer } from "effect"; +import { Effect, Layer, Stream } from "effect"; +import type { Stack } from "@supabase/stack/effect"; import { StackApi } from "../../src/command-internal/stack-api.ts"; import { StackCatalogSetup } from "../../src/command-internal/stack-catalog-setup.ts"; @@ -13,3 +14,6 @@ export const unusedStackServices = Layer.mergeAll( }), Layer.succeed(StackCatalogSetup, { apply: unused }), ); + +/** Fills a fake `Stack`'s gateway log stream for tests that never read it. */ +export const unusedGateway: Stack["gateway"] = { readLogs: () => Stream.die("unused") }; diff --git a/packages/stack/src/HttpProxy.integration.test.ts b/packages/stack/src/HttpProxy.integration.test.ts index c9d6e8b9e9..2d95d4cb1a 100644 --- a/packages/stack/src/HttpProxy.integration.test.ts +++ b/packages/stack/src/HttpProxy.integration.test.ts @@ -1135,6 +1135,12 @@ it.live( expect.objectContaining({ target: "/elsewhere", status: 404, bytes: 9 }), expect.objectContaining({ target: "/down/thing", status: 502, bytes: 11 }), ]); + + // A later sentinel request proves none of the four requests above recorded twice. + const sentinelStatus = yield* get("/api/ok?select=sentinel"); + const sentinel = yield* Queue.take(accesses); + expect(sentinelStatus).toBe(200); + expect(sentinel).toMatchObject({ target: "/api/ok?select=sentinel" }); expect(yield* Queue.size(accesses)).toBe(0); }), ).pipe( diff --git a/packages/stack/src/HttpProxy.ts b/packages/stack/src/HttpProxy.ts index d1f3f6cdbd..1c7ffaacc3 100644 --- a/packages/stack/src/HttpProxy.ts +++ b/packages/stack/src/HttpProxy.ts @@ -1,4 +1,5 @@ import { Clock, Data, Deferred, Effect, Fiber, FiberSet, Ref, Scope } from "effect"; +import { decodeQuery, redactCredentials } from "./internal/redact-credentials.ts"; import { PortError } from "./Ports.ts"; import type { BackendAddress, ProxyError } from "./Proxy.ts"; import { @@ -144,83 +145,6 @@ const upstreamHeadersFor = (headers: IncomingMessage["headers"], route: HttpRout return result; }; -const decodeQuery = (value: string) => { - try { - return decodeURIComponent(value.replace(/\+/gu, " ")); - } catch { - return value; - } -}; - -/** Decodes repeatedly so a nested URL encoded more than once still exposes its delimiters. */ -const decodeNested = (value: string) => { - let current = value; - for (let pass = 0; pass < 4; pass++) { - const next = decodeQuery(current); - if (next === current) break; - current = next; - } - return current; -}; - -// Auth puts PKCE codes, OTP token hashes and OAuth tokens in redirect and verify URLs; S3 -// presigned URLs carry their SigV4 signature, credential scope and session token. -const credentialParameters = [ - "apikey", - "jwt", - "access_token", - "token", - "token_hash", - "code", - "refresh_token", - "id_token", - "provider_token", - "provider_refresh_token", - "x-amz-signature", - "x-amz-credential", - "x-amz-security-token", -]; -const scheme = String.raw`[a-z][a-z\d+.-]*`; - -/** - * A decoded parameter that is a credential pair, nests one (a `redirect_to` URL carrying a - * token), or nests a URL with userinfo. - */ -const sensitive = new RegExp( - String.raw`(?:^|[?&#=])(?:${credentialParameters.join("|")})=|${scheme}:\/\/[^/?#@]*@`, - "iu", -); -const userinfo = new RegExp(String.raw`^(${scheme}:\/\/)[^/?#@]*@`, "iu"); - -const redactPairs = (pairs: string) => - pairs - .split("&") - .map((parameter) => { - const separator = parameter.indexOf("="); - return separator >= 0 && sensitive.test(decodeNested(parameter)) - ? `${parameter.slice(0, separator)}=redacted` - : parameter; - }) - .join("&"); - -/** - * Redacts an absolute URL's userinfo and credential values in its query and fragment, where - * OAuth implicit grants put `access_token` for non-browser clients. Edits the text in place, so - * relative and unparsable URLs work and the rest of the URL keeps its original encoding. - */ -export const redactCredentials = (url: string) => { - const hashAt = url.indexOf("#"); - const beforeHash = hashAt < 0 ? url : url.slice(0, hashAt); - const queryAt = beforeHash.indexOf("?"); - const base = (queryAt < 0 ? beforeHash : beforeHash.slice(0, queryAt)).replace( - userinfo, - "$1redacted@", - ); - const query = queryAt < 0 ? "" : `?${redactPairs(beforeHash.slice(queryAt + 1))}`; - const fragment = hashAt < 0 ? "" : `#${redactPairs(url.slice(hashAt + 1))}`; - return `${base}${query}${fragment}`; -}; - /** Captures a request's access fields while its socket is open; completes them once it settles. */ const accessFor = (request: IncomingMessage, time: number) => { const referer = headerValue(request.headers.referer); @@ -479,6 +403,7 @@ const proxyRequest = Effect.fn("HttpProxy.proxyRequest")( sent: Sent, ) => Effect.gen(function* () { + yield* Effect.annotateCurrentSpan({ route_id: route.id }); const backend = yield* Effect.raceFirst(route.target, disconnected(request, response)); yield* forward( request, @@ -495,7 +420,15 @@ const proxyRequest = Effect.fn("HttpProxy.proxyRequest")( ).pipe(Effect.andThen(forward(request, response, route, backend, false, sent))), ), ); - }), + }).pipe( + Effect.ensuring( + Effect.suspend(() => + response.headersSent + ? Effect.annotateCurrentSpan({ "http.response.status_code": response.statusCode }) + : Effect.void, + ), + ), + ), ); const statusLine = /^HTTP\/\d(?:\.\d)? (\d{3})\b/u; @@ -511,6 +444,7 @@ const upgrade = Effect.fn("HttpProxy.upgrade")( handshake: Deferred.Deferred, ) => Effect.gen(function* () { + yield* Effect.annotateCurrentSpan({ route_id: route.id }); const backend = yield* Effect.raceFirst( route.target, Effect.callback((resume) => { diff --git a/packages/stack/src/Owner.logs.integration.test.ts b/packages/stack/src/Owner.logs.integration.test.ts index 2555e3ba46..53c219aa2b 100644 --- a/packages/stack/src/Owner.logs.integration.test.ts +++ b/packages/stack/src/Owner.logs.integration.test.ts @@ -210,13 +210,11 @@ describe("owner persisted logs", () => { ); if (api === undefined) return yield* Effect.die("the shared API port was not claimed"); - const response = yield* client.get(`http://127.0.0.1:${api.port}/unrouted?apikey=k`); + const response = yield* client.get(`http://127.0.0.1:${api.port}/unrouted`); const [recorded] = yield* firstOutput(first.rpc.readLogs({ id: "gateway", follow: true })); expect(response.status).toBe(404); - expect(recorded?.text).toMatch( - /^127\.0\.0\.1 - - \[[^\]]+\] "GET \/unrouted\?apikey=redacted HTTP\/1\.1" 404 9 "-" "[^"]*" \d+ms$/u, - ); + expect(recorded?.text).toContain("/unrouted"); const directory = path.join(first.logsRoot, "gateway", "gateway"); expect(yield* fs.readDirectory(directory)).toEqual(["0000000001.log"]); yield* Scope.close(firstScope, Exit.void); diff --git a/packages/stack/src/internal/redact-credentials.ts b/packages/stack/src/internal/redact-credentials.ts new file mode 100644 index 0000000000..4d9f257925 --- /dev/null +++ b/packages/stack/src/internal/redact-credentials.ts @@ -0,0 +1,77 @@ +/** Decodes a query component, keeping it as is when it is not valid percent-encoding. */ +export const decodeQuery = (value: string) => { + try { + return decodeURIComponent(value.replace(/\+/gu, " ")); + } catch { + return value; + } +}; + +/** Decodes repeatedly so a nested URL encoded more than once still exposes its delimiters. */ +const decodeNested = (value: string) => { + let current = value; + for (let pass = 0; pass < 4; pass++) { + const next = decodeQuery(current); + if (next === current) break; + current = next; + } + return current; +}; + +// Auth puts PKCE codes, OTP token hashes and OAuth tokens in redirect and verify URLs; S3 +// presigned URLs carry their SigV4 signature, credential scope and session token. +const credentialParameters = [ + "apikey", + "jwt", + "access_token", + "token", + "token_hash", + "code", + "refresh_token", + "id_token", + "provider_token", + "provider_refresh_token", + "x-amz-signature", + "x-amz-credential", + "x-amz-security-token", +]; +const scheme = String.raw`[a-z][a-z\d+.-]*`; + +/** + * A decoded parameter that is a credential pair, nests one (a `redirect_to` URL carrying a + * token), or nests a URL with userinfo. + */ +const sensitive = new RegExp( + String.raw`(?:^|[?&#=])(?:${credentialParameters.join("|")})=|${scheme}:\/\/[^/?#@]*@`, + "iu", +); +const userinfo = new RegExp(String.raw`^(${scheme}:\/\/)[^/?#@]*@`, "iu"); + +const redactPairs = (pairs: string) => + pairs + .split("&") + .map((parameter) => { + const separator = parameter.indexOf("="); + return separator >= 0 && sensitive.test(decodeNested(parameter)) + ? `${parameter.slice(0, separator)}=redacted` + : parameter; + }) + .join("&"); + +/** + * Redacts an absolute URL's userinfo and credential values in its query and fragment, where + * OAuth implicit grants put `access_token` for non-browser clients. Edits the text in place, so + * relative and unparsable URLs work and the rest of the URL keeps its original encoding. + */ +export const redactCredentials = (url: string) => { + const hashAt = url.indexOf("#"); + const beforeHash = hashAt < 0 ? url : url.slice(0, hashAt); + const queryAt = beforeHash.indexOf("?"); + const base = (queryAt < 0 ? beforeHash : beforeHash.slice(0, queryAt)).replace( + userinfo, + "$1redacted@", + ); + const query = queryAt < 0 ? "" : `?${redactPairs(beforeHash.slice(queryAt + 1))}`; + const fragment = hashAt < 0 ? "" : `#${redactPairs(url.slice(hashAt + 1))}`; + return `${base}${query}${fragment}`; +}; diff --git a/packages/stack/src/HttpProxy.unit.test.ts b/packages/stack/src/internal/redact-credentials.unit.test.ts similarity index 98% rename from packages/stack/src/HttpProxy.unit.test.ts rename to packages/stack/src/internal/redact-credentials.unit.test.ts index 8b3c2c84b3..93d44c6a49 100644 --- a/packages/stack/src/HttpProxy.unit.test.ts +++ b/packages/stack/src/internal/redact-credentials.unit.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "@effect/vitest"; -import { redactCredentials } from "./HttpProxy.ts"; +import { redactCredentials } from "./redact-credentials.ts"; describe("redactCredentials", () => { it.each([ From 03da7bfd0ef0840e494306cd9a49d148030d2c3f Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 2 Oct 2026 20:41:22 +0200 Subject: [PATCH 26/35] fix(stack): halve rejected log bodies so only refused lines are skipped A body Analytics rejects is posted again in halves, like one it leaves unstored, so one refused line no longer drops the rest of its body. Warnings use singular counts and name only the reasons that apply, and functions serve reuses the stack logs marker formatting. Co-Authored-By: Claude Opus 5.5 --- .../experimental/stack/logs/logs.format.ts | 3 +- .../functions/serve/serve.stack.handler.ts | 41 +++++-------------- packages/stack/ARCHITECTURE.md | 3 +- .../src/host/LogForwarder.integration.test.ts | 29 +++++++------ packages/stack/src/host/LogForwarder.ts | 30 ++++++++------ packages/stack/src/host/LogStore.ts | 2 +- 6 files changed, 50 insertions(+), 58 deletions(-) diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.format.ts b/apps/cli/src/commands/experimental/stack/logs/logs.format.ts index 087f4a4100..29c2c3b5db 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.format.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.format.ts @@ -237,7 +237,8 @@ const localTime = (iso: string) => }, }); -const markerText = (record: LogRecord) => { +/** The text of a launch or lost marker, as `stack logs` prints it. */ +export const markerText = (record: LogRecord) => { if (record.kind === "launch") return record.launchId === undefined ? "--- launch ---" : `--- launch ${record.launchId} ---`; if (record.count === undefined) return "--- older records were removed by retention ---"; diff --git a/apps/cli/src/commands/functions/serve/serve.stack.handler.ts b/apps/cli/src/commands/functions/serve/serve.stack.handler.ts index f36ae22bae..e42de33d89 100644 --- a/apps/cli/src/commands/functions/serve/serve.stack.handler.ts +++ b/apps/cli/src/commands/functions/serve/serve.stack.handler.ts @@ -14,6 +14,7 @@ import { stackOpenReadyProject } from "../../../command-internal/stack-local-dat import { OutputFlag } from "../../../command-internal/global-flags.ts"; import { TelemetryState } from "../../../telemetry/telemetry-state.service.ts"; import type { FunctionsServeFlags } from "../../../shared/functions/serve.ts"; +import { logEvent, markerText } from "../../experimental/stack/logs/logs.format.ts"; import { FunctionsServeStackError } from "./serve.errors.ts"; type Instance = Effect.Success>; @@ -48,36 +49,16 @@ const follow = Effect.fn("functions.serve.follow")(function* ( // keeps its first output. const since = DateTime.formatIso(yield* DateTime.now); const logs = yield* instance.readLogs({ follow: true, since }).pipe( - Stream.runForEach(({ kind, timestamp, text, stream, count }) => { - if (kind === "lost" && count !== undefined) - return output.format === "stream-json" - ? output.event({ - type: "log-marker", - timestamp, - source: "live", - service: "functions", - instance_id: instance.id, - kind, - ...(stream === undefined ? {} : { stream }), - count, - }) - : output.raw( - `--- ${count} ${stream ?? "output"} ${count === 1 ? "chunk" : "chunks"} lost ---\n`, - "stderr", - ); - if (kind !== "stdout" && kind !== "stderr") return Effect.void; - const line = text ?? ""; - return output.format === "stream-json" - ? output.event({ - type: "log-entry", - timestamp, - source: "live", - service: "functions", - instance_id: instance.id, - stream: kind, - line, - }) - : output.raw(`${line}\n`, kind); + Stream.runForEach((record) => { + const lost = record.kind === "lost" && record.count !== undefined; + if (record.kind !== "stdout" && record.kind !== "stderr" && !lost) return Effect.void; + if (output.format === "stream-json") + return output.event( + logEvent({ ...record, service: "functions", instanceId: instance.id }, "live"), + ); + return record.kind === "stdout" || record.kind === "stderr" + ? output.raw(`${record.text ?? ""}\n`, record.kind) + : output.raw(`${markerText(record)}\n`, "stderr"); }), Effect.forkScoped({ startImmediately: true }), ); diff --git a/packages/stack/ARCHITECTURE.md b/packages/stack/ARCHITECTURE.md index 8bb498a67b..d605f3c0cd 100644 --- a/packages/stack/ARCHITECTURE.md +++ b/packages/stack/ARCHITECTURE.md @@ -712,7 +712,8 @@ launch changes. otherwise it stays pending and is posted alone again, so an Analytics that stores nothing loses nothing. Each round warns once. - **Statuses:** a 401, 403 or 404 response clears the pending body and pauses the instance with its - cursor until the target changes; any other 4xx except 408 and 429 skips the body. + cursor until the target changes; after any other 4xx except 408 and 429 the body is posted again + in halves, and only events still rejected on their own are skipped. - **Sanitizing:** event text and metadata strings have NUL and unpaired surrogates replaced with U+FFFD before posting, because Postgres `jsonb` rejects them and drops the whole batch. - **Recovery and retargets:** a start, restart or retarget reconciles a pending body before later diff --git a/packages/stack/src/host/LogForwarder.integration.test.ts b/packages/stack/src/host/LogForwarder.integration.test.ts index 2700b626a3..bc64c41dcf 100644 --- a/packages/stack/src/host/LogForwarder.integration.test.ts +++ b/packages/stack/src/host/LogForwarder.integration.test.ts @@ -854,7 +854,7 @@ describe("LogForwarder", () => { expect(shipped).toEqual(retained.map((record) => record.text)); expect(warnings).toEqual([ [ - "Logs of database instance database were deleted before Analytics stored them; skipping 1 lines", + "Logs of database instance database were deleted before Analytics stored them; skipping 1 line", ], [ "Logs of database instance database were deleted before shipping; resuming from the next retained record", @@ -901,32 +901,35 @@ describe("LogForwarder", () => { Effect.as("paused"), Effect.raceFirst(manual.sleeping(pollMillis).pipe(Effect.as("retrying"))), ); - yield* manual.advance(stuckMillis); - const postedWhilePaused = yield* logflare.unread; yield* analytics.relaunch(2); const resumed = yield* logflare.next; expect(outcome).toBe("paused"); - expect(postedWhilePaused).toBe(0); expect(messages(refusedPost)).toEqual(["refused line"]); expect(ids(resumed)).toEqual(ids(refusedPost)); }).pipe(Effect.scoped, Effect.provide(layer)), ); - it.live("skips a body Analytics rejects and ships the next one", () => + it.live("halves a body Analytics rejects and skips only the line it still rejects alone", () => Effect.gen(function* () { const { store, logflare, database, analytics } = yield* fixture(); yield* analytics.set(true); yield* startForwarder(store, logflare, [analytics.instance, database.instance]); - yield* logflare.respond(400); + yield* logflare.respond(400, 400); - yield* database.log("rejected line"); - const rejected = yield* logflare.next; - yield* database.log("accepted line"); - const accepted = yield* logflare.next; + yield* database.log("rejected line", "accepted line"); + const posts = [yield* logflare.next, yield* logflare.next, yield* logflare.next]; + yield* database.log("next line"); + const next = yield* logflare.next; - expect(messages(rejected)).toEqual(["rejected line"]); - expect(messages(accepted)).toEqual(["accepted line"]); + expect(posts.map(messages)).toEqual([ + ["rejected line", "accepted line"], + ["rejected line"], + ["accepted line"], + ]); + expect(messages(next)).toEqual(["next line"]); + // The next line is posted only once the earlier body is confirmed stored. + expect((yield* storedMessages(logflare))[0]).toBe("accepted line"); }).pipe(Effect.scoped, Effect.provide(layer)), ); @@ -990,7 +993,7 @@ describe("LogForwarder", () => { "Analytics has not stored 256 postgres.logs lines of database it accepted; posting them in halves", ], [ - "Analytics did not store 1 postgres.logs lines of database posted on their own (0 rejected, 1 unstored while it stored later posts); skipping them", + "Skipping 1 postgres.logs line of database that Analytics did not store while it stored later posts when posted on their own", ], ]); }).pipe(Effect.scoped, Effect.provide(layer)), diff --git a/packages/stack/src/host/LogForwarder.ts b/packages/stack/src/host/LogForwarder.ts index 494060fdf0..3d86eed349 100644 --- a/packages/stack/src/host/LogForwarder.ts +++ b/packages/stack/src/host/LogForwarder.ts @@ -237,6 +237,9 @@ export const bodies = ( return result; }; +const lines = (count: number, source: string) => + `${count} ${source} ${count === 1 ? "line" : "lines"}`; + /** Splits ids into two halves, the first one longer by at most one. */ const halves = (ids: ReadonlyArray): Array> => { const middle = Math.ceil(ids.length / 2); @@ -521,7 +524,7 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar if (isBodyRejection(error)) return warnOnce( warned.rejection, - `Analytics rejected a body of ${body.items.length} ${source} lines of ${session.instanceId}`, + `Analytics rejected a body of ${lines(body.items.length, source)} of ${session.instanceId}`, error, ).pipe(Effect.as("rejected")); if (!mayHaveQueued(error)) @@ -545,7 +548,8 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar * `stuckMillis` for a previous owner's launch. Events the current launch leaves unstored for * `stuckMillis` are posted again in halves; one that still is not stored on its own is skipped * once a later post to the launch was stored, and kept pending otherwise. A refused - * target clears the pending body and pauses; a rejected body is skipped. + * target clears the pending body and pauses; a rejected body is posted in halves until each + * event Analytics still rejects on its own is skipped. */ const settle = Effect.fn("LogForwarder.settle")(function* ( session: Session, @@ -611,7 +615,7 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar const deleted = new Set(missing.filter((id) => !events.has(id))); if (deleted.size > 0) { yield* Effect.logWarning( - `Logs of ${session.service} instance ${session.instanceId} were deleted before Analytics stored them; skipping ${deleted.size} lines`, + `Logs of ${session.service} instance ${session.instanceId} were deleted before Analytics stored them; skipping ${deleted.size === 1 ? "1 line" : `${deleted.size} lines`}`, ); counts.deleted += deleted.size; wanted = wanted.filter((id) => !deleted.has(id)); @@ -686,7 +690,13 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar for (const suspect of suspects) if (proof > suspect.post) skipped.add(suspect.id); if (skipped.size > 0) { yield* Effect.logWarning( - `Analytics did not store ${skipped.size} ${source} lines of ${session.instanceId} posted on their own (${rejected} rejected, ${skipped.size - rejected} unstored while it stored later posts); skipping them`, + `Skipping ${lines(skipped.size, source)} of ${session.instanceId} that Analytics ${ + rejected === skipped.size + ? "rejected" + : rejected === 0 + ? "did not store while it stored later posts" + : `rejected (${rejected}) or did not store while it stored later posts` + } when posted on their own`, ); remaining = remaining.filter((id) => !skipped.has(id)); counts.skipped += skipped.size; @@ -698,7 +708,7 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar already ? Effect.void : Effect.logWarning( - `Analytics stored none of ${groups.length} ${source} lines of ${session.instanceId} posted on their own; waiting until it stores them`, + `Analytics stored none of ${lines(groups.length, source)} of ${session.instanceId} posted on their own; waiting until it stores them`, ), ), ); @@ -707,15 +717,11 @@ export const make = Effect.fn("LogForwarder.make")(function* (options: LogForwar yield* Effect.gen(function* () { const stuck = yield* deliverAll(shipment.ids); - if (stuck === "rejected") { - yield* Effect.logWarning( - `Skipping ${remaining.length} ${source} lines of ${session.instanceId} that Analytics rejected`, - ); - counts.skipped += remaining.length; - } else if (stuck.length > 0) { + if (stuck === "rejected") yield* isolate([...lastPosted]); + else if (stuck.length > 0) { counts.stuck = stuck.length; yield* Effect.logWarning( - `Analytics has not stored ${stuck.length} ${source} lines of ${session.instanceId} it accepted; posting them in halves`, + `Analytics has not stored ${lines(stuck.length, source)} of ${session.instanceId} it accepted; posting them in halves`, ); yield* isolate(stuck); } diff --git a/packages/stack/src/host/LogStore.ts b/packages/stack/src/host/LogStore.ts index 447d7809ca..b06245885d 100644 --- a/packages/stack/src/host/LogStore.ts +++ b/packages/stack/src/host/LogStore.ts @@ -860,7 +860,7 @@ export const make = Effect.fn("LogStore.make")(function* (options: LogStoreOptio if (entry.kind === "lost" && !warnedLaunches.has(entry.launchId)) { warnedLaunches.add(entry.launchId); yield* Effect.logWarning( - `${instance.service} instance ${instance.instanceId} dropped ${entry.count} ${entry.stream} chunks of launch ${entry.launchId} before they were persisted; later drops of this launch are recorded only in its logs`, + `${instance.service} instance ${instance.instanceId} dropped ${entry.count} ${entry.stream} ${entry.count === 1 ? "chunk" : "chunks"} of launch ${entry.launchId} before they were persisted; later drops of this launch are recorded only in its logs`, ); } return; From 1be9e3bf6395a231cc8fc4de627568d131da29c5 Mon Sep 17 00:00:00 2001 From: avallete Date: Fri, 2 Oct 2026 20:57:09 +0200 Subject: [PATCH 27/35] fix(stack): adapt develop's port test and share stack log events Develop's native port test now uses this branch's recipe signatures, the catalog error helper both sides stopped using is removed, and the stack log event and marker formatting moves to command-internal so functions serve no longer imports another command's internals. Co-Authored-By: Claude Opus 5.5 --- .../src/command-internal/stack-log-events.ts | 37 +++++++++++++++++++ .../experimental/stack/logs/logs.format.ts | 36 +----------------- .../experimental/stack/logs/logs.handler.ts | 3 +- .../functions/serve/serve.stack.handler.ts | 2 +- .../ProcessRecipe.integration.test.ts | 2 +- packages/stack/src/services/ProcessRecipe.ts | 9 ----- 6 files changed, 41 insertions(+), 48 deletions(-) create mode 100644 apps/cli/src/command-internal/stack-log-events.ts diff --git a/apps/cli/src/command-internal/stack-log-events.ts b/apps/cli/src/command-internal/stack-log-events.ts new file mode 100644 index 0000000000..f8ef296b09 --- /dev/null +++ b/apps/cli/src/command-internal/stack-log-events.ts @@ -0,0 +1,37 @@ +import type { LogRecord, StackLogRecord } from "@supabase/stack/effect"; +import type { StreamEvent } from "../shared/output/types.ts"; + +/** Whether a record was read as history or followed live. */ +export type LogSource = "history" | "live"; + +/** The `log-entry` or `log-marker` event of a record. */ +export const logEvent = (record: StackLogRecord, source: LogSource): StreamEvent => { + const subject = { service: record.service, instance_id: record.instanceId }; + if (record.kind === "stdout" || record.kind === "stderr") + return { + type: "log-entry", + timestamp: record.timestamp, + source, + ...subject, + stream: record.kind, + line: record.text ?? "", + }; + return { + type: "log-marker", + timestamp: record.timestamp, + source, + ...subject, + kind: record.kind, + ...(record.stream === undefined ? {} : { stream: record.stream }), + ...(record.count === undefined ? {} : { count: record.count }), + }; +}; + +/** The text of a launch or lost marker, as `stack logs` prints it. */ +export const markerText = (record: LogRecord) => { + if (record.kind === "launch") + return record.launchId === undefined ? "--- launch ---" : `--- launch ${record.launchId} ---`; + if (record.count === undefined) return "--- older records were removed by retention ---"; + const unit = record.count === 1 ? "chunk" : "chunks"; + return `--- ${record.count} ${record.stream ?? "output"} ${unit} lost ---`; +}; diff --git a/apps/cli/src/commands/experimental/stack/logs/logs.format.ts b/apps/cli/src/commands/experimental/stack/logs/logs.format.ts index 29c2c3b5db..d1a204c66f 100644 --- a/apps/cli/src/commands/experimental/stack/logs/logs.format.ts +++ b/apps/cli/src/commands/experimental/stack/logs/logs.format.ts @@ -10,9 +10,7 @@ import { yellow, } from "../../../../command-internal/colors.ts"; import { stripControlSequences } from "../../../../shared/output/strip-control-sequences.ts"; -import type { StreamEvent } from "../../../../shared/output/types.ts"; - -export type LogSource = "history" | "live"; +import { markerText } from "../../../../command-internal/stack-log-events.ts"; /** A `--since` bound: an absolute time, or each instance's latest launch. */ export type SinceBound = @@ -202,29 +200,6 @@ export const isAfter = (record: LogRecord, printed: LogPosition | undefined) => record.position === undefined || comparePositions(record.position, printed) > 0; -/** The `log-entry` or `log-marker` event of a record. */ -export const logEvent = (record: StackLogRecord, source: LogSource): StreamEvent => { - const subject = { service: record.service, instance_id: record.instanceId }; - if (record.kind === "stdout" || record.kind === "stderr") - return { - type: "log-entry", - timestamp: record.timestamp, - source, - ...subject, - stream: record.kind, - line: record.text ?? "", - }; - return { - type: "log-marker", - timestamp: record.timestamp, - source, - ...subject, - kind: record.kind, - ...(record.stream === undefined ? {} : { stream: record.stream }), - ...(record.count === undefined ? {} : { count: record.count }), - }; -}; - const palette = [aqua, yellow, green, magenta, blue]; const pad = (value: number, width: number) => String(value).padStart(width, "0"); @@ -237,15 +212,6 @@ const localTime = (iso: string) => }, }); -/** The text of a launch or lost marker, as `stack logs` prints it. */ -export const markerText = (record: LogRecord) => { - if (record.kind === "launch") - return record.launchId === undefined ? "--- launch ---" : `--- launch ${record.launchId} ---`; - if (record.count === undefined) return "--- older records were removed by retention ---"; - const unit = record.count === 1 ? "chunk" : "chunks"; - return `--- ${record.count} ${record.stream ?? "output"} ${unit} lost ---`; -}; - /** Formats records as `