diff --git a/.dockerignore b/.dockerignore index a49dd2c0..7ed41f3d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,8 +19,10 @@ ui/host/node_modules/ ui/host/dist/ # Dev files -.env -.env.* +.env* +**/.env* +secrets/ +.sops.yaml .tmp/ tmp/ diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..f3c44e18 --- /dev/null +++ b/.env.example @@ -0,0 +1,265 @@ +# Generated by cmd/fanout-docgen from internal/config/config.go. +# Regenerate with just docs-generate; verify with just docs-generate-check. +# Fanout does not load dotenv files. Inject through your launcher or deployment. +# Precedence: built-in defaults < nonempty environment values. +# All settings require restart. Empty environment values mean no override. +# Every entry is commented out so copying this template sets nothing, preserving image ENV. +# Uncomment only settings you want to supply; use runtime secret injection for secrets. +# Local authentication requires FANOUT_AUTH_CODE_SECRET (32+ characters). +# Chat is disabled unless FANOUT_AI_PROVIDER selects anthropic or openai; its key is required. + +# ANTHROPIC_API_KEY; string; secret +# Required when FANOUT_AI_PROVIDER is anthropic. +# Default: (empty) +# ANTHROPIC_API_KEY= +# Alternatively set ANTHROPIC_API_KEY_FILE to a secret file path; do not set both. + +# ANTHROPIC_BASE_URL; string +# Defaults to https://api.anthropic.com. Fanout appends /v1/messages, preserving any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. +# Default: (empty) +# ANTHROPIC_BASE_URL= + +# FANOUT_ADDR; string +# Binds the browser, API, MCP, OTLP/gRPC and OTLP/HTTP listener. +# Default: 127.0.0.1:7520 +# FANOUT_ADDR= + +# FANOUT_AI_MODEL; string +# Overrides the provider's default: claude-sonnet-5-5 for Anthropic and gpt-6.1-sol for OpenAI. The OpenAI provider expects a reasoning model supporting reasoning.encrypted_content for stateless Responses replay. A 400 rejecting encrypted content is reported as a server-side configuration error; choose a compatible reasoning model with FANOUT_AI_MODEL. +# Default: (empty) +# FANOUT_AI_MODEL= + +# FANOUT_AI_PROVIDER; string +# Enables chat by selecting exactly anthropic or openai. Empty disables chat. +# Default: (empty) +# FANOUT_AI_PROVIDER= + +# FANOUT_ALERTS_ENABLED; boolean +# Runs the alert evaluation engine. +# Default: true +# FANOUT_ALERTS_ENABLED= + +# FANOUT_ALERTS_EVALUATION_INTERVAL; duration +# Schedules alert evaluation; use a Go duration of at least 1s. +# Default: 30s +# FANOUT_ALERTS_EVALUATION_INTERVAL= + +# FANOUT_ALERTS_HISTORY_DAYS; integer +# Sets alert history retention in days. +# Default: 7 +# FANOUT_ALERTS_HISTORY_DAYS= + +# FANOUT_AUTH_CODE_SECRET; string; secret +# Signs local sign-in codes; required in local mode with at least 32 characters. +# Default: (empty) +# FANOUT_AUTH_CODE_SECRET= +# Alternatively set FANOUT_AUTH_CODE_SECRET_FILE to a secret file path; do not set both. + +# FANOUT_AUTH_MODE; string +# Selects local sign-in or oidc; each mode validates its prerequisites. +# Default: local +# FANOUT_AUTH_MODE= + +# FANOUT_DATA_DIR; string +# Stores telemetry, query state, and the control database. +# Default: ./data +# FANOUT_DATA_DIR= + +# FANOUT_DEFAULT_NAMESPACE; string +# Supplies the namespace for telemetry without one. +# Default: default +# FANOUT_DEFAULT_NAMESPACE= + +# FANOUT_DUCKDB_MAX_CONNECTIONS; integer +# Caps the DuckDB connection pool. Reads scan immutable Parquet concurrently; rollup-cache writes are serialized by the query write gate. Zero means "size it from the machine" — the same spelling DuckDBThreads uses for deferring to a default. Resolution happens in resolveSizing and is reported in the startup configuration log. +# Default: 0 +# FANOUT_DUCKDB_MAX_CONNECTIONS= + +# FANOUT_DUCKDB_MEMORY; string +# Caps DuckDB's memory (e.g. "8GB"). Empty means Fanout sizes it from detected memory, reserving headroom for the Go runtime; see resolveSizing. DuckDB's own default is 80% of detected RAM, which is calculated as though DuckDB owned the machine and has been observed to get the process OOM-killed once the Go heap is added on top. +# Default: (empty) +# FANOUT_DUCKDB_MEMORY= + +# FANOUT_DUCKDB_THREADS; integer +# Caps DuckDB's global query worker pool. Zero leaves DuckDB's own default in place (one worker per core). Set it to leave cores free for ingest on a query-heavy co-tenant host. +# Default: (empty) +# FANOUT_DUCKDB_THREADS= + +# FANOUT_INGEST_BATCH_SIZE; integer +# Limits telemetry rows per batch; valid values are 1–50000. +# Default: 50000 +# FANOUT_INGEST_BATCH_SIZE= + +# FANOUT_INGEST_MAX_IN_FLIGHT_BYTES; integer +# Caps the decoded telemetry the process will hold across concurrent requests. Concurrency is HTTP/2 streams times connections and each handler holds its batch until the commit is durable, so without a ceiling a burst is bounded only by memory. Zero is unbounded, which is the behaviour that predates this setting. +# Default: 268435456 +# FANOUT_INGEST_MAX_IN_FLIGHT_BYTES= + +# FANOUT_MAINTENANCE_INTERVAL; duration +# Controls Parquet retention and compaction, and query-cache checkpointing. +# Default: 1h +# FANOUT_MAINTENANCE_INTERVAL= + +# FANOUT_MCP_ENABLED; boolean +# Serves the MCP endpoint at /mcp. +# Default: true +# FANOUT_MCP_ENABLED= + +# FANOUT_METRICS_PUBLIC; boolean +# Allows unauthenticated Prometheus scrapes. +# Default: false +# FANOUT_METRICS_PUBLIC= + +# FANOUT_METRICS_TOKEN; string; secret +# Authorizes Prometheus scrapes with a bearer token; optional when using an admin session. +# Default: (empty) +# FANOUT_METRICS_TOKEN= +# Alternatively set FANOUT_METRICS_TOKEN_FILE to a secret file path; do not set both. + +# FANOUT_OIDC_ADMIN_GROUPS; string +# Lists identity-provider groups mapped to admin, separated by commas. +# Default: (empty) +# FANOUT_OIDC_ADMIN_GROUPS= + +# FANOUT_OIDC_ALLOWED_DOMAINS; string +# Lists allowed email domains, separated by commas; wildcard * requires verified email. +# Default: (empty) +# FANOUT_OIDC_ALLOWED_DOMAINS= + +# FANOUT_OIDC_ALLOWED_GROUPS; string +# Lists allowed identity-provider groups, separated by commas. +# Default: (empty) +# FANOUT_OIDC_ALLOWED_GROUPS= + +# FANOUT_OIDC_AUTO_PROVISION; boolean +# Creates accounts on OIDC sign-in; requires an allowed group or domain. +# Default: false +# FANOUT_OIDC_AUTO_PROVISION= + +# FANOUT_OIDC_CLIENT_ID; string +# Identifies Fanout's client registration; required in OIDC mode. +# Default: (empty) +# FANOUT_OIDC_CLIENT_ID= + +# FANOUT_OIDC_CLIENT_SECRET; string; secret +# Authenticates Fanout's client registration; required in OIDC mode. +# Default: (empty) +# FANOUT_OIDC_CLIENT_SECRET= +# Alternatively set FANOUT_OIDC_CLIENT_SECRET_FILE to a secret file path; do not set both. + +# FANOUT_OIDC_DEFAULT_ROLE; string +# Assigns viewer, operator, or admin to users without a group role mapping. +# Default: viewer +# FANOUT_OIDC_DEFAULT_ROLE= + +# FANOUT_OIDC_EMAIL_CLAIM; string +# Names the claim containing the user's email address. +# Default: email +# FANOUT_OIDC_EMAIL_CLAIM= + +# FANOUT_OIDC_EMAIL_VERIFICATION; string +# Selects required verified-email claims or issuer trust; issuer trust requires an allowlist. +# Default: required +# FANOUT_OIDC_EMAIL_VERIFICATION= + +# FANOUT_OIDC_ISSUER_URL; string +# Selects the identity provider issuer; required in OIDC mode. +# Default: (empty) +# FANOUT_OIDC_ISSUER_URL= + +# FANOUT_OIDC_OPERATOR_GROUPS; string +# Lists identity-provider groups mapped to operator, separated by commas. +# Default: (empty) +# FANOUT_OIDC_OPERATOR_GROUPS= + +# FANOUT_PPROF_ENABLED; boolean +# Exposes Go's net/http/pprof handlers at /debug/pprof/* for CPU/heap/mutex/goroutine profiling under load. Off by default; the routes require an admin browser session with operations:read. +# Default: false +# FANOUT_PPROF_ENABLED= + +# FANOUT_PUBLIC_URL; string +# Sets the external browser origin. Required for OIDC; HTTPS is required with MCP. +# Default: (empty) +# FANOUT_PUBLIC_URL= + +# FANOUT_RETENTION_DAYS; integer +# Sets telemetry retention in days; zero disables expiry. +# Default: 30 +# FANOUT_RETENTION_DAYS= + +# FANOUT_ROLLUP_INTERVAL; duration +# Schedules analytical rollups; use a Go duration of at least 1s. +# Default: 1m +# FANOUT_ROLLUP_INTERVAL= + +# FANOUT_ROLLUP_SKIP_TO_LATEST; boolean +# RollupSkipToLatest, set once at boot, advances the service and edge rollup watermarks to the current max ingested timestamp. Existing history is not aggregated into those analytical rollups. Completed-batch trace caches still rebuild all retained history in bounded passes; uncached files remain immediately queryable. Useful for large pre-seeded datasets (benchmarks, restores); off in normal operation. +# Default: false +# FANOUT_ROLLUP_SKIP_TO_LATEST= + +# FANOUT_SELF_SIGNUP; boolean +# Allows verified email addresses to create viewer accounts; requires local auth and SMTP. +# Default: false +# FANOUT_SELF_SIGNUP= + +# FANOUT_SESSION_ABSOLUTE_TTL; duration +# Limits total session lifetime; it must be at least FANOUT_SESSION_IDLE_TTL. +# Default: 168h +# FANOUT_SESSION_ABSOLUTE_TTL= + +# FANOUT_SESSION_IDLE_TTL; duration +# Expires inactive sessions; use a Go duration of at least 5m. +# Default: 12h +# FANOUT_SESSION_IDLE_TTL= + +# FANOUT_SMTP_FROM; string +# Sets the email sender address; required with SMTP. +# Default: (empty) +# FANOUT_SMTP_FROM= + +# FANOUT_SMTP_HOST; string +# Selects the mail server. Host, username, password, and from must all be set to enable SMTP. +# Default: (empty) +# FANOUT_SMTP_HOST= + +# FANOUT_SMTP_PASSWORD; string; secret +# Authenticates to the mail server; required with SMTP. +# Default: (empty) +# FANOUT_SMTP_PASSWORD= +# Alternatively set FANOUT_SMTP_PASSWORD_FILE to a secret file path; do not set both. + +# FANOUT_SMTP_PORT; integer +# Sets the mail server's positive TCP port. +# Default: 587 +# FANOUT_SMTP_PORT= + +# FANOUT_SMTP_USERNAME; string +# Authenticates to the mail server; required with SMTP. +# Default: (empty) +# FANOUT_SMTP_USERNAME= + +# FANOUT_TLS_CERT_FILE; string +# Selects the TLS certificate file; requires FANOUT_TLS_KEY_FILE. +# Default: (empty) +# FANOUT_TLS_CERT_FILE= + +# FANOUT_TLS_KEY_FILE; string +# Selects the TLS private key file; requires FANOUT_TLS_CERT_FILE. +# Default: (empty) +# FANOUT_TLS_KEY_FILE= + +# FANOUT_TRUSTED_PROXY_CIDRS; string +# Lists trusted proxy networks, separated by commas; empty trusts no forwarded client addresses. +# Default: (empty) +# FANOUT_TRUSTED_PROXY_CIDRS= + +# OPENAI_API_KEY; string; secret +# Required when FANOUT_AI_PROVIDER is openai. +# Default: (empty) +# OPENAI_API_KEY= +# Alternatively set OPENAI_API_KEY_FILE to a secret file path; do not set both. + +# OPENAI_BASE_URL; string +# Defaults to https://api.openai.com/v1. Fanout appends /responses, preserving the API version and any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. +# Default: (empty) +# OPENAI_BASE_URL= diff --git a/.gitignore b/.gitignore index cd672b0d..1950c6f5 100644 --- a/.gitignore +++ b/.gitignore @@ -39,11 +39,11 @@ coverage.out coverage.html cover.out -# Local environment scratch files. Fanout runtime configuration uses explicit -# YAML documents and process environment; it does not load dotenv files. -.env -.env.* -!.env.example +# Personal environment files and operator secret profiles remain private. +.env* +!/.env.example +/secrets/ +/.sops.yaml # Root-scoped for the same reason as testdata/: a bare `*.png` would silently # swallow committed image assets anywhere in the tree. diff --git a/AGENTS.md b/AGENTS.md index 1a77df95..ac4ba3fe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,39 @@ # Agent instructions +## Configuration conventions + +- Configuration is environment-only. `internal/config/config.go` is the + authoritative schema: built-in defaults, then nonempty process environment. + Custom names use `FANOUT_` plus a clear name, adding the area only when it + clarifies; vendor credentials and endpoints keep `ANTHROPIC_API_KEY`, + `ANTHROPIC_BASE_URL`, `OPENAI_API_KEY`, and `OPENAI_BASE_URL`. + Provider and model selection remain `FANOUT_AI_PROVIDER` and `FANOUT_AI_MODEL`. + An empty provider disables chat; exactly `anthropic` or `openai` requires + that provider's key. Ignore AI credentials and endpoints when chat is disabled; + validate only the selected provider's endpoint. +- Every `secret:"true"` field accepts `_FILE`. Both forms nonempty fail; + secret files lose exactly one trailing LF or CRLF and must remain nonempty. + Errors name variables and paths, never secret contents. No file-mode check. +- `group` tags serve docgen only, using ai, alerts, auth, ingest, mcp, metrics, + server, smtp, and storage. Run `just docs-generate` after schema changes and + verify `just docs-generate-check`. The generator owns `.env.example` and the + site reference; every template value is empty, with defaults and file forms + in comments. Secret defaults must never appear in public artifacts. +- The binary does not load dotenv. `just run` uses Node's native `parseEnv` for + optional ignored `./.env`, then inherited environment values win. Personal + development credentials may live there. Never shell-source dotenv. Only + Fanout's namespace rejects unknown names; tolerate platform service variables + and ignore unrelated names in shared vendor namespaces. +- Native listener default is `127.0.0.1:7520`. The image supplies + `FANOUT_ADDR=:7520` and `FANOUT_DATA_DIR=/var/lib/fanout/data`. +- Repository-managed secret profiles adopt SOPS authority through a verified + migration, with key recovery in 1Password. Existing vault/platform delivery + remains authoritative until all consumers and recovery are verified. Follow + `docs/operations.md` for delivery; never commit credential profiles in plaintext. +- This public product repository ships no operator secrets or recipients. Keep + operator `secrets/`, personal environment files, and private migration sources + ignored and outside Docker inputs; select private SOPS profiles explicitly. + ## Public repository - Keep credentials, private telemetry, local workstation paths, and private AI diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 74f4b3fd..e8bbb304 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,8 +14,15 @@ You need Go 1.27.2 and a C/C++ toolchain on native Linux or macOS (amd64 or arm6 [Bun](https://bun.sh), [just](https://just.systems), [golangci-lint](https://golangci-lint.run/), and [Lefthook](https://github.com/evilmartians/lefthook). Running in local auth mode -also requires a 32-character authentication code secret; SMTP and an AI key are -optional — see [README](README.md#requirements). +also requires a 32-character authentication code secret. SMTP and AI chat are +optional; selecting an AI provider requires its key — see [README](README.md#requirements). + +Copy `.env.example` to ignored `.env` for personal development settings and +credentials. Every entry is commented out; uncomment only values you want to +supply. Copying the template sets nothing. After building, `just run` loads it with Node's native dotenv +parser; inherited environment values win. The binary validates the settings. +Keep this file out of Git and Docker inputs. See +[runtime secret delivery](docs/operations.md#runtime-secret-delivery). The DuckDB wrapper downloads the pinned, checksum-verified engine from [Fanout's dependency artifact mirror](https://github.com/labstack/fanout/releases/tag/duckdb-v2.0.0-alpha43763) @@ -83,10 +90,18 @@ PNG in the same change. The recipe fails when the font is missing rather than shipping a card in a substitute face, which typst would otherwise do without complaint. -**Runtime configuration is strict.** [`fanout.example.yaml`](fanout.example.yaml) -is the complete schema. New settings need a YAML key, a `FANOUT_` environment -name, a test, and matching documentation; unknown names deliberately fail -startup. +**Runtime configuration is strict.** `internal/config/config.go` owns the +schema. New settings need an environment name, a documentation `group` tag, +a purpose comment, a test, and matching documentation. Custom settings use +`FANOUT_` plus a clear name, adding the area only when it clarifies; vendor +credentials and endpoints keep the vendor's documented names. Unknown owned +names fail startup, while unrelated shared-vendor/tool variables are ignored. +Secrets use `secret:"true"` and automatically accept an `_FILE` form; do not +set both forms nonempty. Run `just docs-generate` to regenerate the settings +reference and `.env.example`; `just docs-generate-check` verifies them. The +template leaves all values empty and documents defaults in comments. +Personal environment files and operator profiles stay ignored and outside +Docker. Never shell-source dotenv. **SQLite migrations own the control schema.** Write forward SQL changes in `internal/db/migrations` using Goose annotations. `just db-migrate-create NAME` diff --git a/Dockerfile b/Dockerfile index 34614b4b..3713f60a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,6 @@ LABEL org.opencontainers.image.title="Fanout" \ org.opencontainers.image.version="${VERSION}" COPY --from=build /app/fanout /usr/local/bin/fanout -COPY fanout.docker.yaml /etc/fanout/fanout.yaml COPY LICENSE NOTICE THIRD_PARTY_NOTICES TRADEMARK.md /usr/share/licenses/fanout/ COPY --from=build --chown=nonroot:nonroot /runtime/var/lib/fanout /var/lib/fanout @@ -55,6 +54,8 @@ COPY --from=build --chown=nonroot:nonroot /runtime/var/lib/fanout /var/lib/fanou USER nonroot:nonroot WORKDIR /var/lib/fanout +ENV FANOUT_ADDR=:7520 FANOUT_DATA_DIR=/var/lib/fanout/data + EXPOSE 7520 # Startup does DuckDB catalog attachment and maintenance, so the grace period @@ -64,4 +65,3 @@ HEALTHCHECK --interval=30s --timeout=3s --start-period=40s --retries=3 \ CMD ["fanout", "healthcheck"] ENTRYPOINT ["fanout"] -CMD ["--config", "/etc/fanout/fanout.yaml"] diff --git a/README.md b/README.md index 7ae8a847..9bfefa7b 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ Fanout. - A 32-character **authentication code secret** SMTP and an AI provider are optional. Without SMTP, an operator can mint a -short-lived login link from the local Fanout binary. Without an AI key, ingest, +short-lived login link from the local Fanout binary. With no AI provider selected, ingest, dashboards, traces, logs, metrics, and MCP continue to work; only investigation chat and AI-assisted controls are hidden. @@ -155,17 +155,18 @@ hostname reachable from that Collector. The equivalent OTLP/gRPC endpoint is `localhost:7520` (plaintext locally). Behind a TLS proxy, both transports use the public origin; gRPC requires -HTTP/2 to Fanout. See the [single-port migration notes](docs/operations.md#single-port-migration). +HTTP/2 to Fanout. For later sign-in without SMTP, mint a 15-minute, single-use link against the running container's control database: ```sh -docker exec fanout fanout --config /etc/fanout/fanout.yaml \ - login-link admin@example.com +docker exec fanout fanout login-link admin@example.com ``` -Add `FANOUT_AI_API_KEY` to enable chat. Configure all four SMTP settings +Select `FANOUT_AI_PROVIDER=anthropic` and inject `ANTHROPIC_API_KEY`, or select +`FANOUT_AI_PROVIDER=openai` and inject `OPENAI_API_KEY`, to enable chat. The +selected provider requires its key at startup. Configure all four SMTP settings (`FANOUT_SMTP_HOST`, `FANOUT_SMTP_USERNAME`, `FANOUT_SMTP_PASSWORD`, and `FANOUT_SMTP_FROM`) to enable email-code login. @@ -173,20 +174,10 @@ The distroless image runs unprivileged as UID 65532. A bind-mounted host directo `/var/lib/fanout/data` must be writable by that user; a named volume, as above, needs no such handling. -The image selects `/etc/fanout/fanout.yaml` by default. That file contains only -the container listener and data-directory defaults; it does not contain -credentials. Start a container-specific document from that file so it retains -the shared listener and persistent data-directory settings: - -```sh -cp fanout.docker.yaml fanout.yaml -# Add the remaining settings, then mount it over the image document: -docker run -v ./fanout.yaml:/etc/fanout/fanout.yaml:ro \ - labstack/fanout:latest -``` - -All surfaces use `server.addr` (`:7520` by default). Publish only that -port; OTLP/HTTP and OTLP/gRPC use the same address as the application. +The image sets `FANOUT_ADDR=:7520` and +`FANOUT_DATA_DIR=/var/lib/fanout/data`. Override settings with deployment +environment values or mounted secret files. Publish port `7520`; the UI, API, +MCP, OTLP/HTTP, and OTLP/gRPC share this listener. ### From source @@ -205,8 +196,9 @@ export FANOUT_AUTH_CODE_SECRET=$(openssl rand -hex 32) # must be 32+ characters ./bin/fanout ``` -Optionally set `FANOUT_AI_API_KEY` for chat and the SMTP variables shown above -for email-code login. Without SMTP, run `./bin/fanout login-link +Optionally select `FANOUT_AI_PROVIDER=anthropic` with `ANTHROPIC_API_KEY`, or +`FANOUT_AI_PROVIDER=openai` with `OPENAI_API_KEY`, for chat. Set the SMTP variables shown above for +email-code login. Without SMTP, run `./bin/fanout login-link admin@example.com` from the same configuration and data directory. Fanout serves the UI, API, MCP, OTLP/gRPC and OTLP/HTTP on @@ -220,43 +212,60 @@ capacity tests. ## Configuration -Configuration is resolved once at startup and validated before Fanout opens -data files or listeners. Sources apply in this order: built-in defaults, an -optional YAML document selected with `--config`, then `FANOUT_` environment -variables. Fanout does not search for configuration or load `.env` files. - -[`fanout.example.yaml`](fanout.example.yaml) is the complete commented schema: - -```sh -cp fanout.example.yaml fanout.yaml -./bin/fanout --config ./fanout.yaml -``` - -Environment variables override the corresponding YAML values and are useful -for container injection and secrets. An empty environment value means "no -override"; use YAML for an explicit empty string. Unknown YAML keys and unknown -`FANOUT_` variables are startup errors, except for the service-discovery names -Kubernetes and Docker link-style networking inject for a Service named -`fanout`. All environment variables outside the `FANOUT_` namespace are -ignored. - -YAML null values are rejected. Boolean values must use YAML 1.2 `true` or -`false` (not `yes`, `no`, `on`, or `off`). If a YAML document contains a -credential, Fanout requires that the file not be accessible by group or others -(for example, mode `0600`). The definitions live in -[`internal/config/config.go`](internal/config/config.go); the settings most -operators touch are: - -| YAML key | Environment override | Default | Purpose | -| --- | --- | --- | --- | -| `server.addr` | `FANOUT_ADDR` | `:7520` | UI, API, MCP, and both OTLP transports | -| `storage.data_dir` | `FANOUT_DATA_DIR` | `./data` | Parquet, query state, and control SQLite | -| `auth.mode` | `FANOUT_AUTH_MODE` | `local` | `local` (login link or SMTP) or `oidc` | -| `auth.code_secret` | `FANOUT_AUTH_CODE_SECRET` | — | Required in local mode, 32+ characters | -| `ai.provider` | `FANOUT_AI_PROVIDER` | `anthropic` | `anthropic` or `openai` | -| `ai.api_key` | `FANOUT_AI_API_KEY` | — | Enables AI investigation chat | -| `storage.retention_days` | `FANOUT_RETENTION_DAYS` | `30` | Telemetry retention window | -| `mcp.enabled` | `FANOUT_MCP_ENABLED` | `true` | Serve the MCP endpoint at `/mcp` | +Configuration is resolved once at startup from built-in defaults followed by +nonempty process environment values, and validated before data files or +listeners open. Every change requires a restart. Native installs bind +`127.0.0.1:7520` by default; set `FANOUT_ADDR=:7520` for other hosts to reach it. + +Custom settings use `FANOUT_` plus a clear name, adding the area only when it +clarifies. Vendor credentials and endpoints keep their documented names: +`ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, `OPENAI_API_KEY`, and +`OPENAI_BASE_URL`. Unknown `FANOUT_` variables fail startup, except for +platform-injected service-discovery names. Unrelated vendor/tool variables are +ignored. Empty values mean unset. Bools, integers, and Go durations are parsed +as their declared types. + +[`.env.example`](.env.example) is generated from +[`internal/config/config.go`](internal/config/config.go), with every value +empty and every entry commented out. Types, defaults, and secret file forms +are documented in comments. Copying it sets nothing and preserves image ENV. +Copy it to ignored `.env` and uncomment personal settings and development +credentials as needed. After building, `just run` uses Node's +`parseEnv` to load optional `./.env`, then lets inherited environment values +win, including empty values. The binary itself reads only process environment; +production launchers and deployments supply their own values. + +Every secret also accepts a file path through `_FILE`: +`FANOUT_AUTH_CODE_SECRET_FILE`, `FANOUT_OIDC_CLIENT_SECRET_FILE`, +`FANOUT_SMTP_PASSWORD_FILE`, `FANOUT_METRICS_TOKEN_FILE`, +`ANTHROPIC_API_KEY_FILE`, and `OPENAI_API_KEY_FILE`. Set either the value or +its file path. Setting both nonempty fails startup. Fanout strips exactly one +trailing LF or CRLF from a secret file and rejects an empty result. + +| Environment variable | Default | Purpose | +| --- | --- | --- | +| `FANOUT_ADDR` | `127.0.0.1:7520` | UI, API, MCP, and both OTLP transports | +| `FANOUT_DATA_DIR` | `./data` | Parquet, query state, and control SQLite | +| `FANOUT_AUTH_MODE` | `local` | Login link or SMTP, or `oidc` | +| `FANOUT_AUTH_CODE_SECRET` | — | Required in local mode, 32+ characters | +| `FANOUT_AI_PROVIDER` | — | Empty disables chat; `anthropic` or `openai` enables it | +| `ANTHROPIC_API_KEY` | — | Required for Anthropic chat | +| `OPENAI_API_KEY` | — | Required for OpenAI chat | +| `FANOUT_RETENTION_DAYS` | `30` | Telemetry retention window | +| `FANOUT_MCP_ENABLED` | `true` | Serve the MCP endpoint at `/mcp` | + +Chat requires an explicit provider and its credential. Inherited keys never +enable it. Only the selected provider's endpoint is validated and used. +`FANOUT_AI_MODEL` selects the model. `ANTHROPIC_BASE_URL` defaults to +`https://api.anthropic.com`; Fanout appends `/v1/messages`. `OPENAI_BASE_URL` +defaults to `https://api.openai.com/v1`; Fanout appends `/responses`. +Both preserve gateway prefixes and accept a trailing slash. The selected URL +must use HTTP(S) without credentials, a query, or a fragment. + +See [runtime secret delivery](docs/operations.md#runtime-secret-delivery) for +deployment examples. This public repository excludes operator secret profiles +and private recipient policy. Run `just docs-generate` after changing the +schema; `just docs-generate-check` verifies `.env.example` and the reference. ### Advanced DuckDB sizing @@ -270,14 +279,14 @@ These variables are escape hatches for measured, unusual workloads: | Variable | Automatic behavior | When to override | | --- | --- | --- | -| `storage.duckdb.memory` / `FANOUT_DUCKDB_MEMORY` | 60% of the container or host memory available to Fanout | A measured co-tenant workload needs a different Go/DuckDB split | -| `storage.duckdb.max_connections` / `FANOUT_DUCKDB_MAX_CONNECTIONS` | Available Go CPUs, bounded to 2–16 connections | Query concurrency has been benchmarked for this machine | -| `storage.duckdb.threads` / `FANOUT_DUCKDB_THREADS` | DuckDB chooses its own query worker count | Query-heavy work must leave specific cores free for ingest | +| `FANOUT_DUCKDB_MEMORY` | 60% of the container or host memory available to Fanout | A measured co-tenant workload needs a different Go/DuckDB split | +| `FANOUT_DUCKDB_MAX_CONNECTIONS` | Available Go CPUs, bounded to 2–16 connections | Query concurrency has been benchmarked for this machine | +| `FANOUT_DUCKDB_THREADS` | DuckDB chooses its own query worker count | Query-heavy work must leave specific cores free for ingest | An explicit value always wins. Startup logs and `/readyz` report the detected host and cgroup limits, the selected source, and whether detection was incomplete. If Fanout cannot conclusively inspect a container limit, it warns; -set `storage.duckdb.memory` or `FANOUT_DUCKDB_MEMORY` for a guaranteed bound. +set `FANOUT_DUCKDB_MEMORY` for a guaranteed bound. For TLS and reverse proxies, health checks, backups and restores, upgrades, retention, and recovery, see the [operator runbook](docs/operations.md). diff --git a/SECURITY.md b/SECURITY.md index d3a1bfd4..7eb0bcfa 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -37,9 +37,8 @@ release is supported. There are no backports to earlier tags. ## Operating Fanout safely -Fanout serves plaintext by default. Configure `server.tls.cert_file` and -`server.tls.key_file` in YAML (or `FANOUT_TLS_CERT_FILE` and -`FANOUT_TLS_KEY_FILE`), or place it behind a reverse proxy that terminates TLS. +Fanout serves plaintext by default. Configure `FANOUT_TLS_CERT_FILE` and +`FANOUT_TLS_KEY_FILE`, or place it behind a reverse proxy that terminates TLS. Browser and API reads require an authenticated account, and every OTLP request requires the separately managed ingest token. Demo instances use the same credential paths as production instances. diff --git a/THIRD_PARTY_NOTICES b/THIRD_PARTY_NOTICES index 55c75338..0230def8 100644 --- a/THIRD_PARTY_NOTICES +++ b/THIRD_PARTY_NOTICES @@ -23,7 +23,6 @@ COMPONENT INVENTORY - Go: github.com/duckdb/duckdb-go-bindings v0.10506.0 - Go: github.com/duckdb/duckdb-go/v2 v2.10506.0 - Go: github.com/dustin/go-humanize v1.1.0 -- Go: github.com/fsnotify/fsnotify v1.10.1 - Go: github.com/go-jose/go-jose/v4 v4.1.5 - Go: github.com/go-viper/mapstructure/v2 v2.5.0 - Go: github.com/google/jsonschema-go v0.4.3 @@ -31,16 +30,9 @@ COMPONENT INVENTORY - Go: github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 - Go: github.com/klauspost/compress v1.20.1 - Go: github.com/klauspost/cpuid/v2 v2.4.0 -- Go: github.com/knadh/koanf/maps v0.1.3 -- Go: github.com/knadh/koanf/parsers/yaml v1.1.1 -- Go: github.com/knadh/koanf/providers/confmap v1.0.1 -- Go: github.com/knadh/koanf/providers/file v1.2.1 -- Go: github.com/knadh/koanf/v2 v2.3.7 - Go: github.com/labstack/echo/v5 v5.4.0 - Go: github.com/mattn/go-isatty v0.0.24 - Go: github.com/mfridman/interpolate v0.0.2 -- Go: github.com/mitchellh/copystructure v1.2.0 -- Go: github.com/mitchellh/reflectwalk v1.0.2 - Go: github.com/modelcontextprotocol/go-sdk v1.8.0 - Go: github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 - Go: github.com/ncruces/go-strftime v1.1.0 @@ -1177,39 +1169,8 @@ SOFTWARE. ---- Applies to ------------------------------------------------------------- -- Go: github.com/fsnotify/fsnotify v1.10.1 / LICENSE ----------------------------------------------------------------------------- - -Copyright © 2012 The Go Authors. All rights reserved. -Copyright © fsnotify Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. -* Neither the name of Google Inc. nor the names of its contributors may be used - to endorse or promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --- Applies to ------------------------------------------------------------- - Go: github.com/go-viper/mapstructure/v2 v2.5.0 / LICENSE -- Go: github.com/mitchellh/reflectwalk v1.0.2 / LICENSE ---------------------------------------------------------------------------- The MIT License (MIT) @@ -1659,36 +1620,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ---- Applies to ------------------------------------------------------------- -- Go: github.com/knadh/koanf/maps v0.1.3 / LICENSE -- Go: github.com/knadh/koanf/parsers/yaml v1.1.1 / LICENSE -- Go: github.com/knadh/koanf/providers/confmap v1.0.1 / LICENSE -- Go: github.com/knadh/koanf/providers/file v1.2.1 / LICENSE -- Go: github.com/knadh/koanf/v2 v2.3.7 / LICENSE ----------------------------------------------------------------------------- - -The MIT License - -Copyright (c) 2019, Kailash Nadh. https://github.com/knadh - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - --- Applies to ------------------------------------------------------------- - Go: github.com/labstack/echo/v5 v5.4.0 / LICENSE ---------------------------------------------------------------------------- @@ -1758,32 +1689,6 @@ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ---- Applies to ------------------------------------------------------------- -- Go: github.com/mitchellh/copystructure v1.2.0 / LICENSE ----------------------------------------------------------------------------- - -The MIT License (MIT) - -Copyright (c) 2014 Mitchell Hashimoto - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - --- Applies to ------------------------------------------------------------- - Go: github.com/modelcontextprotocol/go-sdk v1.8.0 / LICENSE ---------------------------------------------------------------------------- diff --git a/cmd/fanout-docgen/main.go b/cmd/fanout-docgen/main.go index 021a2ddb..5ece1e13 100644 --- a/cmd/fanout-docgen/main.go +++ b/cmd/fanout-docgen/main.go @@ -52,18 +52,19 @@ func main() { // list going stale is loud rather than a quietly shorter table. routeDirs = flag.String("route-dirs", "internal/api,internal/agent,cmd/fanout", "comma-separated directories whose files register HTTP routes") - outDir = flag.String("out", "site/src/content/docs/reference", "reference root to write generated pages into") - check = flag.Bool("check", false, "exit non-zero when a written page differs from the one on disk") + outDir = flag.String("out", "site/src/content/docs/reference", "reference root to write generated pages into") + envExample = flag.String("env-example", ".env.example", "path to the generated environment template") + check = flag.Bool("check", false, "exit non-zero when a written page differs from the one on disk") ) flag.Parse() - if err := run(*source, *alerts, *routeDirs, *outDir, *check); err != nil { + if err := run(*source, *alerts, *routeDirs, *outDir, *envExample, *check); err != nil { fmt.Fprintf(os.Stderr, "fanout-docgen: %v\n", err) os.Exit(1) } } -func run(source, alertSource, routeDirs, outDir string, check bool) error { +func run(source, alertSource, routeDirs, outDir, envExample string, check bool) error { fields, err := collect(source) if err != nil { return err @@ -117,9 +118,12 @@ func run(source, alertSource, routeDirs, outDir string, check bool) error { } pages["mcp-tools.mdx"] = toolsPage - var stale []string + outputs := map[string][]byte{envExample: renderEnvironment(fields)} for name, body := range pages { - path := filepath.Join(outDir, name) + outputs[filepath.Join(outDir, name)] = body + } + var stale []string + for path, body := range outputs { if check { existing, err := os.ReadFile(path) if err != nil { @@ -272,7 +276,8 @@ func renderAlertEnv(vars []field) []byte { // field is one setting, assembled from both views of the same declaration. type field struct { GoName string - Key string // koanf key, e.g. "storage.duckdb.memory" + Key string // alert expression variable + Group string // documentation area, e.g. "storage" Env string // e.g. "FANOUT_DUCKDB_MEMORY" Type string Default string @@ -281,8 +286,7 @@ type field struct { } func (f field) group() string { - key, _, _ := strings.Cut(f.Key, ".") - return key + return f.Group } // collect reflects over config.Config for the contract, parses source for the @@ -302,11 +306,11 @@ func collect(source string) ([]field, error) { if !sf.IsExported() { continue } - key := sf.Tag.Get("koanf") + group := sf.Tag.Get("group") env := sf.Tag.Get("env") - if key == "" || env == "" { + if group == "" || env == "" { return nil, fmt.Errorf( - "config.Config field %s is exported but carries no koanf/env tag; "+ + "config.Config field %s is exported but carries no group/env tag; "+ "either tag it so it reaches the reference, or unexport it", sf.Name, ) @@ -314,7 +318,7 @@ func collect(source string) ([]field, error) { seen[sf.Name] = true fields = append(fields, field{ GoName: sf.Name, - Key: key, + Group: group, Env: env, Type: typeName(sf.Type), Default: sf.Tag.Get("default"), @@ -336,7 +340,7 @@ func collect(source string) ([]field, error) { } } - sort.Slice(fields, func(i, j int) bool { return fields[i].Key < fields[j].Key }) + sort.Slice(fields, func(i, j int) bool { return fields[i].Env < fields[j].Env }) return fields, nil } @@ -433,7 +437,7 @@ type groupPage struct { ReadWhen []string } -// Registering a group is deliberate. A new koanf prefix appearing in the type +// Registering a group is deliberate. A new group appearing in the type // is a new section of the product, and it should be named by a person rather // than published under a bare key — so an unregistered group fails the run // instead of shipping a page called "metrics" with no explanation on it. @@ -483,7 +487,7 @@ var groups = map[string]groupPage{ "ai": { Title: "AI provider settings", Description: "The provider, key, model and base URL backing the chat investigator.", - Summary: "Settings for the model behind the chat investigator. Without a key the agent is disabled.", + Summary: "Chat is disabled unless a provider is selected; its API key is then required.", ReadWhen: []string{ "The chat investigator is disabled and you want it on.", "You are pointing Fanout at a self-hosted or proxied model endpoint.", @@ -550,23 +554,33 @@ func renderPage(name string, page groupPage, fields []field) []byte { b.WriteString("{/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */}\n\n") - b.WriteString("Every setting below is read from the environment variable in the second\n") - b.WriteString("column, or from the same key in a YAML configuration file. An unrecognised\n") - b.WriteString("`FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces\n") - b.WriteString("as a refusal to start rather than as a default nobody chose.\n\n") + b.WriteString("Every setting below is read once at startup and requires a restart to change.\n") + b.WriteString("Built-in defaults are overridden by nonempty environment values.\n") + b.WriteString("Unknown `FANOUT_` names fail startup; platform-injected service names are tolerated.\n") + b.WriteString("Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies.\n") + b.WriteString("Vendor credentials and endpoints keep the vendor's documented names.\n\n") + if name == "ai" { + b.WriteString("Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`,\n") + b.WriteString("`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are\n") + b.WriteString("ignored; only the explicitly selected AI provider's key and endpoint are used.\n\n") + } - b.WriteString("| Setting | Environment variable | Type | Default |\n") - b.WriteString("|---|---|---|---|\n") + b.WriteString("| Environment variable | Type | Default |\n") + b.WriteString("|---|---|---|\n") for _, f := range fields { typ := f.Type if f.Secret { typ += " · secret" } def := "—" - if f.Default != "" { + if f.Default != "" && !f.Secret { def = "`" + f.Default + "`" } - fmt.Fprintf(&b, "| `%s` | `%s` | %s | %s |\n", f.Key, f.Env, typ, def) + names := "`" + f.Env + "`" + if f.Secret { + names += " or `" + f.Env + "_FILE`" + } + fmt.Fprintf(&b, "| %s | %s | %s |\n", names, typ, def) } var noted []field @@ -578,21 +592,56 @@ func renderPage(name string, page groupPage, fields []field) []byte { if len(noted) > 0 { b.WriteString("\n## Notes\n") for _, f := range noted { - fmt.Fprintf(&b, "\n### `%s`\n\n%s\n", f.Key, f.Doc) + fmt.Fprintf(&b, "\n### `%s`\n\n%s\n", f.Env, f.Doc) } } if hasSecret(fields) { b.WriteString("\n:::caution[Secrets]\n") b.WriteString("Settings marked `secret` are redacted from Fanout's startup configuration\n") - b.WriteString("log. They are still plain environment variables: anything that can read the\n") - b.WriteString("process environment can read them.\n") + b.WriteString("log. Supply either the value or its `_FILE` path, never both. File contents\n") + b.WriteString("lose exactly one trailing newline (LF or CRLF); empty secrets fail startup.\n") b.WriteString(":::\n") } return []byte(b.String()) } +// renderEnvironment shares the exact reflected schema used by the site pages. +// Entries are commented out; defaults and secret file forms stay in comments. +func renderEnvironment(fields []field) []byte { + var b strings.Builder + b.WriteString("# Generated by cmd/fanout-docgen from internal/config/config.go.\n") + b.WriteString("# Regenerate with just docs-generate; verify with just docs-generate-check.\n") + b.WriteString("# Fanout does not load dotenv files. Inject through your launcher or deployment.\n") + b.WriteString("# Precedence: built-in defaults < nonempty environment values.\n") + b.WriteString("# All settings require restart. Empty environment values mean no override.\n") + b.WriteString("# Every entry is commented out so copying this template sets nothing, preserving image ENV.\n") + b.WriteString("# Uncomment only settings you want to supply; use runtime secret injection for secrets.\n") + b.WriteString("# Local authentication requires FANOUT_AUTH_CODE_SECRET (32+ characters).\n") + b.WriteString("# Chat is disabled unless FANOUT_AI_PROVIDER selects anthropic or openai; its key is required.\n") + for _, f := range fields { + fmt.Fprintf(&b, "\n# %s; %s", f.Env, f.Type) + if f.Secret { + b.WriteString("; secret") + } + b.WriteByte('\n') + if f.Doc != "" { + fmt.Fprintf(&b, "# %s\n", f.Doc) + } + defaultValue := f.Default + if f.Secret || defaultValue == "" { + defaultValue = "(empty)" + } + fmt.Fprintf(&b, "# Default: %s\n", defaultValue) + fmt.Fprintf(&b, "# %s=\n", f.Env) + if f.Secret { + fmt.Fprintf(&b, "# Alternatively set %s_FILE to a secret file path; do not set both.\n", f.Env) + } + } + return []byte(b.String()) +} + func hasSecret(fields []field) bool { for _, f := range fields { if f.Secret { diff --git a/cmd/fanout-docgen/main_test.go b/cmd/fanout-docgen/main_test.go index b53429e6..a3430930 100644 --- a/cmd/fanout-docgen/main_test.go +++ b/cmd/fanout-docgen/main_test.go @@ -5,6 +5,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/labstack/fanout/internal/config" ) const configSource = "../../internal/config/config.go" @@ -22,15 +24,73 @@ func TestCollectCrossChecksReflectionAgainstSource(t *testing.T) { } for _, f := range fields { - if f.Key == "" || f.Env == "" { - t.Errorf("%s: koanf and env tags are both required", f.GoName) - } - if !strings.HasPrefix(f.Env, "FANOUT_") { - t.Errorf("%s: environment variable %q lacks the FANOUT_ prefix the loader requires", f.GoName, f.Env) + if f.Group == "" || f.Env == "" { + t.Errorf("%s: group and env tags are both required", f.GoName) } if f.Type == "" { t.Errorf("%s: no rendered type", f.GoName) } + if f.Doc == "" { + t.Errorf("%s: setting purpose must be documented in the authoritative schema", f.GoName) + } + } +} + +func TestEnvironmentTemplateMatchesSchema(t *testing.T) { + fields, err := collect(configSource) + if err != nil { + t.Fatal(err) + } + body := renderEnvironment(fields) + written, err := os.ReadFile("../../.env.example") + if err != nil { + t.Fatal(err) + } + if string(written) != string(body) { + t.Error(".env.example is stale; run just docs-generate") + } + var env []string + seen := map[string]string{} + for _, line := range strings.Split(string(body), "\n") { + if line == "" { + continue + } + if !strings.HasPrefix(line, "#") { + t.Fatalf("template must contain only comments: %q", line) + } + entry := strings.TrimPrefix(line, "# ") + if !strings.HasSuffix(entry, "=") { + continue + } + name, value, ok := strings.Cut(entry, "=") + if _, duplicate := seen[name]; !ok || duplicate { + t.Fatal("invalid or duplicate template entry") + } + seen[name] = value + } + if len(seen) != len(fields) { + t.Fatalf("template has %d settings, schema has %d", len(seen), len(fields)) + } + for _, f := range fields { + if f.Secret && !strings.Contains(string(body), f.Env+"_FILE") { + t.Errorf("missing file form for %s", f.Env) + } + value, exists := seen[f.Env] + if !exists || value != "" { + t.Errorf("template must leave %s empty so it preserves defaults", f.Env) + } + if f.Default != "" && !f.Secret && !strings.Contains(string(body), "# Default: "+f.Default+"\n# "+f.Env+"=\n") { + t.Errorf("template must document the default for %s", f.Env) + } + } + // A copied template contributes no overrides to the image environment. + env = append(env, "FANOUT_ADDR=:7520", "FANOUT_DATA_DIR=/var/lib/fanout/data", "FANOUT_AUTH_CODE_SECRET="+strings.Repeat("x", 32)) + cfg, err := config.Load(config.LoadOptions{Environ: env}) + if err != nil { + t.Fatalf("template rejected by the runtime loader: %v", err) + } + if cfg.Addr != ":7520" || cfg.DataDir != "/var/lib/fanout/data" { + t.Fatal("template must preserve image environment values") } } @@ -53,7 +113,7 @@ func TestEverySettingGroupIsRegistered(t *testing.T) { t.Errorf( "setting %q is in group %q, which has no entry in the groups registry; "+ "add a title and description for it", - f.Key, f.group(), + f.Env, f.group(), ) } } @@ -74,13 +134,16 @@ func TestRenderProducesOnePageWithFrontmatterPerGroup(t *testing.T) { for name, body := range pages { text := string(body) + if strings.Contains(text, "Fanout also recognizes") != (name == "settings/ai.mdx") { + t.Errorf("%s: vendor-variable paragraph belongs only on the AI settings page", name) + } if !strings.HasPrefix(text, "---\n") { t.Errorf("%s: does not open with frontmatter", name) } if !strings.Contains(text, "generated: true") { t.Errorf("%s: missing the generated marker, so the site cannot tell it apart from an authored page", name) } - if !strings.Contains(text, "| Setting | Environment variable | Type | Default |") { + if !strings.Contains(text, "| Environment variable | Type | Default |") { t.Errorf("%s: missing the settings table", name) } // Every page is indexed in llms.txt, which fails the site build when a diff --git a/cmd/fanout/main.go b/cmd/fanout/main.go index ed769cbc..f83950be 100644 --- a/cmd/fanout/main.go +++ b/cmd/fanout/main.go @@ -2,6 +2,9 @@ package main import ( "context" + "crypto/tls" + "crypto/x509" + "encoding/pem" "errors" "flag" "fmt" @@ -10,6 +13,7 @@ import ( "net" "net/http" "net/http/pprof" + "net/netip" "net/url" "os" "os/signal" @@ -58,7 +62,7 @@ type repairCommand struct { } func main() { - configPath, showVersion, loginEmail, healthURL, repair, err := parseCommandLine(os.Args[1:], os.Stderr) + showVersion, loginEmail, healthURL, derivedProbe, repair, err := parseCommandLine(os.Args[1:], os.Stderr) if errors.Is(err, flag.ErrHelp) { return } @@ -69,8 +73,15 @@ func main() { fmt.Println(version) return } - if healthURL != "" { - if err := checkHealth(healthURL); err != nil { + if derivedProbe { + healthURL, err = healthcheckURL(os.Getenv("FANOUT_ADDR"), os.Getenv("FANOUT_TLS_CERT_FILE")) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + } + if healthURL != "" || derivedProbe { + if err := checkHealth(healthURL, derivedProbe, os.Getenv("FANOUT_TLS_CERT_FILE")); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } @@ -79,7 +90,7 @@ func main() { slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, nil))) - cfg, err := config.Load(config.LoadOptions{Path: configPath}) + cfg, err := config.Load(config.LoadOptions{}) if err != nil { slog.Error("invalid configuration", "err", err) os.Exit(1) @@ -359,7 +370,7 @@ func main() { slog.Info("browser MCP server enabled", "path", "/api/mcp", "auth", "session") } if cfg.AgentConfigured() { - provider, err := agent.NewProvider(cfg.AIProvider, cfg.AIAPIKey, cfg.AIModel, cfg.AIBaseURL) + provider, err := agent.NewProvider(cfg.AIProvider, cfg.AIAPIKey(), cfg.AIModel, cfg.AIBaseURL()) if err != nil { slog.Error("agent provider init failed", "err", err) os.Exit(1) @@ -373,7 +384,7 @@ func main() { agent.NewRuntime(provider, toolRegistry, agent.NewStore(sqlite.DB)).Register(e.Group("/api/agent", api.RequireCapability(api.RunAgent))) slog.Info("AG-UI agent enabled", "path", "/api/agent/runs", "provider", cfg.AIProvider) } else { - slog.Info("AG-UI agent disabled", "reason", "ai.api_key is not configured") + slog.Info("AG-UI agent disabled", "reason", "FANOUT_AI_PROVIDER is not selected") } // The compiled browser client is an embedded asset, not a second runtime. @@ -423,15 +434,15 @@ func main() { } } -func parseCommandLine(args []string, output io.Writer) (configPath string, showVersion bool, loginEmail, healthURL string, repair *repairCommand, err error) { +func parseCommandLine(args []string, output io.Writer) (showVersion bool, loginEmail, healthURL string, derivedProbe bool, repair *repairCommand, err error) { if len(args) == 1 && args[0] == "version" { - return "", true, "", "", nil, nil + return true, "", "", false, nil, nil } if len(args) >= 1 && len(args) <= 2 && args[0] == "healthcheck" { if len(args) == 2 { - return "", false, "", args[1], nil, nil + return false, "", args[1], false, nil, nil } - return "", false, "", "http://127.0.0.1:7520/healthz", nil, nil + return false, "", "", true, nil, nil } flags := flag.NewFlagSet("fanout", flag.ContinueOnError) @@ -439,34 +450,33 @@ func parseCommandLine(args []string, output io.Writer) (configPath string, showV flags.Usage = func() { fmt.Fprintln(output, "Usage of fanout:") fmt.Fprintln(output, " fanout [flags]") - fmt.Fprintln(output, " fanout [--config path] login-link ") - fmt.Fprintln(output, " fanout [--config path] repair verify") - fmt.Fprintln(output, " fanout [--config path] repair quarantine --batch ") + fmt.Fprintln(output, " fanout login-link ") + fmt.Fprintln(output, " fanout repair verify") + fmt.Fprintln(output, " fanout repair quarantine --batch ") fmt.Fprintln(output, " fanout healthcheck [url]") fmt.Fprintln(output, " fanout version") fmt.Fprintln(output, "Flags:") flags.PrintDefaults() } - flags.StringVar(&configPath, "config", "", "path to a Fanout YAML configuration file") flags.BoolVar(&showVersion, "version", false, "print the Fanout version") flags.BoolVar(&showVersion, "v", false, "print the Fanout version") if err := flags.Parse(args); err != nil { - return "", false, "", "", nil, err + return false, "", "", false, nil, err } if flags.NArg() == 2 && flags.Arg(0) == "login-link" { - return configPath, false, flags.Arg(1), "", nil, nil + return false, flags.Arg(1), "", false, nil, nil } if flags.NArg() > 0 && flags.Arg(0) == "repair" { repair, err := parseRepairCommand(flags.Args()[1:], output) - return configPath, false, "", "", repair, err + return false, "", "", false, repair, err } if flags.NArg() != 0 { err := fmt.Errorf("unexpected arguments: %s", strings.Join(flags.Args(), " ")) fmt.Fprintln(output, err) flags.Usage() - return "", false, "", "", nil, err + return false, "", "", false, nil, err } - return configPath, showVersion, "", "", nil, nil + return showVersion, "", "", false, nil, nil } func parseRepairCommand(args []string, output io.Writer) (*repairCommand, error) { @@ -518,8 +528,73 @@ func runRepair(root string, command repairCommand, output io.Writer) error { } } -func checkHealth(healthURL string) error { +func healthcheckURL(addr, certFile string) (string, error) { + if addr == "" { + addr = config.DefaultAddr + } + host, port, err := net.SplitHostPort(addr) + if err != nil { + return "", fmt.Errorf("FANOUT_ADDR must be a host:port address: %w", err) + } + if host == "" { + host = "127.0.0.1" + } + if ip := net.ParseIP(host); ip != nil && ip.IsUnspecified() { + host = "127.0.0.1" + } + scheme := "http" + if certFile != "" { + scheme = "https" + } + return (&url.URL{Scheme: scheme, Host: net.JoinHostPort(host, port), Path: "/healthz"}).String(), nil +} + +func checkHealth(healthURL string, derivedProbe bool, certFile string) error { client := &http.Client{Timeout: 3 * time.Second} + if derivedProbe { + // Derived probes bypass proxies. HTTPS IP literals trust only the configured + // certificate and verify its first DNS or IP SAN instead of the bind IP. + // Hostnames verify normally; explicit URLs retain all default behaviour. + probe, err := url.Parse(healthURL) + if err != nil { + return fmt.Errorf("healthcheck: %w", err) + } + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + if _, err := netip.ParseAddr(probe.Hostname()); probe.Scheme == "https" && err == nil { + data, err := os.ReadFile(certFile) + if err != nil { + return fmt.Errorf("healthcheck: read certificate %q: %w", certFile, err) + } + var block *pem.Block + for len(data) > 0 { + block, data = pem.Decode(data) + if block == nil || block.Type == "CERTIFICATE" { + break + } + } + if block == nil || block.Type != "CERTIFICATE" { + return fmt.Errorf("healthcheck: no certificate in %q", certFile) + } + leaf, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return fmt.Errorf("healthcheck: parse certificate %q: %w", certFile, err) + } + var serverName string + if len(leaf.DNSNames) > 0 { + serverName = leaf.DNSNames[0] + } else if len(leaf.IPAddresses) > 0 { + serverName = leaf.IPAddresses[0].String() + } else { + return fmt.Errorf("healthcheck: certificate %q has no DNS or IP SAN", certFile) + } + roots := x509.NewCertPool() + roots.AddCert(leaf) + transport.TLSClientConfig = &tls.Config{RootCAs: roots, ServerName: serverName} + } + defer transport.CloseIdleConnections() + client.Transport = transport + } response, err := client.Get(healthURL) if err != nil { return fmt.Errorf("healthcheck: %w", err) diff --git a/cmd/fanout/main_test.go b/cmd/fanout/main_test.go index 3fa98739..ce518c6f 100644 --- a/cmd/fanout/main_test.go +++ b/cmd/fanout/main_test.go @@ -3,16 +3,26 @@ package main import ( "bytes" "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "encoding/pem" "errors" "flag" "io" + "math/big" + "net" "net/http" "net/http/httptest" "net/url" "os" + "os/exec" "path/filepath" "strings" "testing" + "time" "github.com/labstack/fanout/internal/auth" "github.com/labstack/fanout/internal/config" @@ -22,27 +32,28 @@ import ( ) func TestParseCommandLine(t *testing.T) { + t.Setenv("FANOUT_ADDR", "invalid") + t.Setenv("FANOUT_TLS_CERT_FILE", "cert.pem") + t.Setenv("FANOUT_AUTH_CODE_SECRET_FILE", "/missing") tests := []struct { name string args []string - wantPath string wantVersion bool wantEmail string wantHealth string + wantDerived bool wantRepair *repairCommand wantErr bool }{ {name: "server defaults", args: nil}, - {name: "config", args: []string{"--config", "/etc/fanout.yaml"}, wantPath: "/etc/fanout.yaml"}, {name: "version command", args: []string{"version"}, wantVersion: true}, {name: "version flag", args: []string{"--version"}, wantVersion: true}, {name: "short version flag", args: []string{"-v"}, wantVersion: true}, {name: "login link", args: []string{"login-link", "admin@example.com"}, wantEmail: "admin@example.com"}, - {name: "login link with config", args: []string{"--config", "/etc/fanout.yaml", "login-link", "admin@example.com"}, wantPath: "/etc/fanout.yaml", wantEmail: "admin@example.com"}, - {name: "default healthcheck", args: []string{"healthcheck"}, wantHealth: "http://127.0.0.1:7520/healthz"}, + {name: "default healthcheck", args: []string{"healthcheck"}, wantDerived: true}, {name: "custom healthcheck", args: []string{"healthcheck", "http://fanout:8080/healthz"}, wantHealth: "http://fanout:8080/healthz"}, {name: "repair verify", args: []string{"repair", "verify"}, wantRepair: &repairCommand{action: "verify"}}, - {name: "repair quarantine with config", args: []string{"--config", "/etc/fanout.yaml", "repair", "quarantine", "--batch", "broken"}, wantPath: "/etc/fanout.yaml", wantRepair: &repairCommand{action: "quarantine", batch: "broken"}}, + {name: "repair quarantine", args: []string{"repair", "quarantine", "--batch", "broken"}, wantRepair: &repairCommand{action: "quarantine", batch: "broken"}}, {name: "repair quarantine missing batch", args: []string{"repair", "quarantine"}, wantErr: true}, {name: "unknown repair action", args: []string{"repair", "delete"}, wantErr: true}, {name: "login link missing email", args: []string{"login-link"}, wantErr: true}, @@ -51,12 +62,12 @@ func TestParseCommandLine(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - path, showVersion, email, healthURL, repair, err := parseCommandLine(test.args, io.Discard) + showVersion, email, healthURL, derivedProbe, repair, err := parseCommandLine(test.args, io.Discard) if (err != nil) != test.wantErr { t.Fatalf("error = %v, wantErr %v", err, test.wantErr) } - if path != test.wantPath || showVersion != test.wantVersion || email != test.wantEmail || healthURL != test.wantHealth { - t.Fatalf("result = (%q, %v, %q, %q), want (%q, %v, %q, %q)", path, showVersion, email, healthURL, test.wantPath, test.wantVersion, test.wantEmail, test.wantHealth) + if showVersion != test.wantVersion || email != test.wantEmail || healthURL != test.wantHealth || derivedProbe != test.wantDerived { + t.Fatalf("result = (%v, %q, %q, %v), want (%v, %q, %q, %v)", showVersion, email, healthURL, derivedProbe, test.wantVersion, test.wantEmail, test.wantHealth, test.wantDerived) } if repair == nil != (test.wantRepair == nil) || repair != nil && *repair != *test.wantRepair { t.Fatalf("repair = %#v, want %#v", repair, test.wantRepair) @@ -110,7 +121,7 @@ func TestCheckHealth(t *testing.T) { writer.WriteHeader(test.statusCode) })) defer server.Close() - if err := checkHealth(server.URL); (err != nil) != test.wantErr { + if err := checkHealth(server.URL, false, ""); (err != nil) != test.wantErr { t.Fatalf("checkHealth() error = %v, wantErr %v", err, test.wantErr) } }) @@ -205,3 +216,305 @@ func TestCreateLoginLink(t *testing.T) { t.Fatalf("issued audit events = %d, want 1", events) } } + +func TestHealthcheckURL(t *testing.T) { + for _, tc := range []struct{ addr, cert, want string }{ + {"", "", "http://127.0.0.1:7520/healthz"}, + {":8080", "", "http://127.0.0.1:8080/healthz"}, + {"0.0.0.0:8080", "", "http://127.0.0.1:8080/healthz"}, + {"[::]:8080", "", "http://127.0.0.1:8080/healthz"}, + {"[::1]:8080", "", "http://[::1]:8080/healthz"}, + {"[2001:db8::1]:8080", "", "http://[2001:db8::1]:8080/healthz"}, + {"127.0.0.1:9000", "cert.pem", "https://127.0.0.1:9000/healthz"}, + {"localhost:9000", "", "http://localhost:9000/healthz"}, + {"example.com:9000", "", "http://example.com:9000/healthz"}, + } { + got, err := healthcheckURL(tc.addr, tc.cert) + if err != nil || got != tc.want { + t.Fatalf("healthcheckURL(%q) = %q, %v; want %q", tc.addr, got, err, tc.want) + } + } + if _, err := healthcheckURL("invalid", ""); err == nil { + t.Fatal("expected invalid address error") + } +} + +func TestHealthcheckUsesEnvironmentWithoutValidatingSecrets(t *testing.T) { + t.Setenv("FANOUT_ADDR", "[::]:9090") + t.Setenv("FANOUT_TLS_CERT_FILE", "cert.pem") + t.Setenv("FANOUT_AUTH_CODE_SECRET_FILE", "/missing") + probe, err := healthcheckURL(os.Getenv("FANOUT_ADDR"), os.Getenv("FANOUT_TLS_CERT_FILE")) + if err != nil || probe != "https://127.0.0.1:9090/healthz" { + t.Fatalf("probe = %s, %v", probe, err) + } +} + +func TestCheckHealthWithLocalTLS(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) + defer server.Close() + certFile := writeHealthcheckCertificate(t, server.Certificate().Raw) + if err := checkHealth(server.URL+"/healthz", true, certFile); err != nil { + t.Fatal(err) + } + var verificationError *tls.CertificateVerificationError + if err := checkHealth(server.URL+"/healthz", false, certFile); !errors.As(err, &verificationError) { + t.Fatalf("explicit URL must verify the certificate, got %v", err) + } + other := *server.Certificate() + other.SerialNumber = big.NewInt(2) + other.SignatureAlgorithm = x509.UnknownSignatureAlgorithm + other.PublicKey = nil + otherCertificate := newHealthcheckCertificate(t, &other) + otherFile := writeHealthcheckCertificate(t, otherCertificate.Certificate[0]) + // Only the first certificate is trusted, even when the listener's leaf follows it. + chain := append(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: otherCertificate.Certificate[0]}), + pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})...) + if err := os.WriteFile(otherFile, chain, 0o600); err != nil { + t.Fatal(err) + } + if err := checkHealth(server.URL+"/healthz", true, otherFile); !errors.As(err, &verificationError) { + t.Fatalf("derived IP probe must reject a different certificate, got %v", err) + } + probe := strings.Replace(server.URL, "127.0.0.1", "localhost", 1) + "/healthz" + if err := checkHealth(probe, true, certFile); !errors.As(err, &verificationError) { + t.Fatalf("derived hostname probe must verify the certificate, got %v", err) + } +} + +func writeHealthcheckCertificate(t *testing.T, der []byte) string { + t.Helper() + path := filepath.Join(t.TempDir(), "cert.pem") + if err := os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func newHealthcheckCertificate(t *testing.T, certificate *x509.Certificate) tls.Certificate { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + der, err := x509.CreateCertificate(rand.Reader, certificate, certificate, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} +} + +func TestCheckHealthCertificateFileErrors(t *testing.T) { + const contents = "private file contents must not appear in errors" + for _, test := range []struct { + name string + data []byte + }{ + {name: "unreadable"}, + {name: "non-certificate", data: []byte(contents)}, + {name: "other PEM block", data: pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: []byte(contents)})}, + {name: "invalid certificate", data: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: []byte(contents)})}, + } { + t.Run(test.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "cert.pem") + if test.data != nil { + if err := os.WriteFile(path, test.data, 0o600); err != nil { + t.Fatal(err) + } + } + err := checkHealth("https://127.0.0.1:7520/healthz", true, path) + if err == nil || !strings.HasPrefix(err.Error(), "healthcheck:") || !strings.Contains(err.Error(), path) || strings.Contains(err.Error(), contents) { + t.Fatalf("certificate file error must name its path without contents, got %v", err) + } + }) + } +} + +func TestCheckHealthTLSCertificate(t *testing.T) { + for _, test := range []struct { + name string + dnsNames []string + ips []net.IP + expired bool + clientOnly bool + wantErr bool + }{ + {name: "IP SAN", ips: []net.IP{net.ParseIP("192.0.2.10")}}, + {name: "DNS before IP SAN", dnsNames: []string{"fanout.example.com", "other.example.com"}, ips: []net.IP{net.ParseIP("192.0.2.10")}}, + {name: "no SAN", wantErr: true}, + {name: "expired", dnsNames: []string{"fanout.example.com"}, expired: true, wantErr: true}, + {name: "client usage", dnsNames: []string{"fanout.example.com"}, clientOnly: true, wantErr: true}, + } { + t.Run(test.name, func(t *testing.T) { + certificate := &x509.Certificate{ + SerialNumber: big.NewInt(1), DNSNames: test.dnsNames, IPAddresses: test.ips, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + if test.expired { + certificate.NotAfter = time.Now().Add(-time.Minute) + } + if test.clientOnly { + certificate.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} + } + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if len(test.dnsNames) > 0 && request.TLS.ServerName != test.dnsNames[0] { + t.Errorf("TLS server name = %q, want %q", request.TLS.ServerName, test.dnsNames[0]) + } + w.WriteHeader(http.StatusOK) + })) + server.TLS = &tls.Config{Certificates: []tls.Certificate{newHealthcheckCertificate(t, certificate)}} + server.StartTLS() + defer server.Close() + certFile := writeHealthcheckCertificate(t, server.Certificate().Raw) + err := checkHealth(server.URL+"/healthz", true, certFile) + if (err != nil) != test.wantErr { + t.Fatalf("checkHealth() error = %v, wantErr %v", err, test.wantErr) + } + if test.name == "no SAN" && (!strings.HasPrefix(err.Error(), "healthcheck:") || !strings.Contains(err.Error(), certFile)) { + t.Fatalf("missing SAN error must name the certificate path, got %v", err) + } + }) + } +} + +func TestCheckHealthWithInterfaceTLS(t *testing.T) { + certificate := &x509.Certificate{ + SerialNumber: big.NewInt(1), DNSNames: []string{"fanout.example.com"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/healthz" { + t.Errorf("probe path = %q, want /healthz", request.URL.Path) + } + w.WriteHeader(http.StatusOK) + })) + server.TLS = &tls.Config{Certificates: []tls.Certificate{newHealthcheckCertificate(t, certificate)}} + server.StartTLS() + defer server.Close() + certFile := writeHealthcheckCertificate(t, server.Certificate().Raw) + for _, addr := range []string{"192.0.2.10:7520", "[2001:db8::10]:7520", "[fe80::1%eth0]:7520", "fanout.example.com:7520", "wrong.example.com:7520"} { + t.Run(addr, func(t *testing.T) { + original := http.DefaultTransport + transport := original.(*http.Transport).Clone() + // Route binds to a local test listener to exercise real TLS without + // requiring those addresses or DNS entries on the test host. + transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { + if address != addr { + t.Errorf("probe address = %q, want %q", address, addr) + } + var dialer net.Dialer + return dialer.DialContext(ctx, network, server.Listener.Addr().String()) + } + http.DefaultTransport = transport + t.Cleanup(func() { + http.DefaultTransport = original + transport.CloseIdleConnections() + }) + t.Setenv("FANOUT_ADDR", addr) + t.Setenv("FANOUT_TLS_CERT_FILE", certFile) + probe, err := healthcheckURL(os.Getenv("FANOUT_ADDR"), os.Getenv("FANOUT_TLS_CERT_FILE")) + if err != nil { + t.Fatal(err) + } + err = checkHealth(probe, true, certFile) + var verificationError *tls.CertificateVerificationError + if strings.HasSuffix(addr, "example.com:7520") { + if !errors.As(err, &verificationError) { + t.Fatalf("derived hostname probe must use system verification, got %v", err) + } + } else if err != nil { + t.Fatalf("derived TLS probe: %v", err) + } + _, _, explicitProbe, derivedProbe, _, err := parseCommandLine([]string{"healthcheck", probe}, io.Discard) + if err != nil { + t.Fatal(err) + } + err = checkHealth(explicitProbe, derivedProbe, certFile) + if !errors.As(err, &verificationError) { + t.Fatalf("explicit TLS probe must use system verification, got %v", err) + } + }) + } +} + +func TestCheckHealthDerivedIgnoresEnvironmentProxy(t *testing.T) { + // ProxyFromEnvironment caches settings on first use, so use a fresh process + // to ensure earlier tests cannot hide the configured proxies. + if os.Getenv("FANOUT_TEST_PROXY_CHILD") != "1" { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + proxy := "http://" + listener.Addr().String() + if err := listener.Close(); err != nil { + t.Fatal(err) + } + t.Setenv("HTTP_PROXY", proxy) + t.Setenv("HTTPS_PROXY", proxy) + t.Setenv("NO_PROXY", "") + t.Setenv("no_proxy", "") + t.Setenv("REQUEST_METHOD", "") + t.Setenv("FANOUT_TEST_PROXY_CHILD", "1") + command := exec.Command(os.Args[0], "-test.run=^TestCheckHealthDerivedIgnoresEnvironmentProxy$", "-test.v") + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("proxy regression subprocess: %v\n%s", err, output) + } + return + } + for _, tc := range []struct{ scheme, addr string }{ + {"http", "192.0.2.10:7520"}, {"https", "192.0.2.10:7520"}, + {"http", "fanout.example.com:7520"}, {"https", "example.com:7520"}, + } { + t.Run(tc.scheme+"/"+tc.addr, func(t *testing.T) { + scheme, addr := tc.scheme, tc.addr + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/healthz" { + t.Errorf("probe path = %q, want /healthz", request.URL.Path) + } + w.WriteHeader(http.StatusOK) + })) + certFile := "" + if scheme == "https" { + server.StartTLS() + certFile = writeHealthcheckCertificate(t, server.Certificate().Raw) + } else { + server.Start() + } + defer server.Close() + original := http.DefaultTransport + transport := original.(*http.Transport).Clone() + if scheme == "https" && addr == "example.com:7520" { + roots := x509.NewCertPool() + roots.AddCert(server.Certificate()) + transport.TLSClientConfig = &tls.Config{RootCAs: roots} + } + transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { + if address == addr { + address = server.Listener.Addr().String() + } else if "http://"+address != os.Getenv("HTTP_PROXY") { + return nil, errors.New("unexpected dial address: " + address) + } + var dialer net.Dialer + return dialer.DialContext(ctx, network, address) + } + http.DefaultTransport = transport + t.Cleanup(func() { + http.DefaultTransport = original + transport.CloseIdleConnections() + }) + t.Setenv("FANOUT_ADDR", addr) + t.Setenv("FANOUT_TLS_CERT_FILE", certFile) + probe, err := healthcheckURL(os.Getenv("FANOUT_ADDR"), os.Getenv("FANOUT_TLS_CERT_FILE")) + if err != nil { + t.Fatal(err) + } + if err := checkHealth(probe, true, certFile); err != nil { + t.Fatalf("derived %s probe with unreachable proxy: %v", scheme, err) + } + if err := checkHealth(probe, false, certFile); err == nil || !strings.Contains(err.Error(), strings.TrimPrefix(os.Getenv("HTTP_PROXY"), "http://")) { + t.Fatalf("explicit %s probe must use the unreachable proxy, got %v", scheme, err) + } + }) + } +} diff --git a/context7.json b/context7.json index fe906ee4..69a99477 100644 --- a/context7.json +++ b/context7.json @@ -15,6 +15,6 @@ "Telemetry arrives over OTLP/gRPC or OTLP/HTTP on port 7520. Point an OpenTelemetry SDK or Collector at it; do not invent a bespoke ingest protocol.", "Parquet is the authoritative telemetry store. DuckDB query state and rollups are rebuildable from it, and SQLite is reserved for transactional product state.", "Fanout exposes an MCP server, and the browser client, the in-process agent and any external MCP host all reach the same typed observability contract rather than issuing raw SQL.", - "Configuration is by `FANOUT_*` environment variables or a YAML file; see fanout.example.yaml. The Go module path is `github.com/labstack/fanout`." + "Configuration uses process environment over built-in defaults; see .env.example. Custom names use `FANOUT_`; AI keys and endpoints use documented vendor names. Secret variables also accept `_FILE` paths. The Go module path is `github.com/labstack/fanout`." ] } diff --git a/docs/authentication.md b/docs/authentication.md index 4dd4ea40..9308ab86 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -1,8 +1,7 @@ # Authentication and access operations -Operator reference for running Fanout's authentication safely. Every setting -below is available as an environment variable and as a key in the layered -configuration file. +Operator reference for running Fanout's authentication safely. Settings are +environment variables; see the [settings reference](https://fanout.run/reference/configuration). ## First run @@ -55,10 +54,10 @@ login link remains available when delivery fails. ### Local self-signup Local accounts are invitation-only by default. To let any verified email -address create an account, enable `auth.self_signup` or -`FANOUT_SELF_SIGNUP=true`. Fanout sends the existing email code and -creates an active `viewer` only after that code is successfully verified. The -role is fixed: self-signup can never create an operator or administrator. +address create an account, set `FANOUT_SELF_SIGNUP=true`. Fanout sends the +existing email code and creates an active `viewer` only after that code is +successfully verified. The role is fixed: self-signup can never create an +operator or administrator. The setting requires complete SMTP configuration and local auth mode. Self-signup remains unavailable until the first administrator completes setup. @@ -72,13 +71,13 @@ If you are locked out in local mode, run the command with the same configuration and data directory as the server: ```sh -fanout --config /etc/fanout/fanout.yaml login-link admin@example.com +fanout login-link admin@example.com ``` -For a container using the bundled configuration: +For a running container: ```sh -docker exec fanout --config /etc/fanout/fanout.yaml \ +docker exec fanout \ login-link admin@example.com ``` @@ -167,19 +166,17 @@ asserts a verified email. This keeps sessions, conversations, dashboards, and audit attribution isolated without shared credentials or an anonymous authorization bypass: -```yaml -auth: - mode: oidc - oidc: - email_verification: required - auto_provision: true - allowed_domains: "*" - default_role: viewer +```sh +FANOUT_AUTH_MODE=oidc +FANOUT_OIDC_EMAIL_VERIFICATION=required +FANOUT_OIDC_AUTO_PROVISION=true +FANOUT_OIDC_ALLOWED_DOMAINS="*" +FANOUT_OIDC_DEFAULT_ROLE=viewer ``` The wildcard is deliberately explicit and is rejected with -`email_verification: issuer`. Use a specific domain or group allowlist for a -private installation. A wildcard makes the installation available to every +`FANOUT_OIDC_EMAIL_VERIFICATION=issuer`. Use a specific domain or group allowlist +for a private installation. A wildcard makes the installation available to every identity accepted by the configured provider, so rate-limit public traffic and budget AI usage at the deployment edge. diff --git a/docs/diagrams/architecture.d2 b/docs/diagrams/architecture.d2 index 1aa4d40f..0d8c5fbd 100644 --- a/docs/diagrams/architecture.d2 +++ b/docs/diagrams/architecture.d2 @@ -44,9 +44,9 @@ fanout: "fanout — one Go process" { store: Storage { style.fill: transparent - telemetry: "Parquet batches + trace indexes\nstorage.data_dir/telemetry" {shape: cylinder} - qstate: "DuckDB query state\nstorage.data_dir/query" {shape: cylinder} - control: "Control SQLite\nstorage.data_dir/control" {shape: cylinder} + telemetry: "Parquet batches + trace indexes\nFANOUT_DATA_DIR/telemetry" {shape: cylinder} + qstate: "DuckDB query state\nFANOUT_DATA_DIR/query" {shape: cylinder} + control: "Control SQLite\nFANOUT_DATA_DIR/control" {shape: cylinder} } model: "Anthropic or OpenAI" {style.stroke-dash: 3} diff --git a/docs/diagrams/architecture.svg b/docs/diagrams/architecture.svg index 01633b3e..18433e2a 100644 --- a/docs/diagrams/architecture.svg +++ b/docs/diagrams/architecture.svg @@ -1,24 +1,24 @@ -Clientsfanout — one Go processStorageAnthropic or OpenAIOTLP collector or SDKBrowserExternal MCP hostHTTP :7520routes + auth middlewareIngestOTLP gRPC + HTTP :7520Embedded React assetsAgent runtimemodel + tool loopMCP server5 typed + 4 dashboard toolsTyped observability contractTelemetry commit workersatomic Parquet batchesQuery kernelDuckDB + rollupsStorage maintenanceretention + native compactionAlert enginerule evaluation + webhooksParquet batches + trace indexesstorage.data_dir/telemetryDuckDB query statestorage.data_dir/queryControl SQLitestorage.data_dir/control serves embedded assetsAG-UI streamin-memory transporttyped HTTP APIevaluates rollupsOTLP/gRPC or OTLP/HTTPHTTPS/mcp — OAuthindexed traces + SQL scansatomic replacementusers, sessions, settings, dashboards, threadsrules and fired alertsHTTPS - + .d2-37886755 .fill-N1{fill:#0A0F25;} + .d2-37886755 .fill-N2{fill:#676C7E;} + .d2-37886755 .fill-N3{fill:#9499AB;} + .d2-37886755 .fill-N4{fill:#CFD2DD;} + .d2-37886755 .fill-N5{fill:#DEE1EB;} + .d2-37886755 .fill-N6{fill:#EEF1F8;} + .d2-37886755 .fill-N7{fill:#FFFFFF;} + .d2-37886755 .fill-B1{fill:#000536;} + .d2-37886755 .fill-B2{fill:#0F66B7;} + .d2-37886755 .fill-B3{fill:#4393DD;} + .d2-37886755 .fill-B4{fill:#87BFF3;} + .d2-37886755 .fill-B5{fill:#BCDDFB;} + .d2-37886755 .fill-B6{fill:#E5F3FF;} + .d2-37886755 .fill-AA2{fill:#7639C5;} + .d2-37886755 .fill-AA4{fill:#C1A2F3;} + .d2-37886755 .fill-AA5{fill:#DACEFB;} + .d2-37886755 .fill-AB4{fill:#EA99C6;} + .d2-37886755 .fill-AB5{fill:#FFDEF1;} + .d2-37886755 .stroke-N1{stroke:#0A0F25;} + .d2-37886755 .stroke-N2{stroke:#676C7E;} + .d2-37886755 .stroke-N3{stroke:#9499AB;} + .d2-37886755 .stroke-N4{stroke:#CFD2DD;} + .d2-37886755 .stroke-N5{stroke:#DEE1EB;} + .d2-37886755 .stroke-N6{stroke:#EEF1F8;} + .d2-37886755 .stroke-N7{stroke:#FFFFFF;} + .d2-37886755 .stroke-B1{stroke:#000536;} + .d2-37886755 .stroke-B2{stroke:#0F66B7;} + .d2-37886755 .stroke-B3{stroke:#4393DD;} + .d2-37886755 .stroke-B4{stroke:#87BFF3;} + .d2-37886755 .stroke-B5{stroke:#BCDDFB;} + .d2-37886755 .stroke-B6{stroke:#E5F3FF;} + .d2-37886755 .stroke-AA2{stroke:#7639C5;} + .d2-37886755 .stroke-AA4{stroke:#C1A2F3;} + .d2-37886755 .stroke-AA5{stroke:#DACEFB;} + .d2-37886755 .stroke-AB4{stroke:#EA99C6;} + .d2-37886755 .stroke-AB5{stroke:#FFDEF1;} + .d2-37886755 .background-color-N1{background-color:#0A0F25;} + .d2-37886755 .background-color-N2{background-color:#676C7E;} + .d2-37886755 .background-color-N3{background-color:#9499AB;} + .d2-37886755 .background-color-N4{background-color:#CFD2DD;} + .d2-37886755 .background-color-N5{background-color:#DEE1EB;} + .d2-37886755 .background-color-N6{background-color:#EEF1F8;} + .d2-37886755 .background-color-N7{background-color:#FFFFFF;} + .d2-37886755 .background-color-B1{background-color:#000536;} + .d2-37886755 .background-color-B2{background-color:#0F66B7;} + .d2-37886755 .background-color-B3{background-color:#4393DD;} + .d2-37886755 .background-color-B4{background-color:#87BFF3;} + .d2-37886755 .background-color-B5{background-color:#BCDDFB;} + .d2-37886755 .background-color-B6{background-color:#E5F3FF;} + .d2-37886755 .background-color-AA2{background-color:#7639C5;} + .d2-37886755 .background-color-AA4{background-color:#C1A2F3;} + .d2-37886755 .background-color-AA5{background-color:#DACEFB;} + .d2-37886755 .background-color-AB4{background-color:#EA99C6;} + .d2-37886755 .background-color-AB5{background-color:#FFDEF1;} + .d2-37886755 .color-N1{color:#0A0F25;} + .d2-37886755 .color-N2{color:#676C7E;} + .d2-37886755 .color-N3{color:#9499AB;} + .d2-37886755 .color-N4{color:#CFD2DD;} + .d2-37886755 .color-N5{color:#DEE1EB;} + .d2-37886755 .color-N6{color:#EEF1F8;} + .d2-37886755 .color-N7{color:#FFFFFF;} + .d2-37886755 .color-B1{color:#000536;} + .d2-37886755 .color-B2{color:#0F66B7;} + .d2-37886755 .color-B3{color:#4393DD;} + .d2-37886755 .color-B4{color:#87BFF3;} + .d2-37886755 .color-B5{color:#BCDDFB;} + .d2-37886755 .color-B6{color:#E5F3FF;} + .d2-37886755 .color-AA2{color:#7639C5;} + .d2-37886755 .color-AA4{color:#C1A2F3;} + .d2-37886755 .color-AA5{color:#DACEFB;} + .d2-37886755 .color-AB4{color:#EA99C6;} + .d2-37886755 .color-AB5{color:#FFDEF1;}.appendix text.text{fill:#0A0F25}.md{--color-fg-default:#0A0F25;--color-fg-muted:#676C7E;--color-fg-subtle:#9499AB;--color-canvas-default:#FFFFFF;--color-canvas-subtle:#EEF1F8;--color-border-default:#000536;--color-border-muted:#0F66B7;--color-neutral-muted:#EEF1F8;--color-accent-fg:#0F66B7;--color-accent-emphasis:#0F66B7;--color-attention-subtle:#676C7E;--color-danger-fg:red;}.sketch-overlay-B1{fill:url(#streaks-darker-d2-37886755);mix-blend-mode:lighten}.sketch-overlay-B2{fill:url(#streaks-dark-d2-37886755);mix-blend-mode:overlay}.sketch-overlay-B3{fill:url(#streaks-dark-d2-37886755);mix-blend-mode:overlay}.sketch-overlay-B4{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-B5{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-B6{fill:url(#streaks-bright-d2-37886755);mix-blend-mode:darken}.sketch-overlay-AA2{fill:url(#streaks-dark-d2-37886755);mix-blend-mode:overlay}.sketch-overlay-AA4{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-AA5{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-AB4{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-AB5{fill:url(#streaks-bright-d2-37886755);mix-blend-mode:darken}.sketch-overlay-N1{fill:url(#streaks-darker-d2-37886755);mix-blend-mode:lighten}.sketch-overlay-N2{fill:url(#streaks-dark-d2-37886755);mix-blend-mode:overlay}.sketch-overlay-N3{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-N4{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-N5{fill:url(#streaks-bright-d2-37886755);mix-blend-mode:darken}.sketch-overlay-N6{fill:url(#streaks-bright-d2-37886755);mix-blend-mode:darken}.sketch-overlay-N7{fill:url(#streaks-bright-d2-37886755);mix-blend-mode:darken}.light-code{display: block}.dark-code{display: none}]]>Clientsfanout — one Go processStorageAnthropic or OpenAIOTLP collector or SDKBrowserExternal MCP hostHTTP :7520routes + auth middlewareIngestOTLP gRPC + HTTP :7520Embedded React assetsAgent runtimemodel + tool loopMCP server5 typed + 4 dashboard toolsTyped observability contractTelemetry commit workersatomic Parquet batchesQuery kernelDuckDB + rollupsStorage maintenanceretention + native compactionAlert enginerule evaluation + webhooksParquet batches + trace indexesFANOUT_DATA_DIR/telemetryDuckDB query stateFANOUT_DATA_DIR/queryControl SQLiteFANOUT_DATA_DIR/control serves embedded assetsAG-UI streamin-memory transporttyped HTTP APIevaluates rollupsOTLP/gRPC or OTLP/HTTPHTTPS/mcp — OAuthindexed traces + SQL scansatomic replacementusers, sessions, settings, dashboards, threadsrules and fired alertsHTTPS + @@ -113,9 +113,9 @@ - - + + - + diff --git a/docs/diagrams/persistence.d2 b/docs/diagrams/persistence.d2 index d5b508b5..b0d74d6f 100644 --- a/docs/diagrams/persistence.d2 +++ b/docs/diagrams/persistence.d2 @@ -49,7 +49,7 @@ maintenance: "Bounded maintenance" { merge -> swap } -querystate: "Rebuildable query state\nstorage.data_dir/query" { +querystate: "Rebuildable query state\nFANOUT_DATA_DIR/query" { shape: cylinder tooltip: DuckDB rollups and temporary spill. Not authoritative telemetry. } diff --git a/docs/diagrams/persistence.svg b/docs/diagrams/persistence.svg index 50c353fc..13d27eb5 100644 --- a/docs/diagrams/persistence.svg +++ b/docs/diagrams/persistence.svg @@ -1,27 +1,27 @@ -Authenticated OTLP requestsspans, logs, metricsDurable write pathAuthoritative telemetryimmutable *.batch directoriesOrdinary Parquet files are authoritativeReadersBounded maintenanceRebuildable query statestorage.data_dir/queryDuckDB rollups and temporary spill. Not authoritative telemetry.Bounded group admissionconfigured row target + 20 ms windowUp to four commit workersindependent batches encode in parallelStaging batch directorymetadata + signal Parquet + trace.fidx for spansDurable publicationfsync → atomic rename → fsync parentpublished batch directories are discovered directly from the filesystemIndexed trace readbinary-search .fidx → Parquet rowsDuckDB SQLfilters, ordering, broad scansDuckDB rollupsrebuildable analytical cacheRetention + compaction admissionbounded batch and row groupsNative Parquet mergeprepared outside the reader gateCrash-safe namespace swapmarker + bounded renames + fsync acknowledge only after publicationOrdinary Parquet files are authoritative - + .d2-2701310579 .fill-N1{fill:#0A0F25;} + .d2-2701310579 .fill-N2{fill:#676C7E;} + .d2-2701310579 .fill-N3{fill:#9499AB;} + .d2-2701310579 .fill-N4{fill:#CFD2DD;} + .d2-2701310579 .fill-N5{fill:#DEE1EB;} + .d2-2701310579 .fill-N6{fill:#EEF1F8;} + .d2-2701310579 .fill-N7{fill:#FFFFFF;} + .d2-2701310579 .fill-B1{fill:#000536;} + .d2-2701310579 .fill-B2{fill:#0F66B7;} + .d2-2701310579 .fill-B3{fill:#4393DD;} + .d2-2701310579 .fill-B4{fill:#87BFF3;} + .d2-2701310579 .fill-B5{fill:#BCDDFB;} + .d2-2701310579 .fill-B6{fill:#E5F3FF;} + .d2-2701310579 .fill-AA2{fill:#7639C5;} + .d2-2701310579 .fill-AA4{fill:#C1A2F3;} + .d2-2701310579 .fill-AA5{fill:#DACEFB;} + .d2-2701310579 .fill-AB4{fill:#EA99C6;} + .d2-2701310579 .fill-AB5{fill:#FFDEF1;} + .d2-2701310579 .stroke-N1{stroke:#0A0F25;} + .d2-2701310579 .stroke-N2{stroke:#676C7E;} + .d2-2701310579 .stroke-N3{stroke:#9499AB;} + .d2-2701310579 .stroke-N4{stroke:#CFD2DD;} + .d2-2701310579 .stroke-N5{stroke:#DEE1EB;} + .d2-2701310579 .stroke-N6{stroke:#EEF1F8;} + .d2-2701310579 .stroke-N7{stroke:#FFFFFF;} + .d2-2701310579 .stroke-B1{stroke:#000536;} + .d2-2701310579 .stroke-B2{stroke:#0F66B7;} + .d2-2701310579 .stroke-B3{stroke:#4393DD;} + .d2-2701310579 .stroke-B4{stroke:#87BFF3;} + .d2-2701310579 .stroke-B5{stroke:#BCDDFB;} + .d2-2701310579 .stroke-B6{stroke:#E5F3FF;} + .d2-2701310579 .stroke-AA2{stroke:#7639C5;} + .d2-2701310579 .stroke-AA4{stroke:#C1A2F3;} + .d2-2701310579 .stroke-AA5{stroke:#DACEFB;} + .d2-2701310579 .stroke-AB4{stroke:#EA99C6;} + .d2-2701310579 .stroke-AB5{stroke:#FFDEF1;} + .d2-2701310579 .background-color-N1{background-color:#0A0F25;} + .d2-2701310579 .background-color-N2{background-color:#676C7E;} + .d2-2701310579 .background-color-N3{background-color:#9499AB;} + .d2-2701310579 .background-color-N4{background-color:#CFD2DD;} + .d2-2701310579 .background-color-N5{background-color:#DEE1EB;} + .d2-2701310579 .background-color-N6{background-color:#EEF1F8;} + .d2-2701310579 .background-color-N7{background-color:#FFFFFF;} + .d2-2701310579 .background-color-B1{background-color:#000536;} + .d2-2701310579 .background-color-B2{background-color:#0F66B7;} + .d2-2701310579 .background-color-B3{background-color:#4393DD;} + .d2-2701310579 .background-color-B4{background-color:#87BFF3;} + .d2-2701310579 .background-color-B5{background-color:#BCDDFB;} + .d2-2701310579 .background-color-B6{background-color:#E5F3FF;} + .d2-2701310579 .background-color-AA2{background-color:#7639C5;} + .d2-2701310579 .background-color-AA4{background-color:#C1A2F3;} + .d2-2701310579 .background-color-AA5{background-color:#DACEFB;} + .d2-2701310579 .background-color-AB4{background-color:#EA99C6;} + .d2-2701310579 .background-color-AB5{background-color:#FFDEF1;} + .d2-2701310579 .color-N1{color:#0A0F25;} + .d2-2701310579 .color-N2{color:#676C7E;} + .d2-2701310579 .color-N3{color:#9499AB;} + .d2-2701310579 .color-N4{color:#CFD2DD;} + .d2-2701310579 .color-N5{color:#DEE1EB;} + .d2-2701310579 .color-N6{color:#EEF1F8;} + .d2-2701310579 .color-N7{color:#FFFFFF;} + .d2-2701310579 .color-B1{color:#000536;} + .d2-2701310579 .color-B2{color:#0F66B7;} + .d2-2701310579 .color-B3{color:#4393DD;} + .d2-2701310579 .color-B4{color:#87BFF3;} + .d2-2701310579 .color-B5{color:#BCDDFB;} + .d2-2701310579 .color-B6{color:#E5F3FF;} + .d2-2701310579 .color-AA2{color:#7639C5;} + .d2-2701310579 .color-AA4{color:#C1A2F3;} + .d2-2701310579 .color-AA5{color:#DACEFB;} + .d2-2701310579 .color-AB4{color:#EA99C6;} + .d2-2701310579 .color-AB5{color:#FFDEF1;}.appendix text.text{fill:#0A0F25}.md{--color-fg-default:#0A0F25;--color-fg-muted:#676C7E;--color-fg-subtle:#9499AB;--color-canvas-default:#FFFFFF;--color-canvas-subtle:#EEF1F8;--color-border-default:#000536;--color-border-muted:#0F66B7;--color-neutral-muted:#EEF1F8;--color-accent-fg:#0F66B7;--color-accent-emphasis:#0F66B7;--color-attention-subtle:#676C7E;--color-danger-fg:red;}.sketch-overlay-B1{fill:url(#streaks-darker-d2-2701310579);mix-blend-mode:lighten}.sketch-overlay-B2{fill:url(#streaks-dark-d2-2701310579);mix-blend-mode:overlay}.sketch-overlay-B3{fill:url(#streaks-dark-d2-2701310579);mix-blend-mode:overlay}.sketch-overlay-B4{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-B5{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-B6{fill:url(#streaks-bright-d2-2701310579);mix-blend-mode:darken}.sketch-overlay-AA2{fill:url(#streaks-dark-d2-2701310579);mix-blend-mode:overlay}.sketch-overlay-AA4{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-AA5{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-AB4{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-AB5{fill:url(#streaks-bright-d2-2701310579);mix-blend-mode:darken}.sketch-overlay-N1{fill:url(#streaks-darker-d2-2701310579);mix-blend-mode:lighten}.sketch-overlay-N2{fill:url(#streaks-dark-d2-2701310579);mix-blend-mode:overlay}.sketch-overlay-N3{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-N4{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-N5{fill:url(#streaks-bright-d2-2701310579);mix-blend-mode:darken}.sketch-overlay-N6{fill:url(#streaks-bright-d2-2701310579);mix-blend-mode:darken}.sketch-overlay-N7{fill:url(#streaks-bright-d2-2701310579);mix-blend-mode:darken}.light-code{display: block}.dark-code{display: none}]]>Authenticated OTLP requestsspans, logs, metricsDurable write pathAuthoritative telemetryimmutable *.batch directoriesOrdinary Parquet files are authoritativeReadersBounded maintenanceRebuildable query stateFANOUT_DATA_DIR/queryDuckDB rollups and temporary spill. Not authoritative telemetry.Bounded group admissionconfigured row target + 20 ms windowUp to four commit workersindependent batches encode in parallelStaging batch directorymetadata + signal Parquet + trace.fidx for spansDurable publicationfsync → atomic rename → fsync parentpublished batch directories are discovered directly from the filesystemIndexed trace readbinary-search .fidx → Parquet rowsDuckDB SQLfilters, ordering, broad scansDuckDB rollupsrebuildable analytical cacheRetention + compaction admissionbounded batch and row groupsNative Parquet mergeprepared outside the reader gateCrash-safe namespace swapmarker + bounded renames + fsync acknowledge only after publicationOrdinary Parquet files are authoritative + - + -DuckDB rollups and temporary spill. Not authoritative telemetry. - +DuckDB rollups and temporary spill. Not authoritative telemetry. + - + - + diff --git a/docs/operations.md b/docs/operations.md index d1cac7ca..55f78a0d 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -4,14 +4,136 @@ Fanout is one process and one persistent data directory. Operate those two things as a unit: preserve the directory, pin the binary or image version, and give ingest credentials only to trusted services and collectors. +## Runtime secret delivery + +Runtime configuration uses built-in defaults followed by nonempty process +environment values. Every setting requires a restart. Fanout reads no +configuration document or dotenv file. Generated `.env.example` lists the +settings as commented-out empty entries, with types, defaults, and secret file +forms in comments. Copying the template sets nothing and preserves image ENV; +uncomment only settings you want to supply. For development, `just run` +loads optional ignored `./.env` with Node's native `parseEnv`; inherited values +win, including empty values. Personal development credentials may live there. +Personal environment files and operator profiles are excluded from Git and Docker. + +Custom names use `FANOUT_` plus a clear name, adding the area only when it +clarifies. Vendor credentials and endpoints keep their documented names. +Unknown `FANOUT_` names fail startup, except platform service-discovery names; +unrelated names in shared vendor namespaces are ignored. + +### Environment and mounted secret files + +Deliver credentials through deployment environment injection or managed mounts. +Every secret accepts either its value or its file path: + +| Value variable | File path variable | +| --- | --- | +| `FANOUT_AUTH_CODE_SECRET` | `FANOUT_AUTH_CODE_SECRET_FILE` | +| `FANOUT_OIDC_CLIENT_SECRET` | `FANOUT_OIDC_CLIENT_SECRET_FILE` | +| `FANOUT_SMTP_PASSWORD` | `FANOUT_SMTP_PASSWORD_FILE` | +| `FANOUT_METRICS_TOKEN` | `FANOUT_METRICS_TOKEN_FILE` | +| `ANTHROPIC_API_KEY` | `ANTHROPIC_API_KEY_FILE` | +| `OPENAI_API_KEY` | `OPENAI_API_KEY_FILE` | + +Both forms nonempty fail startup. Fanout strips exactly one trailing LF or CRLF +from file contents, then rejects an empty result. Errors identify the variable +and path, never the contents. Fanout performs no file-mode check; the deployment +controls mount access. Do not bake credentials into images or frontend bundles. + +For Docker, mount a provisioned secret and inject its path: + +```sh +docker run --name fanout -p 7520:7520 \ + -v fanout-data:/var/lib/fanout/data \ + --mount type=bind,src="$AUTH_SECRET_PATH",dst=/run/secrets/auth_code,readonly \ + -e FANOUT_AUTH_CODE_SECRET_FILE=/run/secrets/auth_code \ + labstack/fanout:latest +``` + +The source secret must be readable by the image's UID 65532. In Compose, grant +only the required secret to the service: + +```yaml +services: + fanout: + image: labstack/fanout:latest + ports: ["7520:7520"] + volumes: ["fanout-data:/var/lib/fanout/data"] + environment: + FANOUT_AUTH_CODE_SECRET_FILE: /run/secrets/auth_code + secrets: [auth_code] +secrets: + auth_code: + file: ${AUTH_SECRET_PATH} +volumes: + fanout-data: +``` + +For Kubernetes, put ordinary settings in a ConfigMap and credentials in a +Secret provisioned through your platform's secret manager. This Pod fragment +injects both without putting credential values in the manifest: + +```yaml +containers: + - name: fanout + image: labstack/fanout:latest + envFrom: + - configMapRef: + name: fanout-settings + env: + - name: FANOUT_AUTH_CODE_SECRET + valueFrom: + secretKeyRef: + name: fanout-credentials + key: auth_code_secret +``` + +Alternatively mount the Secret as files and set the corresponding `_FILE` +paths. Use the existing persistent-volume and access policy for telemetry data. + +### SOPS runtime injection + +Use one authoritative SOPS-encrypted profile per actual workload/credential +set, such as ignored `secrets/fanout.env`. Select a complete profile explicitly; +never layer different credential sets or fall back on a missing profile. +Platform-owned credentials remain platform-owned. This public repository ships +no operator secrets, recipient policy, or decryption keys. + +With a provisioned SOPS key and existing encrypted profile, launch through +[SOPS runtime injection](https://getsops.io/docs/usage/advanced/#passing-secrets-to-other-processes) +without writing decrypted credentials to disk: + +```sh +sops exec-env secrets/fanout.env './bin/fanout' +# For local development with optional personal settings: +sops exec-env secrets/fanout.env 'just run' +``` + +The profile supplies environment values to the child; ordinary settings may +also come from deployment inputs. A dotenv parser neither decrypts SOPS nor +resolves 1Password references. Verify synthetic values containing dollars, +quotes, spaces, and newlines through the actual deployment parser before +using a profile. Keep private age identities out of Git, restrict their +local files to mode `0600`, and document key recovery in the existing 1Password +item. 1Password holds recovery rather than a second maintained copy of +SOPS-owned application secrets. + +AI chat is disabled by default. Select exactly `anthropic` or `openai` with +`FANOUT_AI_PROVIDER`, then deliver its required `ANTHROPIC_API_KEY` or +`OPENAI_API_KEY`. A selected provider without its key fails startup. Inherited +keys never enable chat. Only the selected endpoint is validated and used. +Overrides are `ANTHROPIC_BASE_URL` and `OPENAI_BASE_URL`; the OpenAI base +includes `/v1`, while Anthropic's excludes it. + ## Network surface | Default | Purpose | Exposure | |---|---|---| -| `:7520` | Browser, API, MCP, health, metrics, OTLP/gRPC and OTLP/HTTP | Users, trusted services or a reverse proxy | +| `127.0.0.1:7520` (native), `:7520` (image) | Browser, API, MCP, health, metrics, OTLP/gRPC and OTLP/HTTP | Users, trusted services or a reverse proxy | -Both native and container deployments use one listener. Set -`FANOUT_ADDR=127.0.0.1:7520` for loopback-only access. Every OTLP request +Both native and container deployments use one listener. Native installs default +to loopback; set `FANOUT_ADDR=:7520` for remote access. The image supplies +`FANOUT_ADDR=:7520` and `FANOUT_DATA_DIR=/var/lib/fanout/data`. Every OTLP request requires the ingest token created during first-admin setup; browser sessions and MCP tokens do not authorize ingest. Do not embed the ingest token in a browser or shipped mobile application. @@ -51,6 +173,15 @@ original host. ## Health and monitoring +`fanout healthcheck` derives `/healthz` from `FANOUT_ADDR` or its built-in +default. Empty, `0.0.0.0`, and `::` bind hosts become `127.0.0.1`. +`FANOUT_TLS_CERT_FILE` selects HTTPS. Derived probes bypass environment proxies. +HTTPS IP-literal probes trust only the configured certificate and verify its +first DNS or IP SAN, which may differ from the bind IP; hostname probes verify +normally. Healthcheck reads only these two settings and does not require valid +secrets. An explicit `fanout healthcheck ` uses default proxy and +certificate verification behaviour. + - `/healthz` is the liveness check. - `/readyz` includes storage readiness and resolved runtime sizing. - `/metrics` exposes Prometheus metrics. Keep it private or configure @@ -67,16 +198,6 @@ Fanout uses SQLite with the modernc driver, sqlc query bindings, and embedded Goose SQL migrations. Pending migrations run forward at startup. PostgreSQL and pgx are not required. -This preview replaces the Atlas migration history with a fresh Goose baseline. -Existing Atlas-managed control databases are unsupported; startup rejects them -without converting or resetting their data. To start fresh, stop Fanout, back -up the complete data directory, and move `data/control` aside (under your -configured data directory). Startup creates a new control database. Complete -first-admin setup again and reconfigure the ingest token, users, alert rules, -and dashboards; prior sessions, OAuth grants, and agent history remain only in -the saved control database. Telemetry and query files stay in place. Restore -the saved directory with the previous binary if you need the previous state. - ## Backup The supported portable baseline is a **cold backup** of the complete @@ -112,44 +233,6 @@ Never recover an account by deleting its row from the control database; that cascades to sessions, OAuth grants, and owned dashboards. Use `fanout login-link ` as documented in [authentication.md](authentication.md). -## Single-port migration - -This is a breaking configuration change: - -1. Remove `FANOUT_OTLP_GRPC_ADDR` and `FANOUT_OTLP_HTTP_ADDR`, or YAML - `ingest.otlp_grpc_addr` and `ingest.otlp_http_addr`. They are rejected at startup. -2. Rename `FANOUT_HTTP_ADDR` to `FANOUT_ADDR`, or YAML `server.http_addr` - to `server.addr` (default `:7520`). Publish only that container port. -3. Change exporters from ports `4317`/`4318` to `7520`, or to the public HTTPS - origin and port. The ingest token and OTLP signal paths do not change. -4. Enable HTTP/2 backend forwarding for gRPC at any proxy. Remove obsolete - ingest port mappings and proxy upstreams. A separate ingest hostname may - still route to the same Fanout port if required by the edge configuration. -5. Set `FANOUT_PUBLIC_URL` to the external origin so setup instructions start - with the reachable endpoint. If collectors use a different private hostname, - edit the endpoint on **Connect telemetry** before copying the configuration. - Remove `FANOUT_INGEST_ADVERTISED_ENDPOINT` or YAML - `ingest.advertised_endpoint`; these guidance-only settings are rejected. - -Local comparison results are in [the single-port benchmark report](benchmarks/single-port-2026-09-08.md). - -### Network isolation changes - -Ingest can no longer bind to a different interface from the application. -For an internet-facing dashboard with LAN-only ingest, keep the backend port -private and configure the public proxy to deny `/v1`, `/v1/*`, and the OTLP -gRPC service paths under `/opentelemetry.proto.collector.*`. Allow collectors -through the private network or an explicitly restricted proxy route. Verify -that both OTLP transports are inaccessible through the public origin; the -ingest token is still required on the private path. - -Single-port removes the need for a proxy solely to combine listeners. Splitting -public application access from private ingest still requires network-aware -routing. The mandatory default is deliberate for this pre-release product: -there is one existing operator, so keeping two listener topologies would add -configuration and testing costs without an installed compatibility base. The -address rename is a separate breaking change, bundled into this migration. - ## Upgrade and rollback 1. Read the release notes and pin the target image tag or binary version. @@ -169,7 +252,7 @@ SMTP is optional. Creating a local user without it succeeds and returns operator with access to the same configuration and data directory then runs: ```sh -fanout --config /etc/fanout/fanout.yaml login-link user@example.com +fanout login-link user@example.com ``` The resulting link expires after 15 minutes, works once, and is recorded in the diff --git a/fanout.docker.yaml b/fanout.docker.yaml deleted file mode 100644 index ad49d6e6..00000000 --- a/fanout.docker.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# One listener for the browser, API, MCP and both OTLP transports. -# Runtime environment variables override these container defaults. -server: - addr: ":7520" - -storage: - data_dir: /var/lib/fanout/data diff --git a/fanout.example.yaml b/fanout.example.yaml deleted file mode 100644 index 7b6a8053..00000000 --- a/fanout.example.yaml +++ /dev/null @@ -1,87 +0,0 @@ -# Fanout runtime configuration. -# -# Start with: fanout --config /path/to/fanout.yaml -# Every value can be overridden by its documented FANOUT_ environment variable. -# Secrets are intentionally omitted below; inject them through the environment -# or add them to a suitably protected copy of this file. -# Use YAML 1.2 true/false for booleans; yes/no/on/off are strings and rejected. -# Null values (a key with nothing after the colon) are rejected. - -server: - addr: ":7520" # FANOUT_ADDR - public_url: "" # FANOUT_PUBLIC_URL - trusted_proxy_cidrs: "" # FANOUT_TRUSTED_PROXY_CIDRS - pprof_enabled: false # FANOUT_PPROF_ENABLED - tls: - cert_file: "" # FANOUT_TLS_CERT_FILE - key_file: "" # FANOUT_TLS_KEY_FILE - -ingest: - default_namespace: default # FANOUT_DEFAULT_NAMESPACE - batch_size: 50000 # FANOUT_INGEST_BATCH_SIZE - # Decoded telemetry held across concurrent requests before new ones are - # refused with a retryable status. 0 disables the ceiling. - max_in_flight_bytes: 268435456 # FANOUT_INGEST_MAX_IN_FLIGHT_BYTES - -storage: - data_dir: ./data # FANOUT_DATA_DIR - retention_days: 30 # FANOUT_RETENTION_DAYS - rollup_interval: 1m # FANOUT_ROLLUP_INTERVAL - rollup_skip_to_latest: false # FANOUT_ROLLUP_SKIP_TO_LATEST - maintenance_interval: 1h # FANOUT_MAINTENANCE_INTERVAL - duckdb: - # Empty/zero values let Fanout size these from the machine. - memory: "" # FANOUT_DUCKDB_MEMORY - threads: 0 # FANOUT_DUCKDB_THREADS - max_connections: 0 # FANOUT_DUCKDB_MAX_CONNECTIONS - -alerts: - enabled: true # FANOUT_ALERTS_ENABLED - evaluation_interval: 30s # FANOUT_ALERTS_EVALUATION_INTERVAL - history_days: 7 # FANOUT_ALERTS_HISTORY_DAYS - -ai: - provider: anthropic # FANOUT_AI_PROVIDER - model: "" # FANOUT_AI_MODEL - base_url: "" # FANOUT_AI_BASE_URL - # Optional; enables investigation chat: FANOUT_AI_API_KEY - # api_key: "" - -smtp: - host: "" # FANOUT_SMTP_HOST - port: 587 # FANOUT_SMTP_PORT - username: "" # FANOUT_SMTP_USERNAME - from: "" # FANOUT_SMTP_FROM - # Optional with the other SMTP fields; enables email-code login: FANOUT_SMTP_PASSWORD - # password: "" - -auth: - mode: local # FANOUT_AUTH_MODE - session_idle_ttl: 12h # FANOUT_SESSION_IDLE_TTL - session_absolute_ttl: 168h # FANOUT_SESSION_ABSOLUTE_TTL - # Required in local auth mode: FANOUT_AUTH_CODE_SECRET (32+ characters) - # code_secret: "" - # Allow verified email addresses to create viewer accounts. - self_signup: false # FANOUT_SELF_SIGNUP - oidc: - issuer_url: "" # FANOUT_OIDC_ISSUER_URL - client_id: "" # FANOUT_OIDC_CLIENT_ID - email_claim: email # FANOUT_OIDC_EMAIL_CLAIM - email_verification: required # FANOUT_OIDC_EMAIL_VERIFICATION - auto_provision: false # FANOUT_OIDC_AUTO_PROVISION - allowed_groups: "" # FANOUT_OIDC_ALLOWED_GROUPS - # Use "*" only with email_verification: required to admit any verified identity. - allowed_domains: "" # FANOUT_OIDC_ALLOWED_DOMAINS - default_role: viewer # FANOUT_OIDC_DEFAULT_ROLE - operator_groups: "" # FANOUT_OIDC_OPERATOR_GROUPS - admin_groups: "" # FANOUT_OIDC_ADMIN_GROUPS - # Required in OIDC mode: FANOUT_OIDC_CLIENT_SECRET - # client_secret: "" - -metrics: - public: false # FANOUT_METRICS_PUBLIC - # Optional bearer token: FANOUT_METRICS_TOKEN - # token: "" - -mcp: - enabled: true # FANOUT_MCP_ENABLED diff --git a/go.mod b/go.mod index e8fa1808..1a85b188 100644 --- a/go.mod +++ b/go.mod @@ -9,14 +9,9 @@ require ( github.com/alexedwards/scs/v2 v2.9.0 github.com/coreos/go-oidc/v3 v3.21.0 github.com/duckdb/duckdb-go/v2 v2.10506.0 - github.com/go-viper/mapstructure/v2 v2.5.0 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/jsonschema-go v0.4.3 github.com/google/uuid v1.6.0 - github.com/knadh/koanf/parsers/yaml v1.1.1 - github.com/knadh/koanf/providers/confmap v1.0.1 - github.com/knadh/koanf/providers/file v1.2.1 - github.com/knadh/koanf/v2 v2.3.7 github.com/labstack/echo/v5 v5.4.0 github.com/modelcontextprotocol/go-sdk v1.8.0 github.com/parquet-go/parquet-go v0.32.0 @@ -50,19 +45,16 @@ require ( github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10506.0 // indirect github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10506.0 // indirect github.com/dustin/go-humanize v1.1.0 // indirect - github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/go-jose/go-jose/v4 v4.1.5 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/goccy/go-json v0.11.2 // indirect github.com/google/flatbuffers v25.12.19+incompatible // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 // indirect github.com/klauspost/compress v1.20.1 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect - github.com/knadh/koanf/maps v0.1.3 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-isatty v0.0.24 // indirect github.com/mfridman/interpolate v0.0.2 // indirect - github.com/mitchellh/copystructure v1.2.0 // indirect - github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v1.1.0 // indirect github.com/parquet-go/bitpack v1.1.0 // indirect diff --git a/go.sum b/go.sum index 742ca040..f1edba6e 100644 --- a/go.sum +++ b/go.sum @@ -42,8 +42,6 @@ github.com/duckdb/duckdb-go/v2 v2.10506.0 h1:mcZjUQ/kSNeZdtGCqyj5/JEDUdj7/kvILfI github.com/duckdb/duckdb-go/v2 v2.10506.0/go.mod h1:6pLO+4GMcb0q36+z+RlQ7h+MMxum1lL0eAVf4GsPAxs= github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg= github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI= -github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= -github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/go-jose/go-jose/v4 v4.1.5 h1:RjgjO2LOtWOJKUC5wpwY9LR3B3vwVAz6JS2YHfYU6eA= github.com/go-jose/go-jose/v4 v4.1.5/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= @@ -75,16 +73,6 @@ github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLM github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= -github.com/knadh/koanf/maps v0.1.3 h1:P1z7EvTqdFBrPYbzSvorvrpib+sjkUMxf0FVvA5NKK4= -github.com/knadh/koanf/maps v0.1.3/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= -github.com/knadh/koanf/parsers/yaml v1.1.1 h1:u70vV5IyaM0HvONh8HoqBC97oTgO33KcpZbTLiKVinU= -github.com/knadh/koanf/parsers/yaml v1.1.1/go.mod h1:HHmcHXUrp9cOPcuC+2wrr44GTUB0EC+PyfN3HZD9tFg= -github.com/knadh/koanf/providers/confmap v1.0.1 h1:L15hbvMqlvhwUuCtL9BkL+rqiMAjk6cZc8O9XoDtE3A= -github.com/knadh/koanf/providers/confmap v1.0.1/go.mod h1:txHYHiI2hAtF0/0sCmcuol4IDcuQbKTybiB1nOcUo1A= -github.com/knadh/koanf/providers/file v1.2.1 h1:bEWbtQwYrA+W2DtdBrQWyXqJaJSG3KrP3AESOJYp9wM= -github.com/knadh/koanf/providers/file v1.2.1/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA= -github.com/knadh/koanf/v2 v2.3.7 h1:amceufOeoQcq6VFKjm7/ggJ3t0Dkqaxy5fza4j3YgTA= -github.com/knadh/koanf/v2 v2.3.7/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/labstack/echo/v5 v5.4.0 h1:iY674460IvSmUcj7MziL3YrgyTClghz5BpdZoKiGxR4= @@ -93,10 +81,6 @@ github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsRe github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= -github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= -github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= -github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= -github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/modelcontextprotocol/go-sdk v1.8.0 h1:KIvahhYqwtbeniWVPs3TcXEA7b8jEtwfBpOTAI+Urx4= github.com/modelcontextprotocol/go-sdk v1.8.0/go.mod h1:dL7u98E/zjJTGzEq+j30jQ8K2k1mb6LeAH4inEcSGts= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= diff --git a/internal/agent/provider_contract_test.go b/internal/agent/provider_contract_test.go index 2e62b9b3..25306982 100644 --- a/internal/agent/provider_contract_test.go +++ b/internal/agent/provider_contract_test.go @@ -55,8 +55,8 @@ func TestOpenAIOutputItemsReplayVerbatim(t *testing.T) { // Test request replay independently of parser failures: retained output must // replace rebuilt text/calls and calls without executed results must vanish. unsafe := json.RawMessage(`{"type":"function_call","call_id":"not_executed","name":"status","arguments":"{}"}`) - body := captureRequestBody(t, lines(`data: {"type":"response.completed","response":{}}`), func(url string) error { - p, err := NewProvider("openai", "test", "", url) + body := captureRequestBody(t, lines(`data: {"type":"response.completed","response":{}}`), "/v1/responses", func(url string) error { + p, err := NewProvider("openai", "test", "", url+"/v1") if err != nil { return err } @@ -134,13 +134,13 @@ func TestOpenAIEncryptedContentConfigurationError(t *testing.T) { _, _ = io.WriteString(w, `{"error":{"message":"reasoning.encrypted_content is unsupported by this model"}}`) })) defer server.Close() - p, err := NewProvider("openai", "test", "custom-model", server.URL) + p, err := NewProvider("openai", "test", "custom-model", server.URL+"/v1") if err != nil { t.Fatal(err) } err = p.Stream(context.Background(), StreamParams{}, func(StreamEvent) error { return nil }) var apiErr *APIError - if err == nil || !strings.Contains(err.Error(), "reasoning model") || !strings.Contains(err.Error(), "ai.model") || !errors.As(err, &apiErr) || apiErr.StatusCode != 400 { + if err == nil || !strings.Contains(err.Error(), "reasoning model") || !strings.Contains(err.Error(), "FANOUT_AI_MODEL") || !errors.As(err, &apiErr) || apiErr.StatusCode != 400 { t.Errorf("configuration error = %v", err) } if got := clientErrorMessage(err); got != "Fanout could not reach the model provider. Please try again." { @@ -149,8 +149,8 @@ func TestOpenAIEncryptedContentConfigurationError(t *testing.T) { } func TestNewProviderNewestDefaults(t *testing.T) { - for _, kind := range []string{"", "anthropic", "openai"} { - p, err := NewProvider(kind, "test", "", "") + for _, kind := range []string{"anthropic", "openai"} { + p, err := NewProvider(kind, "test", "", "https://provider.example/gateway") if err != nil { t.Fatal(err) } @@ -167,6 +167,14 @@ func TestNewProviderNewestDefaults(t *testing.T) { } } +func TestNewProviderRejectsUnsupportedNames(t *testing.T) { + for _, kind := range []string{"", "OpenAI", " OpenAI ", " openai ", "Anthropic", "unsupported"} { + if _, err := NewProvider(kind, "test", "", "https://provider.example"); err == nil { + t.Errorf("provider %q must be rejected", kind) + } + } +} + func TestAnthropicCacheBreakpoints(t *testing.T) { for _, finalRole := range []Role{RoleUser, RoleTool} { t.Run(string(finalRole), func(t *testing.T) { @@ -176,7 +184,7 @@ func TestAnthropicCacheBreakpoints(t *testing.T) { } else { messages = append(messages, ProviderMessage{Role: RoleTool, ToolResult: &ToolResult{ToolCallID: "b", Content: "last"}}) } - body := captureRequestBody(t, lines(`data: {"type":"message_delta","delta":{"stop_reason":"end_turn"}}`), func(url string) error { + body := captureRequestBody(t, lines(`data: {"type":"message_delta","delta":{"stop_reason":"end_turn"}}`), "/v1/messages", func(url string) error { p, err := NewProvider("anthropic", "test", "", url) if err != nil { return err @@ -268,7 +276,7 @@ func TestRuntimeOpenAITerminalNotices(t *testing.T) { t.Run(tc.name, func(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, tc.sse) })) defer server.Close() - p, err := NewProvider("openai", "test", "", server.URL) + p, err := NewProvider("openai", "test", "", server.URL+"/v1") if err != nil { t.Fatal(err) } diff --git a/internal/agent/provider_http.go b/internal/agent/provider_http.go index 3af87726..d634db5e 100644 --- a/internal/agent/provider_http.go +++ b/internal/agent/provider_http.go @@ -15,23 +15,17 @@ import ( ) func NewProvider(kind, apiKey, model, baseURL string) (Provider, error) { - switch strings.ToLower(strings.TrimSpace(kind)) { - case "", "anthropic": + switch kind { + case "anthropic": if model == "" { model = "claude-sonnet-5-5" } - if baseURL == "" { - baseURL = "https://api.anthropic.com" - } - return &anthropicProvider{apiKey: apiKey, model: model, baseURL: strings.TrimRight(baseURL, "/"), client: modelHTTPClient()}, nil + return &anthropicProvider{apiKey: apiKey, model: model, baseURL: baseURL, client: modelHTTPClient()}, nil case "openai": if model == "" { model = "gpt-6.1-sol" } - if baseURL == "" { - baseURL = "https://api.openai.com" - } - return &openAIProvider{apiKey: apiKey, model: model, baseURL: strings.TrimRight(baseURL, "/"), client: modelHTTPClient()}, nil + return &openAIProvider{apiKey: apiKey, model: model, baseURL: baseURL, client: modelHTTPClient()}, nil default: return nil, fmt.Errorf("unsupported AI provider %q", kind) } @@ -106,11 +100,11 @@ func (p *openAIProvider) Stream(ctx context.Context, params StreamParams, cb fun } body["tools"] = tools } - response, err := postModel(ctx, p.client, p.baseURL+"/v1/responses", p.apiKey, "openai", body) + response, err := postModel(ctx, p.client, p.baseURL+"/responses", p.apiKey, "openai", body) if err != nil { var apiErr *APIError if errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusBadRequest && strings.Contains(strings.ToLower(apiErr.Body), "encrypted") { - return fmt.Errorf("OpenAI configuration error: ai.model must select a reasoning model supporting reasoning.encrypted_content (default gpt-6.1-sol): %w", err) + return fmt.Errorf("OpenAI configuration error: FANOUT_AI_MODEL must select a reasoning model supporting reasoning.encrypted_content (default gpt-6.1-sol): %w", err) } return err } diff --git a/internal/agent/provider_http_test.go b/internal/agent/provider_http_test.go index 8bf0079c..04feac58 100644 --- a/internal/agent/provider_http_test.go +++ b/internal/agent/provider_http_test.go @@ -12,6 +12,7 @@ import ( "testing" agtypes "github.com/ag-ui-protocol/ag-ui/sdks/community/go/pkg/core/types" + "github.com/labstack/fanout/internal/config" ) // lines joins SSE lines with \n. @@ -19,6 +20,39 @@ func lines(ss ...string) string { return strings.Join(ss, "\n") + "\n" } +func TestProviderBaseURLPreservesGatewayPath(t *testing.T) { + for _, tc := range []struct { + provider, basePath, requestPath, response string + }{ + {"openai", "/gateway/v1", "/gateway/v1/responses", lines(`data: {"type":"response.completed","response":{}}`)}, + {"anthropic", "/gateway", "/gateway/v1/messages", lines(`data: {"type":"message_delta","delta":{"stop_reason":"end_turn"}}`, `data: {"type":"message_stop"}`)}, + } { + for _, slash := range []string{"", "/"} { + t.Run(tc.provider+slash, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != tc.requestPath { + t.Errorf("request = %s %s, want POST %s", r.Method, r.URL.Path, tc.requestPath) + } + if tc.provider == "openai" && r.Header.Get("Authorization") != "Bearer test-key" || tc.provider == "anthropic" && r.Header.Get("X-Api-Key") != "test-key" { + t.Error("provider credential was not mapped to the correct header") + } + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, tc.response) + })) + defer server.Close() + cfg := config.Config{AIProvider: tc.provider, AnthropicBaseURL: server.URL + tc.basePath + slash, OpenAIBaseURL: server.URL + tc.basePath + slash} + p, err := NewProvider(tc.provider, "test-key", "", cfg.AIBaseURL()) + if err != nil { + t.Fatal(err) + } + if err := p.Stream(context.Background(), StreamParams{}, func(StreamEvent) error { return nil }); err != nil { + t.Fatal(err) + } + }) + } + } +} + // collect runs a parser over SSE input and buckets the emitted events. func collect(t *testing.T, parse func(io.Reader, func(StreamEvent) error) error, input string) (texts []string, toolCalls []ToolCall, stopReasons []string, errs []string, parseErr error) { t.Helper() @@ -53,7 +87,7 @@ func openAIStream(t *testing.T) func(io.Reader, func(StreamEvent) error) error { _, _ = io.Copy(w, reader) })) defer server.Close() - provider := &openAIProvider{apiKey: "sk-test", model: "gpt-6.1-sol", baseURL: server.URL, client: server.Client()} + provider := &openAIProvider{apiKey: "sk-test", model: "gpt-6.1-sol", baseURL: server.URL + "/v1", client: server.Client()} return provider.Stream(context.Background(), StreamParams{}, cb) } } @@ -317,10 +351,13 @@ func TestParseAnthropicSkipsNonDataLines(t *testing.T) { // captureRequestBody serves one canned SSE response and returns the decoded // request body the provider sent. -func captureRequestBody(t *testing.T, sseBody string, run func(baseURL string) error) map[string]any { +func captureRequestBody(t *testing.T, sseBody, requestPath string, run func(baseURL string) error) map[string]any { t.Helper() var body map[string]any server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != requestPath { + t.Errorf("request path = %q, want %q", r.URL.Path, requestPath) + } if err := json.NewDecoder(r.Body).Decode(&body); err != nil { t.Errorf("decode request body: %v", err) } @@ -335,8 +372,8 @@ func captureRequestBody(t *testing.T, sseBody string, run func(baseURL string) e } func TestOpenAIResponsesRequestBodyShape(t *testing.T) { - body := captureRequestBody(t, lines(`data: {"type":"response.completed","response":{}}`), func(baseURL string) error { - provider, err := NewProvider("openai", "sk-test", "", baseURL+"/") + body := captureRequestBody(t, lines(`data: {"type":"response.completed","response":{}}`), "/v1/responses", func(baseURL string) error { + provider, err := NewProvider("openai", "sk-test", "", baseURL+"/v1") if err != nil { return err } @@ -414,7 +451,7 @@ func TestOpenAIResponsesReasoningReplayWithinRun(t *testing.T) { } })) defer server.Close() - provider := &openAIProvider{apiKey: "sk-test", model: "gpt-6.1-sol", baseURL: server.URL, client: server.Client()} + provider := &openAIProvider{apiKey: "sk-test", model: "gpt-6.1-sol", baseURL: server.URL + "/v1", client: server.Client()} runtime := NewRuntime(provider, &fakeTools{execution: ToolExecution{Content: `{"ok":true}`}}, nil) emitter, output := newTestEmitter() messages := []agtypes.Message{{ID: "user-1", Role: agtypes.RoleUser, Content: "status?"}} @@ -459,7 +496,7 @@ func TestAnthropicRequestBodyShape(t *testing.T) { `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ok"}}`, `data: {"type":"message_delta","delta":{"stop_reason":"end_turn"}}`, ) - body := captureRequestBody(t, sseBody, func(baseURL string) error { + body := captureRequestBody(t, sseBody, "/v1/messages", func(baseURL string) error { provider := &anthropicProvider{apiKey: "sk-test", model: "claude-sonnet-5-5", baseURL: baseURL, client: http.DefaultClient} params := StreamParams{ System: "be helpful", diff --git a/internal/api/health_test.go b/internal/api/health_test.go index 02ce36b3..f6b407f0 100644 --- a/internal/api/health_test.go +++ b/internal/api/health_test.go @@ -78,7 +78,7 @@ func TestReadiness_NilDuck(t *testing.T) { func TestReadinessReportsSizingResolvedByLoader(t *testing.T) { environ := []string{ "FANOUT_AUTH_CODE_SECRET=0123456789abcdef0123456789abcdef", - "FANOUT_AI_API_KEY=sk-test", + "ANTHROPIC_API_KEY=sk-test", "FANOUT_SMTP_HOST=smtp.example.com", "FANOUT_SMTP_USERNAME=user", "FANOUT_SMTP_PASSWORD=pass", diff --git a/internal/api/oauth.go b/internal/api/oauth.go index 4f2b7bc3..eb5909a7 100644 --- a/internal/api/oauth.go +++ b/internal/api/oauth.go @@ -71,7 +71,7 @@ func NewMCPAuthorization(store *appauth.OAuthStore, users *appauth.UserStore, re // registration). Bound how many clients one source can create before // the janitor collects the abandoned ones. The budget is per client // IP, which collapses to one bucket when Fanout sits behind a proxy - // and server.trusted_proxy_cidrs is unset, so it is set well above a + // and FANOUT_TRUSTED_PROXY_CIDRS is unset, so it is set well above a // plausible team rollout rather than at the tightest defensible value. registerLimiter: appauth.NewKeyedLimiter(60, 15*time.Minute), }, nil diff --git a/internal/config/config.go b/internal/config/config.go index 54ed8cbb..818cd965 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -14,43 +14,79 @@ import ( const maxIngestBatchSize = 50_000 -// Config is Fanout's canonical configuration schema. Public names use -// FANOUT_ plus the shortest stable term that is clear in a docker run, and the -// same terminology across YAML, environment variables, Go, logs, and docs. +const ( + DefaultAddr = "127.0.0.1:7520" + DefaultAnthropicBaseURL = "https://api.anthropic.com" + DefaultOpenAIBaseURL = "https://api.openai.com/v1" +) + +type providerSettings struct { + keyEnv, baseURLEnv, defaultBaseURL string + key, baseURL func(Config) string +} + +var providers = map[string]providerSettings{ + "anthropic": { + keyEnv: "ANTHROPIC_API_KEY", baseURLEnv: "ANTHROPIC_BASE_URL", defaultBaseURL: DefaultAnthropicBaseURL, + key: func(c Config) string { return c.AnthropicAPIKey }, baseURL: func(c Config) string { return c.AnthropicBaseURL }, + }, + "openai": { + keyEnv: "OPENAI_API_KEY", baseURLEnv: "OPENAI_BASE_URL", defaultBaseURL: DefaultOpenAIBaseURL, + key: func(c Config) string { return c.OpenAIAPIKey }, baseURL: func(c Config) string { return c.OpenAIBaseURL }, + }, +} + +func providerForEnvironment(name string) string { + for provider, settings := range providers { + if name == settings.keyEnv || name == settings.baseURLEnv { + return provider + } + } + return "" +} + +// Config is Fanout's canonical configuration schema. Custom environment names +// use FANOUT_; provider credentials and endpoints use their vendor's names. // Elapsed-time settings use time.Duration with unit-bearing values. type Config struct { // Addr binds the browser, API, MCP, OTLP/gRPC and OTLP/HTTP listener. - Addr string `koanf:"server.addr" env:"FANOUT_ADDR" default:":7520"` - DataDir string `koanf:"storage.data_dir" env:"FANOUT_DATA_DIR" default:"./data"` - IngestBatchSize int `koanf:"ingest.batch_size" env:"FANOUT_INGEST_BATCH_SIZE" default:"50000"` + Addr string `group:"server" env:"FANOUT_ADDR" default:"127.0.0.1:7520"` + // DataDir stores telemetry, query state, and the control database. + DataDir string `group:"storage" env:"FANOUT_DATA_DIR" default:"./data"` + // IngestBatchSize limits telemetry rows per batch; valid values are 1–50000. + IngestBatchSize int `group:"ingest" env:"FANOUT_INGEST_BATCH_SIZE" default:"50000"` // IngestMaxInFlightBytes caps the decoded telemetry the process will hold // across concurrent requests. Concurrency is HTTP/2 streams times // connections and each handler holds its batch until the commit is // durable, so without a ceiling a burst is bounded only by memory. Zero is // unbounded, which is the behaviour that predates this setting. - IngestMaxInFlightBytes int `koanf:"ingest.max_in_flight_bytes" env:"FANOUT_INGEST_MAX_IN_FLIGHT_BYTES" default:"268435456"` - RollupInterval time.Duration `koanf:"storage.rollup_interval" env:"FANOUT_ROLLUP_INTERVAL" default:"1m"` - MCPEnabled bool `koanf:"mcp.enabled" env:"FANOUT_MCP_ENABLED" default:"true"` - RetentionDays int `koanf:"storage.retention_days" env:"FANOUT_RETENTION_DAYS" default:"30"` + IngestMaxInFlightBytes int `group:"ingest" env:"FANOUT_INGEST_MAX_IN_FLIGHT_BYTES" default:"268435456"` + // RollupInterval schedules analytical rollups; use a Go duration of at least 1s. + RollupInterval time.Duration `group:"storage" env:"FANOUT_ROLLUP_INTERVAL" default:"1m"` + // MCPEnabled serves the MCP endpoint at /mcp. + MCPEnabled bool `group:"mcp" env:"FANOUT_MCP_ENABLED" default:"true"` + // RetentionDays sets telemetry retention in days; zero disables expiry. + RetentionDays int `group:"storage" env:"FANOUT_RETENTION_DAYS" default:"30"` // MaintenanceInterval controls Parquet retention and compaction, and // query-cache checkpointing. - MaintenanceInterval time.Duration `koanf:"storage.maintenance_interval" env:"FANOUT_MAINTENANCE_INTERVAL" default:"1h"` + MaintenanceInterval time.Duration `group:"storage" env:"FANOUT_MAINTENANCE_INTERVAL" default:"1h"` // RollupSkipToLatest, set once at boot, advances the service and edge rollup // watermarks to the current max ingested timestamp. Existing history is not // aggregated into those analytical rollups. Completed-batch trace caches // still rebuild all retained history in bounded passes; // uncached files remain immediately queryable. Useful for large pre-seeded // datasets (benchmarks, restores); off in normal operation. - RollupSkipToLatest bool `koanf:"storage.rollup_skip_to_latest" env:"FANOUT_ROLLUP_SKIP_TO_LATEST" default:"false"` - DefaultNamespace string `koanf:"ingest.default_namespace" env:"FANOUT_DEFAULT_NAMESPACE" default:"default"` + RollupSkipToLatest bool `group:"storage" env:"FANOUT_ROLLUP_SKIP_TO_LATEST" default:"false"` + // DefaultNamespace supplies the namespace for telemetry without one. + DefaultNamespace string `group:"ingest" env:"FANOUT_DEFAULT_NAMESPACE" default:"default"` // PprofEnabled exposes Go's net/http/pprof handlers at /debug/pprof/* for // CPU/heap/mutex/goroutine profiling under load. Off by default; the routes // require an admin browser session with operations:read. - PprofEnabled bool `koanf:"server.pprof_enabled" env:"FANOUT_PPROF_ENABLED" default:"false"` + PprofEnabled bool `group:"server" env:"FANOUT_PPROF_ENABLED" default:"false"` // The DuckDB knobs below self-size where possible and otherwise default to // values validated on the reference deployment target, a small shared VM // (Hetzner CPX32: 4 vCPU, 8 GB RAM, 160 GB disk). There the self-sizing - // resolves to a ~6.4 GB memory cap and 4 query threads (deterministic from + // resolves to a ~4.8 GB memory cap and 4 query threads (deterministic from // 8 GB / 4 vCPU). Measure the target host with the external fanout-bench // harness before setting // production limits; throughput and memory use vary with CPU, disk, and the @@ -66,57 +102,98 @@ type Config struct { // resolveSizing. DuckDB's own default is 80% of detected RAM, which is // calculated as though DuckDB owned the machine and has been observed to // get the process OOM-killed once the Go heap is added on top. - DuckDBMemory string `koanf:"storage.duckdb.memory" env:"FANOUT_DUCKDB_MEMORY"` + DuckDBMemory string `group:"storage" env:"FANOUT_DUCKDB_MEMORY"` // DuckDBThreads caps DuckDB's global query worker pool. Zero leaves // DuckDB's own default in place (one worker per core). Set it to leave // cores free for ingest on a query-heavy co-tenant host. - DuckDBThreads int `koanf:"storage.duckdb.threads" env:"FANOUT_DUCKDB_THREADS"` + DuckDBThreads int `group:"storage" env:"FANOUT_DUCKDB_THREADS"` // DuckDBMaxConns caps the DuckDB connection pool. Reads scan immutable Parquet // concurrently; rollup-cache writes are serialized by the query write gate. // Zero means "size it from the machine" — the same spelling DuckDBThreads // uses for deferring to a default. Resolution happens in resolveSizing and // is reported in the startup configuration log. - DuckDBMaxConns int `koanf:"storage.duckdb.max_connections" env:"FANOUT_DUCKDB_MAX_CONNECTIONS" default:"0"` - AlertEnabled bool `koanf:"alerts.enabled" env:"FANOUT_ALERTS_ENABLED" default:"true"` - AlertEvaluationInterval time.Duration `koanf:"alerts.evaluation_interval" env:"FANOUT_ALERTS_EVALUATION_INTERVAL" default:"30s"` - AlertHistoryDays int `koanf:"alerts.history_days" env:"FANOUT_ALERTS_HISTORY_DAYS" default:"7"` - AIProvider string `koanf:"ai.provider" env:"FANOUT_AI_PROVIDER" default:"anthropic"` - AIAPIKey string `koanf:"ai.api_key" env:"FANOUT_AI_API_KEY" secret:"true"` + DuckDBMaxConns int `group:"storage" env:"FANOUT_DUCKDB_MAX_CONNECTIONS" default:"0"` + // AlertEnabled runs the alert evaluation engine. + AlertEnabled bool `group:"alerts" env:"FANOUT_ALERTS_ENABLED" default:"true"` + // AlertEvaluationInterval schedules alert evaluation; use a Go duration of at least 1s. + AlertEvaluationInterval time.Duration `group:"alerts" env:"FANOUT_ALERTS_EVALUATION_INTERVAL" default:"30s"` + // AlertHistoryDays sets alert history retention in days. + AlertHistoryDays int `group:"alerts" env:"FANOUT_ALERTS_HISTORY_DAYS" default:"7"` + // AIProvider enables chat by selecting exactly anthropic or openai. Empty disables chat. + AIProvider string `group:"ai" env:"FANOUT_AI_PROVIDER" default:""` // AIModel overrides the provider's default: claude-sonnet-5-5 for Anthropic // and gpt-6.1-sol for OpenAI. The OpenAI provider expects a reasoning model // supporting reasoning.encrypted_content for stateless Responses replay. // A 400 rejecting encrypted content is reported as a server-side configuration - // error; choose a compatible reasoning model with ai.model. - AIModel string `koanf:"ai.model" env:"FANOUT_AI_MODEL"` - AIBaseURL string `koanf:"ai.base_url" env:"FANOUT_AI_BASE_URL"` - SMTPHost string `koanf:"smtp.host" env:"FANOUT_SMTP_HOST"` - SMTPPort int `koanf:"smtp.port" env:"FANOUT_SMTP_PORT" default:"587"` - SMTPUser string `koanf:"smtp.username" env:"FANOUT_SMTP_USERNAME"` - SMTPPass string `koanf:"smtp.password" env:"FANOUT_SMTP_PASSWORD" secret:"true"` - SMTPFrom string `koanf:"smtp.from" env:"FANOUT_SMTP_FROM"` - AuthMode string `koanf:"auth.mode" env:"FANOUT_AUTH_MODE" default:"local"` - PublicURL string `koanf:"server.public_url" env:"FANOUT_PUBLIC_URL"` - AuthCodeSecret string `koanf:"auth.code_secret" env:"FANOUT_AUTH_CODE_SECRET" secret:"true"` - SelfSignup bool `koanf:"auth.self_signup" env:"FANOUT_SELF_SIGNUP" default:"false"` - SessionIdleTTL time.Duration `koanf:"auth.session_idle_ttl" env:"FANOUT_SESSION_IDLE_TTL" default:"12h"` - SessionAbsoluteTTL time.Duration `koanf:"auth.session_absolute_ttl" env:"FANOUT_SESSION_ABSOLUTE_TTL" default:"168h"` - OIDCIssuerURL string `koanf:"auth.oidc.issuer_url" env:"FANOUT_OIDC_ISSUER_URL"` - OIDCClientID string `koanf:"auth.oidc.client_id" env:"FANOUT_OIDC_CLIENT_ID"` - OIDCClientSecret string `koanf:"auth.oidc.client_secret" env:"FANOUT_OIDC_CLIENT_SECRET" secret:"true"` - OIDCEmailClaim string `koanf:"auth.oidc.email_claim" env:"FANOUT_OIDC_EMAIL_CLAIM" default:"email"` - OIDCEmailVerification string `koanf:"auth.oidc.email_verification" env:"FANOUT_OIDC_EMAIL_VERIFICATION" default:"required"` - OIDCAutoProvision bool `koanf:"auth.oidc.auto_provision" env:"FANOUT_OIDC_AUTO_PROVISION" default:"false"` - OIDCAllowedGroups string `koanf:"auth.oidc.allowed_groups" env:"FANOUT_OIDC_ALLOWED_GROUPS"` - OIDCAllowedDomains string `koanf:"auth.oidc.allowed_domains" env:"FANOUT_OIDC_ALLOWED_DOMAINS"` - OIDCDefaultRole string `koanf:"auth.oidc.default_role" env:"FANOUT_OIDC_DEFAULT_ROLE" default:"viewer"` - OIDCOperatorGroups string `koanf:"auth.oidc.operator_groups" env:"FANOUT_OIDC_OPERATOR_GROUPS"` - OIDCAdminGroups string `koanf:"auth.oidc.admin_groups" env:"FANOUT_OIDC_ADMIN_GROUPS"` - MetricsToken string `koanf:"metrics.token" env:"FANOUT_METRICS_TOKEN" secret:"true"` - MetricsPublic bool `koanf:"metrics.public" env:"FANOUT_METRICS_PUBLIC" default:"false"` - TrustedProxyCIDRs string `koanf:"server.trusted_proxy_cidrs" env:"FANOUT_TRUSTED_PROXY_CIDRS"` - TLSCertFile string `koanf:"server.tls.cert_file" env:"FANOUT_TLS_CERT_FILE"` - TLSKeyFile string `koanf:"server.tls.key_file" env:"FANOUT_TLS_KEY_FILE"` - resolvedSizing sizingSource + // error; choose a compatible reasoning model with FANOUT_AI_MODEL. + AIModel string `group:"ai" env:"FANOUT_AI_MODEL"` + // AnthropicAPIKey is required when FANOUT_AI_PROVIDER is anthropic. + AnthropicAPIKey string `group:"ai" env:"ANTHROPIC_API_KEY" secret:"true"` + // AnthropicBaseURL defaults to https://api.anthropic.com. Fanout appends + // /v1/messages, preserving any gateway path prefix. Use an HTTP(S) base URL + // without embedded credentials, a query, or a fragment. + AnthropicBaseURL string `group:"ai" env:"ANTHROPIC_BASE_URL"` + // OpenAIAPIKey is required when FANOUT_AI_PROVIDER is openai. + OpenAIAPIKey string `group:"ai" env:"OPENAI_API_KEY" secret:"true"` + // OpenAIBaseURL defaults to https://api.openai.com/v1. Fanout appends + // /responses, preserving the API version and any gateway path prefix. + // Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. + OpenAIBaseURL string `group:"ai" env:"OPENAI_BASE_URL"` + // SMTPHost selects the mail server. Host, username, password, and from must all be set to enable SMTP. + SMTPHost string `group:"smtp" env:"FANOUT_SMTP_HOST"` + // SMTPPort sets the mail server's positive TCP port. + SMTPPort int `group:"smtp" env:"FANOUT_SMTP_PORT" default:"587"` + // SMTPUser authenticates to the mail server; required with SMTP. + SMTPUser string `group:"smtp" env:"FANOUT_SMTP_USERNAME"` + // SMTPPass authenticates to the mail server; required with SMTP. + SMTPPass string `group:"smtp" env:"FANOUT_SMTP_PASSWORD" secret:"true"` + // SMTPFrom sets the email sender address; required with SMTP. + SMTPFrom string `group:"smtp" env:"FANOUT_SMTP_FROM"` + // AuthMode selects local sign-in or oidc; each mode validates its prerequisites. + AuthMode string `group:"auth" env:"FANOUT_AUTH_MODE" default:"local"` + // PublicURL sets the external browser origin. Required for OIDC; HTTPS is required with MCP. + PublicURL string `group:"server" env:"FANOUT_PUBLIC_URL"` + // AuthCodeSecret signs local sign-in codes; required in local mode with at least 32 characters. + AuthCodeSecret string `group:"auth" env:"FANOUT_AUTH_CODE_SECRET" secret:"true"` + // SelfSignup allows verified email addresses to create viewer accounts; requires local auth and SMTP. + SelfSignup bool `group:"auth" env:"FANOUT_SELF_SIGNUP" default:"false"` + // SessionIdleTTL expires inactive sessions; use a Go duration of at least 5m. + SessionIdleTTL time.Duration `group:"auth" env:"FANOUT_SESSION_IDLE_TTL" default:"12h"` + // SessionAbsoluteTTL limits total session lifetime; it must be at least FANOUT_SESSION_IDLE_TTL. + SessionAbsoluteTTL time.Duration `group:"auth" env:"FANOUT_SESSION_ABSOLUTE_TTL" default:"168h"` + // OIDCIssuerURL selects the identity provider issuer; required in OIDC mode. + OIDCIssuerURL string `group:"auth" env:"FANOUT_OIDC_ISSUER_URL"` + // OIDCClientID identifies Fanout's client registration; required in OIDC mode. + OIDCClientID string `group:"auth" env:"FANOUT_OIDC_CLIENT_ID"` + // OIDCClientSecret authenticates Fanout's client registration; required in OIDC mode. + OIDCClientSecret string `group:"auth" env:"FANOUT_OIDC_CLIENT_SECRET" secret:"true"` + // OIDCEmailClaim names the claim containing the user's email address. + OIDCEmailClaim string `group:"auth" env:"FANOUT_OIDC_EMAIL_CLAIM" default:"email"` + // OIDCEmailVerification selects required verified-email claims or issuer trust; issuer trust requires an allowlist. + OIDCEmailVerification string `group:"auth" env:"FANOUT_OIDC_EMAIL_VERIFICATION" default:"required"` + // OIDCAutoProvision creates accounts on OIDC sign-in; requires an allowed group or domain. + OIDCAutoProvision bool `group:"auth" env:"FANOUT_OIDC_AUTO_PROVISION" default:"false"` + // OIDCAllowedGroups lists allowed identity-provider groups, separated by commas. + OIDCAllowedGroups string `group:"auth" env:"FANOUT_OIDC_ALLOWED_GROUPS"` + // OIDCAllowedDomains lists allowed email domains, separated by commas; wildcard * requires verified email. + OIDCAllowedDomains string `group:"auth" env:"FANOUT_OIDC_ALLOWED_DOMAINS"` + // OIDCDefaultRole assigns viewer, operator, or admin to users without a group role mapping. + OIDCDefaultRole string `group:"auth" env:"FANOUT_OIDC_DEFAULT_ROLE" default:"viewer"` + // OIDCOperatorGroups lists identity-provider groups mapped to operator, separated by commas. + OIDCOperatorGroups string `group:"auth" env:"FANOUT_OIDC_OPERATOR_GROUPS"` + // OIDCAdminGroups lists identity-provider groups mapped to admin, separated by commas. + OIDCAdminGroups string `group:"auth" env:"FANOUT_OIDC_ADMIN_GROUPS"` + // MetricsToken authorizes Prometheus scrapes with a bearer token; optional when using an admin session. + MetricsToken string `group:"metrics" env:"FANOUT_METRICS_TOKEN" secret:"true"` + // MetricsPublic allows unauthenticated Prometheus scrapes. + MetricsPublic bool `group:"metrics" env:"FANOUT_METRICS_PUBLIC" default:"false"` + // TrustedProxyCIDRs lists trusted proxy networks, separated by commas; empty trusts no forwarded client addresses. + TrustedProxyCIDRs string `group:"server" env:"FANOUT_TRUSTED_PROXY_CIDRS"` + // TLSCertFile selects the TLS certificate file; requires FANOUT_TLS_KEY_FILE. + TLSCertFile string `group:"server" env:"FANOUT_TLS_CERT_FILE"` + // TLSKeyFile selects the TLS private key file; requires FANOUT_TLS_CERT_FILE. + TLSKeyFile string `group:"server" env:"FANOUT_TLS_KEY_FILE"` + resolvedSizing sizingSource } // LogStartup reports the effective non-secret sizing and security-sensitive @@ -124,10 +201,10 @@ type Config struct { func (c Config) LogStartup() { c.logResolvedSizing(c.resolvedSizing) if !c.SecureCookies() { - slog.Warn("browser session cookies are not Secure; configure server.public_url or local TLS before exposing Fanout") + slog.Warn("browser session cookies are not Secure; configure FANOUT_PUBLIC_URL or local TLS before exposing Fanout") } if strings.TrimSpace(c.PublicURL) != "" && !c.TLSEnabled() && strings.TrimSpace(c.TrustedProxyCIDRs) == "" { - slog.Warn("reverse-proxy client IPs are not trusted; configure server.trusted_proxy_cidrs so audit and rate limits use end-client addresses") + slog.Warn("reverse-proxy client IPs are not trusted; configure FANOUT_TRUSTED_PROXY_CIDRS so audit and rate limits use end-client addresses") } if c.MetricsPublic { slog.Warn("Prometheus metrics are publicly accessible", "path", "/metrics") @@ -166,7 +243,7 @@ func (c Config) ControlSQLitePath() string { } // TLSEnabled reports whether a cert/key pair is configured. When true, HTTP -// serves HTTPS and OTLP gRPC over TLS on server.addr. +// serves HTTPS and OTLP gRPC over TLS on FANOUT_ADDR. func (c Config) TLSEnabled() bool { return strings.TrimSpace(c.TLSCertFile) != "" && strings.TrimSpace(c.TLSKeyFile) != "" @@ -175,113 +252,120 @@ func (c Config) TLSEnabled() bool { // Validate checks that config values are sane. func (c Config) Validate() error { if strings.TrimSpace(c.Addr) == "" { - return fmt.Errorf("server.addr must not be empty") + return fmt.Errorf("FANOUT_ADDR must not be empty") } if strings.TrimSpace(c.DataDir) == "" { - return fmt.Errorf("storage.data_dir must not be empty") + return fmt.Errorf("FANOUT_DATA_DIR must not be empty") } if c.IngestBatchSize <= 0 || c.IngestBatchSize > maxIngestBatchSize { - return fmt.Errorf("ingest.batch_size must be between 1 and %d, got %d", maxIngestBatchSize, c.IngestBatchSize) + return fmt.Errorf("FANOUT_INGEST_BATCH_SIZE must be between 1 and %d, got %d", maxIngestBatchSize, c.IngestBatchSize) } if c.RollupInterval < time.Second { - return fmt.Errorf("storage.rollup_interval must be at least 1s, got %s", c.RollupInterval) + return fmt.Errorf("FANOUT_ROLLUP_INTERVAL must be at least 1s, got %s", c.RollupInterval) } if c.RetentionDays < 0 { - return fmt.Errorf("storage.retention_days must be >= 0, got %d", c.RetentionDays) + return fmt.Errorf("FANOUT_RETENTION_DAYS must be >= 0, got %d", c.RetentionDays) } if c.MaintenanceInterval < time.Second { - return fmt.Errorf("storage.maintenance_interval must be at least 1s, got %s", c.MaintenanceInterval) + return fmt.Errorf("FANOUT_MAINTENANCE_INTERVAL must be at least 1s, got %s", c.MaintenanceInterval) } if c.DuckDBThreads < 0 { - return fmt.Errorf("storage.duckdb.threads must be >= 0, got %d", c.DuckDBThreads) + return fmt.Errorf("FANOUT_DUCKDB_THREADS must be >= 0, got %d", c.DuckDBThreads) } if c.DuckDBMaxConns <= 0 { - return fmt.Errorf("storage.duckdb.max_connections must resolve to > 0, got %d", c.DuckDBMaxConns) + return fmt.Errorf("FANOUT_DUCKDB_MAX_CONNECTIONS must resolve to > 0, got %d", c.DuckDBMaxConns) } if c.AlertEvaluationInterval < time.Second { - return fmt.Errorf("alerts.evaluation_interval must be at least 1s, got %s", c.AlertEvaluationInterval) + return fmt.Errorf("FANOUT_ALERTS_EVALUATION_INTERVAL must be at least 1s, got %s", c.AlertEvaluationInterval) } if c.AlertHistoryDays < 0 { - return fmt.Errorf("alerts.history_days must be >= 0, got %d", c.AlertHistoryDays) + return fmt.Errorf("FANOUT_ALERTS_HISTORY_DAYS must be >= 0, got %d", c.AlertHistoryDays) } if c.MCPEnabled { u, err := url.Parse(c.MCPResourceURL()) if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.Path != "/mcp" || u.RawQuery != "" || u.Fragment != "" { - return fmt.Errorf("server.public_url must be an HTTPS origin when MCP is enabled") + return fmt.Errorf("FANOUT_PUBLIC_URL must be an HTTPS origin when MCP is enabled") } } authMode := strings.ToLower(strings.TrimSpace(c.AuthMode)) if authMode == "" { - return fmt.Errorf("auth.mode must not be empty") + return fmt.Errorf("FANOUT_AUTH_MODE must not be empty") } if authMode != "local" && authMode != "oidc" { - return fmt.Errorf("auth.mode must be local or oidc") + return fmt.Errorf("FANOUT_AUTH_MODE must be local or oidc") } idleTTL := c.SessionIdleTTL absoluteTTL := c.SessionAbsoluteTTL if idleTTL < 5*time.Minute { - return fmt.Errorf("auth.session_idle_ttl must be at least 5m") + return fmt.Errorf("FANOUT_SESSION_IDLE_TTL must be at least 5m") } if absoluteTTL <= 0 || idleTTL > absoluteTTL { - return fmt.Errorf("auth.session_absolute_ttl must be positive and at least auth.session_idle_ttl") + return fmt.Errorf("FANOUT_SESSION_ABSOLUTE_TTL must be positive and at least FANOUT_SESSION_IDLE_TTL") } if authMode == "local" { if len(strings.TrimSpace(c.AuthCodeSecret)) < 32 { - return fmt.Errorf("auth.code_secret must be at least 32 characters") + return fmt.Errorf("FANOUT_AUTH_CODE_SECRET must be at least 32 characters") } } if anySet(c.SMTPHost, c.SMTPUser, c.SMTPPass, c.SMTPFrom) { if !c.SMTPConfigured() { - return fmt.Errorf("smtp.host, smtp.username, smtp.password, and smtp.from must all be set when SMTP is configured") + return fmt.Errorf("FANOUT_SMTP_HOST, FANOUT_SMTP_USERNAME, FANOUT_SMTP_PASSWORD, and FANOUT_SMTP_FROM must all be set when SMTP is configured") } if c.SMTPPort <= 0 { - return fmt.Errorf("smtp.port must be > 0") + return fmt.Errorf("FANOUT_SMTP_PORT must be > 0") } } if c.SelfSignup { if authMode != "local" { - return fmt.Errorf("auth.self_signup requires auth.mode=local") + return fmt.Errorf("FANOUT_SELF_SIGNUP requires FANOUT_AUTH_MODE=local") } if !c.SMTPConfigured() { - return fmt.Errorf("auth.self_signup requires SMTP") + return fmt.Errorf("FANOUT_SELF_SIGNUP requires SMTP") } } if authMode == "oidc" { issuer, err := url.Parse(strings.TrimSpace(c.OIDCIssuerURL)) if err != nil || issuer.Scheme != "https" || issuer.Host == "" { - return fmt.Errorf("auth.oidc.issuer_url must be an HTTPS URL") + return fmt.Errorf("FANOUT_OIDC_ISSUER_URL must be an HTTPS URL") } if strings.TrimSpace(c.OIDCClientID) == "" || strings.TrimSpace(c.OIDCClientSecret) == "" { - return fmt.Errorf("auth.oidc.client_id and auth.oidc.client_secret are required in oidc mode") + return fmt.Errorf("FANOUT_OIDC_CLIENT_ID and FANOUT_OIDC_CLIENT_SECRET are required in oidc mode") } publicURL, err := url.Parse(strings.TrimSpace(c.PublicURL)) if err != nil || publicURL.Scheme != "https" || publicURL.Host == "" { - return fmt.Errorf("server.public_url must be an HTTPS URL in oidc mode") + return fmt.Errorf("FANOUT_PUBLIC_URL must be an HTTPS URL in oidc mode") } if c.OIDCEmailVerification != "required" && c.OIDCEmailVerification != "issuer" { - return fmt.Errorf("auth.oidc.email_verification must be required or issuer") + return fmt.Errorf("FANOUT_OIDC_EMAIL_VERIFICATION must be required or issuer") } if c.OIDCEmailVerification != "required" && csvContains(c.OIDCAllowedDomains, "*") { - return fmt.Errorf("auth.oidc.allowed_domains wildcard requires auth.oidc.email_verification=required") + return fmt.Errorf("FANOUT_OIDC_ALLOWED_DOMAINS wildcard requires FANOUT_OIDC_EMAIL_VERIFICATION=required") } if strings.TrimSpace(c.OIDCEmailClaim) == "" { - return fmt.Errorf("auth.oidc.email_claim must not be empty") + return fmt.Errorf("FANOUT_OIDC_EMAIL_CLAIM must not be empty") } if c.OIDCEmailVerification == "issuer" && strings.TrimSpace(c.OIDCAllowedGroups) == "" && strings.TrimSpace(c.OIDCAllowedDomains) == "" { - return fmt.Errorf("OIDC issuer email verification requires auth.oidc.allowed_groups or auth.oidc.allowed_domains") + return fmt.Errorf("OIDC issuer email verification requires FANOUT_OIDC_ALLOWED_GROUPS or FANOUT_OIDC_ALLOWED_DOMAINS") } if !appauth.ValidRole(c.OIDCDefaultRole) { - return fmt.Errorf("auth.oidc.default_role must be viewer, operator, or admin") + return fmt.Errorf("FANOUT_OIDC_DEFAULT_ROLE must be viewer, operator, or admin") } if c.OIDCAutoProvision && strings.TrimSpace(c.OIDCAllowedGroups) == "" && strings.TrimSpace(c.OIDCAllowedDomains) == "" { - return fmt.Errorf("OIDC auto-provisioning requires auth.oidc.allowed_groups or auth.oidc.allowed_domains") + return fmt.Errorf("OIDC auto-provisioning requires FANOUT_OIDC_ALLOWED_GROUPS or FANOUT_OIDC_ALLOWED_DOMAINS") } } + provider, supported := providers[c.AIProvider] + if c.AIProvider != "" && !supported { + return fmt.Errorf("FANOUT_AI_PROVIDER must be anthropic or openai") + } if c.AgentConfigured() { - switch strings.ToLower(strings.TrimSpace(c.AIProvider)) { - case "", "anthropic", "openai": - default: - return fmt.Errorf("ai.provider must be anthropic or openai") + if strings.TrimSpace(c.AIAPIKey()) == "" { + return fmt.Errorf("%s is required when FANOUT_AI_PROVIDER is %s", provider.keyEnv, c.AIProvider) + } + baseURL := c.AIBaseURL() + u, err := url.Parse(baseURL) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || strings.Contains(baseURL, "#") { + return fmt.Errorf("%s must be an HTTP(S) base URL without credentials, query, or fragment", provider.baseURLEnv) } } for _, raw := range strings.Split(c.TrustedProxyCIDRs, ",") { @@ -290,11 +374,11 @@ func (c Config) Validate() error { continue } if _, _, err := net.ParseCIDR(value); err != nil { - return fmt.Errorf("server.trusted_proxy_cidrs contains invalid CIDR %q", value) + return fmt.Errorf("FANOUT_TRUSTED_PROXY_CIDRS contains invalid CIDR %q", value) } } if anySet(c.TLSCertFile, c.TLSKeyFile) && !c.TLSEnabled() { - return fmt.Errorf("server.tls requires cert_file and key_file") + return fmt.Errorf("FANOUT_TLS_CERT_FILE and FANOUT_TLS_KEY_FILE must both be set") } return nil } @@ -308,7 +392,7 @@ func (c Config) SecureCookies() bool { } // MCPResourceURL returns the canonical OAuth resource URI for the MCP route. -// MCP is served by the main HTTP server, so it always shares server.public_url. +// MCP is served by the main HTTP server, so it always shares FANOUT_PUBLIC_URL. // The localhost value preserves zero-configuration development when no public // origin is configured. func (c Config) MCPResourceURL() string { @@ -329,7 +413,28 @@ func (c Config) SMTPConfigured() bool { // AgentConfigured reports whether the in-process AI agent should be started. func (c Config) AgentConfigured() bool { - return strings.TrimSpace(c.AIAPIKey) != "" + return c.AIProvider != "" +} + +// AIAPIKey returns only the explicitly selected provider's credential. +func (c Config) AIAPIKey() string { + if provider, ok := providers[c.AIProvider]; ok { + return provider.key(c) + } + return "" +} + +// AIBaseURL returns the selected provider's API base, including its default. +func (c Config) AIBaseURL() string { + provider, ok := providers[c.AIProvider] + if !ok { + return "" + } + baseURL := provider.baseURL(c) + if strings.TrimSpace(baseURL) == "" { + return provider.defaultBaseURL + } + return strings.TrimRight(strings.TrimSpace(baseURL), "/") } func csvContains(value, target string) bool { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 30e911fc..42a14c84 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -7,16 +7,12 @@ import ( "strings" "testing" "time" - - "github.com/knadh/koanf/parsers/yaml" - "github.com/knadh/koanf/providers/file" - "github.com/knadh/koanf/v2" ) func validEnvironment() []string { return []string{ "FANOUT_AUTH_CODE_SECRET=0123456789abcdef0123456789abcdef", - "FANOUT_AI_API_KEY=sk-test", + "ANTHROPIC_API_KEY=sk-test", "FANOUT_SMTP_HOST=smtp.example.com", "FANOUT_SMTP_USERNAME=user", "FANOUT_SMTP_PASSWORD=pass", @@ -27,541 +23,197 @@ func validEnvironment() []string { func TestLoadReturnsDefaults(t *testing.T) { cfg, err := Load(LoadOptions{Environ: validEnvironment()}) if err != nil { - t.Fatalf("Load: %v", err) - } - - if cfg.Addr != ":7520" { - t.Errorf("Addr = %q, want %q", cfg.Addr, ":7520") - } - - if cfg.DataDir != "./data" { - t.Errorf("DataDir = %q, want %q", cfg.DataDir, "./data") - } - if cfg.IngestBatchSize != 50000 { - t.Errorf("IngestBatchSize = %d, want %d", cfg.IngestBatchSize, 50000) - } - if cfg.RollupInterval != time.Minute { - t.Errorf("RollupInterval = %s, want %s", cfg.RollupInterval, time.Minute) - } - if cfg.RetentionDays != 30 { - t.Errorf("RetentionDays = %d, want %d", cfg.RetentionDays, 30) - } - if cfg.DefaultNamespace != "default" { - t.Errorf("DefaultNamespace = %q, want %q", cfg.DefaultNamespace, "default") - } - // DuckDBMemory is no longer asserted to be empty: Load resolves it from - // detected memory, and what it resolves to depends on the machine running - // the test. On a host where detection declines it stays empty, which is the - // documented fallback rather than a failure. The resolution rules - // themselves are covered in sizing_test.go. - if cfg.DuckDBThreads != 0 { - t.Errorf("DuckDBThreads = %d, want 0 (DuckDB self-sizes to core count)", cfg.DuckDBThreads) - } - // Sized from the machine rather than fixed at 4, so assert the contract: - // above the write-gate invariant and within the auto-sizing ceiling. - if cfg.DuckDBMaxConns < minDuckDBConns || cfg.DuckDBMaxConns > maxAutoDuckDBConns { - t.Errorf("DuckDBMaxConns = %d, want between %d and %d", - cfg.DuckDBMaxConns, minDuckDBConns, maxAutoDuckDBConns) - } -} - -func TestLoadLayering(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte(`server: - addr: ":1111" -mcp: - enabled: false -metrics: - public: true -auth: - session_idle_ttl: 10h - self_signup: true -`), 0o600); err != nil { - t.Fatalf("write config: %v", err) - } - - cfg, err := Load(LoadOptions{ - Path: path, - Environ: append(validEnvironment(), "FANOUT_ADDR=:2222"), - }) - if err != nil { - t.Fatalf("Load: %v", err) + t.Fatal(err) } - if cfg.Addr != ":2222" { - t.Fatalf("Addr = %q, want environment override", cfg.Addr) + if cfg.Addr != "127.0.0.1:7520" || cfg.DataDir != "./data" || cfg.IngestBatchSize != 50000 || cfg.RollupInterval != time.Minute || cfg.RetentionDays != 30 || cfg.DefaultNamespace != "default" || !cfg.MCPEnabled || !cfg.AlertEnabled || cfg.AIProvider != "" { + t.Fatal("built-in defaults were not resolved") } - if cfg.MCPEnabled || !cfg.MetricsPublic || cfg.SessionIdleTTL != 10*time.Hour || !cfg.SelfSignup { - t.Fatalf("YAML values were not merged: %+v", cfg) + if cfg.DuckDBThreads != 0 || cfg.DuckDBMaxConns < minDuckDBConns || cfg.DuckDBMaxConns > maxAutoDuckDBConns { + t.Fatal("machine sizing did not preserve the connection and thread defaults") } } func TestLoadTypedEnvironmentValues(t *testing.T) { cfg, err := Load(LoadOptions{Environ: append(validEnvironment(), - "FANOUT_MCP_ENABLED=false", - "FANOUT_METRICS_PUBLIC=true", - "FANOUT_SESSION_IDLE_TTL=10h", - "FANOUT_SELF_SIGNUP=true", + "FANOUT_ADDR=:9000", "FANOUT_MCP_ENABLED=false", "FANOUT_METRICS_PUBLIC=true", + "FANOUT_SESSION_IDLE_TTL=10h", "FANOUT_SELF_SIGNUP=true", "FANOUT_RETENTION_DAYS=0", + "FANOUT_ROLLUP_INTERVAL=5m", "FANOUT_MAINTENANCE_INTERVAL=2h", "FANOUT_ALERTS_EVALUATION_INTERVAL=45s", )}) if err != nil { - t.Fatalf("Load: %v", err) + t.Fatal(err) } - if cfg.MCPEnabled || !cfg.MetricsPublic || cfg.SessionIdleTTL != 10*time.Hour || !cfg.SelfSignup { - t.Fatalf("environment values were not decoded: %+v", cfg) + if cfg.Addr != ":9000" || cfg.MCPEnabled || !cfg.MetricsPublic || cfg.SessionIdleTTL != 10*time.Hour || !cfg.SelfSignup || cfg.RetentionDays != 0 || cfg.RollupInterval != 5*time.Minute || cfg.MaintenanceInterval != 2*time.Hour || cfg.AlertEvaluationInterval != 45*time.Second { + t.Fatal("environment values were not decoded") } } func TestLoadTreatsEmptyEnvironmentValuesAsAbsent(t *testing.T) { - cfg, err := Load(LoadOptions{Environ: append(validEnvironment(), - "FANOUT_ADDR=", - "FANOUT_MCP_ENABLED=", - "FANOUT_RETENTION_DAYS=", - )}) - if err != nil { - t.Fatalf("Load: %v", err) - } - if cfg.Addr != ":7520" || !cfg.MCPEnabled || cfg.RetentionDays != 30 { - t.Fatalf("empty environment values erased defaults: %+v", cfg) - } -} - -func TestMCPResourceURL(t *testing.T) { - tests := []struct { - name string - publicURL string - want string - }{ - {name: "local default", want: "https://localhost:7520/mcp"}, - {name: "public origin", publicURL: "https://fanout.example.com", want: "https://fanout.example.com/mcp"}, - {name: "trimmed origin", publicURL: " https://fanout.example.com/ ", want: "https://fanout.example.com/mcp"}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - cfg := Config{PublicURL: tc.publicURL} - if got := cfg.MCPResourceURL(); got != tc.want { - t.Fatalf("MCPResourceURL() = %q, want %q", got, tc.want) - } - }) - } -} - -func TestExampleConfigurationMatchesSchema(t *testing.T) { - path := filepath.Join("..", "..", "fanout.example.yaml") - cfg, err := Load(LoadOptions{ - Path: path, - Environ: validEnvironment(), - }) - if err != nil { - t.Fatalf("Load example: %v", err) - } - if cfg.Addr != ":7520" || cfg.DataDir != "./data" { - t.Fatalf("example defaults = HTTP %q data %q", cfg.Addr, cfg.DataDir) - } - defaults, err := Load(LoadOptions{Environ: validEnvironment()}) + cfg, err := Load(LoadOptions{Environ: append(validEnvironment(), "FANOUT_ADDR=", "FANOUT_MCP_ENABLED=", "FANOUT_RETENTION_DAYS=", "FANOUT_AUTH_CODE_SECRET_FILE=")}) if err != nil { - t.Fatalf("Load defaults: %v", err) - } - if !reflect.DeepEqual(cfg, defaults) { - t.Error("example values do not match built-in defaults") - } - - raw, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read example: %v", err) - } - example := koanf.New(".") - if err := example.Load(file.Provider(path), yaml.Parser()); err != nil { - t.Fatalf("parse example: %v", err) - } - active := make(map[string]bool, len(example.Keys())) - for _, key := range example.Keys() { - active[key] = true - } - - specs, err := configFieldSpecs() - if err != nil { - t.Fatalf("field specs: %v", err) - } - wantSecrets := map[string]bool{ - "ai.api_key": true, - "smtp.password": true, - "auth.code_secret": true, - "auth.oidc.client_secret": true, - "metrics.token": true, - } - foundSecrets := make(map[string]bool, len(wantSecrets)) - rawText := string(raw) - for _, spec := range specs { - if !strings.Contains(rawText, spec.env) { - t.Errorf("example does not document %s", spec.env) - } - if spec.secret != wantSecrets[spec.key] { - t.Errorf("secret classification for %s = %t, want %t", spec.key, spec.secret, wantSecrets[spec.key]) - } - if spec.secret { - foundSecrets[spec.key] = true - } - if spec.secret && active[spec.key] { - t.Errorf("secret key %s must be documented but inactive in the example", spec.key) - } - if spec.secret { - leaf := spec.key[strings.LastIndex(spec.key, ".")+1:] - if !strings.Contains(rawText, "# "+leaf+":") { - t.Errorf("secret key %s is not documented as a commented YAML key", spec.key) - } - } - if !spec.secret && !active[spec.key] { - t.Errorf("non-secret key %s must be active in the example", spec.key) - } + t.Fatal(err) } - if !reflect.DeepEqual(foundSecrets, wantSecrets) { - t.Errorf("secret settings = %v, want exactly %v", foundSecrets, wantSecrets) + if cfg.Addr != "127.0.0.1:7520" || !cfg.MCPEnabled || cfg.RetentionDays != 30 { + t.Fatal("empty values erased defaults") } } func TestConfigurationSchemaUsesUnitBearingDurations(t *testing.T) { specs, err := configFieldSpecs() if err != nil { - t.Fatalf("field specs: %v", err) + t.Fatal(err) } - durationType := reflect.TypeOf(time.Duration(0)) for _, spec := range specs { - key := strings.ToLower(spec.key) - env := strings.ToLower(spec.env) - if strings.Contains(key, "_seconds") || strings.Contains(key, "_ms") || - strings.Contains(env, "_seconds") || strings.Contains(env, "_ms") { - t.Errorf("elapsed-time setting uses a unit suffix: %s / %s", spec.key, spec.env) + if strings.HasSuffix(spec.env, "_SECONDS") || strings.HasSuffix(spec.env, "_MS") { + t.Errorf("unit suffix in %s", spec.env) } - if spec.typ == durationType && !strings.HasSuffix(key, "_interval") && !strings.HasSuffix(key, "_ttl") && !strings.HasSuffix(key, "_retention") { - t.Errorf("duration setting %s must end in _interval or _ttl", spec.key) + if spec.typ == reflect.TypeOf(time.Duration(0)) && !strings.HasSuffix(spec.env, "_INTERVAL") && !strings.HasSuffix(spec.env, "_TTL") { + t.Errorf("duration name %s must end in _INTERVAL or _TTL", spec.env) } } } -func TestLoadDurationIntervals(t *testing.T) { - t.Run("YAML", func(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - document := `storage: - rollup_interval: 5m - maintenance_interval: 1h -alerts: - evaluation_interval: 45s -` - if err := os.WriteFile(path, []byte(document), 0o600); err != nil { - t.Fatalf("write config: %v", err) - } - cfg, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}) - if err != nil { - t.Fatalf("Load: %v", err) - } - if cfg.RollupInterval != 5*time.Minute || - cfg.MaintenanceInterval != time.Hour || - cfg.AlertEvaluationInterval != 45*time.Second { - t.Fatalf("duration values were not decoded: %+v", cfg) - } - }) - - t.Run("environment", func(t *testing.T) { - cfg, err := Load(LoadOptions{Environ: append(validEnvironment(), - "FANOUT_ROLLUP_INTERVAL=5m", - "FANOUT_MAINTENANCE_INTERVAL=1h", - "FANOUT_ALERTS_EVALUATION_INTERVAL=45s", - )}) - if err != nil { - t.Fatalf("Load: %v", err) - } - if cfg.RollupInterval != 5*time.Minute || - cfg.MaintenanceInterval != time.Hour || - cfg.AlertEvaluationInterval != 45*time.Second { - t.Fatalf("duration values were not decoded: %+v", cfg) - } - }) -} - -func TestDockerConfigurationMatchesSchema(t *testing.T) { - path := filepath.Join("..", "..", "fanout.docker.yaml") - cfg, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}) - if err != nil { - t.Fatalf("Load Docker config: %v", err) - } - if cfg.Addr != ":7520" || cfg.DataDir != "/var/lib/fanout/data" { - t.Fatalf("Docker config values = HTTP %q data %q", cfg.Addr, cfg.DataDir) - } -} - -func TestLoadRejectsUnknownInputs(t *testing.T) { - t.Run("YAML key", func(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte("server:\n htpp_addr: ':1111'\n"), 0o600); err != nil { - t.Fatalf("write config: %v", err) - } - _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}) - if err == nil || !strings.Contains(err.Error(), "server.htpp_addr") { - t.Fatalf("error = %v, want unknown YAML key", err) - } - }) - - t.Run("environment variable", func(t *testing.T) { - _, err := Load(LoadOptions{Environ: append(validEnvironment(), "FANOUT_HTPP_ADDR=:1111")}) - if err == nil || !strings.Contains(err.Error(), "FANOUT_HTPP_ADDR") { - t.Fatalf("error = %v, want unknown environment variable", err) - } - }) - - t.Run("removed ingest YAML key", func(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte("ingest:\n public_endpoint: https://ingest.example.com\n"), 0o600); err != nil { - t.Fatalf("write config: %v", err) - } - _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}) - if err == nil || !strings.Contains(err.Error(), "ingest.public_endpoint") { - t.Fatalf("error = %v, want removed YAML key rejected", err) - } - }) - - t.Run("removed ingest environment variable", func(t *testing.T) { - _, err := Load(LoadOptions{Environ: append(validEnvironment(), "FANOUT_INGEST_ENDPOINT=https://ingest.example.com")}) - if err == nil || !strings.Contains(err.Error(), "FANOUT_INGEST_ENDPOINT") { - t.Fatalf("error = %v, want removed environment variable rejected", err) - } - }) - - t.Run("removed YAML keys", func(t *testing.T) { - for _, test := range []struct { - key, document string - }{ - {"agent.provider", "agent:\n provider: anthropic\n"}, - {"agent.api_key", "agent:\n api_key: secret\n"}, - {"agent.model", "agent:\n model: test\n"}, - {"agent.base_url", "agent:\n base_url: https://ai.example.com\n"}, - {"ingest.flush_seconds", "ingest:\n flush_seconds: 15\n"}, - {"storage.rollup_every_seconds", "storage:\n rollup_every_seconds: 60\n"}, - {"storage.merge_every_seconds", "storage:\n merge_every_seconds: 60\n"}, - {"storage.maintenance_every_seconds", "storage:\n maintenance_every_seconds: 3600\n"}, - {"alerts.evaluation_interval_seconds", "alerts:\n evaluation_interval_seconds: 30\n"}, - {"mcp.public_url", "mcp:\n public_url: https://fanout.example.com/mcp\n"}, - } { - t.Run(test.key, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte(test.document), 0o600); err != nil { - t.Fatalf("write config: %v", err) - } - _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}) - if err == nil || !strings.Contains(err.Error(), test.key) { - t.Fatalf("error = %v, want removed key %s rejected", err, test.key) - } - }) - } - }) - - t.Run("removed environment variables", func(t *testing.T) { - for _, name := range []string{ - "FANOUT_ROLLUP_EVERY_SECONDS", - "FANOUT_MERGE_EVERY_SECONDS", - "FANOUT_MERGE_INTERVAL", - "FANOUT_MAINTENANCE_EVERY_SECONDS", - "FANOUT_ALERTS_EVALUATION_INTERVAL_SECONDS", - "FANOUT_MCP_PUBLIC_URL", - } { - t.Run(name, func(t *testing.T) { - _, err := Load(LoadOptions{Environ: append(validEnvironment(), name+"=1")}) - if err == nil || !strings.Contains(err.Error(), name) { - t.Fatalf("error = %v, want removed environment variable %s rejected", err, name) - } - }) - } - }) - - t.Run("near platform variable", func(t *testing.T) { - _, err := Load(LoadOptions{Environ: append(validEnvironment(), "FANOUT_SERVICE_PORTAL=7520")}) - if err == nil || !strings.Contains(err.Error(), "FANOUT_SERVICE_PORTAL") { - t.Fatalf("error = %v, want unknown environment variable", err) - } - }) - - t.Run("malformed environment entry", func(t *testing.T) { - _, err := Load(LoadOptions{Environ: append(validEnvironment(), "FANOUT_ADDR")}) - if err == nil || !strings.Contains(err.Error(), "NAME=value") { - t.Fatalf("error = %v, want malformed environment error", err) +func TestLoadRejectsUnknownAndMalformedEnvironment(t *testing.T) { + for _, entry := range []string{"FANOUT_HTPP_ADDR=:1111", "FANOUT_SERVICE_PORTAL=7520", "FANOUT_ADDR", "FANOUT_ADDR_FILE=/unused", "FANOUT_UNKNOWN_SETTING=synthetic-private-value"} { + _, err := Load(LoadOptions{Environ: append(validEnvironment(), entry)}) + name, _, _ := strings.Cut(entry, "=") + if err == nil || !strings.Contains(err.Error(), name) || strings.Contains(err.Error(), "synthetic-private-value") { + t.Fatalf("expected redacted error naming %s", name) } - }) -} - -func TestLoadIgnoresKubernetesServiceEnvironment(t *testing.T) { - environ := append(validEnvironment(), - "FANOUT_SERVICE_HOST=10.0.0.1", - "FANOUT_SERVICE_PORT=7520", - "FANOUT_SERVICE_PORT_HTTP=7520", - "FANOUT_PORT=tcp://10.0.0.1:7520", - "FANOUT_PORT_7520_TCP_ADDR=10.0.0.1", - ) - if _, err := Load(LoadOptions{Environ: environ}); err != nil { - t.Fatalf("Load with Kubernetes service variables: %v", err) - } -} - -func TestLoadIgnoresEnvironmentOutsideNamespace(t *testing.T) { - if _, err := Load(LoadOptions{Environ: append(validEnvironment(), - "OTHER_APP_SETTING=value", - )}); err != nil { - t.Fatalf("Load with unrelated environment variable: %v", err) } } -func TestLoadAllowsEmptyYAMLSections(t *testing.T) { - for _, document := range []string{ - "metrics:\n", - "auth:\n oidc:\n # client_id: fanout\n", - "server: {}\n", - } { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte(document), 0o600); err != nil { - t.Fatal(err) - } - if _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}); err != nil { - t.Fatalf("Load %q: %v", document, err) +func TestLoadRejectsDuplicateEnvironmentNames(t *testing.T) { + for _, entry := range []string{"FANOUT_ADDR=", "ANTHROPIC_API_KEY=test", "FANOUT_METRICS_TOKEN_FILE="} { + name, _, _ := strings.Cut(entry, "=") + _, err := Load(LoadOptions{Environ: []string{entry, entry}}) + if err == nil || !strings.Contains(err.Error(), "duplicate") || !strings.Contains(err.Error(), name) { + t.Fatalf("expected duplicate error for %s", name) } } } -func TestLoadRejectsYAMLNullValues(t *testing.T) { - for _, key := range []struct { - name, document string - }{ - {"HTTP address", "server:\n addr:\n"}, - {"MCP enabled", "mcp:\n enabled:\n"}, - {"retention", "storage:\n retention_days:\n"}, - } { - t.Run(key.name, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte(key.document), 0o600); err != nil { - t.Fatal(err) - } - _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}) - if err == nil || !strings.Contains(err.Error(), "must not be null") { - t.Fatalf("error = %v, want null rejection", err) - } - }) +func TestLoadIgnoresPlatformAndUnrelatedEnvironment(t *testing.T) { + env := append(validEnvironment(), "FANOUT_SERVICE_HOST=10.0.0.1", "FANOUT_SERVICE_PORT=7520", "FANOUT_SERVICE_PORT_HTTP=7520", "FANOUT_PORT=tcp://10.0.0.1:7520", "FANOUT_PORT_7520_TCP_ADDR=10.0.0.1", "OTHER_APP_SETTING=value", "OPENAI_MODEL=other-model") + if _, err := Load(LoadOptions{Environ: env}); err != nil { + t.Fatal(err) } } -func TestLoadRejectsInvalidFilesAndValues(t *testing.T) { - t.Run("missing file", func(t *testing.T) { - _, err := Load(LoadOptions{Path: filepath.Join(t.TempDir(), "missing.yaml"), Environ: validEnvironment()}) - if err == nil { - t.Fatal("expected missing file error") - } - }) - - for _, test := range []struct { - name, document string - }{ - {"invalid duration", "storage:\n rollup_interval: never\n"}, - {"fractional integer", "smtp:\n port: 25.9\n"}, - {"integer as boolean", "mcp:\n enabled: 2\n"}, - {"YAML keyword as boolean", "alerts:\n enabled: off\n"}, +func TestLoadRejectsInvalidTypedValuesWithoutLeakingValues(t *testing.T) { + for _, entry := range []string{ + "FANOUT_ROLLUP_INTERVAL=private-value", "FANOUT_SMTP_PORT=25.9", "FANOUT_MCP_ENABLED=off", "FANOUT_RETENTION_DAYS=999999999999999999999999999", + "FANOUT_INGEST_BATCH_SIZE=0", "FANOUT_DUCKDB_MAX_CONNECTIONS=-3", "FANOUT_DUCKDB_THREADS=-4", + "FANOUT_ROLLUP_INTERVAL=999ms", "FANOUT_MAINTENANCE_INTERVAL=-5s", "FANOUT_ALERTS_EVALUATION_INTERVAL=0s", } { - t.Run(test.name, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte(test.document), 0o600); err != nil { - t.Fatalf("write config: %v", err) - } - _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}) - if err == nil { - t.Fatal("expected type error") + env := validEnvironment() + name, _, _ := strings.Cut(entry, "=") + for i, existing := range env { + if strings.HasPrefix(existing, name+"=") { + env = append(env[:i], env[i+1:]...) + break } - }) - } - - t.Run("invalid merged config", func(t *testing.T) { - _, err := Load(LoadOptions{Environ: append(validEnvironment(), "FANOUT_INGEST_BATCH_SIZE=0")}) - if err == nil || !strings.Contains(err.Error(), "batch_size") { - t.Fatalf("error = %v, want validation error", err) - } - }) - - t.Run("explicit empty auth mode", func(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte("auth:\n mode: \"\"\n"), 0o600); err != nil { - t.Fatal(err) } - _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}) - if err == nil || !strings.Contains(err.Error(), "auth.mode must not be empty") { - t.Fatalf("error = %v, want explicit empty auth-mode rejection", err) + _, err := Load(LoadOptions{Environ: append(env, entry)}) + if err == nil || !strings.Contains(err.Error(), name) || strings.Contains(err.Error(), "private-value") { + t.Fatalf("expected redacted validation error for %s", name) } - }) - - for _, test := range []struct { - name, assignment, key string - }{ - {"negative max connections is not auto", "FANOUT_DUCKDB_MAX_CONNECTIONS=-3", "max_connections"}, - {"zero alert interval", "FANOUT_ALERTS_EVALUATION_INTERVAL=0s", "evaluation_interval"}, - {"negative maintenance interval", "FANOUT_MAINTENANCE_INTERVAL=-5s", "maintenance_interval"}, - {"negative DuckDB threads", "FANOUT_DUCKDB_THREADS=-4", "duckdb.threads"}, - } { - t.Run(test.name, func(t *testing.T) { - _, err := Load(LoadOptions{Environ: append(validEnvironment(), test.assignment)}) - if err == nil || !strings.Contains(err.Error(), test.key) { - t.Fatalf("error = %v, want validation error for %s", err, test.key) - } - }) - } - - for _, test := range []struct { - name, assignment, key string - }{ - {"subsecond rollup interval", "FANOUT_ROLLUP_INTERVAL=999ms", "rollup_interval"}, - {"subsecond maintenance interval", "FANOUT_MAINTENANCE_INTERVAL=500ms", "maintenance_interval"}, - {"subsecond alert interval", "FANOUT_ALERTS_EVALUATION_INTERVAL=999ms", "evaluation_interval"}, - } { - t.Run(test.name, func(t *testing.T) { - _, err := Load(LoadOptions{Environ: append(validEnvironment(), test.assignment)}) - if err == nil || !strings.Contains(err.Error(), test.key) { - t.Fatalf("error = %v, want lower-bound validation for %s", err, test.key) - } - }) } } func TestLoadIgnoresDotenvInputs(t *testing.T) { dir := t.TempDir() if err := os.WriteFile(filepath.Join(dir, ".env"), []byte("FANOUT_ADDR=:9999\n"), 0o600); err != nil { - t.Fatalf("write .env: %v", err) + t.Fatal(err) } t.Chdir(dir) - cfg, err := Load(LoadOptions{Environ: validEnvironment()}) if err != nil { - t.Fatalf("Load: %v", err) + t.Fatal(err) } - if cfg.Addr != ":7520" { - t.Fatalf("Addr = %q, want default", cfg.Addr) + if cfg.Addr != "127.0.0.1:7520" { + t.Fatal("binary loaded dotenv") } } -func TestLoadErrorsDoNotContainSecrets(t *testing.T) { - const secret = "do-not-print-this-password" - path := filepath.Join(t.TempDir(), "fanout.yaml") - document := "smtp:\n password: " + secret + "\n port: 25.9\n" - if err := os.WriteFile(path, []byte(document), 0o600); err != nil { +func TestLoadSecretFiles(t *testing.T) { + names := []string{"FANOUT_AUTH_CODE_SECRET", "FANOUT_OIDC_CLIENT_SECRET", "FANOUT_SMTP_PASSWORD", "FANOUT_METRICS_TOKEN", "ANTHROPIC_API_KEY", "OPENAI_API_KEY"} + specs, err := configFieldSpecs() + if err != nil { t.Fatal(err) } - _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}) - if err == nil { - t.Fatal("expected decode error") - } - if strings.Contains(err.Error(), secret) { - t.Fatalf("error leaked secret: %v", err) + for _, name := range names { + for _, tc := range []struct{ name, suffix, retained string }{ + {"no newline", "", ""}, + {"LF", "\n", ""}, + {"CRLF", "\r\n", ""}, + {"two LF", "\n\n", "\n"}, + {"two CRLF", "\r\n\r\n", "\r\n"}, + {"bare CR", "\r", "\r"}, + {"spaces", " \n", " "}, + } { + t.Run(name+"/"+tc.name, func(t *testing.T) { + secret := strings.Repeat("x", 32) + path := filepath.Join(t.TempDir(), "secret") + if err := os.WriteFile(path, []byte(secret+tc.suffix), 0o644); err != nil { + t.Fatal(err) + } + env := []string{} + for _, entry := range validEnvironment() { + if !strings.HasPrefix(entry, name+"=") { + env = append(env, entry) + } + } + env = append(env, name+"=", name+"_FILE="+path) + if provider := providerForEnvironment(name); provider != "" { + env = append(env, "FANOUT_AI_PROVIDER="+provider) + } + cfg, err := Load(LoadOptions{Environ: env}) + if err != nil { + t.Fatal(err) + } + want := secret + tc.retained + for _, spec := range specs { + if spec.env == name && reflect.ValueOf(cfg).Field(spec.index).String() != want { + t.Fatalf("%s did not preserve exact file contents", name) + } + } + }) + } } } -func TestLoadRequiresPrivatePermissionsForConfigSecrets(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte("ai:\n api_key: secret-from-file\n"), 0o644); err != nil { - t.Fatal(err) +func TestLoadRejectsConflictingOrInvalidSecretFiles(t *testing.T) { + const name = "FANOUT_METRICS_TOKEN" + const secret = "synthetic-private-value" + path := filepath.Join(t.TempDir(), "secret") + for _, content := range []string{"", "\n", "\r\n", secret} { + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + env := append(validEnvironment(), name+"_FILE="+path) + if content == secret { + env = append(env, name+"="+secret) + } + _, err := Load(LoadOptions{Environ: env}) + if err == nil || !strings.Contains(err.Error(), name+"_FILE") || !strings.Contains(err.Error(), path) || strings.Contains(err.Error(), secret) { + t.Fatal("expected redacted file error naming variable and path") + } } - _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}) - if err == nil || !strings.Contains(err.Error(), "must not be accessible by group or others") { - t.Fatalf("error = %v, want file permission rejection", err) + for _, invalidPath := range []string{path + "-missing", filepath.Dir(path)} { + _, err := Load(LoadOptions{Environ: append(validEnvironment(), name+"_FILE="+invalidPath)}) + if err == nil || !strings.Contains(err.Error(), name+"_FILE") || !strings.Contains(err.Error(), invalidPath) { + t.Fatal("expected file read error naming variable and path") + } + } +} + +func TestMCPResourceURL(t *testing.T) { + for _, tc := range []struct{ publicURL, want string }{ + {"", "https://localhost:7520/mcp"}, {"https://fanout.example.com", "https://fanout.example.com/mcp"}, {" https://fanout.example.com/ ", "https://fanout.example.com/mcp"}, + } { + if got := (Config{PublicURL: tc.publicURL}).MCPResourceURL(); got != tc.want { + t.Fatalf("MCPResourceURL = %s, want %s", got, tc.want) + } } } @@ -577,7 +229,7 @@ func TestValidate(t *testing.T) { AlertEvaluationInterval: 30 * time.Second, AlertHistoryDays: 7, AIProvider: "anthropic", - AIAPIKey: "sk-test", + AnthropicAPIKey: "sk-test", SMTPHost: "smtp.example.com", SMTPPort: 587, SMTPUser: "user", @@ -659,8 +311,8 @@ func TestValidate(t *testing.T) { t.Run("local mode allows absent SMTP and agent", func(t *testing.T) { c := valid c.SMTPHost, c.SMTPUser, c.SMTPPass, c.SMTPFrom = "", "", "", "" - c.AIAPIKey = "" - c.AIProvider = "ignored-without-a-key" + c.AnthropicAPIKey = "" + c.AIProvider = "" if err := c.Validate(); err != nil { t.Fatalf("minimal local config should pass: %v", err) } @@ -728,7 +380,7 @@ func TestValidate(t *testing.T) { } { c.PublicURL = invalid if err := c.Validate(); err == nil { - t.Errorf("server.public_url %q should be rejected with MCP enabled", invalid) + t.Errorf("FANOUT_PUBLIC_URL %q should be rejected with MCP enabled", invalid) } } }) @@ -753,34 +405,34 @@ func TestFieldSpecsRejectInvalidSchemas(t *testing.T) { want string }{ { - name: "duplicate key", + name: "secret file name collision", typ: reflect.TypeOf(struct { - A string `koanf:"same" env:"FANOUT_A"` - B string `koanf:"same" env:"FANOUT_B"` + A string `env:"FANOUT_SECRET" secret:"true"` + B string `env:"FANOUT_SECRET_FILE"` }{}), - want: "share key", + want: "share environment variable", }, { name: "duplicate environment", typ: reflect.TypeOf(struct { - A string `koanf:"a" env:"FANOUT_SAME"` - B string `koanf:"b" env:"FANOUT_SAME"` + A string `env:"FANOUT_SAME"` + B string `env:"FANOUT_SAME"` }{}), want: "share environment variable", }, { name: "missing tag", typ: reflect.TypeOf(struct { - A string `koanf:"a"` + A string }{}), want: "must declare", }, { name: "missing prefix", typ: reflect.TypeOf(struct { - A string `koanf:"a" env:"A"` + A string `env:"A"` }{}), - want: "lacks FANOUT_ prefix", + want: "outside the supported configuration contract", }, } for _, test := range tests { @@ -812,31 +464,3 @@ func TestSecureCookies(t *testing.T) { }) } } - -func TestRemovedListenerAndEndpointSettingsRejected(t *testing.T) { - t.Run("old server address", func(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte("server:\n http_addr: ':7520'\n"), 0o600); err != nil { - t.Fatal(err) - } - if _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}); err == nil || !strings.Contains(err.Error(), "server.http_addr") { - t.Fatalf("removed key: %v", err) - } - }) - for _, key := range []string{"otlp_grpc_addr", "otlp_http_addr", "advertised_endpoint"} { - t.Run(key, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte("ingest:\n "+key+": ':4317'\n"), 0o600); err != nil { - t.Fatal(err) - } - if _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}); err == nil || !strings.Contains(err.Error(), key) { - t.Fatalf("removed key: %v", err) - } - }) - } - for _, name := range []string{"FANOUT_HTTP_ADDR", "FANOUT_OTLP_GRPC_ADDR", "FANOUT_OTLP_HTTP_ADDR", "FANOUT_INGEST_ADVERTISED_ENDPOINT"} { - if _, err := Load(LoadOptions{Environ: append(validEnvironment(), name+"=:4317")}); err == nil || !strings.Contains(err.Error(), name) { - t.Fatalf("removed variable: %v", err) - } - } -} diff --git a/internal/config/loader.go b/internal/config/loader.go index 6c6945d3..1398f19a 100644 --- a/internal/config/loader.go +++ b/internal/config/loader.go @@ -8,23 +8,16 @@ import ( "strconv" "strings" "time" - - "github.com/go-viper/mapstructure/v2" - "github.com/knadh/koanf/parsers/yaml" - "github.com/knadh/koanf/providers/confmap" - "github.com/knadh/koanf/providers/file" - "github.com/knadh/koanf/v2" ) -// LoadOptions selects the explicit configuration inputs. A nil Environ uses -// os.Environ; an empty slice deliberately supplies no environment variables. +// LoadOptions selects the environment. A nil Environ uses os.Environ; +// an empty slice deliberately supplies no environment variables. type LoadOptions struct { - Path string Environ []string } type fieldSpec struct { - key string + index int env string typ reflect.Type defaultValue any @@ -32,79 +25,71 @@ type fieldSpec struct { secret bool } -// Load resolves defaults, an optional YAML document, and process environment -// in that order, then sizes and validates the effective configuration. +// Load resolves built-in defaults and process environment, then sizes and +// validates the effective configuration. func Load(options LoadOptions) (Config, error) { specs, err := configFieldSpecs() if err != nil { return Config{}, err } - - specByKey := make(map[string]fieldSpec, len(specs)) - specByEnv := make(map[string]fieldSpec, len(specs)) - defaults := make(map[string]any) + known := make(map[string]bool, len(specs)) + var cfg Config + value := reflect.ValueOf(&cfg).Elem() for _, spec := range specs { - specByKey[spec.key] = spec - specByEnv[spec.env] = spec - if spec.hasDefault { - defaults[spec.key] = spec.defaultValue - } - } - - k := koanf.New(".") - if err := k.Load(confmap.Provider(defaults, "."), nil); err != nil { - return Config{}, fmt.Errorf("load configuration defaults: %w", err) - } - - if options.Path != "" { - info, statErr := os.Stat(options.Path) - if statErr != nil { - return Config{}, fmt.Errorf("load config %q: %w", options.Path, statErr) - } - if !info.Mode().IsRegular() { - return Config{}, fmt.Errorf("load config %q: not a regular file", options.Path) + known[spec.env] = true + if spec.secret { + known[spec.env+"_FILE"] = true } - fromFile := koanf.New(".") - if err := fromFile.Load(file.Provider(options.Path), yaml.Parser()); err != nil { - return Config{}, fmt.Errorf("load config %q: %w", options.Path, err) - } - if err := validateYAMLKeys(fromFile, specByKey); err != nil { - return Config{}, fmt.Errorf("config %q: %w", options.Path, err) - } - if containsSecretValues(fromFile, specs) && info.Mode().Perm()&0o077 != 0 { - return Config{}, fmt.Errorf("config %q contains secrets and must not be accessible by group or others (mode is %04o)", options.Path, info.Mode().Perm()) - } - if err := k.Merge(fromFile); err != nil { - return Config{}, fmt.Errorf("merge config %q: %w", options.Path, err) + if spec.hasDefault { + value.Field(spec.index).Set(reflect.ValueOf(spec.defaultValue)) } } - environ := options.Environ if environ == nil { environ = os.Environ() } - overrides, err := environmentOverrides(environ, specByEnv) + values, err := environmentValues(environ, known) if err != nil { return Config{}, err } - if err := k.Load(confmap.Provider(overrides, "."), nil); err != nil { - return Config{}, fmt.Errorf("load environment configuration: %w", err) - } - - var cfg Config - if err := k.UnmarshalWithConf("", &cfg, koanf.UnmarshalConf{ - Tag: "koanf", - FlatPaths: true, - DecoderConfig: &mapstructure.DecoderConfig{ - DecodeHook: mapstructure.StringToTimeDurationHookFunc(), - ErrorUnused: true, - WeaklyTypedInput: false, - }, - }); err != nil { - return Config{}, fmt.Errorf("decode configuration: %w", err) + for _, spec := range specs { + // Shared vendor settings, including secret mounts, belong only to the + // explicitly selected provider. Fanout-owned secrets always resolve. + if provider := providerForEnvironment(spec.env); provider != "" && provider != values["FANOUT_AI_PROVIDER"] { + continue + } + text := values[spec.env] + if spec.secret { + name := spec.env + "_FILE" + if path := values[name]; path != "" { + if text != "" { + return Config{}, fmt.Errorf("%s and %s (%q) must not both be set", spec.env, name, path) + } + content, err := os.ReadFile(path) + if err != nil { + return Config{}, fmt.Errorf("read %s file %q: %w", name, path, err) + } + text = string(content) + if strings.HasSuffix(text, "\r\n") { + text = strings.TrimSuffix(text, "\r\n") + } else { + text = strings.TrimSuffix(text, "\n") + } + if text == "" { + return Config{}, fmt.Errorf("%s file %q is empty", name, path) + } + } + } + if text == "" { + continue + } + parsed, err := parseTextValue(text, spec.typ) + if err != nil { + return Config{}, fmt.Errorf("environment variable %s %w", spec.env, err) + } + value.Field(spec.index).Set(reflect.ValueOf(parsed)) } - // Size before validating: validation should judge the configuration the - // process will actually run with, not the holes left for it to fill. + // Validation judges the sizing the process will actually use. cfg.resolveSizing() if err := cfg.Validate(); err != nil { return Config{}, fmt.Errorf("validate configuration: %w", err) @@ -121,26 +106,29 @@ func fieldSpecs(typ reflect.Type) ([]fieldSpec, error) { return nil, fmt.Errorf("configuration schema must be a struct, got %s", typ) } specs := make([]fieldSpec, 0, typ.NumField()) - keys := make(map[string]string, typ.NumField()) envs := make(map[string]string, typ.NumField()) for index := 0; index < typ.NumField(); index++ { field := typ.Field(index) if !field.IsExported() { continue } - key := field.Tag.Get("koanf") - envName := field.Tag.Get("env") - if key == "" || envName == "" { - return nil, fmt.Errorf("configuration field %s must declare koanf and env tags", field.Name) + name := field.Tag.Get("env") + if name == "" { + return nil, fmt.Errorf("configuration field %s must declare an env tag", field.Name) } - if previous, exists := keys[key]; exists { - return nil, fmt.Errorf("configuration fields %s and %s share key %q", previous, field.Name, key) + if !strings.HasPrefix(name, "FANOUT_") && providerForEnvironment(name) == "" { + return nil, fmt.Errorf("environment variable %s is outside the supported configuration contract", name) } - if previous, exists := envs[envName]; exists { - return nil, fmt.Errorf("configuration fields %s and %s share environment variable %q", previous, field.Name, envName) + secret := field.Tag.Get("secret") == "true" + names := []string{name} + if secret { + names = append(names, name+"_FILE") } - if !strings.HasPrefix(envName, "FANOUT_") { - return nil, fmt.Errorf("configuration field %s environment variable %q lacks FANOUT_ prefix", field.Name, envName) + for _, env := range names { + if previous, exists := envs[env]; exists { + return nil, fmt.Errorf("configuration fields %s and %s share environment variable %s", previous, field.Name, env) + } + envs[env] = field.Name } defaultText, hasDefault := field.Tag.Lookup("default") var defaultValue any @@ -148,158 +136,47 @@ func fieldSpecs(typ reflect.Type) ([]fieldSpec, error) { var err error defaultValue, err = parseTextValue(defaultText, field.Type) if err != nil { - return nil, fmt.Errorf("configuration field %s has invalid default: %w", field.Name, err) + return nil, fmt.Errorf("environment variable %s has invalid default: %w", name, err) } } specs = append(specs, fieldSpec{ - key: key, - env: envName, + index: index, + env: name, typ: field.Type, defaultValue: defaultValue, hasDefault: hasDefault, - secret: field.Tag.Get("secret") == "true", + secret: secret, }) - keys[key] = field.Name - envs[envName] = field.Name } return specs, nil } -func validateYAMLKeys(k *koanf.Koanf, known map[string]fieldSpec) error { - var unknown []string - var emptyContainers []string - for _, key := range k.Keys() { - value := k.Get(key) - if spec, exists := known[key]; exists { - if value == nil { - return fmt.Errorf("key %s must not be null", key) - } - if err := validateYAMLValue(value, spec.typ); err != nil { - return fmt.Errorf("key %s %w", key, err) - } - continue - } - if isSchemaContainer(key, known) && isEmptyContainer(value) { - emptyContainers = append(emptyContainers, key) - continue - } - unknown = append(unknown, key) - } - if len(unknown) == 0 { - for _, key := range emptyContainers { - k.Delete(key) - } - return nil - } - sort.Strings(unknown) - return fmt.Errorf("contains unknown keys: %s", strings.Join(unknown, ", ")) -} - -func isSchemaContainer(key string, known map[string]fieldSpec) bool { - prefix := key + "." - for candidate := range known { - if strings.HasPrefix(candidate, prefix) { - return true - } - } - return false -} - -func validateYAMLValue(value any, typ reflect.Type) error { - if typ == reflect.TypeOf(time.Duration(0)) { - text, ok := value.(string) - if !ok { - return fmt.Errorf("must be a duration string") - } - if _, err := time.ParseDuration(text); err != nil { - return fmt.Errorf("must be a Go duration: %w", err) - } - return nil - } - kind := reflect.TypeOf(value).Kind() - switch typ.Kind() { - case reflect.String: - if kind != reflect.String { - return fmt.Errorf("must be a string") - } - case reflect.Bool: - if kind != reflect.Bool { - return fmt.Errorf("must be true or false") - } - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - if kind < reflect.Int || kind > reflect.Int64 { - return fmt.Errorf("must be an integer") - } - default: - return fmt.Errorf("uses unsupported configuration type %s", typ) - } - return nil -} - -func isEmptyContainer(value any) bool { - if value == nil { - return true - } - rv := reflect.ValueOf(value) - return (rv.Kind() == reflect.Map || rv.Kind() == reflect.Slice) && rv.Len() == 0 -} - -func containsSecretValues(k *koanf.Koanf, specs []fieldSpec) bool { - for _, spec := range specs { - if !spec.secret { - continue - } - value := k.Get(spec.key) - if value == nil { - continue - } - if text, ok := value.(string); ok && strings.TrimSpace(text) == "" { - continue - } - return true - } - return false -} - -func environmentOverrides(environ []string, known map[string]fieldSpec) (map[string]any, error) { - overrides := make(map[string]any) +func environmentValues(environ []string, known map[string]bool) (map[string]string, error) { + values := make(map[string]string) var unknown []string for _, assignment := range environ { name, value, found := strings.Cut(assignment, "=") - _, isKnown := known[name] - if !found { - if isKnown || strings.HasPrefix(name, "FANOUT_") { - return nil, fmt.Errorf("environment entry %s must use NAME=value form", name) - } + isKnown := known[name] + if !isKnown && (!strings.HasPrefix(name, "FANOUT_") || isPlatformInjectedEnvironment(name)) { continue } - if !strings.HasPrefix(name, "FANOUT_") { - continue + if !found { + return nil, fmt.Errorf("environment entry %s must use NAME=value form", name) } - spec, exists := known[name] - if !exists { - if isPlatformInjectedEnvironment(name) { - continue - } + if !isKnown { unknown = append(unknown, name) continue } - // An explicitly empty environment value means "no override", so it cannot - // erase a safe default or silently disable a feature. - if value == "" { - continue - } - parsed, err := parseTextValue(value, spec.typ) - if err != nil { - return nil, fmt.Errorf("environment variable %s has an invalid value for %s: %w", name, spec.key, err) + if _, exists := values[name]; exists { + return nil, fmt.Errorf("duplicate environment variable %s", name) } - overrides[spec.key] = parsed + values[name] = value } if len(unknown) > 0 { sort.Strings(unknown) return nil, fmt.Errorf("unknown Fanout environment variables: %s", strings.Join(unknown, ", ")) } - return overrides, nil + return values, nil } func isPlatformInjectedEnvironment(name string) bool { @@ -314,7 +191,7 @@ func parseTextValue(value string, typ reflect.Type) (any, error) { if typ == reflect.TypeOf(time.Duration(0)) { parsed, err := time.ParseDuration(value) if err != nil { - return nil, fmt.Errorf("must be a Go duration: %w", err) + return nil, fmt.Errorf("must be a Go duration") } return parsed, nil } @@ -324,13 +201,13 @@ func parseTextValue(value string, typ reflect.Type) (any, error) { case reflect.Bool: parsed, err := strconv.ParseBool(value) if err != nil { - return nil, fmt.Errorf("must be true or false: %w", err) + return nil, fmt.Errorf("must be true or false") } return parsed, nil case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: parsed, err := strconv.ParseInt(value, 10, typ.Bits()) if err != nil { - return nil, fmt.Errorf("must be an integer: %w", err) + return nil, fmt.Errorf("must be an integer") } result := reflect.New(typ).Elem() result.SetInt(parsed) diff --git a/internal/config/pinned_memory_test.go b/internal/config/pinned_memory_test.go index 74de5a80..29826aa4 100644 --- a/internal/config/pinned_memory_test.go +++ b/internal/config/pinned_memory_test.go @@ -2,7 +2,7 @@ package config import "testing" -// Pinning storage.duckdb.memory skips detection entirely, so nothing compares +// Pinning FANOUT_DUCKDB_MEMORY skips detection entirely, so nothing compares // the value to the machine. Both production deployments pinned 4GB on 8 GiB // hosts and were OOM-killed repeatedly: DuckDB's budget is not the process's // budget, and the Go runtime, the ingest appender and compaction's merges all diff --git a/internal/config/provider_test.go b/internal/config/provider_test.go new file mode 100644 index 00000000..5096a330 --- /dev/null +++ b/internal/config/provider_test.go @@ -0,0 +1,162 @@ +package config + +import ( + "fmt" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func TestDefaultAddrMatchesSchema(t *testing.T) { + field, ok := reflect.TypeOf(Config{}).FieldByName("Addr") + if !ok || field.Tag.Get("default") != DefaultAddr { + t.Fatalf("Addr default tag = %q, want %q", field.Tag.Get("default"), DefaultAddr) + } +} + +func TestLoadIgnoresInactiveProviderSecretFiles(t *testing.T) { + emptyPath := filepath.Join(t.TempDir(), "empty") + if err := os.WriteFile(emptyPath, nil, 0o644); err != nil { + t.Fatal(err) + } + for _, provider := range []string{"", "anthropic", "openai"} { + for _, vendor := range []string{"anthropic", "openai"} { + if vendor == provider { + continue + } + for _, file := range []struct{ name, path string }{ + {"missing", "/missing"}, {"empty", emptyPath}, + } { + for _, bothSet := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%s/%s/both_set=%t", provider, vendor, file.name, bothSet), func(t *testing.T) { + env := []string{"FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), "FANOUT_AI_PROVIDER=" + provider} + if provider != "" { + env = append(env, strings.ToUpper(provider)+"_API_KEY=selected-key") + } + name := strings.ToUpper(vendor) + env = append(env, name+"_API_KEY_FILE="+file.path, name+"_BASE_URL=unused-invalid-url") + if bothSet { + env = append(env, name+"_API_KEY=unused-key") + } + cfg, err := Load(LoadOptions{Environ: env}) + if err != nil { + t.Fatal(err) + } + if vendor == "anthropic" && (cfg.AnthropicAPIKey != "" || cfg.AnthropicBaseURL != "") || + vendor == "openai" && (cfg.OpenAIAPIKey != "" || cfg.OpenAIBaseURL != "") { + t.Fatal("inactive provider settings must be ignored") + } + }) + } + } + } + } +} + +func TestLoadRejectsSelectedProviderSecretFileErrors(t *testing.T) { + for _, provider := range []string{"anthropic", "openai"} { + for _, bothSet := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/both_set=%t", provider, bothSet), func(t *testing.T) { + name := strings.ToUpper(provider) + "_API_KEY" + path := filepath.Join(t.TempDir(), "missing") + env := []string{ + "FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), "FANOUT_AI_PROVIDER=" + provider, + name + "_FILE=" + path, + } + if bothSet { + env = append(env, name+"=synthetic-private-value") + } + _, err := Load(LoadOptions{Environ: env}) + if err == nil || !strings.Contains(err.Error(), name+"_FILE") || !strings.Contains(err.Error(), path) || strings.Contains(err.Error(), "synthetic-private-value") { + t.Fatalf("expected redacted file error naming variable and path, got %v", err) + } + if bothSet && !strings.Contains(err.Error(), "must not both be set") { + t.Fatalf("expected selected provider conflict, got %v", err) + } + }) + } + } +} + +func TestLoadSelectsProviderCredentials(t *testing.T) { + for _, tc := range []struct { + name, provider, key, baseURL string + credentials []string + }{ + {"anthropic", "anthropic", "anthropic-test", "https://anthropic.example/gateway", []string{"ANTHROPIC_API_KEY=anthropic-test", "ANTHROPIC_BASE_URL=https://anthropic.example/gateway/", "OPENAI_API_KEY=openai-test", "OPENAI_BASE_URL=https://openai.example/v1"}}, + {"openai", "openai", "openai-test", "https://openai.example/gateway/v1", []string{"ANTHROPIC_API_KEY=anthropic-test", "ANTHROPIC_BASE_URL=not-used", "OPENAI_API_KEY=openai-test", "OPENAI_BASE_URL=https://openai.example/gateway/v1/"}}, + {"inherited credentials do not enable chat", "", "", "", []string{"ANTHROPIC_API_KEY=anthropic-test", "OPENAI_API_KEY=openai-test", "ANTHROPIC_BASE_URL=not-used", "OPENAI_BASE_URL=not-used"}}, + {"anthropic default endpoint", "anthropic", "anthropic-test", DefaultAnthropicBaseURL, []string{"ANTHROPIC_API_KEY=anthropic-test", "OPENAI_BASE_URL=not-used"}}, + {"openai default endpoint", "openai", "openai-test", DefaultOpenAIBaseURL, []string{"OPENAI_API_KEY=openai-test", "ANTHROPIC_BASE_URL=not-used"}}, + } { + t.Run(tc.name, func(t *testing.T) { + env := []string{"FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), "FANOUT_AI_PROVIDER=" + tc.provider} + cfg, err := Load(LoadOptions{Environ: append(env, tc.credentials...)}) + if err != nil { + t.Fatal(err) + } + if cfg.AIProvider != tc.provider || cfg.AIAPIKey() != tc.key || cfg.AIBaseURL() != tc.baseURL || cfg.AgentConfigured() != (tc.provider != "") { + t.Fatal("selected provider credential, endpoint, or enabled state is incorrect") + } + }) + } +} + +func TestLoadIgnoresOtherVendorEnvironment(t *testing.T) { + cfg, err := Load(LoadOptions{Environ: []string{ + "FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), + "FANOUT_AI_PROVIDER=openai", "OPENAI_API_KEY=openai-env", + "OPENAI_BASE_URL=https://gateway.example/api/v1/", "FANOUT_AI_MODEL=configured-model", + "ANTHROPIC_API_KEY=", "ANTHROPIC_BASE_URL=", + "OPENAI_MODEL=unrelated-tool", "ANTHROPIC_MODEL=unrelated-tool", + "OPENAI_PROJECT_ID=another-consumer", "ANTHROPIC_AUTH_TOKEN=another-consumer", + }}) + if err != nil { + t.Fatal(err) + } + if cfg.AIAPIKey() != "openai-env" || cfg.AIBaseURL() != "https://gateway.example/api/v1" || cfg.AIModel != "configured-model" || cfg.AnthropicAPIKey != "" || cfg.AnthropicBaseURL != "" { + t.Fatal("provider environment or shared consumer isolation is incorrect") + } +} + +func TestLoadRejectsInvalidProviderURLWithoutLeakingValues(t *testing.T) { + for _, provider := range []string{"anthropic", "openai"} { + name := strings.ToUpper(provider) + "_BASE_URL" + for _, value := range []string{"file:///private", "https://user:private-value@example.com", "https://example.com?key=private-value", "https://example.com#private-value", "https://example.com?", "https://example.com#", "https://", "https://example.com/%zz"} { + _, err := Load(LoadOptions{Environ: []string{"FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), "FANOUT_AI_PROVIDER=" + provider, strings.ToUpper(provider) + "_API_KEY=test-key", name + "=" + value}}) + if err == nil || !strings.Contains(err.Error(), name) || strings.Contains(err.Error(), value) || strings.Contains(err.Error(), "private-value") { + t.Fatalf("expected redacted validation error naming %s", name) + } + } + } +} + +func TestLoadRejectsUnsupportedProviderWithoutAKey(t *testing.T) { + for _, provider := range []string{"unsupported", "OpenAI", " OpenAI ", " openai ", "Anthropic"} { + _, err := Load(LoadOptions{Environ: []string{"FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), "FANOUT_AI_PROVIDER=" + provider}}) + if err == nil || !strings.Contains(err.Error(), "FANOUT_AI_PROVIDER") { + t.Fatal("unsupported provider must fail at startup, even without a key") + } + } +} + +func TestLoadRequiresSelectedProviderKey(t *testing.T) { + for _, provider := range []string{"anthropic", "openai"} { + otherKey := "OPENAI_API_KEY" + if provider == "openai" { + otherKey = "ANTHROPIC_API_KEY" + } + for _, value := range []string{"", " "} { + name := strings.ToUpper(provider) + "_API_KEY" + _, err := Load(LoadOptions{Environ: []string{ + "FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), "FANOUT_AI_PROVIDER=" + provider, + name + "=" + value, otherKey + "=other-provider-key", + }}) + if err == nil || !strings.Contains(err.Error(), name) { + t.Fatalf("expected missing key error naming the environment variable, got %v", err) + } + } + } +} diff --git a/internal/config/sizing.go b/internal/config/sizing.go index 5aa1f490..b8fa0312 100644 --- a/internal/config/sizing.go +++ b/internal/config/sizing.go @@ -134,13 +134,13 @@ func (c Config) logResolvedSizing(src sizingSource) { "gomaxprocs", runtime.GOMAXPROCS(0), ) if src.MemoryAuto && src.MemoryDetectionIncomplete { - slog.Warn("memory limit detection was incomplete; automatic DuckDB sizing may exceed a container limit — configure storage.duckdb.memory explicitly", + slog.Warn("memory limit detection was incomplete; automatic DuckDB sizing may exceed a container limit — configure FANOUT_DUCKDB_MEMORY explicitly", "memory_source", src.MemorySource, "detected_memory_bytes", src.DetectedRAM, ) } if src.MemoryAuto && c.DuckDBMemory == "" { - slog.Warn("could not detect available memory; DuckDB will size itself to 80% of RAM, which can exceed the machine once the Go runtime is added — configure storage.duckdb.memory explicitly") + slog.Warn("could not detect available memory; DuckDB will size itself to 80% of RAM, which can exceed the machine once the Go runtime is added — configure FANOUT_DUCKDB_MEMORY explicitly") } // A pinned budget skipped detection, so this is the only place it is ever // compared to the machine it will run on. @@ -174,7 +174,7 @@ func resolveDuckDBMemory(available uint64) string { // pinnedMemoryConcern reports why an operator-pinned DuckDB budget looks wrong // for this machine, or "" when it looks fine or cannot be judged. // -// Pinning storage.duckdb.memory skips detection entirely (resolveSizing only +// Pinning FANOUT_DUCKDB_MEMORY skips detection entirely (resolveSizing only // fills a value that is empty), so without this nothing ever compares the // figure to the machine it will run on. DuckDB's budget is not the process's // budget: the Go heap, the ingest appender and compaction's merges all sit on @@ -192,14 +192,14 @@ func pinnedMemoryConcern(pinned string, detectedRAM uint64) string { } budget, ok := parseByteSize(pinned) if !ok { - return fmt.Sprintf("storage.duckdb.memory %q could not be parsed as a size; DuckDB may reject it or fall back to its own default", pinned) + return fmt.Sprintf("FANOUT_DUCKDB_MEMORY %q could not be parsed as a size; DuckDB may reject it or fall back to its own default", pinned) } if budget >= detectedRAM { - return fmt.Sprintf("storage.duckdb.memory %q is at or above the %d bytes detected for this machine; the Go runtime and compaction allocate on top of it", pinned, detectedRAM) + return fmt.Sprintf("FANOUT_DUCKDB_MEMORY %q is at or above the %d bytes detected for this machine; the Go runtime and compaction allocate on top of it", pinned, detectedRAM) } share := detectedRAM / 100 * duckDBMemoryPercent if budget > share { - return fmt.Sprintf("storage.duckdb.memory %q takes more than the %d%% of %d bytes automatic sizing would leave DuckDB; the remainder has to cover the Go heap, the ingest appender and compaction merges", pinned, duckDBMemoryPercent, detectedRAM) + return fmt.Sprintf("FANOUT_DUCKDB_MEMORY %q takes more than the %d%% of %d bytes automatic sizing would leave DuckDB; the remainder has to cover the Go heap, the ingest appender and compaction merges", pinned, duckDBMemoryPercent, detectedRAM) } return "" } diff --git a/internal/query/duck.go b/internal/query/duck.go index 1d29989b..7e8458ce 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -153,7 +153,7 @@ func duckDBPoolSize(cfg config.Config) int { // of a constant that's wrong everywhere but the box it was tuned for. func duckDSN(dbPath, mem string, threads int) (string, error) { if strings.ContainsAny(mem, "&?'\"\\; ") { - return "", fmt.Errorf("invalid storage.duckdb.memory value: %q", mem) + return "", fmt.Errorf("invalid FANOUT_DUCKDB_MEMORY value: %q", mem) } params := make([]string, 0, 2) if threads > 0 { @@ -292,7 +292,7 @@ func NewDuck(ctx context.Context, cfg config.Config, repository *telemetrystore. return tags }) if cfg.DuckDBMemory == "" { - // Only when the operator hasn't pinned storage.duckdb.memory: keep DuckDB's + // Only when the operator hasn't pinned FANOUT_DUCKDB_MEMORY: keep DuckDB's // cgroup-aware auto limit on big boxes but leave absolute RAM headroom on // small ones, so the kernel doesn't OOM-kill the whole process. if err := applyMemoryHeadroom(ctx, db); err != nil { diff --git a/justfile b/justfile index 98504e14..25c9f315 100644 --- a/justfile +++ b/justfile @@ -32,6 +32,10 @@ embedded := "internal/ui/dist internal/mcp/apps" default: @just --list +# Load optional ignored .env, then inherited values. Build with `just build` first. +run *ARGS='': + node scripts/run-fanout.mjs -- ./bin/fanout {{ARGS}} + # Split out so CI installs exactly what a contributor does, without also # writing git hooks into the runner. diff --git a/scripts/run-fanout.mjs b/scripts/run-fanout.mjs new file mode 100644 index 00000000..12da133a --- /dev/null +++ b/scripts/run-fanout.mjs @@ -0,0 +1,54 @@ +import { spawn } from "node:child_process"; +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { parseEnv } from "node:util"; + +/** @param {string} root @param {Record} inherited */ +export function prepareEnvironment(root, inherited) { + let content = ""; + try { + content = readFileSync(resolve(root, ".env"), "utf8"); + } catch (error) { + if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") { + throw new Error("Cannot read dotenv file: .env"); + } + } + // Node's parser preserves literal dollars and never executes shell code. + const values = parseEnv(content); + for (const [name, value] of Object.entries(inherited)) { + if (value !== undefined) values[name] = value; + } + return values; +} + +/** @param {string[]} args @returns {Promise} */ +export async function main(args) { + if (args.length < 2 || args[0] !== "--") { + throw new Error("Usage: node scripts/run-fanout.mjs -- command [args]"); + } + const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + const env = prepareEnvironment(root, process.env); + const child = spawn(args[1], args.slice(2), { cwd: root, env, stdio: "inherit" }); + const handlers = new Map(["SIGINT", "SIGTERM", "SIGHUP"].map(signal => [signal, () => child.kill(signal)])); + for (const [signal, handler] of handlers) process.on(signal, handler); + try { + return await new Promise((resolveExit, reject) => { + child.once("error", reject); + child.once("exit", (code, signal) => resolveExit(code ?? ({ SIGINT: 130, SIGTERM: 143, SIGHUP: 129 }[signal] ?? 1))); + }); + } finally { + for (const [signal, handler] of handlers) process.off(signal, handler); + } +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + try { + process.exitCode = await main(process.argv.slice(2)); + } catch (error) { + const message = error instanceof Error ? error.message : "Environment launch failed"; + console.error(message.startsWith("Cannot read dotenv") || message.startsWith("Usage:") + ? message : "Cannot launch command; ensure it is executable (for Fanout, run just build)"); + process.exitCode = 1; + } +} diff --git a/scripts/run-fanout.test.ts b/scripts/run-fanout.test.ts new file mode 100644 index 00000000..73aa8990 --- /dev/null +++ b/scripts/run-fanout.test.ts @@ -0,0 +1,113 @@ +import { afterEach, expect, test } from "bun:test"; +import { copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { spawn } from "node:child_process"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { pathToFileURL } from "node:url"; +import { main, prepareEnvironment } from "./run-fanout.mjs"; + +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + +function fixture(): string { + const root = mkdtempSync(join(tmpdir(), "fanout-env-")); + roots.push(root); + return root; +} + +test("dotenv is optional and inherited values win, including empty values", () => { + const root = fixture(); + expect(prepareEnvironment(root, { PATH: "/synthetic/path" })).toEqual({ PATH: "/synthetic/path" }); + writeFileSync(join(root, ".env"), "FANOUT_ADDR=:9000\nFANOUT_AI_MODEL=personal\nANTHROPIC_API_KEY=synthetic-only\nFANOUT_UNKNOWN=value\n"); + const env = prepareEnvironment(root, { FANOUT_ADDR: "", FANOUT_AI_MODEL: "injected", PATH: "/synthetic/path" }); + expect(env.FANOUT_ADDR).toBe(""); + expect(env.FANOUT_AI_MODEL).toBe("injected"); + expect(env.ANTHROPIC_API_KEY).toBe("synthetic-only"); + expect(env.FANOUT_UNKNOWN).toBe("value"); + expect(env.PATH).toBe("/synthetic/path"); +}); + +test("dotenv parsing preserves literal shell text, CRLF, and multiline values", () => { + const root = fixture(); + writeFileSync(join(root, ".env"), `FANOUT_DATA_DIR=' $HOME $(printf unsafe) "quoted" ' +FANOUT_AI_MODEL="line one +line two" +`.replace(/\n/g, "\r\n")); + const env = prepareEnvironment(root, {}); + expect(env.FANOUT_DATA_DIR).toBe(' $HOME $(printf unsafe) "quoted" '); + expect(env.FANOUT_AI_MODEL).toBe("line one\nline two"); + const result = Bun.spawnSync(["node", "--input-type=module", "-e", + "const {prepareEnvironment}=await import(process.argv[2]); process.stdout.write(JSON.stringify(prepareEnvironment(process.argv[1], {})));", + root, pathToFileURL(join(import.meta.dir, "run-fanout.mjs")).href], { cwd: root }); + expect(result.exitCode).toBe(0); + expect(JSON.parse(result.stdout.toString())).toEqual(env); +}); + +test("unreadable dotenv fails and launcher requires a command", async () => { + const root = fixture(); + mkdirSync(join(root, ".env")); + expect(() => prepareEnvironment(root, {})).toThrow("Cannot read dotenv"); + await expect(main([])).rejects.toThrow("Usage:"); + await expect(main(["--"])).rejects.toThrow("Usage:"); +}); + +function launchFixture(childCode: string) { + const root = fixture(); + mkdirSync(join(root, "scripts")); + copyFileSync(join(import.meta.dir, "run-fanout.mjs"), join(root, "scripts/run-fanout.mjs")); + const harness = ` +import { main } from './scripts/run-fanout.mjs'; +const signals = ['SIGINT', 'SIGTERM', 'SIGHUP']; +const before = signals.map(signal => process.listenerCount(signal)); +const code = await main(['--', process.execPath, '-e', process.argv[2]]); +console.log('cleanup:' + signals.every((signal, index) => process.listenerCount(signal) === before[index])); +process.exitCode = code; +`; + writeFileSync(join(root, "harness.mjs"), harness); + return spawn("node", [join(root, "harness.mjs"), childCode], { env: { PATH: process.env.PATH }, stdio: ["ignore", "pipe", "pipe"] }); +} + +test("launcher forwards repeated SIGINT, SIGTERM, and SIGHUP until exit and removes handlers", async () => { + const signals = ["SIGINT", "SIGTERM", "SIGHUP", "SIGINT", "SIGTERM", "SIGHUP"] as const; + const launcher = launchFixture(` +let count = 0; +setTimeout(() => process.exit(1), 10000); +for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) process.on(signal, () => { + console.log(signal); + if (++count === 6) process.exit(0); +}); +console.log('ready'); +`); + let output = ""; + let index = 0; + const result = new Promise((resolveExit, reject) => { + launcher.once("error", reject); + launcher.once("close", code => resolveExit(code)); + // Acknowledgement of each delivery avoids timing-dependent signal coalescing. + launcher.stdout.on("data", data => { + output += data.toString(); + const lines = output.trimEnd().split("\n"); + if (lines.length === index + 1 && index < signals.length) launcher.kill(signals[index++]); + }); + }); + try { + expect(await result).toBe(0); + expect(output.trimEnd().split("\n")).toEqual(["ready", ...signals, "cleanup:true"]); + } finally { + if (launcher.exitCode === null) launcher.kill("SIGKILL"); + } +}, 15000); + +test("launcher maps child signal exits to conventional codes and removes handlers", async () => { + for (const [signal, code] of [["SIGINT", 130], ["SIGTERM", 143], ["SIGHUP", 129]] as const) { + const launcher = launchFixture(`process.kill(process.pid, '${signal}');`); + let output = ""; + launcher.stdout.on("data", data => { output += data.toString(); }); + const exitCode = await new Promise((resolveExit, reject) => { + launcher.once("error", reject); + launcher.once("close", code => resolveExit(code)); + }); + expect(exitCode).toBe(code); + expect(output.trim()).toBe("cleanup:true"); + } +}); diff --git a/site/public/diagrams/architecture.svg b/site/public/diagrams/architecture.svg index 01633b3e..18433e2a 100644 --- a/site/public/diagrams/architecture.svg +++ b/site/public/diagrams/architecture.svg @@ -1,24 +1,24 @@ -Clientsfanout — one Go processStorageAnthropic or OpenAIOTLP collector or SDKBrowserExternal MCP hostHTTP :7520routes + auth middlewareIngestOTLP gRPC + HTTP :7520Embedded React assetsAgent runtimemodel + tool loopMCP server5 typed + 4 dashboard toolsTyped observability contractTelemetry commit workersatomic Parquet batchesQuery kernelDuckDB + rollupsStorage maintenanceretention + native compactionAlert enginerule evaluation + webhooksParquet batches + trace indexesstorage.data_dir/telemetryDuckDB query statestorage.data_dir/queryControl SQLitestorage.data_dir/control serves embedded assetsAG-UI streamin-memory transporttyped HTTP APIevaluates rollupsOTLP/gRPC or OTLP/HTTPHTTPS/mcp — OAuthindexed traces + SQL scansatomic replacementusers, sessions, settings, dashboards, threadsrules and fired alertsHTTPS - + .d2-37886755 .fill-N1{fill:#0A0F25;} + .d2-37886755 .fill-N2{fill:#676C7E;} + .d2-37886755 .fill-N3{fill:#9499AB;} + .d2-37886755 .fill-N4{fill:#CFD2DD;} + .d2-37886755 .fill-N5{fill:#DEE1EB;} + .d2-37886755 .fill-N6{fill:#EEF1F8;} + .d2-37886755 .fill-N7{fill:#FFFFFF;} + .d2-37886755 .fill-B1{fill:#000536;} + .d2-37886755 .fill-B2{fill:#0F66B7;} + .d2-37886755 .fill-B3{fill:#4393DD;} + .d2-37886755 .fill-B4{fill:#87BFF3;} + .d2-37886755 .fill-B5{fill:#BCDDFB;} + .d2-37886755 .fill-B6{fill:#E5F3FF;} + .d2-37886755 .fill-AA2{fill:#7639C5;} + .d2-37886755 .fill-AA4{fill:#C1A2F3;} + .d2-37886755 .fill-AA5{fill:#DACEFB;} + .d2-37886755 .fill-AB4{fill:#EA99C6;} + .d2-37886755 .fill-AB5{fill:#FFDEF1;} + .d2-37886755 .stroke-N1{stroke:#0A0F25;} + .d2-37886755 .stroke-N2{stroke:#676C7E;} + .d2-37886755 .stroke-N3{stroke:#9499AB;} + .d2-37886755 .stroke-N4{stroke:#CFD2DD;} + .d2-37886755 .stroke-N5{stroke:#DEE1EB;} + .d2-37886755 .stroke-N6{stroke:#EEF1F8;} + .d2-37886755 .stroke-N7{stroke:#FFFFFF;} + .d2-37886755 .stroke-B1{stroke:#000536;} + .d2-37886755 .stroke-B2{stroke:#0F66B7;} + .d2-37886755 .stroke-B3{stroke:#4393DD;} + .d2-37886755 .stroke-B4{stroke:#87BFF3;} + .d2-37886755 .stroke-B5{stroke:#BCDDFB;} + .d2-37886755 .stroke-B6{stroke:#E5F3FF;} + .d2-37886755 .stroke-AA2{stroke:#7639C5;} + .d2-37886755 .stroke-AA4{stroke:#C1A2F3;} + .d2-37886755 .stroke-AA5{stroke:#DACEFB;} + .d2-37886755 .stroke-AB4{stroke:#EA99C6;} + .d2-37886755 .stroke-AB5{stroke:#FFDEF1;} + .d2-37886755 .background-color-N1{background-color:#0A0F25;} + .d2-37886755 .background-color-N2{background-color:#676C7E;} + .d2-37886755 .background-color-N3{background-color:#9499AB;} + .d2-37886755 .background-color-N4{background-color:#CFD2DD;} + .d2-37886755 .background-color-N5{background-color:#DEE1EB;} + .d2-37886755 .background-color-N6{background-color:#EEF1F8;} + .d2-37886755 .background-color-N7{background-color:#FFFFFF;} + .d2-37886755 .background-color-B1{background-color:#000536;} + .d2-37886755 .background-color-B2{background-color:#0F66B7;} + .d2-37886755 .background-color-B3{background-color:#4393DD;} + .d2-37886755 .background-color-B4{background-color:#87BFF3;} + .d2-37886755 .background-color-B5{background-color:#BCDDFB;} + .d2-37886755 .background-color-B6{background-color:#E5F3FF;} + .d2-37886755 .background-color-AA2{background-color:#7639C5;} + .d2-37886755 .background-color-AA4{background-color:#C1A2F3;} + .d2-37886755 .background-color-AA5{background-color:#DACEFB;} + .d2-37886755 .background-color-AB4{background-color:#EA99C6;} + .d2-37886755 .background-color-AB5{background-color:#FFDEF1;} + .d2-37886755 .color-N1{color:#0A0F25;} + .d2-37886755 .color-N2{color:#676C7E;} + .d2-37886755 .color-N3{color:#9499AB;} + .d2-37886755 .color-N4{color:#CFD2DD;} + .d2-37886755 .color-N5{color:#DEE1EB;} + .d2-37886755 .color-N6{color:#EEF1F8;} + .d2-37886755 .color-N7{color:#FFFFFF;} + .d2-37886755 .color-B1{color:#000536;} + .d2-37886755 .color-B2{color:#0F66B7;} + .d2-37886755 .color-B3{color:#4393DD;} + .d2-37886755 .color-B4{color:#87BFF3;} + .d2-37886755 .color-B5{color:#BCDDFB;} + .d2-37886755 .color-B6{color:#E5F3FF;} + .d2-37886755 .color-AA2{color:#7639C5;} + .d2-37886755 .color-AA4{color:#C1A2F3;} + .d2-37886755 .color-AA5{color:#DACEFB;} + .d2-37886755 .color-AB4{color:#EA99C6;} + .d2-37886755 .color-AB5{color:#FFDEF1;}.appendix text.text{fill:#0A0F25}.md{--color-fg-default:#0A0F25;--color-fg-muted:#676C7E;--color-fg-subtle:#9499AB;--color-canvas-default:#FFFFFF;--color-canvas-subtle:#EEF1F8;--color-border-default:#000536;--color-border-muted:#0F66B7;--color-neutral-muted:#EEF1F8;--color-accent-fg:#0F66B7;--color-accent-emphasis:#0F66B7;--color-attention-subtle:#676C7E;--color-danger-fg:red;}.sketch-overlay-B1{fill:url(#streaks-darker-d2-37886755);mix-blend-mode:lighten}.sketch-overlay-B2{fill:url(#streaks-dark-d2-37886755);mix-blend-mode:overlay}.sketch-overlay-B3{fill:url(#streaks-dark-d2-37886755);mix-blend-mode:overlay}.sketch-overlay-B4{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-B5{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-B6{fill:url(#streaks-bright-d2-37886755);mix-blend-mode:darken}.sketch-overlay-AA2{fill:url(#streaks-dark-d2-37886755);mix-blend-mode:overlay}.sketch-overlay-AA4{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-AA5{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-AB4{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-AB5{fill:url(#streaks-bright-d2-37886755);mix-blend-mode:darken}.sketch-overlay-N1{fill:url(#streaks-darker-d2-37886755);mix-blend-mode:lighten}.sketch-overlay-N2{fill:url(#streaks-dark-d2-37886755);mix-blend-mode:overlay}.sketch-overlay-N3{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-N4{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-N5{fill:url(#streaks-bright-d2-37886755);mix-blend-mode:darken}.sketch-overlay-N6{fill:url(#streaks-bright-d2-37886755);mix-blend-mode:darken}.sketch-overlay-N7{fill:url(#streaks-bright-d2-37886755);mix-blend-mode:darken}.light-code{display: block}.dark-code{display: none}]]>Clientsfanout — one Go processStorageAnthropic or OpenAIOTLP collector or SDKBrowserExternal MCP hostHTTP :7520routes + auth middlewareIngestOTLP gRPC + HTTP :7520Embedded React assetsAgent runtimemodel + tool loopMCP server5 typed + 4 dashboard toolsTyped observability contractTelemetry commit workersatomic Parquet batchesQuery kernelDuckDB + rollupsStorage maintenanceretention + native compactionAlert enginerule evaluation + webhooksParquet batches + trace indexesFANOUT_DATA_DIR/telemetryDuckDB query stateFANOUT_DATA_DIR/queryControl SQLiteFANOUT_DATA_DIR/control serves embedded assetsAG-UI streamin-memory transporttyped HTTP APIevaluates rollupsOTLP/gRPC or OTLP/HTTPHTTPS/mcp — OAuthindexed traces + SQL scansatomic replacementusers, sessions, settings, dashboards, threadsrules and fired alertsHTTPS + @@ -113,9 +113,9 @@ - - + + - + diff --git a/site/public/diagrams/persistence.svg b/site/public/diagrams/persistence.svg index 50c353fc..13d27eb5 100644 --- a/site/public/diagrams/persistence.svg +++ b/site/public/diagrams/persistence.svg @@ -1,27 +1,27 @@ -Authenticated OTLP requestsspans, logs, metricsDurable write pathAuthoritative telemetryimmutable *.batch directoriesOrdinary Parquet files are authoritativeReadersBounded maintenanceRebuildable query statestorage.data_dir/queryDuckDB rollups and temporary spill. Not authoritative telemetry.Bounded group admissionconfigured row target + 20 ms windowUp to four commit workersindependent batches encode in parallelStaging batch directorymetadata + signal Parquet + trace.fidx for spansDurable publicationfsync → atomic rename → fsync parentpublished batch directories are discovered directly from the filesystemIndexed trace readbinary-search .fidx → Parquet rowsDuckDB SQLfilters, ordering, broad scansDuckDB rollupsrebuildable analytical cacheRetention + compaction admissionbounded batch and row groupsNative Parquet mergeprepared outside the reader gateCrash-safe namespace swapmarker + bounded renames + fsync acknowledge only after publicationOrdinary Parquet files are authoritative - + .d2-2701310579 .fill-N1{fill:#0A0F25;} + .d2-2701310579 .fill-N2{fill:#676C7E;} + .d2-2701310579 .fill-N3{fill:#9499AB;} + .d2-2701310579 .fill-N4{fill:#CFD2DD;} + .d2-2701310579 .fill-N5{fill:#DEE1EB;} + .d2-2701310579 .fill-N6{fill:#EEF1F8;} + .d2-2701310579 .fill-N7{fill:#FFFFFF;} + .d2-2701310579 .fill-B1{fill:#000536;} + .d2-2701310579 .fill-B2{fill:#0F66B7;} + .d2-2701310579 .fill-B3{fill:#4393DD;} + .d2-2701310579 .fill-B4{fill:#87BFF3;} + .d2-2701310579 .fill-B5{fill:#BCDDFB;} + .d2-2701310579 .fill-B6{fill:#E5F3FF;} + .d2-2701310579 .fill-AA2{fill:#7639C5;} + .d2-2701310579 .fill-AA4{fill:#C1A2F3;} + .d2-2701310579 .fill-AA5{fill:#DACEFB;} + .d2-2701310579 .fill-AB4{fill:#EA99C6;} + .d2-2701310579 .fill-AB5{fill:#FFDEF1;} + .d2-2701310579 .stroke-N1{stroke:#0A0F25;} + .d2-2701310579 .stroke-N2{stroke:#676C7E;} + .d2-2701310579 .stroke-N3{stroke:#9499AB;} + .d2-2701310579 .stroke-N4{stroke:#CFD2DD;} + .d2-2701310579 .stroke-N5{stroke:#DEE1EB;} + .d2-2701310579 .stroke-N6{stroke:#EEF1F8;} + .d2-2701310579 .stroke-N7{stroke:#FFFFFF;} + .d2-2701310579 .stroke-B1{stroke:#000536;} + .d2-2701310579 .stroke-B2{stroke:#0F66B7;} + .d2-2701310579 .stroke-B3{stroke:#4393DD;} + .d2-2701310579 .stroke-B4{stroke:#87BFF3;} + .d2-2701310579 .stroke-B5{stroke:#BCDDFB;} + .d2-2701310579 .stroke-B6{stroke:#E5F3FF;} + .d2-2701310579 .stroke-AA2{stroke:#7639C5;} + .d2-2701310579 .stroke-AA4{stroke:#C1A2F3;} + .d2-2701310579 .stroke-AA5{stroke:#DACEFB;} + .d2-2701310579 .stroke-AB4{stroke:#EA99C6;} + .d2-2701310579 .stroke-AB5{stroke:#FFDEF1;} + .d2-2701310579 .background-color-N1{background-color:#0A0F25;} + .d2-2701310579 .background-color-N2{background-color:#676C7E;} + .d2-2701310579 .background-color-N3{background-color:#9499AB;} + .d2-2701310579 .background-color-N4{background-color:#CFD2DD;} + .d2-2701310579 .background-color-N5{background-color:#DEE1EB;} + .d2-2701310579 .background-color-N6{background-color:#EEF1F8;} + .d2-2701310579 .background-color-N7{background-color:#FFFFFF;} + .d2-2701310579 .background-color-B1{background-color:#000536;} + .d2-2701310579 .background-color-B2{background-color:#0F66B7;} + .d2-2701310579 .background-color-B3{background-color:#4393DD;} + .d2-2701310579 .background-color-B4{background-color:#87BFF3;} + .d2-2701310579 .background-color-B5{background-color:#BCDDFB;} + .d2-2701310579 .background-color-B6{background-color:#E5F3FF;} + .d2-2701310579 .background-color-AA2{background-color:#7639C5;} + .d2-2701310579 .background-color-AA4{background-color:#C1A2F3;} + .d2-2701310579 .background-color-AA5{background-color:#DACEFB;} + .d2-2701310579 .background-color-AB4{background-color:#EA99C6;} + .d2-2701310579 .background-color-AB5{background-color:#FFDEF1;} + .d2-2701310579 .color-N1{color:#0A0F25;} + .d2-2701310579 .color-N2{color:#676C7E;} + .d2-2701310579 .color-N3{color:#9499AB;} + .d2-2701310579 .color-N4{color:#CFD2DD;} + .d2-2701310579 .color-N5{color:#DEE1EB;} + .d2-2701310579 .color-N6{color:#EEF1F8;} + .d2-2701310579 .color-N7{color:#FFFFFF;} + .d2-2701310579 .color-B1{color:#000536;} + .d2-2701310579 .color-B2{color:#0F66B7;} + .d2-2701310579 .color-B3{color:#4393DD;} + .d2-2701310579 .color-B4{color:#87BFF3;} + .d2-2701310579 .color-B5{color:#BCDDFB;} + .d2-2701310579 .color-B6{color:#E5F3FF;} + .d2-2701310579 .color-AA2{color:#7639C5;} + .d2-2701310579 .color-AA4{color:#C1A2F3;} + .d2-2701310579 .color-AA5{color:#DACEFB;} + .d2-2701310579 .color-AB4{color:#EA99C6;} + .d2-2701310579 .color-AB5{color:#FFDEF1;}.appendix text.text{fill:#0A0F25}.md{--color-fg-default:#0A0F25;--color-fg-muted:#676C7E;--color-fg-subtle:#9499AB;--color-canvas-default:#FFFFFF;--color-canvas-subtle:#EEF1F8;--color-border-default:#000536;--color-border-muted:#0F66B7;--color-neutral-muted:#EEF1F8;--color-accent-fg:#0F66B7;--color-accent-emphasis:#0F66B7;--color-attention-subtle:#676C7E;--color-danger-fg:red;}.sketch-overlay-B1{fill:url(#streaks-darker-d2-2701310579);mix-blend-mode:lighten}.sketch-overlay-B2{fill:url(#streaks-dark-d2-2701310579);mix-blend-mode:overlay}.sketch-overlay-B3{fill:url(#streaks-dark-d2-2701310579);mix-blend-mode:overlay}.sketch-overlay-B4{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-B5{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-B6{fill:url(#streaks-bright-d2-2701310579);mix-blend-mode:darken}.sketch-overlay-AA2{fill:url(#streaks-dark-d2-2701310579);mix-blend-mode:overlay}.sketch-overlay-AA4{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-AA5{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-AB4{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-AB5{fill:url(#streaks-bright-d2-2701310579);mix-blend-mode:darken}.sketch-overlay-N1{fill:url(#streaks-darker-d2-2701310579);mix-blend-mode:lighten}.sketch-overlay-N2{fill:url(#streaks-dark-d2-2701310579);mix-blend-mode:overlay}.sketch-overlay-N3{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-N4{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-N5{fill:url(#streaks-bright-d2-2701310579);mix-blend-mode:darken}.sketch-overlay-N6{fill:url(#streaks-bright-d2-2701310579);mix-blend-mode:darken}.sketch-overlay-N7{fill:url(#streaks-bright-d2-2701310579);mix-blend-mode:darken}.light-code{display: block}.dark-code{display: none}]]>Authenticated OTLP requestsspans, logs, metricsDurable write pathAuthoritative telemetryimmutable *.batch directoriesOrdinary Parquet files are authoritativeReadersBounded maintenanceRebuildable query stateFANOUT_DATA_DIR/queryDuckDB rollups and temporary spill. Not authoritative telemetry.Bounded group admissionconfigured row target + 20 ms windowUp to four commit workersindependent batches encode in parallelStaging batch directorymetadata + signal Parquet + trace.fidx for spansDurable publicationfsync → atomic rename → fsync parentpublished batch directories are discovered directly from the filesystemIndexed trace readbinary-search .fidx → Parquet rowsDuckDB SQLfilters, ordering, broad scansDuckDB rollupsrebuildable analytical cacheRetention + compaction admissionbounded batch and row groupsNative Parquet mergeprepared outside the reader gateCrash-safe namespace swapmarker + bounded renames + fsync acknowledge only after publicationOrdinary Parquet files are authoritative + - + -DuckDB rollups and temporary spill. Not authoritative telemetry. - +DuckDB rollups and temporary spill. Not authoritative telemetry. + - + - + diff --git a/site/src/content/docs/explanation/storage-model.mdx b/site/src/content/docs/explanation/storage-model.mdx index 8a7d6b51..526d7d7d 100644 --- a/site/src/content/docs/explanation/storage-model.mdx +++ b/site/src/content/docs/explanation/storage-model.mdx @@ -107,11 +107,8 @@ Discarded hashes force decoding; they do not establish a stored checksum. Verification failures such as memory exhaustion or cancellation never authorize quarantine. -Telemetry batches use format 4. An older or unknown batch format makes startup -fail before cleanup or schema rewriting, leaving the files intact. This build -has no reader or automatic conversion for previous formats. Start with an -empty telemetry directory; retain previous data separately for its original -reader. SQLite control state is independent of this telemetry format change. +Telemetry batches use format 4. An unsupported batch format makes startup +fail before cleanup or schema rewriting, leaving the files intact. ## Small files are bounded by compaction @@ -175,7 +172,7 @@ Narrow file selections use explicit lists from footer bounds. Broad selections use a glob with a captured-batch-ID filter, which excludes files published after snapshot capture. Only signals referenced by a query are bound. -`storage.rollup_skip_to_latest` skips historical service and edge aggregation. +`FANOUT_ROLLUP_SKIP_TO_LATEST` skips historical service and edge aggregation. Completed-batch trace caches still rebuild retained history in bounded passes; uncached files remain immediately queryable. diff --git a/site/src/content/docs/guides/back-up-and-restore.mdx b/site/src/content/docs/guides/back-up-and-restore.mdx index 12e88954..ef257adf 100644 --- a/site/src/content/docs/guides/back-up-and-restore.mdx +++ b/site/src/content/docs/guides/back-up-and-restore.mdx @@ -56,15 +56,6 @@ Migrations run forward at startup. A rollback to the previous binary after they have run is not assumed safe — restore the pre-upgrade backup with the previous version instead. -The Goose migration baseline requires a fresh control database. Existing -Atlas-managed control databases are rejected without conversion or reset. For -this transition, stop Fanout, back up the complete directory, and move its -`control` subdirectory aside before starting the new version. Telemetry and -query files remain in place. Complete first-admin setup again, then reconfigure -the ingest token, users, alert rules, and dashboards. Prior sessions, OAuth -grants, and agent history remain in the saved control database; recover that -state using the previous version and its backup. - ## Never delete an account from the control database Removing a user row cascades to sessions, OAuth grants and owned dashboards. To diff --git a/site/src/content/docs/guides/troubleshoot.mdx b/site/src/content/docs/guides/troubleshoot.mdx index b813a75b..3dd0405d 100644 --- a/site/src/content/docs/guides/troubleshoot.mdx +++ b/site/src/content/docs/guides/troubleshoot.mdx @@ -83,7 +83,7 @@ trace index. First stop Fanout and back up `telemetry/`, then verify the full authoritative set: ```sh -fanout --config /etc/fanout/fanout.yaml repair verify +fanout repair verify ``` The command is read-only and names every unreadable batch. Restore each named @@ -92,7 +92,7 @@ only its telemetry is preferable to leaving the instance offline, set it aside explicitly: ```sh -fanout --config /etc/fanout/fanout.yaml repair quarantine --batch +fanout repair quarantine --batch ``` Quarantine is an atomic rename beside the authoritative batch set, not a @@ -137,7 +137,7 @@ operator with shell access to the same configuration and data directory then mints a link directly: ```sh -fanout --config /etc/fanout/fanout.yaml login-link user@example.com +fanout login-link user@example.com ``` The link expires after 15 minutes, works once, and is recorded in the diff --git a/site/src/content/docs/reference/configuration.mdx b/site/src/content/docs/reference/configuration.mdx index 923354de..9804d1f1 100644 --- a/site/src/content/docs/reference/configuration.mdx +++ b/site/src/content/docs/reference/configuration.mdx @@ -1,55 +1,85 @@ --- title: Configuration description: How Fanout is configured, where settings come from, and how the reference stays true to the binary. -summary: The rules that govern every setting — precedence, naming, secrets, and what happens to a name that changes. +summary: The rules that govern every setting — precedence, naming, secret files, and validation. read_when: - You are configuring an instance and want the rules before the list. - A setting you set appears to have been ignored. status: preview --- -Fanout is configured from the environment, or from a YAML file using the same -keys. Every setting is listed under [Settings](/reference/settings/server), +Fanout resolves built-in defaults followed by nonempty process environment +values. Every setting is listed under [Settings](/reference/settings/server), grouped by the part of the product it belongs to. -Those pages are generated from `internal/config` by `cmd/fanout-docgen`, and -`just check` fails when a committed page no longer matches the type the loader -binds. That is the whole point of generating them: the reference cannot describe -a setting the binary does not accept, and cannot omit one it does. +Those pages and `.env.example` are generated from `internal/config` by +`cmd/fanout-docgen`. `just docs-generate-check` verifies that they match the +schema the binary uses. ## Where a value comes from -An environment variable wins over the YAML file, and the file wins over the -built-in default. A setting with no default and no value is empty, which for -most settings means the feature it configures is off. +All settings are read once at startup and require a restart. An empty +environment value means unset and preserves the built-in default. Native +installs listen on `127.0.0.1:7520`; the image supplies `FANOUT_ADDR=:7520` +and `FANOUT_DATA_DIR=/var/lib/fanout/data`. + +The binary reads only process environment. `.env.example` comments out every +empty entry and documents types, defaults, and secret file forms in comments. +Copying the template sets nothing and preserves image ENV. For a source +checkout, copy it to ignored `.env` and uncomment personal settings and +development credentials as needed. Build the binary, then use `just run`: +Node's native `parseEnv` loads optional `./.env`, and inherited environment values win, including empty +values. Values are never executed as shell code or interpolated. The binary +validates unknown names and types. Production deployments supply their own +environment values or managed secret mounts. ## Naming -Every environment variable is `FANOUT_`-prefixed. The YAML key is the dotted -path in the first column of each settings table — `storage.duckdb.memory` is -`FANOUT_DUCKDB_MEMORY`, and the two are the same setting. +Custom names use `FANOUT_` plus a clear name, adding the area only when it +clarifies. Vendor credentials and endpoints keep `ANTHROPIC_API_KEY`, +`ANTHROPIC_BASE_URL`, `OPENAI_API_KEY`, and `OPENAI_BASE_URL`. -## An unknown variable is an error +`FANOUT_AI_PROVIDER` defaults to empty, disabling chat. Exactly `anthropic` +or `openai` enables chat and requires that provider's key. Only the selected +provider's endpoint is validated and used. Inherited credentials never select +a provider. `FANOUT_AI_MODEL` selects the model; unrelated vendor/tool settings +such as `ANTHROPIC_MODEL` and `OPENAI_MODEL` are ignored. See +[AI settings](/reference/settings/ai) for base URL meanings. -Fanout rejects an unrecognised `FANOUT_`-prefixed variable at startup rather -than ignoring it. This matters more than it looks: Fanout is pre-release and -settings get renamed without aliases, so the alternative is a variable you -believe is in effect, silently replaced by a default nobody chose, discovered -months later. A refusal to start is the version of that failure you can act on. +## Unknown names -The consequence is that upgrading can require a configuration change, and the -process tells you which one at the first boot after the upgrade. +Unknown `FANOUT_` names fail startup. Platform-injected service-discovery names +are tolerated, and unrelated names in shared vendor namespaces are ignored. +Errors name the relevant environment variable. Boolean, integer, and Go +duration values must parse as their declared types. ## Secrets -Settings marked `secret` are redacted from the startup configuration log. They -are ordinary environment variables otherwise: anything that can read the process -environment can read them. Fanout does not encrypt them at rest and does not -integrate with a secret manager — mount them the way you would for any other -process. +Every secret accepts either its value or its file path: + +| Value variable | File path variable | +|---|---| +| `FANOUT_AUTH_CODE_SECRET` | `FANOUT_AUTH_CODE_SECRET_FILE` | +| `FANOUT_OIDC_CLIENT_SECRET` | `FANOUT_OIDC_CLIENT_SECRET_FILE` | +| `FANOUT_SMTP_PASSWORD` | `FANOUT_SMTP_PASSWORD_FILE` | +| `FANOUT_METRICS_TOKEN` | `FANOUT_METRICS_TOKEN_FILE` | +| `ANTHROPIC_API_KEY` | `ANTHROPIC_API_KEY_FILE` | +| `OPENAI_API_KEY` | `OPENAI_API_KEY_FILE` | + +Setting both forms nonempty fails startup. File contents lose exactly one +trailing LF or CRLF; an empty result fails. Errors name variables and paths, +never contents. The deployment controls file access; Fanout performs no +file-mode check. Secrets are redacted from startup logs. + +Use deployment environment injection, Docker/Compose secrets, Kubernetes +Secrets, or a private selected SOPS profile. This public repository ships no +operator profiles or recipients. See +[runtime secret delivery](https://github.com/labstack/fanout/blob/main/docs/operations.md#runtime-secret-delivery) +for examples. ## Validation -The whole configuration is validated before any listener opens, so a value the -binary will not accept stops it at startup. [First boot](/start/first-boot) -lists what is checked. +The whole configuration is validated before any listener opens. A value the +binary cannot accept stops startup. [First boot](/start/first-boot) lists what +is checked. The healthcheck command derives its probe from the listener and +TLS environment settings without validating unrelated settings or secrets. diff --git a/site/src/content/docs/reference/settings/ai.mdx b/site/src/content/docs/reference/settings/ai.mdx index d8efe765..5ed983de 100644 --- a/site/src/content/docs/reference/settings/ai.mdx +++ b/site/src/content/docs/reference/settings/ai.mdx @@ -1,7 +1,7 @@ --- title: "AI provider settings" description: "The provider, key, model and base URL backing the chat investigator." -summary: "Settings for the model behind the chat investigator. Without a key the agent is disabled." +summary: "Chat is disabled unless a provider is selected; its API key is then required." read_when: - "The chat investigator is disabled and you want it on." - "You are pointing Fanout at a self-hosted or proxied model endpoint." @@ -11,26 +11,53 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} -Every setting below is read from the environment variable in the second -column, or from the same key in a YAML configuration file. An unrecognised -`FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces -as a refusal to start rather than as a default nobody chose. +Every setting below is read once at startup and requires a restart to change. +Built-in defaults are overridden by nonempty environment values. +Unknown `FANOUT_` names fail startup; platform-injected service names are tolerated. +Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. +Vendor credentials and endpoints keep the vendor's documented names. -| Setting | Environment variable | Type | Default | -|---|---|---|---| -| `ai.api_key` | `FANOUT_AI_API_KEY` | string · secret | — | -| `ai.base_url` | `FANOUT_AI_BASE_URL` | string | — | -| `ai.model` | `FANOUT_AI_MODEL` | string | — | -| `ai.provider` | `FANOUT_AI_PROVIDER` | string | `anthropic` | +Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, +`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are +ignored; only the explicitly selected AI provider's key and endpoint are used. + +| Environment variable | Type | Default | +|---|---|---| +| `ANTHROPIC_API_KEY` or `ANTHROPIC_API_KEY_FILE` | string · secret | — | +| `ANTHROPIC_BASE_URL` | string | — | +| `FANOUT_AI_MODEL` | string | — | +| `FANOUT_AI_PROVIDER` | string | — | +| `OPENAI_API_KEY` or `OPENAI_API_KEY_FILE` | string · secret | — | +| `OPENAI_BASE_URL` | string | — | ## Notes -### `ai.model` +### `ANTHROPIC_API_KEY` + +Required when FANOUT_AI_PROVIDER is anthropic. + +### `ANTHROPIC_BASE_URL` + +Defaults to https://api.anthropic.com. Fanout appends /v1/messages, preserving any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. + +### `FANOUT_AI_MODEL` + +Overrides the provider's default: claude-sonnet-5-5 for Anthropic and gpt-6.1-sol for OpenAI. The OpenAI provider expects a reasoning model supporting reasoning.encrypted_content for stateless Responses replay. A 400 rejecting encrypted content is reported as a server-side configuration error; choose a compatible reasoning model with FANOUT_AI_MODEL. + +### `FANOUT_AI_PROVIDER` + +Enables chat by selecting exactly anthropic or openai. Empty disables chat. + +### `OPENAI_API_KEY` + +Required when FANOUT_AI_PROVIDER is openai. + +### `OPENAI_BASE_URL` -Overrides the provider's default: claude-sonnet-5-5 for Anthropic and gpt-6.1-sol for OpenAI. The OpenAI provider expects a reasoning model supporting reasoning.encrypted_content for stateless Responses replay. A 400 rejecting encrypted content is reported as a server-side configuration error; choose a compatible reasoning model with ai.model. +Defaults to https://api.openai.com/v1. Fanout appends /responses, preserving the API version and any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. :::caution[Secrets] Settings marked `secret` are redacted from Fanout's startup configuration -log. They are still plain environment variables: anything that can read the -process environment can read them. +log. Supply either the value or its `_FILE` path, never both. File contents +lose exactly one trailing newline (LF or CRLF); empty secrets fail startup. ::: diff --git a/site/src/content/docs/reference/settings/alerts.mdx b/site/src/content/docs/reference/settings/alerts.mdx index 6daeb6c9..8836fe5c 100644 --- a/site/src/content/docs/reference/settings/alerts.mdx +++ b/site/src/content/docs/reference/settings/alerts.mdx @@ -10,13 +10,28 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} -Every setting below is read from the environment variable in the second -column, or from the same key in a YAML configuration file. An unrecognised -`FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces -as a refusal to start rather than as a default nobody chose. - -| Setting | Environment variable | Type | Default | -|---|---|---|---| -| `alerts.enabled` | `FANOUT_ALERTS_ENABLED` | boolean | `true` | -| `alerts.evaluation_interval` | `FANOUT_ALERTS_EVALUATION_INTERVAL` | duration | `30s` | -| `alerts.history_days` | `FANOUT_ALERTS_HISTORY_DAYS` | integer | `7` | +Every setting below is read once at startup and requires a restart to change. +Built-in defaults are overridden by nonempty environment values. +Unknown `FANOUT_` names fail startup; platform-injected service names are tolerated. +Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. +Vendor credentials and endpoints keep the vendor's documented names. + +| Environment variable | Type | Default | +|---|---|---| +| `FANOUT_ALERTS_ENABLED` | boolean | `true` | +| `FANOUT_ALERTS_EVALUATION_INTERVAL` | duration | `30s` | +| `FANOUT_ALERTS_HISTORY_DAYS` | integer | `7` | + +## Notes + +### `FANOUT_ALERTS_ENABLED` + +Runs the alert evaluation engine. + +### `FANOUT_ALERTS_EVALUATION_INTERVAL` + +Schedules alert evaluation; use a Go duration of at least 1s. + +### `FANOUT_ALERTS_HISTORY_DAYS` + +Sets alert history retention in days. diff --git a/site/src/content/docs/reference/settings/auth.mdx b/site/src/content/docs/reference/settings/auth.mdx index 0d04de24..40086be4 100644 --- a/site/src/content/docs/reference/settings/auth.mdx +++ b/site/src/content/docs/reference/settings/auth.mdx @@ -11,32 +11,99 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} -Every setting below is read from the environment variable in the second -column, or from the same key in a YAML configuration file. An unrecognised -`FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces -as a refusal to start rather than as a default nobody chose. - -| Setting | Environment variable | Type | Default | -|---|---|---|---| -| `auth.code_secret` | `FANOUT_AUTH_CODE_SECRET` | string · secret | — | -| `auth.mode` | `FANOUT_AUTH_MODE` | string | `local` | -| `auth.oidc.admin_groups` | `FANOUT_OIDC_ADMIN_GROUPS` | string | — | -| `auth.oidc.allowed_domains` | `FANOUT_OIDC_ALLOWED_DOMAINS` | string | — | -| `auth.oidc.allowed_groups` | `FANOUT_OIDC_ALLOWED_GROUPS` | string | — | -| `auth.oidc.auto_provision` | `FANOUT_OIDC_AUTO_PROVISION` | boolean | `false` | -| `auth.oidc.client_id` | `FANOUT_OIDC_CLIENT_ID` | string | — | -| `auth.oidc.client_secret` | `FANOUT_OIDC_CLIENT_SECRET` | string · secret | — | -| `auth.oidc.default_role` | `FANOUT_OIDC_DEFAULT_ROLE` | string | `viewer` | -| `auth.oidc.email_claim` | `FANOUT_OIDC_EMAIL_CLAIM` | string | `email` | -| `auth.oidc.email_verification` | `FANOUT_OIDC_EMAIL_VERIFICATION` | string | `required` | -| `auth.oidc.issuer_url` | `FANOUT_OIDC_ISSUER_URL` | string | — | -| `auth.oidc.operator_groups` | `FANOUT_OIDC_OPERATOR_GROUPS` | string | — | -| `auth.self_signup` | `FANOUT_SELF_SIGNUP` | boolean | `false` | -| `auth.session_absolute_ttl` | `FANOUT_SESSION_ABSOLUTE_TTL` | duration | `168h` | -| `auth.session_idle_ttl` | `FANOUT_SESSION_IDLE_TTL` | duration | `12h` | +Every setting below is read once at startup and requires a restart to change. +Built-in defaults are overridden by nonempty environment values. +Unknown `FANOUT_` names fail startup; platform-injected service names are tolerated. +Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. +Vendor credentials and endpoints keep the vendor's documented names. + +| Environment variable | Type | Default | +|---|---|---| +| `FANOUT_AUTH_CODE_SECRET` or `FANOUT_AUTH_CODE_SECRET_FILE` | string · secret | — | +| `FANOUT_AUTH_MODE` | string | `local` | +| `FANOUT_OIDC_ADMIN_GROUPS` | string | — | +| `FANOUT_OIDC_ALLOWED_DOMAINS` | string | — | +| `FANOUT_OIDC_ALLOWED_GROUPS` | string | — | +| `FANOUT_OIDC_AUTO_PROVISION` | boolean | `false` | +| `FANOUT_OIDC_CLIENT_ID` | string | — | +| `FANOUT_OIDC_CLIENT_SECRET` or `FANOUT_OIDC_CLIENT_SECRET_FILE` | string · secret | — | +| `FANOUT_OIDC_DEFAULT_ROLE` | string | `viewer` | +| `FANOUT_OIDC_EMAIL_CLAIM` | string | `email` | +| `FANOUT_OIDC_EMAIL_VERIFICATION` | string | `required` | +| `FANOUT_OIDC_ISSUER_URL` | string | — | +| `FANOUT_OIDC_OPERATOR_GROUPS` | string | — | +| `FANOUT_SELF_SIGNUP` | boolean | `false` | +| `FANOUT_SESSION_ABSOLUTE_TTL` | duration | `168h` | +| `FANOUT_SESSION_IDLE_TTL` | duration | `12h` | + +## Notes + +### `FANOUT_AUTH_CODE_SECRET` + +Signs local sign-in codes; required in local mode with at least 32 characters. + +### `FANOUT_AUTH_MODE` + +Selects local sign-in or oidc; each mode validates its prerequisites. + +### `FANOUT_OIDC_ADMIN_GROUPS` + +Lists identity-provider groups mapped to admin, separated by commas. + +### `FANOUT_OIDC_ALLOWED_DOMAINS` + +Lists allowed email domains, separated by commas; wildcard * requires verified email. + +### `FANOUT_OIDC_ALLOWED_GROUPS` + +Lists allowed identity-provider groups, separated by commas. + +### `FANOUT_OIDC_AUTO_PROVISION` + +Creates accounts on OIDC sign-in; requires an allowed group or domain. + +### `FANOUT_OIDC_CLIENT_ID` + +Identifies Fanout's client registration; required in OIDC mode. + +### `FANOUT_OIDC_CLIENT_SECRET` + +Authenticates Fanout's client registration; required in OIDC mode. + +### `FANOUT_OIDC_DEFAULT_ROLE` + +Assigns viewer, operator, or admin to users without a group role mapping. + +### `FANOUT_OIDC_EMAIL_CLAIM` + +Names the claim containing the user's email address. + +### `FANOUT_OIDC_EMAIL_VERIFICATION` + +Selects required verified-email claims or issuer trust; issuer trust requires an allowlist. + +### `FANOUT_OIDC_ISSUER_URL` + +Selects the identity provider issuer; required in OIDC mode. + +### `FANOUT_OIDC_OPERATOR_GROUPS` + +Lists identity-provider groups mapped to operator, separated by commas. + +### `FANOUT_SELF_SIGNUP` + +Allows verified email addresses to create viewer accounts; requires local auth and SMTP. + +### `FANOUT_SESSION_ABSOLUTE_TTL` + +Limits total session lifetime; it must be at least FANOUT_SESSION_IDLE_TTL. + +### `FANOUT_SESSION_IDLE_TTL` + +Expires inactive sessions; use a Go duration of at least 5m. :::caution[Secrets] Settings marked `secret` are redacted from Fanout's startup configuration -log. They are still plain environment variables: anything that can read the -process environment can read them. +log. Supply either the value or its `_FILE` path, never both. File contents +lose exactly one trailing newline (LF or CRLF); empty secrets fail startup. ::: diff --git a/site/src/content/docs/reference/settings/ingest.mdx b/site/src/content/docs/reference/settings/ingest.mdx index 39cc5e1b..02271e66 100644 --- a/site/src/content/docs/reference/settings/ingest.mdx +++ b/site/src/content/docs/reference/settings/ingest.mdx @@ -11,19 +11,28 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} -Every setting below is read from the environment variable in the second -column, or from the same key in a YAML configuration file. An unrecognised -`FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces -as a refusal to start rather than as a default nobody chose. +Every setting below is read once at startup and requires a restart to change. +Built-in defaults are overridden by nonempty environment values. +Unknown `FANOUT_` names fail startup; platform-injected service names are tolerated. +Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. +Vendor credentials and endpoints keep the vendor's documented names. -| Setting | Environment variable | Type | Default | -|---|---|---|---| -| `ingest.batch_size` | `FANOUT_INGEST_BATCH_SIZE` | integer | `50000` | -| `ingest.default_namespace` | `FANOUT_DEFAULT_NAMESPACE` | string | `default` | -| `ingest.max_in_flight_bytes` | `FANOUT_INGEST_MAX_IN_FLIGHT_BYTES` | integer | `268435456` | +| Environment variable | Type | Default | +|---|---|---| +| `FANOUT_DEFAULT_NAMESPACE` | string | `default` | +| `FANOUT_INGEST_BATCH_SIZE` | integer | `50000` | +| `FANOUT_INGEST_MAX_IN_FLIGHT_BYTES` | integer | `268435456` | ## Notes -### `ingest.max_in_flight_bytes` +### `FANOUT_DEFAULT_NAMESPACE` + +Supplies the namespace for telemetry without one. + +### `FANOUT_INGEST_BATCH_SIZE` + +Limits telemetry rows per batch; valid values are 1–50000. + +### `FANOUT_INGEST_MAX_IN_FLIGHT_BYTES` Caps the decoded telemetry the process will hold across concurrent requests. Concurrency is HTTP/2 streams times connections and each handler holds its batch until the commit is durable, so without a ceiling a burst is bounded only by memory. Zero is unbounded, which is the behaviour that predates this setting. diff --git a/site/src/content/docs/reference/settings/mcp.mdx b/site/src/content/docs/reference/settings/mcp.mdx index 1b8d2ef2..799c2e83 100644 --- a/site/src/content/docs/reference/settings/mcp.mdx +++ b/site/src/content/docs/reference/settings/mcp.mdx @@ -10,11 +10,18 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} -Every setting below is read from the environment variable in the second -column, or from the same key in a YAML configuration file. An unrecognised -`FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces -as a refusal to start rather than as a default nobody chose. +Every setting below is read once at startup and requires a restart to change. +Built-in defaults are overridden by nonempty environment values. +Unknown `FANOUT_` names fail startup; platform-injected service names are tolerated. +Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. +Vendor credentials and endpoints keep the vendor's documented names. -| Setting | Environment variable | Type | Default | -|---|---|---|---| -| `mcp.enabled` | `FANOUT_MCP_ENABLED` | boolean | `true` | +| Environment variable | Type | Default | +|---|---|---| +| `FANOUT_MCP_ENABLED` | boolean | `true` | + +## Notes + +### `FANOUT_MCP_ENABLED` + +Serves the MCP endpoint at /mcp. diff --git a/site/src/content/docs/reference/settings/metrics.mdx b/site/src/content/docs/reference/settings/metrics.mdx index 0bfeceae..10cf1d1c 100644 --- a/site/src/content/docs/reference/settings/metrics.mdx +++ b/site/src/content/docs/reference/settings/metrics.mdx @@ -10,18 +10,29 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} -Every setting below is read from the environment variable in the second -column, or from the same key in a YAML configuration file. An unrecognised -`FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces -as a refusal to start rather than as a default nobody chose. +Every setting below is read once at startup and requires a restart to change. +Built-in defaults are overridden by nonempty environment values. +Unknown `FANOUT_` names fail startup; platform-injected service names are tolerated. +Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. +Vendor credentials and endpoints keep the vendor's documented names. -| Setting | Environment variable | Type | Default | -|---|---|---|---| -| `metrics.public` | `FANOUT_METRICS_PUBLIC` | boolean | `false` | -| `metrics.token` | `FANOUT_METRICS_TOKEN` | string · secret | — | +| Environment variable | Type | Default | +|---|---|---| +| `FANOUT_METRICS_PUBLIC` | boolean | `false` | +| `FANOUT_METRICS_TOKEN` or `FANOUT_METRICS_TOKEN_FILE` | string · secret | — | + +## Notes + +### `FANOUT_METRICS_PUBLIC` + +Allows unauthenticated Prometheus scrapes. + +### `FANOUT_METRICS_TOKEN` + +Authorizes Prometheus scrapes with a bearer token; optional when using an admin session. :::caution[Secrets] Settings marked `secret` are redacted from Fanout's startup configuration -log. They are still plain environment variables: anything that can read the -process environment can read them. +log. Supply either the value or its `_FILE` path, never both. File contents +lose exactly one trailing newline (LF or CRLF); empty secrets fail startup. ::: diff --git a/site/src/content/docs/reference/settings/server.mdx b/site/src/content/docs/reference/settings/server.mdx index 3cbc4b5d..eaf93934 100644 --- a/site/src/content/docs/reference/settings/server.mdx +++ b/site/src/content/docs/reference/settings/server.mdx @@ -11,26 +11,43 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} -Every setting below is read from the environment variable in the second -column, or from the same key in a YAML configuration file. An unrecognised -`FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces -as a refusal to start rather than as a default nobody chose. - -| Setting | Environment variable | Type | Default | -|---|---|---|---| -| `server.addr` | `FANOUT_ADDR` | string | `:7520` | -| `server.pprof_enabled` | `FANOUT_PPROF_ENABLED` | boolean | `false` | -| `server.public_url` | `FANOUT_PUBLIC_URL` | string | — | -| `server.tls.cert_file` | `FANOUT_TLS_CERT_FILE` | string | — | -| `server.tls.key_file` | `FANOUT_TLS_KEY_FILE` | string | — | -| `server.trusted_proxy_cidrs` | `FANOUT_TRUSTED_PROXY_CIDRS` | string | — | +Every setting below is read once at startup and requires a restart to change. +Built-in defaults are overridden by nonempty environment values. +Unknown `FANOUT_` names fail startup; platform-injected service names are tolerated. +Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. +Vendor credentials and endpoints keep the vendor's documented names. + +| Environment variable | Type | Default | +|---|---|---| +| `FANOUT_ADDR` | string | `127.0.0.1:7520` | +| `FANOUT_PPROF_ENABLED` | boolean | `false` | +| `FANOUT_PUBLIC_URL` | string | — | +| `FANOUT_TLS_CERT_FILE` | string | — | +| `FANOUT_TLS_KEY_FILE` | string | — | +| `FANOUT_TRUSTED_PROXY_CIDRS` | string | — | ## Notes -### `server.addr` +### `FANOUT_ADDR` Binds the browser, API, MCP, OTLP/gRPC and OTLP/HTTP listener. -### `server.pprof_enabled` +### `FANOUT_PPROF_ENABLED` Exposes Go's net/http/pprof handlers at /debug/pprof/* for CPU/heap/mutex/goroutine profiling under load. Off by default; the routes require an admin browser session with operations:read. + +### `FANOUT_PUBLIC_URL` + +Sets the external browser origin. Required for OIDC; HTTPS is required with MCP. + +### `FANOUT_TLS_CERT_FILE` + +Selects the TLS certificate file; requires FANOUT_TLS_KEY_FILE. + +### `FANOUT_TLS_KEY_FILE` + +Selects the TLS private key file; requires FANOUT_TLS_CERT_FILE. + +### `FANOUT_TRUSTED_PROXY_CIDRS` + +Lists trusted proxy networks, separated by commas; empty trusts no forwarded client addresses. diff --git a/site/src/content/docs/reference/settings/smtp.mdx b/site/src/content/docs/reference/settings/smtp.mdx index f259359e..ea1d09db 100644 --- a/site/src/content/docs/reference/settings/smtp.mdx +++ b/site/src/content/docs/reference/settings/smtp.mdx @@ -10,21 +10,44 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} -Every setting below is read from the environment variable in the second -column, or from the same key in a YAML configuration file. An unrecognised -`FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces -as a refusal to start rather than as a default nobody chose. - -| Setting | Environment variable | Type | Default | -|---|---|---|---| -| `smtp.from` | `FANOUT_SMTP_FROM` | string | — | -| `smtp.host` | `FANOUT_SMTP_HOST` | string | — | -| `smtp.password` | `FANOUT_SMTP_PASSWORD` | string · secret | — | -| `smtp.port` | `FANOUT_SMTP_PORT` | integer | `587` | -| `smtp.username` | `FANOUT_SMTP_USERNAME` | string | — | +Every setting below is read once at startup and requires a restart to change. +Built-in defaults are overridden by nonempty environment values. +Unknown `FANOUT_` names fail startup; platform-injected service names are tolerated. +Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. +Vendor credentials and endpoints keep the vendor's documented names. + +| Environment variable | Type | Default | +|---|---|---| +| `FANOUT_SMTP_FROM` | string | — | +| `FANOUT_SMTP_HOST` | string | — | +| `FANOUT_SMTP_PASSWORD` or `FANOUT_SMTP_PASSWORD_FILE` | string · secret | — | +| `FANOUT_SMTP_PORT` | integer | `587` | +| `FANOUT_SMTP_USERNAME` | string | — | + +## Notes + +### `FANOUT_SMTP_FROM` + +Sets the email sender address; required with SMTP. + +### `FANOUT_SMTP_HOST` + +Selects the mail server. Host, username, password, and from must all be set to enable SMTP. + +### `FANOUT_SMTP_PASSWORD` + +Authenticates to the mail server; required with SMTP. + +### `FANOUT_SMTP_PORT` + +Sets the mail server's positive TCP port. + +### `FANOUT_SMTP_USERNAME` + +Authenticates to the mail server; required with SMTP. :::caution[Secrets] Settings marked `secret` are redacted from Fanout's startup configuration -log. They are still plain environment variables: anything that can read the -process environment can read them. +log. Supply either the value or its `_FILE` path, never both. File contents +lose exactly one trailing newline (LF or CRLF); empty secrets fail startup. ::: diff --git a/site/src/content/docs/reference/settings/storage.mdx b/site/src/content/docs/reference/settings/storage.mdx index 5905ae30..b852f70d 100644 --- a/site/src/content/docs/reference/settings/storage.mdx +++ b/site/src/content/docs/reference/settings/storage.mdx @@ -11,40 +11,53 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} -Every setting below is read from the environment variable in the second -column, or from the same key in a YAML configuration file. An unrecognised -`FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces -as a refusal to start rather than as a default nobody chose. - -| Setting | Environment variable | Type | Default | -|---|---|---|---| -| `storage.data_dir` | `FANOUT_DATA_DIR` | string | `./data` | -| `storage.duckdb.max_connections` | `FANOUT_DUCKDB_MAX_CONNECTIONS` | integer | `0` | -| `storage.duckdb.memory` | `FANOUT_DUCKDB_MEMORY` | string | — | -| `storage.duckdb.threads` | `FANOUT_DUCKDB_THREADS` | integer | — | -| `storage.maintenance_interval` | `FANOUT_MAINTENANCE_INTERVAL` | duration | `1h` | -| `storage.retention_days` | `FANOUT_RETENTION_DAYS` | integer | `30` | -| `storage.rollup_interval` | `FANOUT_ROLLUP_INTERVAL` | duration | `1m` | -| `storage.rollup_skip_to_latest` | `FANOUT_ROLLUP_SKIP_TO_LATEST` | boolean | `false` | +Every setting below is read once at startup and requires a restart to change. +Built-in defaults are overridden by nonempty environment values. +Unknown `FANOUT_` names fail startup; platform-injected service names are tolerated. +Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. +Vendor credentials and endpoints keep the vendor's documented names. + +| Environment variable | Type | Default | +|---|---|---| +| `FANOUT_DATA_DIR` | string | `./data` | +| `FANOUT_DUCKDB_MAX_CONNECTIONS` | integer | `0` | +| `FANOUT_DUCKDB_MEMORY` | string | — | +| `FANOUT_DUCKDB_THREADS` | integer | — | +| `FANOUT_MAINTENANCE_INTERVAL` | duration | `1h` | +| `FANOUT_RETENTION_DAYS` | integer | `30` | +| `FANOUT_ROLLUP_INTERVAL` | duration | `1m` | +| `FANOUT_ROLLUP_SKIP_TO_LATEST` | boolean | `false` | ## Notes -### `storage.duckdb.max_connections` +### `FANOUT_DATA_DIR` + +Stores telemetry, query state, and the control database. + +### `FANOUT_DUCKDB_MAX_CONNECTIONS` Caps the DuckDB connection pool. Reads scan immutable Parquet concurrently; rollup-cache writes are serialized by the query write gate. Zero means "size it from the machine" — the same spelling DuckDBThreads uses for deferring to a default. Resolution happens in resolveSizing and is reported in the startup configuration log. -### `storage.duckdb.memory` +### `FANOUT_DUCKDB_MEMORY` Caps DuckDB's memory (e.g. "8GB"). Empty means Fanout sizes it from detected memory, reserving headroom for the Go runtime; see resolveSizing. DuckDB's own default is 80% of detected RAM, which is calculated as though DuckDB owned the machine and has been observed to get the process OOM-killed once the Go heap is added on top. -### `storage.duckdb.threads` +### `FANOUT_DUCKDB_THREADS` Caps DuckDB's global query worker pool. Zero leaves DuckDB's own default in place (one worker per core). Set it to leave cores free for ingest on a query-heavy co-tenant host. -### `storage.maintenance_interval` +### `FANOUT_MAINTENANCE_INTERVAL` Controls Parquet retention and compaction, and query-cache checkpointing. -### `storage.rollup_skip_to_latest` +### `FANOUT_RETENTION_DAYS` + +Sets telemetry retention in days; zero disables expiry. + +### `FANOUT_ROLLUP_INTERVAL` + +Schedules analytical rollups; use a Go duration of at least 1s. + +### `FANOUT_ROLLUP_SKIP_TO_LATEST` RollupSkipToLatest, set once at boot, advances the service and edge rollup watermarks to the current max ingested timestamp. Existing history is not aggregated into those analytical rollups. Completed-batch trace caches still rebuild all retained history in bounded passes; uncached files remain immediately queryable. Useful for large pre-seeded datasets (benchmarks, restores); off in normal operation. diff --git a/site/src/content/docs/start/install.mdx b/site/src/content/docs/start/install.mdx index b6d46919..264f4751 100644 --- a/site/src/content/docs/start/install.mdx +++ b/site/src/content/docs/start/install.mdx @@ -32,9 +32,11 @@ pick whichever registry your hosts already pull from. `FANOUT_AUTH_CODE_SECRET` is required in the default `local` auth mode and must be at least 32 characters — the container refuses to start without it. It is the only setting the command above adds; the image ships listener and data-directory -defaults of its own. Add `FANOUT_AI_API_KEY` to enable the chat investigator, -which is optional: without a key the process starts normally and logs that the -agent is disabled. +defaults of its own. Select `FANOUT_AI_PROVIDER=anthropic` with `ANTHROPIC_API_KEY`, +or `FANOUT_AI_PROVIDER=openai` with `OPENAI_API_KEY`, to enable the chat +investigator. Chat is optional and disabled when no provider is selected, even +with inherited vendor credentials or endpoints. Selecting a provider requires +its key at startup and validates only its endpoint. :::caution[Use a named volume, or fix the ownership yourself] The image is distroless and runs unprivileged as UID/GID 65532. A **named @@ -43,8 +45,9 @@ is not: `-v "$PWD/data:/var/lib/fanout/data"` fails to open its data directory unless that directory is already writable by 65532. ::: -Both the image and native binary bind the shared listener on all interfaces. -Set `FANOUT_ADDR=127.0.0.1:7520` for a local-only instance. +The image sets `FANOUT_ADDR=:7520` and `FANOUT_DATA_DIR=/var/lib/fanout/data`. +Native installs default to `127.0.0.1:7520`; set `FANOUT_ADDR=:7520` to accept +connections from other hosts. @@ -70,6 +73,7 @@ run it: ```sh chmod +x fanout +export FANOUT_AUTH_CODE_SECRET=$(openssl rand -hex 32) ./fanout ``` @@ -114,24 +118,19 @@ reasoning is in the repository's release contract. | Port or path | Purpose | Native default | |---|---|---| -| `7520` | Browser, API, chat, MCP, OTLP/gRPC and OTLP/HTTP | `:7520`, all interfaces | +| `7520` | Browser, API, chat, MCP, OTLP/gRPC and OTLP/HTTP | `127.0.0.1:7520`, loopback | | `./data` | Telemetry, query state, users, sessions, dashboards, alerts and agent history | `./data` | Every OTLP export requires the ingest token, including on the shared port. Ingest rejects requests before setup completes. Read [authenticate exporters](/start/send-telemetry) before pointing a service at the instance. -Upgrading from separate ports? Remove `FANOUT_OTLP_GRPC_ADDR` and -`FANOUT_OTLP_HTTP_ADDR` (or YAML `ingest.otlp_grpc_addr` and -`ingest.otlp_http_addr`), update exporters to port `7520` or your public HTTPS -origin, and remove the old port mappings. Rename `FANOUT_HTTP_ADDR` to -`FANOUT_ADDR` (YAML `server.http_addr` to `server.addr`). Removed settings are -startup errors. Also remove `FANOUT_INGEST_ADVERTISED_ENDPOINT` or YAML -`ingest.advertised_endpoint`; change a private collector hostname directly on -**Connect telemetry** before copying its generated configuration. - -Separate interface bindings are also removed. If the dashboard is public but -ingest must remain LAN-only, keep Fanout's backend port private and deny +Exporters use the shared port `7520` or the public HTTPS origin. Every OTLP +export requires the ingest token. A private collector hostname can be entered +on **Connect telemetry** before copying its generated configuration. + +If the dashboard is public but ingest must remain LAN-only, keep Fanout's +backend port private and deny `/v1`, `/v1/*`, and `/opentelemetry.proto.collector.*` on the public proxy. Allow collectors through the private network or a restricted proxy route, and verify that neither OTLP transport is reachable through the public origin. diff --git a/site/src/pages/llms.txt.ts b/site/src/pages/llms.txt.ts index 96174879..3e0a5d8f 100644 --- a/site/src/pages/llms.txt.ts +++ b/site/src/pages/llms.txt.ts @@ -63,7 +63,7 @@ export const GET: APIRoute = async ({ site }) => { "", "## Operating Fanout from an agent", "", - "- Fanout is a server. Its whole command line is three forms: `fanout [flags]` to run it, `fanout version`, and `fanout [--config path] login-link ` — a local-auth-mode recovery path that needs shell access on the host. Everything else is done over HTTP or MCP; there are no operational subcommands to script.", + "- Fanout reads built-in defaults and process environment. Native installs listen on 127.0.0.1:7520; the image supplies FANOUT_ADDR=:7520. Secret variables accept _FILE paths. Commands: `fanout [flags]`, `fanout version`, `fanout login-link ` (local recovery with host access), `fanout healthcheck [url]`, `fanout repair verify`, and `fanout repair quarantine --batch `. Product operations use HTTP or MCP.", "- MCP is the intended agent interface. The server exposes its tools at `/mcp`.", "- MCP and the HTTP API authenticate with an API key, which is not the same credential as the ingest token. Both are `fo_`-prefixed; the routes tell them apart by prefix.", "- Telemetry goes in over OTLP only: gRPC and HTTP on the same port as the application (7520 by default). There is no bespoke ingest API to script against.", diff --git a/site/src/styles/fanout.css b/site/src/styles/fanout.css index cb9070e4..9704db7e 100644 --- a/site/src/styles/fanout.css +++ b/site/src/styles/fanout.css @@ -262,7 +262,7 @@ h1#_top { /* Code inside a heading is set smaller than the heading's own size. The generated settings reference titles every note with a setting key, and a key is one long unbreakable token: at Starlight's h3 size, - `storage.duckdb.max_connections` is wider than the content column and breaks + `FANOUT_DUCKDB_MAX_CONNECTIONS` is wider than the content column and breaks mid-identifier, rendering as "max_connection" above a lone "s". */ .sl-markdown-content :is(h1, h2, h3, h4, h5, h6) code { font-size: 0.78em; diff --git a/ui/host/e2e/global-setup.ts b/ui/host/e2e/global-setup.ts index 6bf3d7de..24b9113a 100644 --- a/ui/host/e2e/global-setup.ts +++ b/ui/host/e2e/global-setup.ts @@ -73,13 +73,15 @@ export default async function globalSetup() { try { const base = `http://127.0.0.1:${await freePort()}`; // Ignore inherited product configuration and keys; this instance is disposable. - const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith("FANOUT_") && !/(?:API_KEY|TOKEN|SECRET)$/.test(key))); + const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => + !key.startsWith("FANOUT_") && !key.startsWith("ANTHROPIC_") && !key.startsWith("OPENAI_") && + !/(?:API_KEY|TOKEN|SECRET)$/.test(key))); const startDeadline = Date.now() + 30_000; const fd = openSync(serverLog, "a", 0o600); child = spawn(resolve(process.env.FANOUT_E2E_BINARY ?? join(root, "bin/fanout")), [], { cwd: dir, env: { ...env, FANOUT_DATA_DIR: join(dir, "data"), FANOUT_ADDR: new URL(base).host, - FANOUT_AUTH_CODE_SECRET: randomBytes(32).toString("hex"), FANOUT_AI_API_KEY: "", + FANOUT_AUTH_CODE_SECRET: randomBytes(32).toString("hex"), FANOUT_ROLLUP_INTERVAL: "1s" }, stdio: ["ignore", fd, fd], });