From 7467441b327f3057fd34a56f901c9b1ae30fa2a2 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Tue, 13 Oct 2026 19:16:44 -0700 Subject: [PATCH 1/6] feat(config)!: adopt provider env names and dotenv defaults Use vendor credential and endpoint contracts with schema-generated public defaults so deployment configuration has explicit ownership. Keep secrets in runtime injection and exclude personal files from production launches. BREAKING CHANGE: Replace the generic AI key and base URL environment names and YAML keys with the selected provider's settings. OpenAI base URLs now include the API version path. Retired names have no aliases or fallback. --- .dockerignore | 4 + .env | 181 +++++++++++++++ .env.example | 208 ++++++++++++++++++ .env.production | 7 + .gitignore | 10 +- AGENTS.md | 27 +++ CONTRIBUTING.md | 17 +- README.md | 71 +++++- cmd/fanout-docgen/main.go | 136 +++++++++++- cmd/fanout-docgen/main_test.go | 110 ++++++++- cmd/fanout/main.go | 4 +- docs/operations.md | 83 +++++++ fanout.example.yaml | 15 +- internal/agent/provider_contract_test.go | 6 +- internal/agent/provider_http.go | 8 +- internal/agent/provider_http_test.go | 38 +++- internal/api/health_test.go | 2 +- internal/config/config.go | 192 +++++++++++----- internal/config/config_test.go | 19 +- internal/config/loader.go | 17 +- internal/config/provider_test.go | 71 ++++++ justfile | 9 + scripts/run-fanout.mjs | 113 ++++++++++ scripts/run-fanout.test.ts | 103 +++++++++ .../content/docs/reference/configuration.mdx | 41 +++- .../content/docs/reference/settings/ai.mdx | 33 ++- .../docs/reference/settings/alerts.mdx | 21 +- .../content/docs/reference/settings/auth.mdx | 73 +++++- .../docs/reference/settings/ingest.mdx | 15 +- .../content/docs/reference/settings/mcp.mdx | 13 +- .../docs/reference/settings/metrics.mdx | 17 +- .../docs/reference/settings/server.mdx | 23 +- .../content/docs/reference/settings/smtp.mdx | 29 ++- .../docs/reference/settings/storage.mdx | 19 +- site/src/content/docs/start/install.mdx | 18 +- ui/host/e2e/global-setup.ts | 6 +- 36 files changed, 1634 insertions(+), 125 deletions(-) create mode 100644 .env create mode 100644 .env.example create mode 100644 .env.production create mode 100644 internal/config/provider_test.go create mode 100644 scripts/run-fanout.mjs create mode 100644 scripts/run-fanout.test.ts diff --git a/.dockerignore b/.dockerignore index a49dd2c0..437e7221 100644 --- a/.dockerignore +++ b/.dockerignore @@ -21,6 +21,10 @@ ui/host/dist/ # Dev files .env .env.* +**/.env +**/.env.* +secrets/ +.sops.yaml .tmp/ tmp/ diff --git a/.env b/.env new file mode 100644 index 00000000..7788ec6b --- /dev/null +++ b/.env @@ -0,0 +1,181 @@ +# Generated nonsecret configuration by cmd/fanout-docgen. +# Regenerate with just docs-generate. Credentials never belong in this file. +# Shared local defaults; .env.production overrides them for production. +# Use just run, or inject these values explicitly; Fanout does not load dotenv. +# Personal nonsecret overrides belong in ignored .env.local. + +# ai.anthropic.base_url; string +# Defaults to https://api.anthropic.com. Fanout appends /v1/messages, preserving any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. +ANTHROPIC_BASE_URL= + +# ai.model; string +# Overrides the provider's default: claude-sonnet-5-5 for Anthropic and gpt-6.1-sol for OpenAI. The OpenAI provider expects a reasoning model supporting reasoning.encrypted_content for stateless Responses replay. A 400 rejecting encrypted content is reported as a server-side configuration error; choose a compatible reasoning model with ai.model. +FANOUT_AI_MODEL= + +# ai.openai.base_url; string +# Defaults to https://api.openai.com/v1. Fanout appends /responses, preserving the API version and any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. +OPENAI_BASE_URL= + +# ai.provider; string +# Selects anthropic or openai independently of which keys are present. +FANOUT_AI_PROVIDER=anthropic + +# alerts.enabled; boolean +# Runs the alert evaluation engine. +FANOUT_ALERTS_ENABLED=true + +# alerts.evaluation_interval; duration +# Schedules alert evaluation; use a Go duration of at least 1s. +FANOUT_ALERTS_EVALUATION_INTERVAL=30s + +# alerts.history_days; integer +# Sets alert history retention in days. +FANOUT_ALERTS_HISTORY_DAYS=7 + +# auth.mode; string +# Selects local sign-in or oidc; each mode validates its prerequisites. +FANOUT_AUTH_MODE=local + +# auth.oidc.admin_groups; string +# Lists identity-provider groups mapped to admin, separated by commas. +FANOUT_OIDC_ADMIN_GROUPS= + +# auth.oidc.allowed_domains; string +# Lists allowed email domains, separated by commas; wildcard * requires verified email. +FANOUT_OIDC_ALLOWED_DOMAINS= + +# auth.oidc.allowed_groups; string +# Lists allowed identity-provider groups, separated by commas. +FANOUT_OIDC_ALLOWED_GROUPS= + +# auth.oidc.auto_provision; boolean +# Creates accounts on OIDC sign-in; requires an allowed group or domain. +FANOUT_OIDC_AUTO_PROVISION=false + +# auth.oidc.client_id; string +# Identifies Fanout's client registration; required in OIDC mode. +FANOUT_OIDC_CLIENT_ID= + +# auth.oidc.default_role; string +# Assigns viewer, operator, or admin to users without a group role mapping. +FANOUT_OIDC_DEFAULT_ROLE=viewer + +# auth.oidc.email_claim; string +# Names the claim containing the user's email address. +FANOUT_OIDC_EMAIL_CLAIM=email + +# auth.oidc.email_verification; string +# Selects required verified-email claims or issuer trust; issuer trust requires an allowlist. +FANOUT_OIDC_EMAIL_VERIFICATION=required + +# auth.oidc.issuer_url; string +# Selects the identity provider issuer; required in OIDC mode. +FANOUT_OIDC_ISSUER_URL= + +# auth.oidc.operator_groups; string +# Lists identity-provider groups mapped to operator, separated by commas. +FANOUT_OIDC_OPERATOR_GROUPS= + +# auth.self_signup; boolean +# Allows verified email addresses to create viewer accounts; requires local auth and SMTP. +FANOUT_SELF_SIGNUP=false + +# auth.session_absolute_ttl; duration +# Limits total session lifetime; it must be at least session_idle_ttl. +FANOUT_SESSION_ABSOLUTE_TTL=168h + +# auth.session_idle_ttl; duration +# Expires inactive sessions; use a Go duration of at least 5m. +FANOUT_SESSION_IDLE_TTL=12h + +# ingest.batch_size; integer +# Limits telemetry rows per batch; valid values are 1–50000. +FANOUT_INGEST_BATCH_SIZE=50000 + +# ingest.default_namespace; string +# Supplies the namespace for telemetry without one. +FANOUT_DEFAULT_NAMESPACE=default + +# ingest.max_in_flight_bytes; integer +# Caps the decoded telemetry the process will hold across concurrent requests. Concurrency is HTTP/2 streams times connections and each handler holds its batch until the commit is durable, so without a ceiling a burst is bounded only by memory. Zero is unbounded, which is the behaviour that predates this setting. +FANOUT_INGEST_MAX_IN_FLIGHT_BYTES=268435456 + +# mcp.enabled; boolean +# Serves the MCP endpoint at /mcp. +FANOUT_MCP_ENABLED=true + +# metrics.public; boolean +# Allows unauthenticated Prometheus scrapes. +FANOUT_METRICS_PUBLIC=false + +# server.addr; string +# Binds the browser, API, MCP, OTLP/gRPC and OTLP/HTTP listener. +FANOUT_ADDR=127.0.0.1:7520 + +# server.pprof_enabled; boolean +# Exposes Go's net/http/pprof handlers at /debug/pprof/* for CPU/heap/mutex/goroutine profiling under load. Off by default; the routes require an admin browser session with operations:read. +FANOUT_PPROF_ENABLED=false + +# server.public_url; string +# Sets the external browser origin. Required for OIDC; HTTPS is required with MCP. +FANOUT_PUBLIC_URL= + +# server.tls.cert_file; string +# Selects the TLS certificate file; requires tls.key_file. +FANOUT_TLS_CERT_FILE= + +# server.tls.key_file; string +# Selects the TLS private key file; requires tls.cert_file. +FANOUT_TLS_KEY_FILE= + +# server.trusted_proxy_cidrs; string +# Lists trusted proxy networks, separated by commas; empty trusts no forwarded client addresses. +FANOUT_TRUSTED_PROXY_CIDRS= + +# smtp.from; string +# Sets the email sender address; required with SMTP. +FANOUT_SMTP_FROM= + +# smtp.host; string +# Selects the mail server. Host, username, password, and from must all be set to enable SMTP. +FANOUT_SMTP_HOST= + +# smtp.port; integer +# Sets the mail server's positive TCP port. +FANOUT_SMTP_PORT=587 + +# smtp.username; string +# Authenticates to the mail server; required with SMTP. +FANOUT_SMTP_USERNAME= + +# storage.data_dir; string +# Stores telemetry, query state, and the control database. +FANOUT_DATA_DIR=./data + +# storage.duckdb.max_connections; integer +# Caps the DuckDB connection pool. Reads scan immutable Parquet concurrently; rollup-cache writes are serialized by the query write gate. Zero means "size it from the machine" — the same spelling DuckDBThreads uses for deferring to a default. Resolution happens in resolveSizing and is reported in the startup configuration log. +FANOUT_DUCKDB_MAX_CONNECTIONS=0 + +# storage.duckdb.memory; string +# Caps DuckDB's memory (e.g. "8GB"). Empty means Fanout sizes it from detected memory, reserving headroom for the Go runtime; see resolveSizing. DuckDB's own default is 80% of detected RAM, which is calculated as though DuckDB owned the machine and has been observed to get the process OOM-killed once the Go heap is added on top. +FANOUT_DUCKDB_MEMORY= + +# storage.duckdb.threads; integer +# Caps DuckDB's global query worker pool. Zero leaves DuckDB's own default in place (one worker per core). Set it to leave cores free for ingest on a query-heavy co-tenant host. +FANOUT_DUCKDB_THREADS= + +# storage.maintenance_interval; duration +# Controls Parquet retention and compaction, and query-cache checkpointing. +FANOUT_MAINTENANCE_INTERVAL=1h + +# storage.retention_days; integer +# Sets telemetry retention in days; zero disables expiry. +FANOUT_RETENTION_DAYS=30 + +# storage.rollup_interval; duration +# Schedules analytical rollups; use a Go duration of at least 1s. +FANOUT_ROLLUP_INTERVAL=1m + +# storage.rollup_skip_to_latest; boolean +# RollupSkipToLatest, set once at boot, advances the service and edge rollup watermarks to the current max ingested timestamp. Existing history is not aggregated into those analytical rollups. Completed-batch trace caches still rebuild all retained history in bounded passes; uncached files remain immediately queryable. Useful for large pre-seeded datasets (benchmarks, restores); off in normal operation. +FANOUT_ROLLUP_SKIP_TO_LATEST=false diff --git a/.env.example b/.env.example new file mode 100644 index 00000000..a0848953 --- /dev/null +++ b/.env.example @@ -0,0 +1,208 @@ +# Generated by cmd/fanout-docgen from internal/config/config.go. +# Regenerate with just docs-generate; verify with just docs-generate-check. +# Fanout does not load dotenv files. Inject through your launcher or deployment. +# Precedence: built-in defaults < --config YAML < nonempty environment values. +# All settings require restart. Empty environment values mean no override. +# Secret entries below are empty placeholders; use runtime secret injection. +# Local authentication requires FANOUT_AUTH_CODE_SECRET (32+ characters). +# Only the AI key/endpoint selected by FANOUT_AI_PROVIDER is used. + +# ai.anthropic.api_key; string; secret +# Enables chat only when ai.provider is anthropic. +ANTHROPIC_API_KEY= + +# ai.anthropic.base_url; string +# Defaults to https://api.anthropic.com. Fanout appends /v1/messages, preserving any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. +ANTHROPIC_BASE_URL= + +# ai.model; string +# Overrides the provider's default: claude-sonnet-5-5 for Anthropic and gpt-6.1-sol for OpenAI. The OpenAI provider expects a reasoning model supporting reasoning.encrypted_content for stateless Responses replay. A 400 rejecting encrypted content is reported as a server-side configuration error; choose a compatible reasoning model with ai.model. +FANOUT_AI_MODEL= + +# ai.openai.api_key; string; secret +# Enables chat only when ai.provider is openai. +OPENAI_API_KEY= + +# ai.openai.base_url; string +# Defaults to https://api.openai.com/v1. Fanout appends /responses, preserving the API version and any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. +OPENAI_BASE_URL= + +# ai.provider; string +# Selects anthropic or openai independently of which keys are present. +FANOUT_AI_PROVIDER=anthropic + +# alerts.enabled; boolean +# Runs the alert evaluation engine. +FANOUT_ALERTS_ENABLED=true + +# alerts.evaluation_interval; duration +# Schedules alert evaluation; use a Go duration of at least 1s. +FANOUT_ALERTS_EVALUATION_INTERVAL=30s + +# alerts.history_days; integer +# Sets alert history retention in days. +FANOUT_ALERTS_HISTORY_DAYS=7 + +# auth.code_secret; string; secret +# Signs local sign-in codes; required in local mode with at least 32 characters. +FANOUT_AUTH_CODE_SECRET= + +# auth.mode; string +# Selects local sign-in or oidc; each mode validates its prerequisites. +FANOUT_AUTH_MODE=local + +# auth.oidc.admin_groups; string +# Lists identity-provider groups mapped to admin, separated by commas. +FANOUT_OIDC_ADMIN_GROUPS= + +# auth.oidc.allowed_domains; string +# Lists allowed email domains, separated by commas; wildcard * requires verified email. +FANOUT_OIDC_ALLOWED_DOMAINS= + +# auth.oidc.allowed_groups; string +# Lists allowed identity-provider groups, separated by commas. +FANOUT_OIDC_ALLOWED_GROUPS= + +# auth.oidc.auto_provision; boolean +# Creates accounts on OIDC sign-in; requires an allowed group or domain. +FANOUT_OIDC_AUTO_PROVISION=false + +# auth.oidc.client_id; string +# Identifies Fanout's client registration; required in OIDC mode. +FANOUT_OIDC_CLIENT_ID= + +# auth.oidc.client_secret; string; secret +# Authenticates Fanout's client registration; required in OIDC mode. +FANOUT_OIDC_CLIENT_SECRET= + +# auth.oidc.default_role; string +# Assigns viewer, operator, or admin to users without a group role mapping. +FANOUT_OIDC_DEFAULT_ROLE=viewer + +# auth.oidc.email_claim; string +# Names the claim containing the user's email address. +FANOUT_OIDC_EMAIL_CLAIM=email + +# auth.oidc.email_verification; string +# Selects required verified-email claims or issuer trust; issuer trust requires an allowlist. +FANOUT_OIDC_EMAIL_VERIFICATION=required + +# auth.oidc.issuer_url; string +# Selects the identity provider issuer; required in OIDC mode. +FANOUT_OIDC_ISSUER_URL= + +# auth.oidc.operator_groups; string +# Lists identity-provider groups mapped to operator, separated by commas. +FANOUT_OIDC_OPERATOR_GROUPS= + +# auth.self_signup; boolean +# Allows verified email addresses to create viewer accounts; requires local auth and SMTP. +FANOUT_SELF_SIGNUP=false + +# auth.session_absolute_ttl; duration +# Limits total session lifetime; it must be at least session_idle_ttl. +FANOUT_SESSION_ABSOLUTE_TTL=168h + +# auth.session_idle_ttl; duration +# Expires inactive sessions; use a Go duration of at least 5m. +FANOUT_SESSION_IDLE_TTL=12h + +# ingest.batch_size; integer +# Limits telemetry rows per batch; valid values are 1–50000. +FANOUT_INGEST_BATCH_SIZE=50000 + +# ingest.default_namespace; string +# Supplies the namespace for telemetry without one. +FANOUT_DEFAULT_NAMESPACE=default + +# ingest.max_in_flight_bytes; integer +# Caps the decoded telemetry the process will hold across concurrent requests. Concurrency is HTTP/2 streams times connections and each handler holds its batch until the commit is durable, so without a ceiling a burst is bounded only by memory. Zero is unbounded, which is the behaviour that predates this setting. +FANOUT_INGEST_MAX_IN_FLIGHT_BYTES=268435456 + +# mcp.enabled; boolean +# Serves the MCP endpoint at /mcp. +FANOUT_MCP_ENABLED=true + +# metrics.public; boolean +# Allows unauthenticated Prometheus scrapes. +FANOUT_METRICS_PUBLIC=false + +# metrics.token; string; secret +# Authorizes Prometheus scrapes with a bearer token; optional when using an admin session. +FANOUT_METRICS_TOKEN= + +# server.addr; string +# Binds the browser, API, MCP, OTLP/gRPC and OTLP/HTTP listener. +FANOUT_ADDR=:7520 + +# server.pprof_enabled; boolean +# Exposes Go's net/http/pprof handlers at /debug/pprof/* for CPU/heap/mutex/goroutine profiling under load. Off by default; the routes require an admin browser session with operations:read. +FANOUT_PPROF_ENABLED=false + +# server.public_url; string +# Sets the external browser origin. Required for OIDC; HTTPS is required with MCP. +FANOUT_PUBLIC_URL= + +# server.tls.cert_file; string +# Selects the TLS certificate file; requires tls.key_file. +FANOUT_TLS_CERT_FILE= + +# server.tls.key_file; string +# Selects the TLS private key file; requires tls.cert_file. +FANOUT_TLS_KEY_FILE= + +# server.trusted_proxy_cidrs; string +# Lists trusted proxy networks, separated by commas; empty trusts no forwarded client addresses. +FANOUT_TRUSTED_PROXY_CIDRS= + +# smtp.from; string +# Sets the email sender address; required with SMTP. +FANOUT_SMTP_FROM= + +# smtp.host; string +# Selects the mail server. Host, username, password, and from must all be set to enable SMTP. +FANOUT_SMTP_HOST= + +# smtp.password; string; secret +# Authenticates to the mail server; required with SMTP. +FANOUT_SMTP_PASSWORD= + +# smtp.port; integer +# Sets the mail server's positive TCP port. +FANOUT_SMTP_PORT=587 + +# smtp.username; string +# Authenticates to the mail server; required with SMTP. +FANOUT_SMTP_USERNAME= + +# storage.data_dir; string +# Stores telemetry, query state, and the control database. +FANOUT_DATA_DIR=./data + +# storage.duckdb.max_connections; integer +# Caps the DuckDB connection pool. Reads scan immutable Parquet concurrently; rollup-cache writes are serialized by the query write gate. Zero means "size it from the machine" — the same spelling DuckDBThreads uses for deferring to a default. Resolution happens in resolveSizing and is reported in the startup configuration log. +FANOUT_DUCKDB_MAX_CONNECTIONS=0 + +# storage.duckdb.memory; string +# Caps DuckDB's memory (e.g. "8GB"). Empty means Fanout sizes it from detected memory, reserving headroom for the Go runtime; see resolveSizing. DuckDB's own default is 80% of detected RAM, which is calculated as though DuckDB owned the machine and has been observed to get the process OOM-killed once the Go heap is added on top. +FANOUT_DUCKDB_MEMORY= + +# storage.duckdb.threads; integer +# Caps DuckDB's global query worker pool. Zero leaves DuckDB's own default in place (one worker per core). Set it to leave cores free for ingest on a query-heavy co-tenant host. +FANOUT_DUCKDB_THREADS= + +# storage.maintenance_interval; duration +# Controls Parquet retention and compaction, and query-cache checkpointing. +FANOUT_MAINTENANCE_INTERVAL=1h + +# storage.retention_days; integer +# Sets telemetry retention in days; zero disables expiry. +FANOUT_RETENTION_DAYS=30 + +# storage.rollup_interval; duration +# Schedules analytical rollups; use a Go duration of at least 1s. +FANOUT_ROLLUP_INTERVAL=1m + +# storage.rollup_skip_to_latest; boolean +# RollupSkipToLatest, set once at boot, advances the service and edge rollup watermarks to the current max ingested timestamp. Existing history is not aggregated into those analytical rollups. Completed-batch trace caches still rebuild all retained history in bounded passes; uncached files remain immediately queryable. Useful for large pre-seeded datasets (benchmarks, restores); off in normal operation. +FANOUT_ROLLUP_SKIP_TO_LATEST=false diff --git a/.env.production b/.env.production new file mode 100644 index 00000000..b8636c7b --- /dev/null +++ b/.env.production @@ -0,0 +1,7 @@ +# Generated nonsecret configuration by cmd/fanout-docgen. +# Regenerate with just docs-generate. Credentials never belong in this file. +# Production overrides for .env. Deployment supplies its public URL and secrets. + +# server.addr; string +# Binds the browser, API, MCP, OTLP/gRPC and OTLP/HTTP listener. +FANOUT_ADDR=:7520 diff --git a/.gitignore b/.gitignore index cd672b0d..5b142896 100644 --- a/.gitignore +++ b/.gitignore @@ -39,11 +39,15 @@ coverage.out coverage.html cover.out -# Local environment scratch files. Fanout runtime configuration uses explicit -# YAML documents and process environment; it does not load dotenv files. +# Only generated public defaults/templates are tracked. Personal overrides, +# credential migration sources, and operator secret profiles remain private. .env .env.* -!.env.example +!/.env +!/.env.production +!/.env.example +/secrets/ +/.sops.yaml # Root-scoped for the same reason as testdata/: a bare `*.png` would silently # swallow committed image assets anywhere in the tree. diff --git a/AGENTS.md b/AGENTS.md index 1a77df95..48f25279 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,32 @@ # Agent instructions +## Configuration conventions + +- Follow Canon's consumer-first environment naming: custom product settings use + `FANOUT_`; AI keys and endpoints use `ANTHROPIC_API_KEY`, + `ANTHROPIC_BASE_URL`, `OPENAI_API_KEY`, and `OPENAI_BASE_URL`. + Provider and model selection remain `FANOUT_AI_PROVIDER` and `FANOUT_AI_MODEL`. +- `internal/config/config.go` is the authoritative schema. Run + `just docs-generate` after configuration changes and verify + `just docs-generate-check`; the generator owns `.env`, `.env.production`, + `.env.example`, and site references. Public defaults exclude all secret fields. +- Preserve the explicit YAML/process-environment loader. Fanout does not load + dotenv files; launchers and deployments own file loading and secret injection. + Only Fanout's owned namespace rejects unknown environment names; shared vendor + namespaces recognize the documented names and ignore unrelated consumers. +- The Node launcher used by `just run` loads `.env`, then optional `.env.local`, + then explicitly injected values. `just run-production` loads `.env` and required + `.env.production`, then injected values, and excludes all personal files. + Use Node's native dotenv parser; never source dotenv as shell code. Secret + entries and unknown names are rejected in these nonsecret files. +- Repository-managed secret profiles adopt SOPS authority through a verified + migration, with key recovery in 1Password. Existing vault/platform delivery + remains authoritative until all consumers and recovery are verified. Follow + `docs/operations.md` for delivery; never commit credential profiles in plaintext. +- This public product repository ships no operator secrets or recipients. Keep + operator `secrets/`, personal files, and private migration sources ignored and + outside Docker inputs; select private SOPS profiles explicitly at launch. + ## Public repository - Keep credentials, private telemetry, local workstation paths, and private AI diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 74f4b3fd..2ec2f0e9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -83,10 +83,19 @@ PNG in the same change. The recipe fails when the font is missing rather than shipping a card in a substitute face, which typst would otherwise do without complaint. -**Runtime configuration is strict.** [`fanout.example.yaml`](fanout.example.yaml) -is the complete schema. New settings need a YAML key, a `FANOUT_` environment -name, a test, and matching documentation; unknown names deliberately fail -startup. +**Runtime configuration is strict.** `internal/config/config.go` owns the +schema, documented in [`fanout.example.yaml`](fanout.example.yaml). New settings +need a YAML key, a consumer-defined environment name (or `FANOUT_` for custom +settings), a purpose comment, a test, and matching documentation. Unknown YAML +keys and owned `FANOUT_` names deliberately fail startup. Recognized provider +names are explicit; unrelated shared-vendor/tool variables are ignored. +Run `just docs-generate` to regenerate the settings reference, `.env`, +`.env.production`, and `.env.example`; `just docs-generate-check` verifies them. +Public defaults exclude secret fields, and templates use empty placeholders. +Private overrides and operator secret profiles stay ignored and outside Docker. +After building, `just run` loads shared/local nonsecret settings with the Node +launcher; `just run-production` excludes personal files. Inject credentials +explicitly. Never introduce aliases for retired names or shell-source dotenv. **SQLite migrations own the control schema.** Write forward SQL changes in `internal/db/migrations` using Goose annotations. `just db-migrate-create NAME` diff --git a/README.md b/README.md index 7ae8a847..7e3e7fef 100644 --- a/README.md +++ b/README.md @@ -165,7 +165,8 @@ docker exec fanout fanout --config /etc/fanout/fanout.yaml \ login-link admin@example.com ``` -Add `FANOUT_AI_API_KEY` to enable chat. Configure all four SMTP settings +Add `ANTHROPIC_API_KEY` (or `OPENAI_API_KEY` with `FANOUT_AI_PROVIDER=openai`) +to enable chat. Configure all four SMTP settings (`FANOUT_SMTP_HOST`, `FANOUT_SMTP_USERNAME`, `FANOUT_SMTP_PASSWORD`, and `FANOUT_SMTP_FROM`) to enable email-code login. @@ -205,8 +206,9 @@ export FANOUT_AUTH_CODE_SECRET=$(openssl rand -hex 32) # must be 32+ characters ./bin/fanout ``` -Optionally set `FANOUT_AI_API_KEY` for chat and the SMTP variables shown above -for email-code login. Without SMTP, run `./bin/fanout login-link +Optionally set `ANTHROPIC_API_KEY` (or `OPENAI_API_KEY` with +`FANOUT_AI_PROVIDER=openai`) for chat and the SMTP variables shown above for +email-code login. Without SMTP, run `./bin/fanout login-link admin@example.com` from the same configuration and data directory. Fanout serves the UI, API, MCP, OTLP/gRPC and OTLP/HTTP on @@ -222,8 +224,9 @@ capacity tests. Configuration is resolved once at startup and validated before Fanout opens data files or listeners. Sources apply in this order: built-in defaults, an -optional YAML document selected with `--config`, then `FANOUT_` environment +optional YAML document selected with `--config`, then recognized environment variables. Fanout does not search for configuration or load `.env` files. +All runtime settings are read once at startup and require a restart to change. [`fanout.example.yaml`](fanout.example.yaml) is the complete commented schema: @@ -237,8 +240,42 @@ for container injection and secrets. An empty environment value means "no override"; use YAML for an explicit empty string. Unknown YAML keys and unknown `FANOUT_` variables are startup errors, except for the service-discovery names Kubernetes and Docker link-style networking inject for a Service named -`fanout`. All environment variables outside the `FANOUT_` namespace are -ignored. +`fanout`. Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, +`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other environment variables, including +other settings in those shared vendor namespaces, are ignored. In particular, +`ANTHROPIC_MODEL` and `OPENAI_MODEL` do not select Fanout's model. + +[`.env.example`](.env.example) is generated from the same configuration schema +as the settings reference, with types, defaults, and empty secret placeholders. +The shared [`.env`](.env) contains nonsecret local defaults; the generated +[`.env.production`](.env.production) overrides the local listener for production. +Personal nonsecret overrides belong in ignored `.env.local`. This public +repository excludes operator `secrets/` profiles and private recipient policy. + +After building, use `just run` locally or `just run-production` for an explicit +production launch. The Node launcher parses dotenv without shell execution: +local loading is `.env`, then optional `.env.local`, then injected values; +production loading is `.env`, then required `.env.production`, then injected +values. Production never reads `.local` files. A missing required file fails; +no other environment is substituted. Credentials must be injected separately, +and secret entries or unknown names in nonsecret dotenv files are rejected. + +For example, after selecting a private, complete credential profile: + +```sh +sops exec-env secrets/fanout.env 'just run' +sops exec-env secrets/fanout.production.env 'just run-production' +``` + +Select only a profile that actually exists for the intended resources. An +ordinary dotenv loader does not resolve 1Password references or decrypt SOPS. +The binary retains its YAML/environment precedence: every nonempty dotenv value +injected by the launcher overrides the corresponding `--config` YAML value. +For YAML-only ordinary settings, launch the binary directly with injected secrets. +See [runtime secret delivery](docs/operations.md#runtime-secret-delivery) for +SOPS injection and migration precautions. Run `just docs-generate` after changing +the schema; `just docs-generate-check` verifies all three dotenv artifacts and +the reference. The generator refuses to overwrite private or unmanaged defaults. YAML null values are rejected. Boolean values must use YAML 1.2 `true` or `false` (not `yes`, `no`, `on`, or `off`). If a YAML document contains a @@ -254,10 +291,30 @@ operators touch are: | `auth.mode` | `FANOUT_AUTH_MODE` | `local` | `local` (login link or SMTP) or `oidc` | | `auth.code_secret` | `FANOUT_AUTH_CODE_SECRET` | — | Required in local mode, 32+ characters | | `ai.provider` | `FANOUT_AI_PROVIDER` | `anthropic` | `anthropic` or `openai` | -| `ai.api_key` | `FANOUT_AI_API_KEY` | — | Enables AI investigation chat | +| `ai.anthropic.api_key` | `ANTHROPIC_API_KEY` | — | Enables chat when `ai.provider=anthropic` | +| `ai.openai.api_key` | `OPENAI_API_KEY` | — | Enables chat when `ai.provider=openai` | | `storage.retention_days` | `FANOUT_RETENTION_DAYS` | `30` | Telemetry retention window | | `mcp.enabled` | `FANOUT_MCP_ENABLED` | `true` | Serve the MCP endpoint at `/mcp` | +Only the selected AI provider's credential and endpoint are used. Without its +key, chat is disabled even if the other provider's key is present. Unsupported +provider names fail at startup, including when no key is configured. +`FANOUT_AI_MODEL` remains the application model selector. + +`ANTHROPIC_BASE_URL` defaults to `https://api.anthropic.com`; Fanout appends +`/v1/messages`. `OPENAI_BASE_URL` defaults to `https://api.openai.com/v1`; +Fanout appends `/responses`. Both retain gateway path prefixes and accept a +trailing slash. The selected base URL must use HTTP(S) without embedded +credentials, a query, or a fragment. + +When upgrading, replace `FANOUT_AI_API_KEY` with the selected vendor's key and +move YAML `ai.api_key` to `ai.anthropic.api_key` or `ai.openai.api_key`. +Replace `FANOUT_AI_BASE_URL` and YAML `ai.base_url` with the matching vendor +setting. For OpenAI, append `/v1` to the old base URL to preserve the request +destination; Anthropic's base URL keeps its previous meaning. Retired settings +are rejected. Coordinate the new names with launchers, deployment inputs, and +secret-store fields before restarting; they have no aliases. + ### Advanced DuckDB sizing Most deployments should not set DuckDB variables. Give the process or diff --git a/cmd/fanout-docgen/main.go b/cmd/fanout-docgen/main.go index 021a2ddb..89bad736 100644 --- a/cmd/fanout-docgen/main.go +++ b/cmd/fanout-docgen/main.go @@ -52,18 +52,21 @@ func main() { // list going stale is loud rather than a quietly shorter table. routeDirs = flag.String("route-dirs", "internal/api,internal/agent,cmd/fanout", "comma-separated directories whose files register HTTP routes") - outDir = flag.String("out", "site/src/content/docs/reference", "reference root to write generated pages into") - check = flag.Bool("check", false, "exit non-zero when a written page differs from the one on disk") + outDir = flag.String("out", "site/src/content/docs/reference", "reference root to write generated pages into") + envExample = flag.String("env-example", ".env.example", "path to the generated environment template") + envDefaults = flag.String("env-defaults", ".env", "path to the generated nonsecret local defaults") + envProduction = flag.String("env-production", ".env.production", "path to the generated nonsecret production overrides") + check = flag.Bool("check", false, "exit non-zero when a written page differs from the one on disk") ) flag.Parse() - if err := run(*source, *alerts, *routeDirs, *outDir, *check); err != nil { + if err := run(*source, *alerts, *routeDirs, *outDir, *envExample, *envDefaults, *envProduction, *check); err != nil { fmt.Fprintf(os.Stderr, "fanout-docgen: %v\n", err) os.Exit(1) } } -func run(source, alertSource, routeDirs, outDir string, check bool) error { +func run(source, alertSource, routeDirs, outDir, envExample, envDefaults, envProduction string, check bool) error { fields, err := collect(source) if err != nil { return err @@ -117,9 +120,25 @@ func run(source, alertSource, routeDirs, outDir string, check bool) error { } pages["mcp-tools.mdx"] = toolsPage - var stale []string + outputs := map[string][]byte{ + envExample: renderEnvironment(fields), + envDefaults: renderDefaults(fields, false), + envProduction: renderDefaults(fields, true), + } + // Never replace an operator's dotenv file or a managed secret mount with + // generated public defaults. Check before writing any artifacts. + if !check { + for _, path := range []string{envDefaults, envProduction} { + if err := validateGeneratedDefaults(path, fields); err != nil { + return err + } + } + } for name, body := range pages { - path := filepath.Join(outDir, name) + outputs[filepath.Join(outDir, name)] = body + } + var stale []string + for path, body := range outputs { if check { existing, err := os.ReadFile(path) if err != nil { @@ -171,6 +190,44 @@ func run(source, alertSource, routeDirs, outDir string, check bool) error { return nil } +func validateGeneratedDefaults(path string, fields []field) error { + info, err := os.Lstat(path) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + refuse := func() error { + return fmt.Errorf("refuse to overwrite private or unmanaged environment file %s; preserve it outside the public defaults before adopting the layout", path) + } + if !info.Mode().IsRegular() { + return refuse() + } + existing, err := os.ReadFile(path) + if err != nil { + return err + } + if !bytes.HasPrefix(existing, []byte("# Generated nonsecret configuration by cmd/fanout-docgen.\n")) { + return refuse() + } + public := make(map[string]bool, len(fields)) + for _, f := range fields { + public[f.Env] = !f.Secret + } + for _, line := range strings.Split(string(existing), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + name, _, assignment := strings.Cut(line, "=") + if !assignment || !public[strings.TrimSpace(name)] { + return refuse() + } + } + return nil +} + // alertEnvCount is set by collectAlertEnv so the summary line can report it // without threading the slice back through run's return. var alertEnvCount []field @@ -550,10 +607,14 @@ func renderPage(name string, page groupPage, fields []field) []byte { b.WriteString("{/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */}\n\n") - b.WriteString("Every setting below is read from the environment variable in the second\n") + b.WriteString("Every setting below is read once at startup and requires a restart to change.\n") + b.WriteString("It is read from the environment variable in the second\n") b.WriteString("column, or from the same key in a YAML configuration file. An unrecognised\n") b.WriteString("`FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces\n") b.WriteString("as a refusal to start rather than as a default nobody chose.\n\n") + b.WriteString("Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`,\n") + b.WriteString("`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are\n") + b.WriteString("ignored; only the explicitly selected AI provider's key and endpoint are used.\n\n") b.WriteString("| Setting | Environment variable | Type | Default |\n") b.WriteString("|---|---|---|---|\n") @@ -593,6 +654,67 @@ func renderPage(name string, page groupPage, fields []field) []byte { return []byte(b.String()) } +// renderEnvironment shares the exact reflected schema used by the site pages. +// Empty secret values are safe placeholders and never enable an integration. +func renderEnvironment(fields []field) []byte { + var b strings.Builder + b.WriteString("# Generated by cmd/fanout-docgen from internal/config/config.go.\n") + b.WriteString("# Regenerate with just docs-generate; verify with just docs-generate-check.\n") + b.WriteString("# Fanout does not load dotenv files. Inject through your launcher or deployment.\n") + b.WriteString("# Precedence: built-in defaults < --config YAML < nonempty environment values.\n") + b.WriteString("# All settings require restart. Empty environment values mean no override.\n") + b.WriteString("# Secret entries below are empty placeholders; use runtime secret injection.\n") + b.WriteString("# Local authentication requires FANOUT_AUTH_CODE_SECRET (32+ characters).\n") + b.WriteString("# Only the AI key/endpoint selected by FANOUT_AI_PROVIDER is used.\n") + for _, f := range fields { + fmt.Fprintf(&b, "\n# %s; %s", f.Key, f.Type) + if f.Secret { + b.WriteString("; secret") + } + b.WriteByte('\n') + if f.Doc != "" { + fmt.Fprintf(&b, "# %s\n", f.Doc) + } + value := f.Default + if f.Secret { + value = "" + } + fmt.Fprintf(&b, "%s=%s\n", f.Env, value) + } + return []byte(b.String()) +} + +// renderDefaults contains no secret fields, even empty ones. Production replaces +// the local-only listener with the schema's shared-listener default; operators +// supply deployment-specific origins and all credentials at launch time. +func renderDefaults(fields []field, production bool) []byte { + var b strings.Builder + b.WriteString("# Generated nonsecret configuration by cmd/fanout-docgen.\n") + b.WriteString("# Regenerate with just docs-generate. Credentials never belong in this file.\n") + if production { + b.WriteString("# Production overrides for .env. Deployment supplies its public URL and secrets.\n") + } else { + b.WriteString("# Shared local defaults; .env.production overrides them for production.\n") + b.WriteString("# Use just run, or inject these values explicitly; Fanout does not load dotenv.\n") + b.WriteString("# Personal nonsecret overrides belong in ignored .env.local.\n") + } + for _, f := range fields { + if f.Secret || production && f.Key != "server.addr" { + continue + } + fmt.Fprintf(&b, "\n# %s; %s\n", f.Key, f.Type) + if f.Doc != "" { + fmt.Fprintf(&b, "# %s\n", f.Doc) + } + value := f.Default + if !production && f.Key == "server.addr" { + value = "127.0.0.1" + value + } + fmt.Fprintf(&b, "%s=%s\n", f.Env, value) + } + return []byte(b.String()) +} + func hasSecret(fields []field) bool { for _, f := range fields { if f.Secret { diff --git a/cmd/fanout-docgen/main_test.go b/cmd/fanout-docgen/main_test.go index b53429e6..9a098e53 100644 --- a/cmd/fanout-docgen/main_test.go +++ b/cmd/fanout-docgen/main_test.go @@ -5,6 +5,8 @@ import ( "path/filepath" "strings" "testing" + + "github.com/labstack/fanout/internal/config" ) const configSource = "../../internal/config/config.go" @@ -25,12 +27,116 @@ func TestCollectCrossChecksReflectionAgainstSource(t *testing.T) { if f.Key == "" || f.Env == "" { t.Errorf("%s: koanf and env tags are both required", f.GoName) } - if !strings.HasPrefix(f.Env, "FANOUT_") { - t.Errorf("%s: environment variable %q lacks the FANOUT_ prefix the loader requires", f.GoName, f.Env) + if !strings.HasPrefix(f.Env, "FANOUT_") && !strings.HasPrefix(f.Key, "ai.anthropic.") && !strings.HasPrefix(f.Key, "ai.openai.") { + t.Errorf("%s: custom environment variable %q lacks FANOUT_ prefix", f.GoName, f.Env) } if f.Type == "" { t.Errorf("%s: no rendered type", f.GoName) } + if f.Doc == "" { + t.Errorf("%s: setting purpose must be documented in the authoritative schema", f.GoName) + } + } +} + +func TestEnvironmentTemplateMatchesSchema(t *testing.T) { + fields, err := collect(configSource) + if err != nil { + t.Fatal(err) + } + body := renderEnvironment(fields) + written, err := os.ReadFile("../../.env.example") + if err != nil { + t.Fatal(err) + } + if string(written) != string(body) { + t.Error(".env.example is stale; run just docs-generate") + } + var env []string + seen := map[string]string{} + for _, line := range strings.Split(string(body), "\n") { + if line == "" || strings.HasPrefix(line, "#") { + continue + } + name, value, ok := strings.Cut(line, "=") + if _, duplicate := seen[name]; !ok || duplicate { + t.Fatal("invalid or duplicate template entry") + } + seen[name] = value + env = append(env, line) + } + if len(seen) != len(fields) { + t.Fatalf("template has %d settings, schema has %d", len(seen), len(fields)) + } + for _, f := range fields { + value, exists := seen[f.Env] + if !exists || value != f.Default || f.Secret && value != "" { + t.Errorf("template does not match the safe schema default for %s", f.Env) + } + } + env = append(env, "FANOUT_AUTH_CODE_SECRET="+strings.Repeat("x", 32)) + if _, err := config.Load(config.LoadOptions{Environ: env}); err != nil { + t.Fatalf("template rejected by the runtime loader: %v", err) + } +} + +func TestPublicEnvironmentDefaultsExcludeSecrets(t *testing.T) { + fields, err := collect(configSource) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + path string + production bool + addr string + }{ + {"../../.env", false, "127.0.0.1:7520"}, + {"../../.env.production", true, ":7520"}, + } { + body := renderDefaults(fields, tc.production) + written, err := os.ReadFile(tc.path) + if err != nil { + t.Fatal(err) + } + if string(written) != string(body) { + t.Errorf("%s is stale; run just docs-generate", tc.path) + } + for _, f := range fields { + if f.Secret && strings.Contains(string(body), f.Env+"=") { + t.Errorf("public defaults must exclude secret setting %s", f.Env) + } + } + if !strings.Contains(string(body), "FANOUT_ADDR="+tc.addr+"\n") { + t.Errorf("%s has the wrong listener scope", tc.path) + } + if err := validateGeneratedDefaults(tc.path, fields); err != nil { + t.Fatal(err) + } + } +} + +func TestPublicDefaultGenerationPreservesPrivateFiles(t *testing.T) { + fields, err := collect(configSource) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), ".env") + for _, content := range []string{ + "AI_API_KEY=synthetic-private-value\n", + "# Generated nonsecret configuration by cmd/fanout-docgen.\nANTHROPIC_API_KEY=synthetic-private-value\n", + "# Generated nonsecret configuration by cmd/fanout-docgen.\nFANOUT_AI_API_KEY=synthetic-private-value\n", + } { + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + err := validateGeneratedDefaults(path, fields) + if err == nil || strings.Contains(err.Error(), "synthetic-private-value") { + t.Fatal("expected redacted refusal to replace a private file") + } + unchanged, err := os.ReadFile(path) + if err != nil || string(unchanged) != content { + t.Fatal("private dotenv file was modified") + } } } diff --git a/cmd/fanout/main.go b/cmd/fanout/main.go index ed769cbc..63addc2c 100644 --- a/cmd/fanout/main.go +++ b/cmd/fanout/main.go @@ -359,7 +359,7 @@ func main() { slog.Info("browser MCP server enabled", "path", "/api/mcp", "auth", "session") } if cfg.AgentConfigured() { - provider, err := agent.NewProvider(cfg.AIProvider, cfg.AIAPIKey, cfg.AIModel, cfg.AIBaseURL) + provider, err := agent.NewProvider(cfg.AIProvider, cfg.AIAPIKey(), cfg.AIModel, cfg.AIBaseURL()) if err != nil { slog.Error("agent provider init failed", "err", err) os.Exit(1) @@ -373,7 +373,7 @@ func main() { agent.NewRuntime(provider, toolRegistry, agent.NewStore(sqlite.DB)).Register(e.Group("/api/agent", api.RequireCapability(api.RunAgent))) slog.Info("AG-UI agent enabled", "path", "/api/agent/runs", "provider", cfg.AIProvider) } else { - slog.Info("AG-UI agent disabled", "reason", "ai.api_key is not configured") + slog.Info("AG-UI agent disabled", "reason", "selected AI provider API key is not configured") } // The compiled browser client is an embedded asset, not a second runtime. diff --git a/docs/operations.md b/docs/operations.md index d1cac7ca..7cf432cb 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -4,6 +4,89 @@ Fanout is one process and one persistent data directory. Operate those two things as a unit: preserve the directory, pin the binary or image version, and give ingest credentials only to trusted services and collectors. +## Runtime secret delivery + +Runtime configuration uses built-in defaults, then an explicit `--config` YAML +document, then recognized nonempty environment values. Every setting requires +a restart. Fanout does not load dotenv files itself. The generated root +`.env.example` documents every setting with empty credential placeholders; +`.env` and `.env.production` contain only nonsecret settings. + +| File | Use | Public Git policy | +| --- | --- | --- | +| `.env` | Shared defaults with a loopback local listener | Committed; generated; no secret fields | +| `.env.production` | Production listener override | Committed; generated; no deployment-specific credentials or origins | +| `.env.example` | Complete configuration template | Committed; generated; empty secret placeholders | +| `.env.local` / `.env.production.local` | Personal nonsecret files | Ignored; production never loads them | +| `secrets/fanout.env` | Private complete SOPS credential profile, when used | Operator-owned; ignored and excluded from Docker | +| `secrets/fanout.production.env` | Private production profile, when credentials differ | Operator-owned; ignored and excluded from Docker | + +Build the binary, then use `just run` locally or `just run-production` for +production. The Node launcher uses `node:util.parseEnv`; dotenv is not shell code +and values are not interpolated. Local order is `.env`, optional `.env.local`, +then inherited/injected values. Production order is `.env`, required +`.env.production`, then inherited/injected values; no personal files are loaded. +Unknown names, secret entries, and unresolved `op://` references in ordinary +dotenv files fail before launch. A missing required file never selects another +profile. Empty values retain Fanout's no-override semantics. + +The launcher injects dotenv settings as environment values, so they override +the native binary's `--config` YAML. Launch the binary directly for YAML-only +ordinary configuration. Supported binary flags are forwarded unchanged. + +Keep ordinary YAML/dotenv configuration nonsecret where possible. Deliver +credentials through deployment environment injection, restricted managed mounts, +or a platform secret service. Do not bake credentials into images or frontend +bundles. Local dotenv scratch files stay ignored and outside Docker inputs. + +For repository-managed application secrets adopting Canon, use one authoritative +SOPS-encrypted profile per workload/credential set, such as +`secrets/fanout.env`, or `secrets/fanout.production.env` when production actually +uses different credentials. Select one complete profile explicitly; do not layer +production credentials over development credentials or fall back on a missing +profile. Platform-owned credentials remain platform-owned. + +Operator profiles and recipient policy are excluded from this public product +repository. Create them only for actual credential sets; do not ship working +development credentials or invent environment profiles. Existing private +credential migration sources remain ignored and are never loaded automatically. + +With a provisioned SOPS key and an existing encrypted profile, use +[SOPS runtime injection](https://getsops.io/docs/usage/advanced/#passing-secrets-to-other-processes) +to launch without writing decrypted credentials to disk: + +```sh +sops exec-env secrets/fanout.env './bin/fanout --config fanout.yaml' +# Or use the shared nonsecret dotenv defaults: +sops exec-env secrets/fanout.env 'just run' +# Select a complete profile for actual production resources: +sops exec-env secrets/fanout.production.env 'just run-production' +``` + +This example injects only secrets; ordinary settings remain in `fanout.yaml` +or deployment inputs. SOPS supplies the profile's environment to the child; +avoid conflicting secret values in inherited shell variables. Verify synthetic +values containing dollars, quotes, spaces, and newlines through the actual +deployment parser before migrating a profile. Keep private age identities out +of Git, restrict their local files to mode `0600`, and document recovery in the +product's existing 1Password item. 1Password holds key recovery rather than a +second independently maintained copy of SOPS-owned application secrets. + +Existing vault-owned delivery remains valid until its migration is verified. +Before retiring a vault field or old variable name, inventory local, CI, +deployment, shared-project, supported-version, and recovery consumers using +names and safe metadata. Update their mappings, verify the real consumers, and +verify recovery from the destination before removing the old authority. This +repository ships no credential profiles or decryption keys. + +For AI, deliver only the selected provider's `ANTHROPIC_API_KEY` or +`OPENAI_API_KEY`. Provider endpoint overrides are `ANTHROPIC_BASE_URL` and +`OPENAI_BASE_URL`; the OpenAI base includes `/v1`, while Anthropic's excludes it. +The old generic key/base URL names and YAML keys are rejected; the +[configuration upgrade instructions](../README.md#configuration) describe +their exact replacements. Verify authentication and request paths against the +configured provider after rollout, without logging credentials. + ## Network surface | Default | Purpose | Exposure | diff --git a/fanout.example.yaml b/fanout.example.yaml index 7b6a8053..086b4043 100644 --- a/fanout.example.yaml +++ b/fanout.example.yaml @@ -1,7 +1,7 @@ # Fanout runtime configuration. # # Start with: fanout --config /path/to/fanout.yaml -# Every value can be overridden by its documented FANOUT_ environment variable. +# Every value can be overridden by its documented environment variable. # Secrets are intentionally omitted below; inject them through the environment # or add them to a suitably protected copy of this file. # Use YAML 1.2 true/false for booleans; yes/no/on/off are strings and rejected. @@ -43,9 +43,16 @@ alerts: ai: provider: anthropic # FANOUT_AI_PROVIDER model: "" # FANOUT_AI_MODEL - base_url: "" # FANOUT_AI_BASE_URL - # Optional; enables investigation chat: FANOUT_AI_API_KEY - # api_key: "" + anthropic: + # Defaults to https://api.anthropic.com; Fanout appends /v1/messages. + base_url: "" # ANTHROPIC_BASE_URL + # Optional; enables chat when provider is anthropic: ANTHROPIC_API_KEY + # api_key: "" + openai: + # Defaults to https://api.openai.com/v1; Fanout appends /responses. + base_url: "" # OPENAI_BASE_URL + # Optional; enables chat when provider is openai: OPENAI_API_KEY + # api_key: "" smtp: host: "" # FANOUT_SMTP_HOST diff --git a/internal/agent/provider_contract_test.go b/internal/agent/provider_contract_test.go index 2e62b9b3..e3edf8d4 100644 --- a/internal/agent/provider_contract_test.go +++ b/internal/agent/provider_contract_test.go @@ -56,7 +56,7 @@ func TestOpenAIOutputItemsReplayVerbatim(t *testing.T) { // replace rebuilt text/calls and calls without executed results must vanish. unsafe := json.RawMessage(`{"type":"function_call","call_id":"not_executed","name":"status","arguments":"{}"}`) body := captureRequestBody(t, lines(`data: {"type":"response.completed","response":{}}`), func(url string) error { - p, err := NewProvider("openai", "test", "", url) + p, err := NewProvider("openai", "test", "", url+"/v1") if err != nil { return err } @@ -134,7 +134,7 @@ func TestOpenAIEncryptedContentConfigurationError(t *testing.T) { _, _ = io.WriteString(w, `{"error":{"message":"reasoning.encrypted_content is unsupported by this model"}}`) })) defer server.Close() - p, err := NewProvider("openai", "test", "custom-model", server.URL) + p, err := NewProvider("openai", "test", "custom-model", server.URL+"/v1") if err != nil { t.Fatal(err) } @@ -268,7 +268,7 @@ func TestRuntimeOpenAITerminalNotices(t *testing.T) { t.Run(tc.name, func(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = io.WriteString(w, tc.sse) })) defer server.Close() - p, err := NewProvider("openai", "test", "", server.URL) + p, err := NewProvider("openai", "test", "", server.URL+"/v1") if err != nil { t.Fatal(err) } diff --git a/internal/agent/provider_http.go b/internal/agent/provider_http.go index 3af87726..2180b08b 100644 --- a/internal/agent/provider_http.go +++ b/internal/agent/provider_http.go @@ -12,6 +12,8 @@ import ( "sort" "strings" "time" + + "github.com/labstack/fanout/internal/config" ) func NewProvider(kind, apiKey, model, baseURL string) (Provider, error) { @@ -21,7 +23,7 @@ func NewProvider(kind, apiKey, model, baseURL string) (Provider, error) { model = "claude-sonnet-5-5" } if baseURL == "" { - baseURL = "https://api.anthropic.com" + baseURL = config.DefaultAnthropicBaseURL } return &anthropicProvider{apiKey: apiKey, model: model, baseURL: strings.TrimRight(baseURL, "/"), client: modelHTTPClient()}, nil case "openai": @@ -29,7 +31,7 @@ func NewProvider(kind, apiKey, model, baseURL string) (Provider, error) { model = "gpt-6.1-sol" } if baseURL == "" { - baseURL = "https://api.openai.com" + baseURL = config.DefaultOpenAIBaseURL } return &openAIProvider{apiKey: apiKey, model: model, baseURL: strings.TrimRight(baseURL, "/"), client: modelHTTPClient()}, nil default: @@ -106,7 +108,7 @@ func (p *openAIProvider) Stream(ctx context.Context, params StreamParams, cb fun } body["tools"] = tools } - response, err := postModel(ctx, p.client, p.baseURL+"/v1/responses", p.apiKey, "openai", body) + response, err := postModel(ctx, p.client, p.baseURL+"/responses", p.apiKey, "openai", body) if err != nil { var apiErr *APIError if errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusBadRequest && strings.Contains(strings.ToLower(apiErr.Body), "encrypted") { diff --git a/internal/agent/provider_http_test.go b/internal/agent/provider_http_test.go index 8bf0079c..1648b8d5 100644 --- a/internal/agent/provider_http_test.go +++ b/internal/agent/provider_http_test.go @@ -19,6 +19,38 @@ func lines(ss ...string) string { return strings.Join(ss, "\n") + "\n" } +func TestProviderBaseURLPreservesGatewayPath(t *testing.T) { + for _, tc := range []struct { + provider, basePath, requestPath, response string + }{ + {"openai", "/gateway/v1", "/gateway/v1/responses", lines(`data: {"type":"response.completed","response":{}}`)}, + {"anthropic", "/gateway", "/gateway/v1/messages", lines(`data: {"type":"message_delta","delta":{"stop_reason":"end_turn"}}`, `data: {"type":"message_stop"}`)}, + } { + for _, slash := range []string{"", "/"} { + t.Run(tc.provider+slash, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != tc.requestPath { + t.Errorf("request = %s %s, want POST %s", r.Method, r.URL.Path, tc.requestPath) + } + if tc.provider == "openai" && r.Header.Get("Authorization") != "Bearer test-key" || tc.provider == "anthropic" && r.Header.Get("X-Api-Key") != "test-key" { + t.Error("provider credential was not mapped to the correct header") + } + w.Header().Set("Content-Type", "text/event-stream") + _, _ = io.WriteString(w, tc.response) + })) + defer server.Close() + p, err := NewProvider(tc.provider, "test-key", "", server.URL+tc.basePath+slash) + if err != nil { + t.Fatal(err) + } + if err := p.Stream(context.Background(), StreamParams{}, func(StreamEvent) error { return nil }); err != nil { + t.Fatal(err) + } + }) + } + } +} + // collect runs a parser over SSE input and buckets the emitted events. func collect(t *testing.T, parse func(io.Reader, func(StreamEvent) error) error, input string) (texts []string, toolCalls []ToolCall, stopReasons []string, errs []string, parseErr error) { t.Helper() @@ -53,7 +85,7 @@ func openAIStream(t *testing.T) func(io.Reader, func(StreamEvent) error) error { _, _ = io.Copy(w, reader) })) defer server.Close() - provider := &openAIProvider{apiKey: "sk-test", model: "gpt-6.1-sol", baseURL: server.URL, client: server.Client()} + provider := &openAIProvider{apiKey: "sk-test", model: "gpt-6.1-sol", baseURL: server.URL + "/v1", client: server.Client()} return provider.Stream(context.Background(), StreamParams{}, cb) } } @@ -336,7 +368,7 @@ func captureRequestBody(t *testing.T, sseBody string, run func(baseURL string) e func TestOpenAIResponsesRequestBodyShape(t *testing.T) { body := captureRequestBody(t, lines(`data: {"type":"response.completed","response":{}}`), func(baseURL string) error { - provider, err := NewProvider("openai", "sk-test", "", baseURL+"/") + provider, err := NewProvider("openai", "sk-test", "", baseURL+"/v1/") if err != nil { return err } @@ -414,7 +446,7 @@ func TestOpenAIResponsesReasoningReplayWithinRun(t *testing.T) { } })) defer server.Close() - provider := &openAIProvider{apiKey: "sk-test", model: "gpt-6.1-sol", baseURL: server.URL, client: server.Client()} + provider := &openAIProvider{apiKey: "sk-test", model: "gpt-6.1-sol", baseURL: server.URL + "/v1", client: server.Client()} runtime := NewRuntime(provider, &fakeTools{execution: ToolExecution{Content: `{"ok":true}`}}, nil) emitter, output := newTestEmitter() messages := []agtypes.Message{{ID: "user-1", Role: agtypes.RoleUser, Content: "status?"}} diff --git a/internal/api/health_test.go b/internal/api/health_test.go index 02ce36b3..f6b407f0 100644 --- a/internal/api/health_test.go +++ b/internal/api/health_test.go @@ -78,7 +78,7 @@ func TestReadiness_NilDuck(t *testing.T) { func TestReadinessReportsSizingResolvedByLoader(t *testing.T) { environ := []string{ "FANOUT_AUTH_CODE_SECRET=0123456789abcdef0123456789abcdef", - "FANOUT_AI_API_KEY=sk-test", + "ANTHROPIC_API_KEY=sk-test", "FANOUT_SMTP_HOST=smtp.example.com", "FANOUT_SMTP_USERNAME=user", "FANOUT_SMTP_PASSWORD=pass", diff --git a/internal/config/config.go b/internal/config/config.go index 54ed8cbb..ff0fdd7c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -14,24 +14,34 @@ import ( const maxIngestBatchSize = 50_000 -// Config is Fanout's canonical configuration schema. Public names use -// FANOUT_ plus the shortest stable term that is clear in a docker run, and the -// same terminology across YAML, environment variables, Go, logs, and docs. +const ( + DefaultAnthropicBaseURL = "https://api.anthropic.com" + DefaultOpenAIBaseURL = "https://api.openai.com/v1" +) + +// Config is Fanout's canonical configuration schema. Custom environment names +// use FANOUT_; provider credentials and endpoints use their vendor's names. +// YAML retains native structured keys, mapped explicitly by the env tags. // Elapsed-time settings use time.Duration with unit-bearing values. type Config struct { // Addr binds the browser, API, MCP, OTLP/gRPC and OTLP/HTTP listener. - Addr string `koanf:"server.addr" env:"FANOUT_ADDR" default:":7520"` - DataDir string `koanf:"storage.data_dir" env:"FANOUT_DATA_DIR" default:"./data"` - IngestBatchSize int `koanf:"ingest.batch_size" env:"FANOUT_INGEST_BATCH_SIZE" default:"50000"` + Addr string `koanf:"server.addr" env:"FANOUT_ADDR" default:":7520"` + // DataDir stores telemetry, query state, and the control database. + DataDir string `koanf:"storage.data_dir" env:"FANOUT_DATA_DIR" default:"./data"` + // IngestBatchSize limits telemetry rows per batch; valid values are 1–50000. + IngestBatchSize int `koanf:"ingest.batch_size" env:"FANOUT_INGEST_BATCH_SIZE" default:"50000"` // IngestMaxInFlightBytes caps the decoded telemetry the process will hold // across concurrent requests. Concurrency is HTTP/2 streams times // connections and each handler holds its batch until the commit is // durable, so without a ceiling a burst is bounded only by memory. Zero is // unbounded, which is the behaviour that predates this setting. - IngestMaxInFlightBytes int `koanf:"ingest.max_in_flight_bytes" env:"FANOUT_INGEST_MAX_IN_FLIGHT_BYTES" default:"268435456"` - RollupInterval time.Duration `koanf:"storage.rollup_interval" env:"FANOUT_ROLLUP_INTERVAL" default:"1m"` - MCPEnabled bool `koanf:"mcp.enabled" env:"FANOUT_MCP_ENABLED" default:"true"` - RetentionDays int `koanf:"storage.retention_days" env:"FANOUT_RETENTION_DAYS" default:"30"` + IngestMaxInFlightBytes int `koanf:"ingest.max_in_flight_bytes" env:"FANOUT_INGEST_MAX_IN_FLIGHT_BYTES" default:"268435456"` + // RollupInterval schedules analytical rollups; use a Go duration of at least 1s. + RollupInterval time.Duration `koanf:"storage.rollup_interval" env:"FANOUT_ROLLUP_INTERVAL" default:"1m"` + // MCPEnabled serves the MCP endpoint at /mcp. + MCPEnabled bool `koanf:"mcp.enabled" env:"FANOUT_MCP_ENABLED" default:"true"` + // RetentionDays sets telemetry retention in days; zero disables expiry. + RetentionDays int `koanf:"storage.retention_days" env:"FANOUT_RETENTION_DAYS" default:"30"` // MaintenanceInterval controls Parquet retention and compaction, and // query-cache checkpointing. MaintenanceInterval time.Duration `koanf:"storage.maintenance_interval" env:"FANOUT_MAINTENANCE_INTERVAL" default:"1h"` @@ -41,8 +51,9 @@ type Config struct { // still rebuild all retained history in bounded passes; // uncached files remain immediately queryable. Useful for large pre-seeded // datasets (benchmarks, restores); off in normal operation. - RollupSkipToLatest bool `koanf:"storage.rollup_skip_to_latest" env:"FANOUT_ROLLUP_SKIP_TO_LATEST" default:"false"` - DefaultNamespace string `koanf:"ingest.default_namespace" env:"FANOUT_DEFAULT_NAMESPACE" default:"default"` + RollupSkipToLatest bool `koanf:"storage.rollup_skip_to_latest" env:"FANOUT_ROLLUP_SKIP_TO_LATEST" default:"false"` + // DefaultNamespace supplies the namespace for telemetry without one. + DefaultNamespace string `koanf:"ingest.default_namespace" env:"FANOUT_DEFAULT_NAMESPACE" default:"default"` // PprofEnabled exposes Go's net/http/pprof handlers at /debug/pprof/* for // CPU/heap/mutex/goroutine profiling under load. Off by default; the routes // require an admin browser session with operations:read. @@ -50,7 +61,7 @@ type Config struct { // The DuckDB knobs below self-size where possible and otherwise default to // values validated on the reference deployment target, a small shared VM // (Hetzner CPX32: 4 vCPU, 8 GB RAM, 160 GB disk). There the self-sizing - // resolves to a ~6.4 GB memory cap and 4 query threads (deterministic from + // resolves to a ~4.8 GB memory cap and 4 query threads (deterministic from // 8 GB / 4 vCPU). Measure the target host with the external fanout-bench // harness before setting // production limits; throughput and memory use vary with CPU, disk, and the @@ -76,47 +87,88 @@ type Config struct { // Zero means "size it from the machine" — the same spelling DuckDBThreads // uses for deferring to a default. Resolution happens in resolveSizing and // is reported in the startup configuration log. - DuckDBMaxConns int `koanf:"storage.duckdb.max_connections" env:"FANOUT_DUCKDB_MAX_CONNECTIONS" default:"0"` - AlertEnabled bool `koanf:"alerts.enabled" env:"FANOUT_ALERTS_ENABLED" default:"true"` + DuckDBMaxConns int `koanf:"storage.duckdb.max_connections" env:"FANOUT_DUCKDB_MAX_CONNECTIONS" default:"0"` + // AlertEnabled runs the alert evaluation engine. + AlertEnabled bool `koanf:"alerts.enabled" env:"FANOUT_ALERTS_ENABLED" default:"true"` + // AlertEvaluationInterval schedules alert evaluation; use a Go duration of at least 1s. AlertEvaluationInterval time.Duration `koanf:"alerts.evaluation_interval" env:"FANOUT_ALERTS_EVALUATION_INTERVAL" default:"30s"` - AlertHistoryDays int `koanf:"alerts.history_days" env:"FANOUT_ALERTS_HISTORY_DAYS" default:"7"` - AIProvider string `koanf:"ai.provider" env:"FANOUT_AI_PROVIDER" default:"anthropic"` - AIAPIKey string `koanf:"ai.api_key" env:"FANOUT_AI_API_KEY" secret:"true"` + // AlertHistoryDays sets alert history retention in days. + AlertHistoryDays int `koanf:"alerts.history_days" env:"FANOUT_ALERTS_HISTORY_DAYS" default:"7"` + // AIProvider selects anthropic or openai independently of which keys are present. + AIProvider string `koanf:"ai.provider" env:"FANOUT_AI_PROVIDER" default:"anthropic"` // AIModel overrides the provider's default: claude-sonnet-5-5 for Anthropic // and gpt-6.1-sol for OpenAI. The OpenAI provider expects a reasoning model // supporting reasoning.encrypted_content for stateless Responses replay. // A 400 rejecting encrypted content is reported as a server-side configuration // error; choose a compatible reasoning model with ai.model. - AIModel string `koanf:"ai.model" env:"FANOUT_AI_MODEL"` - AIBaseURL string `koanf:"ai.base_url" env:"FANOUT_AI_BASE_URL"` - SMTPHost string `koanf:"smtp.host" env:"FANOUT_SMTP_HOST"` - SMTPPort int `koanf:"smtp.port" env:"FANOUT_SMTP_PORT" default:"587"` - SMTPUser string `koanf:"smtp.username" env:"FANOUT_SMTP_USERNAME"` - SMTPPass string `koanf:"smtp.password" env:"FANOUT_SMTP_PASSWORD" secret:"true"` - SMTPFrom string `koanf:"smtp.from" env:"FANOUT_SMTP_FROM"` - AuthMode string `koanf:"auth.mode" env:"FANOUT_AUTH_MODE" default:"local"` - PublicURL string `koanf:"server.public_url" env:"FANOUT_PUBLIC_URL"` - AuthCodeSecret string `koanf:"auth.code_secret" env:"FANOUT_AUTH_CODE_SECRET" secret:"true"` - SelfSignup bool `koanf:"auth.self_signup" env:"FANOUT_SELF_SIGNUP" default:"false"` - SessionIdleTTL time.Duration `koanf:"auth.session_idle_ttl" env:"FANOUT_SESSION_IDLE_TTL" default:"12h"` - SessionAbsoluteTTL time.Duration `koanf:"auth.session_absolute_ttl" env:"FANOUT_SESSION_ABSOLUTE_TTL" default:"168h"` - OIDCIssuerURL string `koanf:"auth.oidc.issuer_url" env:"FANOUT_OIDC_ISSUER_URL"` - OIDCClientID string `koanf:"auth.oidc.client_id" env:"FANOUT_OIDC_CLIENT_ID"` - OIDCClientSecret string `koanf:"auth.oidc.client_secret" env:"FANOUT_OIDC_CLIENT_SECRET" secret:"true"` - OIDCEmailClaim string `koanf:"auth.oidc.email_claim" env:"FANOUT_OIDC_EMAIL_CLAIM" default:"email"` - OIDCEmailVerification string `koanf:"auth.oidc.email_verification" env:"FANOUT_OIDC_EMAIL_VERIFICATION" default:"required"` - OIDCAutoProvision bool `koanf:"auth.oidc.auto_provision" env:"FANOUT_OIDC_AUTO_PROVISION" default:"false"` - OIDCAllowedGroups string `koanf:"auth.oidc.allowed_groups" env:"FANOUT_OIDC_ALLOWED_GROUPS"` - OIDCAllowedDomains string `koanf:"auth.oidc.allowed_domains" env:"FANOUT_OIDC_ALLOWED_DOMAINS"` - OIDCDefaultRole string `koanf:"auth.oidc.default_role" env:"FANOUT_OIDC_DEFAULT_ROLE" default:"viewer"` - OIDCOperatorGroups string `koanf:"auth.oidc.operator_groups" env:"FANOUT_OIDC_OPERATOR_GROUPS"` - OIDCAdminGroups string `koanf:"auth.oidc.admin_groups" env:"FANOUT_OIDC_ADMIN_GROUPS"` - MetricsToken string `koanf:"metrics.token" env:"FANOUT_METRICS_TOKEN" secret:"true"` - MetricsPublic bool `koanf:"metrics.public" env:"FANOUT_METRICS_PUBLIC" default:"false"` - TrustedProxyCIDRs string `koanf:"server.trusted_proxy_cidrs" env:"FANOUT_TRUSTED_PROXY_CIDRS"` - TLSCertFile string `koanf:"server.tls.cert_file" env:"FANOUT_TLS_CERT_FILE"` - TLSKeyFile string `koanf:"server.tls.key_file" env:"FANOUT_TLS_KEY_FILE"` - resolvedSizing sizingSource + AIModel string `koanf:"ai.model" env:"FANOUT_AI_MODEL"` + // AnthropicAPIKey enables chat only when ai.provider is anthropic. + AnthropicAPIKey string `koanf:"ai.anthropic.api_key" env:"ANTHROPIC_API_KEY" secret:"true"` + // AnthropicBaseURL defaults to https://api.anthropic.com. Fanout appends + // /v1/messages, preserving any gateway path prefix. Use an HTTP(S) base URL + // without embedded credentials, a query, or a fragment. + AnthropicBaseURL string `koanf:"ai.anthropic.base_url" env:"ANTHROPIC_BASE_URL"` + // OpenAIAPIKey enables chat only when ai.provider is openai. + OpenAIAPIKey string `koanf:"ai.openai.api_key" env:"OPENAI_API_KEY" secret:"true"` + // OpenAIBaseURL defaults to https://api.openai.com/v1. Fanout appends + // /responses, preserving the API version and any gateway path prefix. + // Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. + OpenAIBaseURL string `koanf:"ai.openai.base_url" env:"OPENAI_BASE_URL"` + // SMTPHost selects the mail server. Host, username, password, and from must all be set to enable SMTP. + SMTPHost string `koanf:"smtp.host" env:"FANOUT_SMTP_HOST"` + // SMTPPort sets the mail server's positive TCP port. + SMTPPort int `koanf:"smtp.port" env:"FANOUT_SMTP_PORT" default:"587"` + // SMTPUser authenticates to the mail server; required with SMTP. + SMTPUser string `koanf:"smtp.username" env:"FANOUT_SMTP_USERNAME"` + // SMTPPass authenticates to the mail server; required with SMTP. + SMTPPass string `koanf:"smtp.password" env:"FANOUT_SMTP_PASSWORD" secret:"true"` + // SMTPFrom sets the email sender address; required with SMTP. + SMTPFrom string `koanf:"smtp.from" env:"FANOUT_SMTP_FROM"` + // AuthMode selects local sign-in or oidc; each mode validates its prerequisites. + AuthMode string `koanf:"auth.mode" env:"FANOUT_AUTH_MODE" default:"local"` + // PublicURL sets the external browser origin. Required for OIDC; HTTPS is required with MCP. + PublicURL string `koanf:"server.public_url" env:"FANOUT_PUBLIC_URL"` + // AuthCodeSecret signs local sign-in codes; required in local mode with at least 32 characters. + AuthCodeSecret string `koanf:"auth.code_secret" env:"FANOUT_AUTH_CODE_SECRET" secret:"true"` + // SelfSignup allows verified email addresses to create viewer accounts; requires local auth and SMTP. + SelfSignup bool `koanf:"auth.self_signup" env:"FANOUT_SELF_SIGNUP" default:"false"` + // SessionIdleTTL expires inactive sessions; use a Go duration of at least 5m. + SessionIdleTTL time.Duration `koanf:"auth.session_idle_ttl" env:"FANOUT_SESSION_IDLE_TTL" default:"12h"` + // SessionAbsoluteTTL limits total session lifetime; it must be at least session_idle_ttl. + SessionAbsoluteTTL time.Duration `koanf:"auth.session_absolute_ttl" env:"FANOUT_SESSION_ABSOLUTE_TTL" default:"168h"` + // OIDCIssuerURL selects the identity provider issuer; required in OIDC mode. + OIDCIssuerURL string `koanf:"auth.oidc.issuer_url" env:"FANOUT_OIDC_ISSUER_URL"` + // OIDCClientID identifies Fanout's client registration; required in OIDC mode. + OIDCClientID string `koanf:"auth.oidc.client_id" env:"FANOUT_OIDC_CLIENT_ID"` + // OIDCClientSecret authenticates Fanout's client registration; required in OIDC mode. + OIDCClientSecret string `koanf:"auth.oidc.client_secret" env:"FANOUT_OIDC_CLIENT_SECRET" secret:"true"` + // OIDCEmailClaim names the claim containing the user's email address. + OIDCEmailClaim string `koanf:"auth.oidc.email_claim" env:"FANOUT_OIDC_EMAIL_CLAIM" default:"email"` + // OIDCEmailVerification selects required verified-email claims or issuer trust; issuer trust requires an allowlist. + OIDCEmailVerification string `koanf:"auth.oidc.email_verification" env:"FANOUT_OIDC_EMAIL_VERIFICATION" default:"required"` + // OIDCAutoProvision creates accounts on OIDC sign-in; requires an allowed group or domain. + OIDCAutoProvision bool `koanf:"auth.oidc.auto_provision" env:"FANOUT_OIDC_AUTO_PROVISION" default:"false"` + // OIDCAllowedGroups lists allowed identity-provider groups, separated by commas. + OIDCAllowedGroups string `koanf:"auth.oidc.allowed_groups" env:"FANOUT_OIDC_ALLOWED_GROUPS"` + // OIDCAllowedDomains lists allowed email domains, separated by commas; wildcard * requires verified email. + OIDCAllowedDomains string `koanf:"auth.oidc.allowed_domains" env:"FANOUT_OIDC_ALLOWED_DOMAINS"` + // OIDCDefaultRole assigns viewer, operator, or admin to users without a group role mapping. + OIDCDefaultRole string `koanf:"auth.oidc.default_role" env:"FANOUT_OIDC_DEFAULT_ROLE" default:"viewer"` + // OIDCOperatorGroups lists identity-provider groups mapped to operator, separated by commas. + OIDCOperatorGroups string `koanf:"auth.oidc.operator_groups" env:"FANOUT_OIDC_OPERATOR_GROUPS"` + // OIDCAdminGroups lists identity-provider groups mapped to admin, separated by commas. + OIDCAdminGroups string `koanf:"auth.oidc.admin_groups" env:"FANOUT_OIDC_ADMIN_GROUPS"` + // MetricsToken authorizes Prometheus scrapes with a bearer token; optional when using an admin session. + MetricsToken string `koanf:"metrics.token" env:"FANOUT_METRICS_TOKEN" secret:"true"` + // MetricsPublic allows unauthenticated Prometheus scrapes. + MetricsPublic bool `koanf:"metrics.public" env:"FANOUT_METRICS_PUBLIC" default:"false"` + // TrustedProxyCIDRs lists trusted proxy networks, separated by commas; empty trusts no forwarded client addresses. + TrustedProxyCIDRs string `koanf:"server.trusted_proxy_cidrs" env:"FANOUT_TRUSTED_PROXY_CIDRS"` + // TLSCertFile selects the TLS certificate file; requires tls.key_file. + TLSCertFile string `koanf:"server.tls.cert_file" env:"FANOUT_TLS_CERT_FILE"` + // TLSKeyFile selects the TLS private key file; requires tls.cert_file. + TLSKeyFile string `koanf:"server.tls.key_file" env:"FANOUT_TLS_KEY_FILE"` + resolvedSizing sizingSource } // LogStartup reports the effective non-secret sizing and security-sensitive @@ -277,12 +329,18 @@ func (c Config) Validate() error { return fmt.Errorf("OIDC auto-provisioning requires auth.oidc.allowed_groups or auth.oidc.allowed_domains") } } - if c.AgentConfigured() { - switch strings.ToLower(strings.TrimSpace(c.AIProvider)) { - case "", "anthropic", "openai": - default: - return fmt.Errorf("ai.provider must be anthropic or openai") - } + switch strings.ToLower(strings.TrimSpace(c.AIProvider)) { + case "", "anthropic", "openai": + default: + return fmt.Errorf("ai.provider must be anthropic or openai") + } + baseURL, key := c.AIBaseURL(), "ai.anthropic.base_url (ANTHROPIC_BASE_URL)" + if strings.EqualFold(strings.TrimSpace(c.AIProvider), "openai") { + key = "ai.openai.base_url (OPENAI_BASE_URL)" + } + u, err := url.Parse(baseURL) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || strings.Contains(baseURL, "#") { + return fmt.Errorf("%s must be an HTTP(S) base URL without credentials, query, or fragment", key) } for _, raw := range strings.Split(c.TrustedProxyCIDRs, ",") { value := strings.TrimSpace(raw) @@ -329,7 +387,31 @@ func (c Config) SMTPConfigured() bool { // AgentConfigured reports whether the in-process AI agent should be started. func (c Config) AgentConfigured() bool { - return strings.TrimSpace(c.AIAPIKey) != "" + return strings.TrimSpace(c.AIAPIKey()) != "" +} + +// AIAPIKey returns only the explicitly selected provider's credential. +func (c Config) AIAPIKey() string { + switch strings.ToLower(strings.TrimSpace(c.AIProvider)) { + case "", "anthropic": + return c.AnthropicAPIKey + case "openai": + return c.OpenAIAPIKey + default: + return "" + } +} + +// AIBaseURL returns the selected provider's API base, including its default. +func (c Config) AIBaseURL() string { + baseURL, fallback := c.AnthropicBaseURL, DefaultAnthropicBaseURL + if strings.EqualFold(strings.TrimSpace(c.AIProvider), "openai") { + baseURL, fallback = c.OpenAIBaseURL, DefaultOpenAIBaseURL + } + if strings.TrimSpace(baseURL) == "" { + return fallback + } + return strings.TrimRight(strings.TrimSpace(baseURL), "/") } func csvContains(value, target string) bool { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 30e911fc..a14bf674 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -16,7 +16,7 @@ import ( func validEnvironment() []string { return []string{ "FANOUT_AUTH_CODE_SECRET=0123456789abcdef0123456789abcdef", - "FANOUT_AI_API_KEY=sk-test", + "ANTHROPIC_API_KEY=sk-test", "FANOUT_SMTP_HOST=smtp.example.com", "FANOUT_SMTP_USERNAME=user", "FANOUT_SMTP_PASSWORD=pass", @@ -182,7 +182,8 @@ func TestExampleConfigurationMatchesSchema(t *testing.T) { t.Fatalf("field specs: %v", err) } wantSecrets := map[string]bool{ - "ai.api_key": true, + "ai.anthropic.api_key": true, + "ai.openai.api_key": true, "smtp.password": true, "auth.code_secret": true, "auth.oidc.client_secret": true, @@ -329,6 +330,8 @@ func TestLoadRejectsUnknownInputs(t *testing.T) { for _, test := range []struct { key, document string }{ + {"ai.api_key", "ai:\n api_key: secret\n"}, + {"ai.base_url", "ai:\n base_url: https://ai.example.com\n"}, {"agent.provider", "agent:\n provider: anthropic\n"}, {"agent.api_key", "agent:\n api_key: secret\n"}, {"agent.model", "agent:\n model: test\n"}, @@ -355,6 +358,8 @@ func TestLoadRejectsUnknownInputs(t *testing.T) { t.Run("removed environment variables", func(t *testing.T) { for _, name := range []string{ + "FANOUT_AI_API_KEY", + "FANOUT_AI_BASE_URL", "FANOUT_ROLLUP_EVERY_SECONDS", "FANOUT_MERGE_EVERY_SECONDS", "FANOUT_MERGE_INTERVAL", @@ -556,7 +561,7 @@ func TestLoadErrorsDoNotContainSecrets(t *testing.T) { func TestLoadRequiresPrivatePermissionsForConfigSecrets(t *testing.T) { path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte("ai:\n api_key: secret-from-file\n"), 0o644); err != nil { + if err := os.WriteFile(path, []byte("ai:\n anthropic:\n api_key: secret-from-file\n"), 0o644); err != nil { t.Fatal(err) } _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}) @@ -577,7 +582,7 @@ func TestValidate(t *testing.T) { AlertEvaluationInterval: 30 * time.Second, AlertHistoryDays: 7, AIProvider: "anthropic", - AIAPIKey: "sk-test", + AnthropicAPIKey: "sk-test", SMTPHost: "smtp.example.com", SMTPPort: 587, SMTPUser: "user", @@ -659,8 +664,8 @@ func TestValidate(t *testing.T) { t.Run("local mode allows absent SMTP and agent", func(t *testing.T) { c := valid c.SMTPHost, c.SMTPUser, c.SMTPPass, c.SMTPFrom = "", "", "", "" - c.AIAPIKey = "" - c.AIProvider = "ignored-without-a-key" + c.AnthropicAPIKey = "" + c.AIProvider = "anthropic" if err := c.Validate(); err != nil { t.Fatalf("minimal local config should pass: %v", err) } @@ -780,7 +785,7 @@ func TestFieldSpecsRejectInvalidSchemas(t *testing.T) { typ: reflect.TypeOf(struct { A string `koanf:"a" env:"A"` }{}), - want: "lacks FANOUT_ prefix", + want: "outside the supported configuration contract", }, } for _, test := range tests { diff --git a/internal/config/loader.go b/internal/config/loader.go index 6c6945d3..659eafc5 100644 --- a/internal/config/loader.go +++ b/internal/config/loader.go @@ -139,8 +139,8 @@ func fieldSpecs(typ reflect.Type) ([]fieldSpec, error) { if previous, exists := envs[envName]; exists { return nil, fmt.Errorf("configuration fields %s and %s share environment variable %q", previous, field.Name, envName) } - if !strings.HasPrefix(envName, "FANOUT_") { - return nil, fmt.Errorf("configuration field %s environment variable %q lacks FANOUT_ prefix", field.Name, envName) + if !strings.HasPrefix(envName, "FANOUT_") && !isProviderEnvironment(envName) { + return nil, fmt.Errorf("configuration field %s environment variable %q is outside the supported configuration contract", field.Name, envName) } defaultText, hasDefault := field.Tag.Lookup("default") var defaultValue any @@ -273,7 +273,7 @@ func environmentOverrides(environ []string, known map[string]fieldSpec) (map[str } continue } - if !strings.HasPrefix(name, "FANOUT_") { + if !isKnown && !strings.HasPrefix(name, "FANOUT_") { continue } spec, exists := known[name] @@ -302,6 +302,17 @@ func environmentOverrides(environ []string, known map[string]fieldSpec) (map[str return overrides, nil } +// Vendor namespaces are shared with other consumers. Accept these exact names; +// only Fanout's own namespace rejects unknown names. +func isProviderEnvironment(name string) bool { + switch name { + case "ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "OPENAI_API_KEY", "OPENAI_BASE_URL": + return true + default: + return false + } +} + func isPlatformInjectedEnvironment(name string) bool { return name == "FANOUT_SERVICE_HOST" || name == "FANOUT_SERVICE_PORT" || diff --git a/internal/config/provider_test.go b/internal/config/provider_test.go new file mode 100644 index 00000000..994bd784 --- /dev/null +++ b/internal/config/provider_test.go @@ -0,0 +1,71 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestLoadSelectsProviderCredentials(t *testing.T) { + for _, tc := range []struct { + name, provider, key, baseURL string + credentials []string + }{ + {"anthropic", "anthropic", "anthropic-test", "https://anthropic.example/gateway", []string{"ANTHROPIC_API_KEY=anthropic-test", "ANTHROPIC_BASE_URL=https://anthropic.example/gateway/", "OPENAI_API_KEY=openai-test", "OPENAI_BASE_URL=https://openai.example/v1"}}, + {"openai", " OpenAI ", "openai-test", "https://openai.example/gateway/v1", []string{"ANTHROPIC_API_KEY=anthropic-test", "OPENAI_API_KEY=openai-test", "OPENAI_BASE_URL=https://openai.example/gateway/v1/"}}, + {"other key does not enable anthropic", "anthropic", "", DefaultAnthropicBaseURL, []string{"OPENAI_API_KEY=openai-test", "OPENAI_BASE_URL=not-used"}}, + {"other key does not enable openai", "openai", "", DefaultOpenAIBaseURL, []string{"ANTHROPIC_API_KEY=anthropic-test", "ANTHROPIC_BASE_URL=not-used"}}, + } { + t.Run(tc.name, func(t *testing.T) { + env := []string{"FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), "FANOUT_AI_PROVIDER=" + tc.provider} + cfg, err := Load(LoadOptions{Environ: append(env, tc.credentials...)}) + if err != nil { + t.Fatal(err) + } + if cfg.AIAPIKey() != tc.key || cfg.AIBaseURL() != tc.baseURL || cfg.AgentConfigured() != (tc.key != "") { + t.Fatal("selected provider credential, endpoint, or enabled state is incorrect") + } + }) + } +} + +func TestLoadProviderEnvironmentOverridesYAML(t *testing.T) { + path := filepath.Join(t.TempDir(), "fanout.yaml") + document := "ai:\n provider: openai\n model: configured-model\n anthropic:\n api_key: anthropic-file\n base_url: https://anthropic.example\n openai:\n api_key: openai-file\n base_url: https://openai.example/v1\n" + if err := os.WriteFile(path, []byte(document), 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(LoadOptions{Path: path, Environ: []string{ + "FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), + "OPENAI_API_KEY=openai-env", "OPENAI_BASE_URL=https://gateway.example/api/v1/", + "ANTHROPIC_API_KEY=", "ANTHROPIC_BASE_URL=", "FANOUT_AI_MODEL=", + "OPENAI_MODEL=unrelated-tool", "ANTHROPIC_MODEL=unrelated-tool", + "OPENAI_PROJECT_ID=another-consumer", "ANTHROPIC_AUTH_TOKEN=another-consumer", + }}) + if err != nil { + t.Fatal(err) + } + if cfg.AIAPIKey() != "openai-env" || cfg.AIBaseURL() != "https://gateway.example/api/v1" || cfg.AIModel != "configured-model" || cfg.AnthropicAPIKey != "anthropic-file" || cfg.AnthropicBaseURL != "https://anthropic.example" { + t.Fatal("provider environment precedence or shared consumer isolation is incorrect") + } +} + +func TestLoadRejectsInvalidProviderURLWithoutLeakingValues(t *testing.T) { + for _, provider := range []string{"anthropic", "openai"} { + name := strings.ToUpper(provider) + "_BASE_URL" + for _, value := range []string{"file:///private", "https://user:private-value@example.com", "https://example.com?key=private-value", "https://example.com#private-value", "https://example.com?", "https://example.com#", "https://", "https://example.com/%zz"} { + _, err := Load(LoadOptions{Environ: []string{"FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), "FANOUT_AI_PROVIDER=" + provider, name + "=" + value}}) + if err == nil || !strings.Contains(err.Error(), name) || strings.Contains(err.Error(), value) || strings.Contains(err.Error(), "private-value") { + t.Fatalf("expected redacted validation error naming %s", name) + } + } + } +} + +func TestLoadRejectsUnsupportedProviderWithoutAKey(t *testing.T) { + _, err := Load(LoadOptions{Environ: []string{"FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), "FANOUT_AI_PROVIDER=unsupported"}}) + if err == nil || !strings.Contains(err.Error(), "ai.provider") { + t.Fatal("unsupported provider must fail at startup, even without a key") + } +} diff --git a/justfile b/justfile index 98504e14..658e2ddf 100644 --- a/justfile +++ b/justfile @@ -32,6 +32,15 @@ embedded := "internal/ui/dist internal/mcp/apps" default: @just --list +# Launch with shared nonsecret defaults, personal overrides, then injected values. +# Build the binary first with `just build`; secrets must be injected explicitly. +run *ARGS='': + node scripts/run-fanout.mjs --environment local -- ./bin/fanout {{ARGS}} + +# Production explicitly excludes every developer-local dotenv file. +run-production *ARGS='': + node scripts/run-fanout.mjs --environment production -- ./bin/fanout {{ARGS}} + # Split out so CI installs exactly what a contributor does, without also # writing git hooks into the runner. diff --git a/scripts/run-fanout.mjs b/scripts/run-fanout.mjs new file mode 100644 index 00000000..e931992f --- /dev/null +++ b/scripts/run-fanout.mjs @@ -0,0 +1,113 @@ +import { spawn } from "node:child_process"; +import { readFileSync, statSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { parseEnv } from "node:util"; + +/** @param {string} path @param {boolean} required */ +function readDotenv(path, required) { + let content; + try { + if (!statSync(path).isFile()) throw new Error("not a regular file"); + content = readFileSync(path, "utf8"); + } catch (error) { + if (!required && error && typeof error === "object" && "code" in error && error.code === "ENOENT") return {}; + throw new Error(`Cannot read required or invalid dotenv file: ${path}`); + } + try { + // Node's dotenv parser preserves literal dollars and never executes shell code. + return parseEnv(content); + } catch { + throw new Error(`Invalid dotenv syntax: ${path}`); + } +} + +// The generated template is the schema-derived inventory, including secret +// classification. There is no independently maintained configuration allowlist. +/** @param {string} root */ +function inventory(root) { + let content; + try { + content = readFileSync(resolve(root, ".env.example"), "utf8"); + } catch { + throw new Error("Missing generated environment inventory; run just docs-generate"); + } + const names = new Set(Object.keys(parseEnv(content))); + const secrets = new Set(); + for (const block of content.split(/\n\s*\n/)) { + if (/^# [^\n]+; secret$/m.test(block)) { + for (const name of Object.keys(parseEnv(block))) secrets.add(name); + } + } + if (names.size === 0 || secrets.size === 0) throw new Error("Missing generated environment inventory; run just docs-generate"); + return { names, secrets }; +} + +/** + * @param {string} root + * @param {"local" | "production"} environment + * @param {Record} inherited + * @returns {Record} + */ +export function prepareEnvironment(root, environment, inherited) { + if (environment !== "local" && environment !== "production") throw new Error("Environment must be local or production"); + const { names, secrets } = inventory(root); + const files = [".env", ...(environment === "production" ? [".env.production"] : [".env.local"])]; + /** @type {Record} */ + const values = {}; + for (const file of files) { + const layer = readDotenv(resolve(root, file), file !== ".env.local"); + for (const [name, value] of Object.entries(layer)) { + if (value === undefined) throw new Error(`Invalid dotenv syntax: ${file}`); + if (!names.has(name)) throw new Error(`Unknown dotenv setting ${name} in ${file}`); + if (secrets.has(name)) throw new Error(`Secret setting ${name} belongs in runtime injection, not ${file}`); + if (value.includes("op://")) throw new Error(`Unresolved secret reference for ${name} in ${file}; use runtime injection`); + if (value !== "") values[name] = value; + } + } + // Explicitly injected values win. An empty recognized value retains Fanout's + // existing "no override" semantics instead of erasing a file setting. + for (const [name, value] of Object.entries(inherited)) { + if (value !== undefined && (value !== "" || !names.has(name))) values[name] = value; + } + return values; +} + +/** @param {string[]} args @returns {Promise} */ +export async function main(args) { + if (args.length < 4 || args[0] !== "--environment" || args[2] !== "--") { + throw new Error("Usage: node scripts/run-fanout.mjs --environment local|production -- command [args]"); + } + const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); + const environment = /** @type {"local" | "production"} */ (args[1]); + const env = prepareEnvironment(root, environment, process.env); + const child = spawn(args[3], args.slice(4), { cwd: root, env, stdio: "inherit" }); + const terminate = () => child.kill("SIGTERM"); + const interrupt = () => child.kill("SIGINT"); + process.once("SIGTERM", terminate); + process.once("SIGINT", interrupt); + try { + return await new Promise((resolveExit, reject) => { + child.once("error", reject); + child.once("exit", (code, signal) => resolveExit(code ?? (signal === "SIGINT" ? 130 : signal === "SIGTERM" ? 143 : 1))); + }); + } finally { + process.off("SIGTERM", terminate); + process.off("SIGINT", interrupt); + } +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + try { + process.exitCode = await main(process.argv.slice(2)); + } catch (error) { + // Launch errors can contain command arguments. Only preparation errors + // below carry safe setting/file names; never print resolved environment values. + const message = error instanceof Error ? error.message : "Environment launch failed"; + console.error(message.startsWith("Unknown dotenv") || message.startsWith("Secret setting") || + message.startsWith("Unresolved secret") || message.startsWith("Cannot read") || + message.startsWith("Invalid dotenv") || message.startsWith("Usage:") || + message.startsWith("Environment must") || message.startsWith("Missing generated") ? message : "Cannot launch command; ensure it is executable (for Fanout, run just build)"); + process.exitCode = 1; + } +} diff --git a/scripts/run-fanout.test.ts b/scripts/run-fanout.test.ts new file mode 100644 index 00000000..4dc4ac20 --- /dev/null +++ b/scripts/run-fanout.test.ts @@ -0,0 +1,103 @@ +import { afterEach, expect, test } from "bun:test"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { pathToFileURL } from "node:url"; +import { main, prepareEnvironment } from "./run-fanout.mjs"; + +const roots: string[] = []; +afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); + +function fixture(): string { + const root = mkdtempSync(join(tmpdir(), "fanout-env-")); + roots.push(root); + writeFileSync(join(root, ".env.example"), `# server.addr; string +FANOUT_ADDR=:7520 + +# ai.model; string +FANOUT_AI_MODEL= + +# storage.data_dir; string +FANOUT_DATA_DIR=./data + +# auth.code_secret; string; secret +FANOUT_AUTH_CODE_SECRET= + +# ai.anthropic.api_key; string; secret +ANTHROPIC_API_KEY= +`); + writeFileSync(join(root, ".env"), "FANOUT_ADDR=127.0.0.1:7520\nFANOUT_AI_MODEL=base\n"); + return root; +} + +test("local dotenv defaults, personal overrides, and explicit injection have ordered precedence", () => { + const root = fixture(); + writeFileSync(join(root, ".env.local"), "FANOUT_ADDR=127.0.0.1:9000\nFANOUT_AI_MODEL=personal\n"); + const env = prepareEnvironment(root, "local", { FANOUT_ADDR: "", FANOUT_AI_MODEL: "injected", FANOUT_AUTH_CODE_SECRET: "synthetic-only", PATH: "/synthetic/path" }); + expect(env.FANOUT_ADDR).toBe("127.0.0.1:9000"); + expect(env.FANOUT_AI_MODEL).toBe("injected"); + expect(env.FANOUT_AUTH_CODE_SECRET).toBe("synthetic-only"); + expect(env.PATH).toBe("/synthetic/path"); +}); + +test("production excludes personal files and requires its declared override file", () => { + const root = fixture(); + writeFileSync(join(root, ".env.local"), "ANTHROPIC_API_KEY=personal-secret\n"); + writeFileSync(join(root, ".env.production.local"), "FANOUT_AI_MODEL=personal-production\n"); + expect(() => prepareEnvironment(root, "production", {})).toThrow("Cannot read required"); + writeFileSync(join(root, ".env.production"), "FANOUT_ADDR=:7520\nFANOUT_AI_MODEL=production\n"); + const env = prepareEnvironment(root, "production", {}); + expect(env.FANOUT_ADDR).toBe(":7520"); + expect(env.FANOUT_AI_MODEL).toBe("production"); + expect(env.ANTHROPIC_API_KEY).toBeUndefined(); + expect(prepareEnvironment(root, "production", { FANOUT_AI_MODEL: "deployment" }).FANOUT_AI_MODEL).toBe("deployment"); +}); + +test("dotenv parsing preserves dollars, quotes, whitespace, and multiline values without shell execution", () => { + const root = fixture(); + writeFileSync(join(root, ".env.local"), `FANOUT_DATA_DIR=' $HOME $(printf unsafe) "quoted" ' +FANOUT_AI_MODEL="line one +line two" +`); + const env = prepareEnvironment(root, "local", {}); + expect(env.FANOUT_DATA_DIR).toBe(' $HOME $(printf unsafe) "quoted" '); + expect(env.FANOUT_AI_MODEL).toBe("line one\nline two"); + // Verify the real Node runtime as well as Bun's test-runner implementation. + const result = Bun.spawnSync(["node", "--input-type=module", "-e", + "const {prepareEnvironment}=await import(process.argv[2]); process.stdout.write(JSON.stringify(prepareEnvironment(process.argv[1], 'local', {})));", + root, pathToFileURL(join(import.meta.dir, "run-fanout.mjs")).href], { cwd: root }); + expect(result.exitCode).toBe(0); + expect(JSON.parse(result.stdout.toString())).toEqual(env); +}); + +test("public and personal dotenv files reject secrets, retired names, and unresolved references without leaking values", () => { + for (const assignment of [ + "ANTHROPIC_API_KEY=synthetic-private-value", + "FANOUT_AUTH_CODE_SECRET=", + "FANOUT_AI_API_KEY=synthetic-private-value", + "FANOUT_AI_BASE_URL=https://example.com/synthetic-private-value", + "AI_API_KEY=synthetic-private-value", + "FANOUT_AI_MODEL=op://synthetic-private-value", + ]) { + for (const file of [".env", ".env.local"]) { + const root = fixture(); + writeFileSync(join(root, file), assignment+"\n"); + let message = ""; + try { prepareEnvironment(root, "local", {}); } catch (error) { message = (error as Error).message; } + expect(message).not.toBe(""); + expect(message).not.toContain("synthetic-private-value"); + } + } +}); + +test("missing defaults and unsupported environments fail rather than fall back", () => { + const root = fixture(); + rmSync(join(root, ".env")); + expect(() => prepareEnvironment(root, "local", {})).toThrow("Cannot read required"); + expect(() => prepareEnvironment(root, "staging" as "local", {})).toThrow("Environment must"); +}); + +test("launcher requires an explicit environment and command", async () => { + await expect(main([])).rejects.toThrow("Usage:"); + await expect(main(["--environment", "local", "--"])).rejects.toThrow("Usage:"); +}); diff --git a/site/src/content/docs/reference/configuration.mdx b/site/src/content/docs/reference/configuration.mdx index 923354de..db71fc27 100644 --- a/site/src/content/docs/reference/configuration.mdx +++ b/site/src/content/docs/reference/configuration.mdx @@ -13,21 +13,48 @@ keys. Every setting is listed under [Settings](/reference/settings/server), grouped by the part of the product it belongs to. Those pages are generated from `internal/config` by `cmd/fanout-docgen`, and -`just check` fails when a committed page no longer matches the type the loader +the root `.env`, `.env.production`, and `.env.example` are generated from the +same schema. `just check` fails +when a committed page or template no longer matches the type the loader binds. That is the whole point of generating them: the reference cannot describe a setting the binary does not accept, and cannot omit one it does. ## Where a value comes from -An environment variable wins over the YAML file, and the file wins over the -built-in default. A setting with no default and no value is empty, which for -most settings means the feature it configures is off. +A nonempty recognized environment variable wins over the YAML file, and the file +wins over the built-in default. Empty environment values mean no override. +All settings are read once at startup and require restart. Fanout does not load +dotenv files itself. The generated `.env.example` documents types, defaults, +and empty credential placeholders; `.env` and `.env.production` omit secret fields. + +For a source checkout, build the binary and use `just run` for local development +or `just run-production` for an explicit production launch. Local order is +`.env`, optional ignored `.env.local`, then injected environment values. +Production order is `.env`, required `.env.production`, then injected values; +personal files are excluded. The Node launcher uses the native dotenv parser +without shell execution or variable interpolation. Missing required files, +unknown names, and secret entries in ordinary dotenv files fail before launch. + +Inject credentials from a private selected SOPS profile or the deployment's +secret service. This public repository ships no operator secrets or recipients. +Dotenv values are injected as environment, so nonempty values override the +corresponding `--config` YAML. Launch the binary directly for YAML-only ordinary +configuration. ## Naming -Every environment variable is `FANOUT_`-prefixed. The YAML key is the dotted -path in the first column of each settings table — `storage.duckdb.memory` is -`FANOUT_DUCKDB_MEMORY`, and the two are the same setting. +Custom environment settings use the `FANOUT_` prefix. Provider keys and endpoints +use `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, `OPENAI_API_KEY`, and +`OPENAI_BASE_URL`. The YAML key is the dotted path in the first column of each +settings table — `storage.duckdb.memory` maps to `FANOUT_DUCKDB_MEMORY`. +Provider settings map to native structured keys under `ai.anthropic` or `ai.openai`. + +`FANOUT_AI_PROVIDER` selects the provider and `FANOUT_AI_MODEL` selects its model. +Only the selected provider's key and endpoint are used. Without that key, chat is +disabled. Other vendor/tool settings, including `ANTHROPIC_MODEL` and +`OPENAI_MODEL`, are ignored. See [AI settings](/reference/settings/ai) for +the different base URL meanings and [upgrade instructions](/start/install#ai-configuration-upgrades) +for migrating the retired generic key and base URL settings. ## An unknown variable is an error diff --git a/site/src/content/docs/reference/settings/ai.mdx b/site/src/content/docs/reference/settings/ai.mdx index d8efe765..42155dde 100644 --- a/site/src/content/docs/reference/settings/ai.mdx +++ b/site/src/content/docs/reference/settings/ai.mdx @@ -11,24 +11,51 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} -Every setting below is read from the environment variable in the second +Every setting below is read once at startup and requires a restart to change. +It is read from the environment variable in the second column, or from the same key in a YAML configuration file. An unrecognised `FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces as a refusal to start rather than as a default nobody chose. +Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, +`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are +ignored; only the explicitly selected AI provider's key and endpoint are used. + | Setting | Environment variable | Type | Default | |---|---|---|---| -| `ai.api_key` | `FANOUT_AI_API_KEY` | string · secret | — | -| `ai.base_url` | `FANOUT_AI_BASE_URL` | string | — | +| `ai.anthropic.api_key` | `ANTHROPIC_API_KEY` | string · secret | — | +| `ai.anthropic.base_url` | `ANTHROPIC_BASE_URL` | string | — | | `ai.model` | `FANOUT_AI_MODEL` | string | — | +| `ai.openai.api_key` | `OPENAI_API_KEY` | string · secret | — | +| `ai.openai.base_url` | `OPENAI_BASE_URL` | string | — | | `ai.provider` | `FANOUT_AI_PROVIDER` | string | `anthropic` | ## Notes +### `ai.anthropic.api_key` + +Enables chat only when ai.provider is anthropic. + +### `ai.anthropic.base_url` + +Defaults to https://api.anthropic.com. Fanout appends /v1/messages, preserving any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. + ### `ai.model` Overrides the provider's default: claude-sonnet-5-5 for Anthropic and gpt-6.1-sol for OpenAI. The OpenAI provider expects a reasoning model supporting reasoning.encrypted_content for stateless Responses replay. A 400 rejecting encrypted content is reported as a server-side configuration error; choose a compatible reasoning model with ai.model. +### `ai.openai.api_key` + +Enables chat only when ai.provider is openai. + +### `ai.openai.base_url` + +Defaults to https://api.openai.com/v1. Fanout appends /responses, preserving the API version and any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. + +### `ai.provider` + +Selects anthropic or openai independently of which keys are present. + :::caution[Secrets] Settings marked `secret` are redacted from Fanout's startup configuration log. They are still plain environment variables: anything that can read the diff --git a/site/src/content/docs/reference/settings/alerts.mdx b/site/src/content/docs/reference/settings/alerts.mdx index 6daeb6c9..86dec738 100644 --- a/site/src/content/docs/reference/settings/alerts.mdx +++ b/site/src/content/docs/reference/settings/alerts.mdx @@ -10,13 +10,32 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} -Every setting below is read from the environment variable in the second +Every setting below is read once at startup and requires a restart to change. +It is read from the environment variable in the second column, or from the same key in a YAML configuration file. An unrecognised `FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces as a refusal to start rather than as a default nobody chose. +Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, +`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are +ignored; only the explicitly selected AI provider's key and endpoint are used. + | Setting | Environment variable | Type | Default | |---|---|---|---| | `alerts.enabled` | `FANOUT_ALERTS_ENABLED` | boolean | `true` | | `alerts.evaluation_interval` | `FANOUT_ALERTS_EVALUATION_INTERVAL` | duration | `30s` | | `alerts.history_days` | `FANOUT_ALERTS_HISTORY_DAYS` | integer | `7` | + +## Notes + +### `alerts.enabled` + +Runs the alert evaluation engine. + +### `alerts.evaluation_interval` + +Schedules alert evaluation; use a Go duration of at least 1s. + +### `alerts.history_days` + +Sets alert history retention in days. diff --git a/site/src/content/docs/reference/settings/auth.mdx b/site/src/content/docs/reference/settings/auth.mdx index 0d04de24..923ae2a1 100644 --- a/site/src/content/docs/reference/settings/auth.mdx +++ b/site/src/content/docs/reference/settings/auth.mdx @@ -11,11 +11,16 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} -Every setting below is read from the environment variable in the second +Every setting below is read once at startup and requires a restart to change. +It is read from the environment variable in the second column, or from the same key in a YAML configuration file. An unrecognised `FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces as a refusal to start rather than as a default nobody chose. +Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, +`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are +ignored; only the explicitly selected AI provider's key and endpoint are used. + | Setting | Environment variable | Type | Default | |---|---|---|---| | `auth.code_secret` | `FANOUT_AUTH_CODE_SECRET` | string · secret | — | @@ -35,6 +40,72 @@ as a refusal to start rather than as a default nobody chose. | `auth.session_absolute_ttl` | `FANOUT_SESSION_ABSOLUTE_TTL` | duration | `168h` | | `auth.session_idle_ttl` | `FANOUT_SESSION_IDLE_TTL` | duration | `12h` | +## Notes + +### `auth.code_secret` + +Signs local sign-in codes; required in local mode with at least 32 characters. + +### `auth.mode` + +Selects local sign-in or oidc; each mode validates its prerequisites. + +### `auth.oidc.admin_groups` + +Lists identity-provider groups mapped to admin, separated by commas. + +### `auth.oidc.allowed_domains` + +Lists allowed email domains, separated by commas; wildcard * requires verified email. + +### `auth.oidc.allowed_groups` + +Lists allowed identity-provider groups, separated by commas. + +### `auth.oidc.auto_provision` + +Creates accounts on OIDC sign-in; requires an allowed group or domain. + +### `auth.oidc.client_id` + +Identifies Fanout's client registration; required in OIDC mode. + +### `auth.oidc.client_secret` + +Authenticates Fanout's client registration; required in OIDC mode. + +### `auth.oidc.default_role` + +Assigns viewer, operator, or admin to users without a group role mapping. + +### `auth.oidc.email_claim` + +Names the claim containing the user's email address. + +### `auth.oidc.email_verification` + +Selects required verified-email claims or issuer trust; issuer trust requires an allowlist. + +### `auth.oidc.issuer_url` + +Selects the identity provider issuer; required in OIDC mode. + +### `auth.oidc.operator_groups` + +Lists identity-provider groups mapped to operator, separated by commas. + +### `auth.self_signup` + +Allows verified email addresses to create viewer accounts; requires local auth and SMTP. + +### `auth.session_absolute_ttl` + +Limits total session lifetime; it must be at least session_idle_ttl. + +### `auth.session_idle_ttl` + +Expires inactive sessions; use a Go duration of at least 5m. + :::caution[Secrets] Settings marked `secret` are redacted from Fanout's startup configuration log. They are still plain environment variables: anything that can read the diff --git a/site/src/content/docs/reference/settings/ingest.mdx b/site/src/content/docs/reference/settings/ingest.mdx index 39cc5e1b..50c75255 100644 --- a/site/src/content/docs/reference/settings/ingest.mdx +++ b/site/src/content/docs/reference/settings/ingest.mdx @@ -11,11 +11,16 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} -Every setting below is read from the environment variable in the second +Every setting below is read once at startup and requires a restart to change. +It is read from the environment variable in the second column, or from the same key in a YAML configuration file. An unrecognised `FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces as a refusal to start rather than as a default nobody chose. +Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, +`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are +ignored; only the explicitly selected AI provider's key and endpoint are used. + | Setting | Environment variable | Type | Default | |---|---|---|---| | `ingest.batch_size` | `FANOUT_INGEST_BATCH_SIZE` | integer | `50000` | @@ -24,6 +29,14 @@ as a refusal to start rather than as a default nobody chose. ## Notes +### `ingest.batch_size` + +Limits telemetry rows per batch; valid values are 1–50000. + +### `ingest.default_namespace` + +Supplies the namespace for telemetry without one. + ### `ingest.max_in_flight_bytes` Caps the decoded telemetry the process will hold across concurrent requests. Concurrency is HTTP/2 streams times connections and each handler holds its batch until the commit is durable, so without a ceiling a burst is bounded only by memory. Zero is unbounded, which is the behaviour that predates this setting. diff --git a/site/src/content/docs/reference/settings/mcp.mdx b/site/src/content/docs/reference/settings/mcp.mdx index 1b8d2ef2..2803d1df 100644 --- a/site/src/content/docs/reference/settings/mcp.mdx +++ b/site/src/content/docs/reference/settings/mcp.mdx @@ -10,11 +10,22 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} -Every setting below is read from the environment variable in the second +Every setting below is read once at startup and requires a restart to change. +It is read from the environment variable in the second column, or from the same key in a YAML configuration file. An unrecognised `FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces as a refusal to start rather than as a default nobody chose. +Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, +`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are +ignored; only the explicitly selected AI provider's key and endpoint are used. + | Setting | Environment variable | Type | Default | |---|---|---|---| | `mcp.enabled` | `FANOUT_MCP_ENABLED` | boolean | `true` | + +## Notes + +### `mcp.enabled` + +Serves the MCP endpoint at /mcp. diff --git a/site/src/content/docs/reference/settings/metrics.mdx b/site/src/content/docs/reference/settings/metrics.mdx index 0bfeceae..f51484c8 100644 --- a/site/src/content/docs/reference/settings/metrics.mdx +++ b/site/src/content/docs/reference/settings/metrics.mdx @@ -10,16 +10,31 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} -Every setting below is read from the environment variable in the second +Every setting below is read once at startup and requires a restart to change. +It is read from the environment variable in the second column, or from the same key in a YAML configuration file. An unrecognised `FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces as a refusal to start rather than as a default nobody chose. +Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, +`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are +ignored; only the explicitly selected AI provider's key and endpoint are used. + | Setting | Environment variable | Type | Default | |---|---|---|---| | `metrics.public` | `FANOUT_METRICS_PUBLIC` | boolean | `false` | | `metrics.token` | `FANOUT_METRICS_TOKEN` | string · secret | — | +## Notes + +### `metrics.public` + +Allows unauthenticated Prometheus scrapes. + +### `metrics.token` + +Authorizes Prometheus scrapes with a bearer token; optional when using an admin session. + :::caution[Secrets] Settings marked `secret` are redacted from Fanout's startup configuration log. They are still plain environment variables: anything that can read the diff --git a/site/src/content/docs/reference/settings/server.mdx b/site/src/content/docs/reference/settings/server.mdx index 3cbc4b5d..0c8674bc 100644 --- a/site/src/content/docs/reference/settings/server.mdx +++ b/site/src/content/docs/reference/settings/server.mdx @@ -11,11 +11,16 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} -Every setting below is read from the environment variable in the second +Every setting below is read once at startup and requires a restart to change. +It is read from the environment variable in the second column, or from the same key in a YAML configuration file. An unrecognised `FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces as a refusal to start rather than as a default nobody chose. +Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, +`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are +ignored; only the explicitly selected AI provider's key and endpoint are used. + | Setting | Environment variable | Type | Default | |---|---|---|---| | `server.addr` | `FANOUT_ADDR` | string | `:7520` | @@ -34,3 +39,19 @@ Binds the browser, API, MCP, OTLP/gRPC and OTLP/HTTP listener. ### `server.pprof_enabled` Exposes Go's net/http/pprof handlers at /debug/pprof/* for CPU/heap/mutex/goroutine profiling under load. Off by default; the routes require an admin browser session with operations:read. + +### `server.public_url` + +Sets the external browser origin. Required for OIDC; HTTPS is required with MCP. + +### `server.tls.cert_file` + +Selects the TLS certificate file; requires tls.key_file. + +### `server.tls.key_file` + +Selects the TLS private key file; requires tls.cert_file. + +### `server.trusted_proxy_cidrs` + +Lists trusted proxy networks, separated by commas; empty trusts no forwarded client addresses. diff --git a/site/src/content/docs/reference/settings/smtp.mdx b/site/src/content/docs/reference/settings/smtp.mdx index f259359e..3cd00537 100644 --- a/site/src/content/docs/reference/settings/smtp.mdx +++ b/site/src/content/docs/reference/settings/smtp.mdx @@ -10,11 +10,16 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} -Every setting below is read from the environment variable in the second +Every setting below is read once at startup and requires a restart to change. +It is read from the environment variable in the second column, or from the same key in a YAML configuration file. An unrecognised `FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces as a refusal to start rather than as a default nobody chose. +Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, +`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are +ignored; only the explicitly selected AI provider's key and endpoint are used. + | Setting | Environment variable | Type | Default | |---|---|---|---| | `smtp.from` | `FANOUT_SMTP_FROM` | string | — | @@ -23,6 +28,28 @@ as a refusal to start rather than as a default nobody chose. | `smtp.port` | `FANOUT_SMTP_PORT` | integer | `587` | | `smtp.username` | `FANOUT_SMTP_USERNAME` | string | — | +## Notes + +### `smtp.from` + +Sets the email sender address; required with SMTP. + +### `smtp.host` + +Selects the mail server. Host, username, password, and from must all be set to enable SMTP. + +### `smtp.password` + +Authenticates to the mail server; required with SMTP. + +### `smtp.port` + +Sets the mail server's positive TCP port. + +### `smtp.username` + +Authenticates to the mail server; required with SMTP. + :::caution[Secrets] Settings marked `secret` are redacted from Fanout's startup configuration log. They are still plain environment variables: anything that can read the diff --git a/site/src/content/docs/reference/settings/storage.mdx b/site/src/content/docs/reference/settings/storage.mdx index 5905ae30..68c6428c 100644 --- a/site/src/content/docs/reference/settings/storage.mdx +++ b/site/src/content/docs/reference/settings/storage.mdx @@ -11,11 +11,16 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} -Every setting below is read from the environment variable in the second +Every setting below is read once at startup and requires a restart to change. +It is read from the environment variable in the second column, or from the same key in a YAML configuration file. An unrecognised `FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces as a refusal to start rather than as a default nobody chose. +Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, +`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are +ignored; only the explicitly selected AI provider's key and endpoint are used. + | Setting | Environment variable | Type | Default | |---|---|---|---| | `storage.data_dir` | `FANOUT_DATA_DIR` | string | `./data` | @@ -29,6 +34,10 @@ as a refusal to start rather than as a default nobody chose. ## Notes +### `storage.data_dir` + +Stores telemetry, query state, and the control database. + ### `storage.duckdb.max_connections` Caps the DuckDB connection pool. Reads scan immutable Parquet concurrently; rollup-cache writes are serialized by the query write gate. Zero means "size it from the machine" — the same spelling DuckDBThreads uses for deferring to a default. Resolution happens in resolveSizing and is reported in the startup configuration log. @@ -45,6 +54,14 @@ Caps DuckDB's global query worker pool. Zero leaves DuckDB's own default in plac Controls Parquet retention and compaction, and query-cache checkpointing. +### `storage.retention_days` + +Sets telemetry retention in days; zero disables expiry. + +### `storage.rollup_interval` + +Schedules analytical rollups; use a Go duration of at least 1s. + ### `storage.rollup_skip_to_latest` RollupSkipToLatest, set once at boot, advances the service and edge rollup watermarks to the current max ingested timestamp. Existing history is not aggregated into those analytical rollups. Completed-batch trace caches still rebuild all retained history in bounded passes; uncached files remain immediately queryable. Useful for large pre-seeded datasets (benchmarks, restores); off in normal operation. diff --git a/site/src/content/docs/start/install.mdx b/site/src/content/docs/start/install.mdx index b6d46919..8b35596e 100644 --- a/site/src/content/docs/start/install.mdx +++ b/site/src/content/docs/start/install.mdx @@ -32,7 +32,8 @@ pick whichever registry your hosts already pull from. `FANOUT_AUTH_CODE_SECRET` is required in the default `local` auth mode and must be at least 32 characters — the container refuses to start without it. It is the only setting the command above adds; the image ships listener and data-directory -defaults of its own. Add `FANOUT_AI_API_KEY` to enable the chat investigator, +defaults of its own. Add `ANTHROPIC_API_KEY` (or `OPENAI_API_KEY` with +`FANOUT_AI_PROVIDER=openai`) to enable the chat investigator, which is optional: without a key the process starts normally and logs that the agent is disabled. @@ -110,6 +111,21 @@ Every tag above also exists under `ghcr.io/labstack/fanout`. Standalone signatures and SBOMs are deliberately not published yet — the reasoning is in the repository's release contract. +## AI configuration upgrades + +Replace `FANOUT_AI_API_KEY` with `ANTHROPIC_API_KEY`, or `OPENAI_API_KEY` when +`FANOUT_AI_PROVIDER=openai`. Move YAML `ai.api_key` to the corresponding +`ai.anthropic.api_key` or `ai.openai.api_key`. The other provider's key does not +enable chat; provider and model choices remain `FANOUT_AI_PROVIDER` and +`FANOUT_AI_MODEL`. + +Replace `FANOUT_AI_BASE_URL` with the matching `ANTHROPIC_BASE_URL` or +`OPENAI_BASE_URL` (YAML `ai.anthropic.base_url` or `ai.openai.base_url`). For +OpenAI, append `/v1` to the old origin/path: Fanout now appends `/responses` +instead of `/v1/responses`. Anthropic still appends `/v1/messages`. Both preserve +gateway prefixes and accept a trailing slash. Retired names are startup errors; +update deployment/secret inputs with the binary. All settings require restart. + ## What an instance opens | Port or path | Purpose | Native default | diff --git a/ui/host/e2e/global-setup.ts b/ui/host/e2e/global-setup.ts index 6bf3d7de..24b9113a 100644 --- a/ui/host/e2e/global-setup.ts +++ b/ui/host/e2e/global-setup.ts @@ -73,13 +73,15 @@ export default async function globalSetup() { try { const base = `http://127.0.0.1:${await freePort()}`; // Ignore inherited product configuration and keys; this instance is disposable. - const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => !key.startsWith("FANOUT_") && !/(?:API_KEY|TOKEN|SECRET)$/.test(key))); + const env = Object.fromEntries(Object.entries(process.env).filter(([key]) => + !key.startsWith("FANOUT_") && !key.startsWith("ANTHROPIC_") && !key.startsWith("OPENAI_") && + !/(?:API_KEY|TOKEN|SECRET)$/.test(key))); const startDeadline = Date.now() + 30_000; const fd = openSync(serverLog, "a", 0o600); child = spawn(resolve(process.env.FANOUT_E2E_BINARY ?? join(root, "bin/fanout")), [], { cwd: dir, env: { ...env, FANOUT_DATA_DIR: join(dir, "data"), FANOUT_ADDR: new URL(base).host, - FANOUT_AUTH_CODE_SECRET: randomBytes(32).toString("hex"), FANOUT_AI_API_KEY: "", + FANOUT_AUTH_CODE_SECRET: randomBytes(32).toString("hex"), FANOUT_ROLLUP_INTERVAL: "1s" }, stdio: ["ignore", fd, fd], }); From f10fd2697bf8c1beeabf142a26d9b310acb6596c Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Tue, 13 Oct 2026 20:15:30 -0700 Subject: [PATCH 2/6] fix(config)!: make AI chat opt-in and keep dotenv from overriding YAML Review fixes for the provider environment names and dotenv layout. - AI chat is enabled only by FANOUT_AI_PROVIDER (YAML ai.provider), which now defaults to empty. A selected provider requires its API key, and only its base URL is validated. Inherited ANTHROPIC_* or OPENAI_* values from other tools no longer enable chat or block startup. - Provider values must be exactly anthropic or openai; no case or space normalization. config.AIBaseURL is the only source of endpoint defaults. - Generated .env and .env.production contain only environment-specific overrides (the listener address, declared with a local struct tag), so launcher-injected values no longer override --config YAML. .env.example lists every setting with an empty value and documents the default in comments. - The launcher forwards SIGINT, SIGTERM and SIGHUP on every delivery, classifies secrets from each block's generated header line only, and accepts CRLF inventories. - The docgen overwrite guard refuses quoted or escaped values. - The loader is the only check of accepted environment names. - Document the tracked .env migration and the AI configuration upgrade. --- .env | 174 +----------------- .env.example | 110 ++++++++--- AGENTS.md | 8 +- CONTRIBUTING.md | 9 +- README.md | 35 ++-- cmd/fanout-docgen/main.go | 40 ++-- cmd/fanout-docgen/main_test.go | 18 +- cmd/fanout/main.go | 2 +- docs/operations.md | 30 ++- fanout.example.yaml | 6 +- internal/agent/provider_contract_test.go | 12 +- internal/agent/provider_http.go | 16 +- internal/agent/provider_http_test.go | 4 +- internal/config/config.go | 47 +++-- internal/config/config_test.go | 2 +- internal/config/launcher_test.go | 71 +++++++ internal/config/provider_test.go | 38 +++- scripts/run-fanout.mjs | 13 +- scripts/run-fanout.test.ts | 113 +++++++++++- .../content/docs/reference/configuration.mdx | 17 +- .../content/docs/reference/settings/ai.mdx | 10 +- site/src/content/docs/start/install.mdx | 20 +- 22 files changed, 460 insertions(+), 335 deletions(-) create mode 100644 internal/config/launcher_test.go diff --git a/.env b/.env index 7788ec6b..2dd7367b 100644 --- a/.env +++ b/.env @@ -1,181 +1,9 @@ # Generated nonsecret configuration by cmd/fanout-docgen. # Regenerate with just docs-generate. Credentials never belong in this file. -# Shared local defaults; .env.production overrides them for production. +# Shared local overrides; .env.production resets them for production. # Use just run, or inject these values explicitly; Fanout does not load dotenv. # Personal nonsecret overrides belong in ignored .env.local. -# ai.anthropic.base_url; string -# Defaults to https://api.anthropic.com. Fanout appends /v1/messages, preserving any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. -ANTHROPIC_BASE_URL= - -# ai.model; string -# Overrides the provider's default: claude-sonnet-5-5 for Anthropic and gpt-6.1-sol for OpenAI. The OpenAI provider expects a reasoning model supporting reasoning.encrypted_content for stateless Responses replay. A 400 rejecting encrypted content is reported as a server-side configuration error; choose a compatible reasoning model with ai.model. -FANOUT_AI_MODEL= - -# ai.openai.base_url; string -# Defaults to https://api.openai.com/v1. Fanout appends /responses, preserving the API version and any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. -OPENAI_BASE_URL= - -# ai.provider; string -# Selects anthropic or openai independently of which keys are present. -FANOUT_AI_PROVIDER=anthropic - -# alerts.enabled; boolean -# Runs the alert evaluation engine. -FANOUT_ALERTS_ENABLED=true - -# alerts.evaluation_interval; duration -# Schedules alert evaluation; use a Go duration of at least 1s. -FANOUT_ALERTS_EVALUATION_INTERVAL=30s - -# alerts.history_days; integer -# Sets alert history retention in days. -FANOUT_ALERTS_HISTORY_DAYS=7 - -# auth.mode; string -# Selects local sign-in or oidc; each mode validates its prerequisites. -FANOUT_AUTH_MODE=local - -# auth.oidc.admin_groups; string -# Lists identity-provider groups mapped to admin, separated by commas. -FANOUT_OIDC_ADMIN_GROUPS= - -# auth.oidc.allowed_domains; string -# Lists allowed email domains, separated by commas; wildcard * requires verified email. -FANOUT_OIDC_ALLOWED_DOMAINS= - -# auth.oidc.allowed_groups; string -# Lists allowed identity-provider groups, separated by commas. -FANOUT_OIDC_ALLOWED_GROUPS= - -# auth.oidc.auto_provision; boolean -# Creates accounts on OIDC sign-in; requires an allowed group or domain. -FANOUT_OIDC_AUTO_PROVISION=false - -# auth.oidc.client_id; string -# Identifies Fanout's client registration; required in OIDC mode. -FANOUT_OIDC_CLIENT_ID= - -# auth.oidc.default_role; string -# Assigns viewer, operator, or admin to users without a group role mapping. -FANOUT_OIDC_DEFAULT_ROLE=viewer - -# auth.oidc.email_claim; string -# Names the claim containing the user's email address. -FANOUT_OIDC_EMAIL_CLAIM=email - -# auth.oidc.email_verification; string -# Selects required verified-email claims or issuer trust; issuer trust requires an allowlist. -FANOUT_OIDC_EMAIL_VERIFICATION=required - -# auth.oidc.issuer_url; string -# Selects the identity provider issuer; required in OIDC mode. -FANOUT_OIDC_ISSUER_URL= - -# auth.oidc.operator_groups; string -# Lists identity-provider groups mapped to operator, separated by commas. -FANOUT_OIDC_OPERATOR_GROUPS= - -# auth.self_signup; boolean -# Allows verified email addresses to create viewer accounts; requires local auth and SMTP. -FANOUT_SELF_SIGNUP=false - -# auth.session_absolute_ttl; duration -# Limits total session lifetime; it must be at least session_idle_ttl. -FANOUT_SESSION_ABSOLUTE_TTL=168h - -# auth.session_idle_ttl; duration -# Expires inactive sessions; use a Go duration of at least 5m. -FANOUT_SESSION_IDLE_TTL=12h - -# ingest.batch_size; integer -# Limits telemetry rows per batch; valid values are 1–50000. -FANOUT_INGEST_BATCH_SIZE=50000 - -# ingest.default_namespace; string -# Supplies the namespace for telemetry without one. -FANOUT_DEFAULT_NAMESPACE=default - -# ingest.max_in_flight_bytes; integer -# Caps the decoded telemetry the process will hold across concurrent requests. Concurrency is HTTP/2 streams times connections and each handler holds its batch until the commit is durable, so without a ceiling a burst is bounded only by memory. Zero is unbounded, which is the behaviour that predates this setting. -FANOUT_INGEST_MAX_IN_FLIGHT_BYTES=268435456 - -# mcp.enabled; boolean -# Serves the MCP endpoint at /mcp. -FANOUT_MCP_ENABLED=true - -# metrics.public; boolean -# Allows unauthenticated Prometheus scrapes. -FANOUT_METRICS_PUBLIC=false - # server.addr; string # Binds the browser, API, MCP, OTLP/gRPC and OTLP/HTTP listener. FANOUT_ADDR=127.0.0.1:7520 - -# server.pprof_enabled; boolean -# Exposes Go's net/http/pprof handlers at /debug/pprof/* for CPU/heap/mutex/goroutine profiling under load. Off by default; the routes require an admin browser session with operations:read. -FANOUT_PPROF_ENABLED=false - -# server.public_url; string -# Sets the external browser origin. Required for OIDC; HTTPS is required with MCP. -FANOUT_PUBLIC_URL= - -# server.tls.cert_file; string -# Selects the TLS certificate file; requires tls.key_file. -FANOUT_TLS_CERT_FILE= - -# server.tls.key_file; string -# Selects the TLS private key file; requires tls.cert_file. -FANOUT_TLS_KEY_FILE= - -# server.trusted_proxy_cidrs; string -# Lists trusted proxy networks, separated by commas; empty trusts no forwarded client addresses. -FANOUT_TRUSTED_PROXY_CIDRS= - -# smtp.from; string -# Sets the email sender address; required with SMTP. -FANOUT_SMTP_FROM= - -# smtp.host; string -# Selects the mail server. Host, username, password, and from must all be set to enable SMTP. -FANOUT_SMTP_HOST= - -# smtp.port; integer -# Sets the mail server's positive TCP port. -FANOUT_SMTP_PORT=587 - -# smtp.username; string -# Authenticates to the mail server; required with SMTP. -FANOUT_SMTP_USERNAME= - -# storage.data_dir; string -# Stores telemetry, query state, and the control database. -FANOUT_DATA_DIR=./data - -# storage.duckdb.max_connections; integer -# Caps the DuckDB connection pool. Reads scan immutable Parquet concurrently; rollup-cache writes are serialized by the query write gate. Zero means "size it from the machine" — the same spelling DuckDBThreads uses for deferring to a default. Resolution happens in resolveSizing and is reported in the startup configuration log. -FANOUT_DUCKDB_MAX_CONNECTIONS=0 - -# storage.duckdb.memory; string -# Caps DuckDB's memory (e.g. "8GB"). Empty means Fanout sizes it from detected memory, reserving headroom for the Go runtime; see resolveSizing. DuckDB's own default is 80% of detected RAM, which is calculated as though DuckDB owned the machine and has been observed to get the process OOM-killed once the Go heap is added on top. -FANOUT_DUCKDB_MEMORY= - -# storage.duckdb.threads; integer -# Caps DuckDB's global query worker pool. Zero leaves DuckDB's own default in place (one worker per core). Set it to leave cores free for ingest on a query-heavy co-tenant host. -FANOUT_DUCKDB_THREADS= - -# storage.maintenance_interval; duration -# Controls Parquet retention and compaction, and query-cache checkpointing. -FANOUT_MAINTENANCE_INTERVAL=1h - -# storage.retention_days; integer -# Sets telemetry retention in days; zero disables expiry. -FANOUT_RETENTION_DAYS=30 - -# storage.rollup_interval; duration -# Schedules analytical rollups; use a Go duration of at least 1s. -FANOUT_ROLLUP_INTERVAL=1m - -# storage.rollup_skip_to_latest; boolean -# RollupSkipToLatest, set once at boot, advances the service and edge rollup watermarks to the current max ingested timestamp. Existing history is not aggregated into those analytical rollups. Completed-batch trace caches still rebuild all retained history in bounded passes; uncached files remain immediately queryable. Useful for large pre-seeded datasets (benchmarks, restores); off in normal operation. -FANOUT_ROLLUP_SKIP_TO_LATEST=false diff --git a/.env.example b/.env.example index a0848953..c2042d6e 100644 --- a/.env.example +++ b/.env.example @@ -5,204 +5,254 @@ # All settings require restart. Empty environment values mean no override. # Secret entries below are empty placeholders; use runtime secret injection. # Local authentication requires FANOUT_AUTH_CODE_SECRET (32+ characters). -# Only the AI key/endpoint selected by FANOUT_AI_PROVIDER is used. +# Chat is disabled unless FANOUT_AI_PROVIDER selects anthropic or openai; its key is required. # ai.anthropic.api_key; string; secret -# Enables chat only when ai.provider is anthropic. +# Required when ai.provider is anthropic. +# Default: (empty) ANTHROPIC_API_KEY= # ai.anthropic.base_url; string # Defaults to https://api.anthropic.com. Fanout appends /v1/messages, preserving any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. +# Default: (empty) ANTHROPIC_BASE_URL= # ai.model; string # Overrides the provider's default: claude-sonnet-5-5 for Anthropic and gpt-6.1-sol for OpenAI. The OpenAI provider expects a reasoning model supporting reasoning.encrypted_content for stateless Responses replay. A 400 rejecting encrypted content is reported as a server-side configuration error; choose a compatible reasoning model with ai.model. +# Default: (empty) FANOUT_AI_MODEL= # ai.openai.api_key; string; secret -# Enables chat only when ai.provider is openai. +# Required when ai.provider is openai. +# Default: (empty) OPENAI_API_KEY= # ai.openai.base_url; string # Defaults to https://api.openai.com/v1. Fanout appends /responses, preserving the API version and any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. +# Default: (empty) OPENAI_BASE_URL= # ai.provider; string -# Selects anthropic or openai independently of which keys are present. -FANOUT_AI_PROVIDER=anthropic +# Enables chat by selecting exactly anthropic or openai. Empty disables chat. +# Default: (empty) +FANOUT_AI_PROVIDER= # alerts.enabled; boolean # Runs the alert evaluation engine. -FANOUT_ALERTS_ENABLED=true +# Default: true +FANOUT_ALERTS_ENABLED= # alerts.evaluation_interval; duration # Schedules alert evaluation; use a Go duration of at least 1s. -FANOUT_ALERTS_EVALUATION_INTERVAL=30s +# Default: 30s +FANOUT_ALERTS_EVALUATION_INTERVAL= # alerts.history_days; integer # Sets alert history retention in days. -FANOUT_ALERTS_HISTORY_DAYS=7 +# Default: 7 +FANOUT_ALERTS_HISTORY_DAYS= # auth.code_secret; string; secret # Signs local sign-in codes; required in local mode with at least 32 characters. +# Default: (empty) FANOUT_AUTH_CODE_SECRET= # auth.mode; string # Selects local sign-in or oidc; each mode validates its prerequisites. -FANOUT_AUTH_MODE=local +# Default: local +FANOUT_AUTH_MODE= # auth.oidc.admin_groups; string # Lists identity-provider groups mapped to admin, separated by commas. +# Default: (empty) FANOUT_OIDC_ADMIN_GROUPS= # auth.oidc.allowed_domains; string # Lists allowed email domains, separated by commas; wildcard * requires verified email. +# Default: (empty) FANOUT_OIDC_ALLOWED_DOMAINS= # auth.oidc.allowed_groups; string # Lists allowed identity-provider groups, separated by commas. +# Default: (empty) FANOUT_OIDC_ALLOWED_GROUPS= # auth.oidc.auto_provision; boolean # Creates accounts on OIDC sign-in; requires an allowed group or domain. -FANOUT_OIDC_AUTO_PROVISION=false +# Default: false +FANOUT_OIDC_AUTO_PROVISION= # auth.oidc.client_id; string # Identifies Fanout's client registration; required in OIDC mode. +# Default: (empty) FANOUT_OIDC_CLIENT_ID= # auth.oidc.client_secret; string; secret # Authenticates Fanout's client registration; required in OIDC mode. +# Default: (empty) FANOUT_OIDC_CLIENT_SECRET= # auth.oidc.default_role; string # Assigns viewer, operator, or admin to users without a group role mapping. -FANOUT_OIDC_DEFAULT_ROLE=viewer +# Default: viewer +FANOUT_OIDC_DEFAULT_ROLE= # auth.oidc.email_claim; string # Names the claim containing the user's email address. -FANOUT_OIDC_EMAIL_CLAIM=email +# Default: email +FANOUT_OIDC_EMAIL_CLAIM= # auth.oidc.email_verification; string # Selects required verified-email claims or issuer trust; issuer trust requires an allowlist. -FANOUT_OIDC_EMAIL_VERIFICATION=required +# Default: required +FANOUT_OIDC_EMAIL_VERIFICATION= # auth.oidc.issuer_url; string # Selects the identity provider issuer; required in OIDC mode. +# Default: (empty) FANOUT_OIDC_ISSUER_URL= # auth.oidc.operator_groups; string # Lists identity-provider groups mapped to operator, separated by commas. +# Default: (empty) FANOUT_OIDC_OPERATOR_GROUPS= # auth.self_signup; boolean # Allows verified email addresses to create viewer accounts; requires local auth and SMTP. -FANOUT_SELF_SIGNUP=false +# Default: false +FANOUT_SELF_SIGNUP= # auth.session_absolute_ttl; duration # Limits total session lifetime; it must be at least session_idle_ttl. -FANOUT_SESSION_ABSOLUTE_TTL=168h +# Default: 168h +FANOUT_SESSION_ABSOLUTE_TTL= # auth.session_idle_ttl; duration # Expires inactive sessions; use a Go duration of at least 5m. -FANOUT_SESSION_IDLE_TTL=12h +# Default: 12h +FANOUT_SESSION_IDLE_TTL= # ingest.batch_size; integer # Limits telemetry rows per batch; valid values are 1–50000. -FANOUT_INGEST_BATCH_SIZE=50000 +# Default: 50000 +FANOUT_INGEST_BATCH_SIZE= # ingest.default_namespace; string # Supplies the namespace for telemetry without one. -FANOUT_DEFAULT_NAMESPACE=default +# Default: default +FANOUT_DEFAULT_NAMESPACE= # ingest.max_in_flight_bytes; integer # Caps the decoded telemetry the process will hold across concurrent requests. Concurrency is HTTP/2 streams times connections and each handler holds its batch until the commit is durable, so without a ceiling a burst is bounded only by memory. Zero is unbounded, which is the behaviour that predates this setting. -FANOUT_INGEST_MAX_IN_FLIGHT_BYTES=268435456 +# Default: 268435456 +FANOUT_INGEST_MAX_IN_FLIGHT_BYTES= # mcp.enabled; boolean # Serves the MCP endpoint at /mcp. -FANOUT_MCP_ENABLED=true +# Default: true +FANOUT_MCP_ENABLED= # metrics.public; boolean # Allows unauthenticated Prometheus scrapes. -FANOUT_METRICS_PUBLIC=false +# Default: false +FANOUT_METRICS_PUBLIC= # metrics.token; string; secret # Authorizes Prometheus scrapes with a bearer token; optional when using an admin session. +# Default: (empty) FANOUT_METRICS_TOKEN= # server.addr; string # Binds the browser, API, MCP, OTLP/gRPC and OTLP/HTTP listener. -FANOUT_ADDR=:7520 +# Default: :7520 +FANOUT_ADDR= # server.pprof_enabled; boolean # Exposes Go's net/http/pprof handlers at /debug/pprof/* for CPU/heap/mutex/goroutine profiling under load. Off by default; the routes require an admin browser session with operations:read. -FANOUT_PPROF_ENABLED=false +# Default: false +FANOUT_PPROF_ENABLED= # server.public_url; string # Sets the external browser origin. Required for OIDC; HTTPS is required with MCP. +# Default: (empty) FANOUT_PUBLIC_URL= # server.tls.cert_file; string # Selects the TLS certificate file; requires tls.key_file. +# Default: (empty) FANOUT_TLS_CERT_FILE= # server.tls.key_file; string # Selects the TLS private key file; requires tls.cert_file. +# Default: (empty) FANOUT_TLS_KEY_FILE= # server.trusted_proxy_cidrs; string # Lists trusted proxy networks, separated by commas; empty trusts no forwarded client addresses. +# Default: (empty) FANOUT_TRUSTED_PROXY_CIDRS= # smtp.from; string # Sets the email sender address; required with SMTP. +# Default: (empty) FANOUT_SMTP_FROM= # smtp.host; string # Selects the mail server. Host, username, password, and from must all be set to enable SMTP. +# Default: (empty) FANOUT_SMTP_HOST= # smtp.password; string; secret # Authenticates to the mail server; required with SMTP. +# Default: (empty) FANOUT_SMTP_PASSWORD= # smtp.port; integer # Sets the mail server's positive TCP port. -FANOUT_SMTP_PORT=587 +# Default: 587 +FANOUT_SMTP_PORT= # smtp.username; string # Authenticates to the mail server; required with SMTP. +# Default: (empty) FANOUT_SMTP_USERNAME= # storage.data_dir; string # Stores telemetry, query state, and the control database. -FANOUT_DATA_DIR=./data +# Default: ./data +FANOUT_DATA_DIR= # storage.duckdb.max_connections; integer # Caps the DuckDB connection pool. Reads scan immutable Parquet concurrently; rollup-cache writes are serialized by the query write gate. Zero means "size it from the machine" — the same spelling DuckDBThreads uses for deferring to a default. Resolution happens in resolveSizing and is reported in the startup configuration log. -FANOUT_DUCKDB_MAX_CONNECTIONS=0 +# Default: 0 +FANOUT_DUCKDB_MAX_CONNECTIONS= # storage.duckdb.memory; string # Caps DuckDB's memory (e.g. "8GB"). Empty means Fanout sizes it from detected memory, reserving headroom for the Go runtime; see resolveSizing. DuckDB's own default is 80% of detected RAM, which is calculated as though DuckDB owned the machine and has been observed to get the process OOM-killed once the Go heap is added on top. +# Default: (empty) FANOUT_DUCKDB_MEMORY= # storage.duckdb.threads; integer # Caps DuckDB's global query worker pool. Zero leaves DuckDB's own default in place (one worker per core). Set it to leave cores free for ingest on a query-heavy co-tenant host. +# Default: (empty) FANOUT_DUCKDB_THREADS= # storage.maintenance_interval; duration # Controls Parquet retention and compaction, and query-cache checkpointing. -FANOUT_MAINTENANCE_INTERVAL=1h +# Default: 1h +FANOUT_MAINTENANCE_INTERVAL= # storage.retention_days; integer # Sets telemetry retention in days; zero disables expiry. -FANOUT_RETENTION_DAYS=30 +# Default: 30 +FANOUT_RETENTION_DAYS= # storage.rollup_interval; duration # Schedules analytical rollups; use a Go duration of at least 1s. -FANOUT_ROLLUP_INTERVAL=1m +# Default: 1m +FANOUT_ROLLUP_INTERVAL= # storage.rollup_skip_to_latest; boolean # RollupSkipToLatest, set once at boot, advances the service and edge rollup watermarks to the current max ingested timestamp. Existing history is not aggregated into those analytical rollups. Completed-batch trace caches still rebuild all retained history in bounded passes; uncached files remain immediately queryable. Useful for large pre-seeded datasets (benchmarks, restores); off in normal operation. -FANOUT_ROLLUP_SKIP_TO_LATEST=false +# Default: false +FANOUT_ROLLUP_SKIP_TO_LATEST= diff --git a/AGENTS.md b/AGENTS.md index 48f25279..6228e685 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,14 +2,20 @@ ## Configuration conventions -- Follow Canon's consumer-first environment naming: custom product settings use +- Use consumer-first environment naming: custom product settings use `FANOUT_`; AI keys and endpoints use `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, `OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Provider and model selection remain `FANOUT_AI_PROVIDER` and `FANOUT_AI_MODEL`. + An empty provider disables chat; exactly `anthropic` or `openai` enables it and + requires that provider's API key. Ignore AI credentials and endpoints when chat + is disabled; validate only the selected provider's endpoint. - `internal/config/config.go` is the authoritative schema. Run `just docs-generate` after configuration changes and verify `just docs-generate-check`; the generator owns `.env`, `.env.production`, `.env.example`, and site references. Public defaults exclude all secret fields. + `.env` contains only schema-declared `local` overrides; `.env.production` resets + those fields to their built-in defaults. `.env.example` keeps all values empty + and documents defaults in comments so copying it never overrides YAML. - Preserve the explicit YAML/process-environment loader. Fanout does not load dotenv files; launchers and deployments own file loading and secret injection. Only Fanout's owned namespace rejects unknown environment names; shared vendor diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2ec2f0e9..4d2d73f1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,8 +14,13 @@ You need Go 1.27.2 and a C/C++ toolchain on native Linux or macOS (amd64 or arm6 [Bun](https://bun.sh), [just](https://just.systems), [golangci-lint](https://golangci-lint.run/), and [Lefthook](https://github.com/evilmartians/lefthook). Running in local auth mode -also requires a 32-character authentication code secret; SMTP and an AI key are -optional — see [README](README.md#requirements). +also requires a 32-character authentication code secret. SMTP and AI chat are +optional; selecting an AI provider requires its key — see [README](README.md#requirements). + +`.env` is now tracked and generated. Before pulling, move nonsecret personal +values from an older private `.env` to ignored `.env.local` and credentials to +runtime injection, then preserve the old file outside the checkout. Git refuses +to overwrite an untracked `.env`. See [runtime secret delivery](docs/operations.md#runtime-secret-delivery). The DuckDB wrapper downloads the pinned, checksum-verified engine from [Fanout's dependency artifact mirror](https://github.com/labstack/fanout/releases/tag/duckdb-v2.0.0-alpha43763) diff --git a/README.md b/README.md index 7e3e7fef..b25d9f62 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ Fanout. - A 32-character **authentication code secret** SMTP and an AI provider are optional. Without SMTP, an operator can mint a -short-lived login link from the local Fanout binary. Without an AI key, ingest, +short-lived login link from the local Fanout binary. With no AI provider selected, ingest, dashboards, traces, logs, metrics, and MCP continue to work; only investigation chat and AI-assisted controls are hidden. @@ -165,8 +165,9 @@ docker exec fanout fanout --config /etc/fanout/fanout.yaml \ login-link admin@example.com ``` -Add `ANTHROPIC_API_KEY` (or `OPENAI_API_KEY` with `FANOUT_AI_PROVIDER=openai`) -to enable chat. Configure all four SMTP settings +Select `FANOUT_AI_PROVIDER=anthropic` and inject `ANTHROPIC_API_KEY`, or select +`FANOUT_AI_PROVIDER=openai` and inject `OPENAI_API_KEY`, to enable chat. The +selected provider requires its key at startup. Configure all four SMTP settings (`FANOUT_SMTP_HOST`, `FANOUT_SMTP_USERNAME`, `FANOUT_SMTP_PASSWORD`, and `FANOUT_SMTP_FROM`) to enable email-code login. @@ -206,8 +207,8 @@ export FANOUT_AUTH_CODE_SECRET=$(openssl rand -hex 32) # must be 32+ characters ./bin/fanout ``` -Optionally set `ANTHROPIC_API_KEY` (or `OPENAI_API_KEY` with -`FANOUT_AI_PROVIDER=openai`) for chat and the SMTP variables shown above for +Optionally select `FANOUT_AI_PROVIDER=anthropic` with `ANTHROPIC_API_KEY`, or +`FANOUT_AI_PROVIDER=openai` with `OPENAI_API_KEY`, for chat. Set the SMTP variables shown above for email-code login. Without SMTP, run `./bin/fanout login-link admin@example.com` from the same configuration and data directory. @@ -246,9 +247,10 @@ other settings in those shared vendor namespaces, are ignored. In particular, `ANTHROPIC_MODEL` and `OPENAI_MODEL` do not select Fanout's model. [`.env.example`](.env.example) is generated from the same configuration schema -as the settings reference, with types, defaults, and empty secret placeholders. -The shared [`.env`](.env) contains nonsecret local defaults; the generated -[`.env.production`](.env.production) overrides the local listener for production. +as the settings reference, with types and defaults in comments and every value +empty, so copying the template never overrides YAML. The shared [`.env`](.env) +contains only the nonsecret local listener override; the generated +[`.env.production`](.env.production) resets it to the schema default for production. Personal nonsecret overrides belong in ignored `.env.local`. This public repository excludes operator `secrets/` profiles and private recipient policy. @@ -271,7 +273,8 @@ Select only a profile that actually exists for the intended resources. An ordinary dotenv loader does not resolve 1Password references or decrypt SOPS. The binary retains its YAML/environment precedence: every nonempty dotenv value injected by the launcher overrides the corresponding `--config` YAML value. -For YAML-only ordinary settings, launch the binary directly with injected secrets. +Generated dotenv files override only the listener, so other YAML settings survive +both launch modes. See [runtime secret delivery](docs/operations.md#runtime-secret-delivery) for SOPS injection and migration precautions. Run `just docs-generate` after changing the schema; `just docs-generate-check` verifies all three dotenv artifacts and @@ -290,15 +293,17 @@ operators touch are: | `storage.data_dir` | `FANOUT_DATA_DIR` | `./data` | Parquet, query state, and control SQLite | | `auth.mode` | `FANOUT_AUTH_MODE` | `local` | `local` (login link or SMTP) or `oidc` | | `auth.code_secret` | `FANOUT_AUTH_CODE_SECRET` | — | Required in local mode, 32+ characters | -| `ai.provider` | `FANOUT_AI_PROVIDER` | `anthropic` | `anthropic` or `openai` | -| `ai.anthropic.api_key` | `ANTHROPIC_API_KEY` | — | Enables chat when `ai.provider=anthropic` | -| `ai.openai.api_key` | `OPENAI_API_KEY` | — | Enables chat when `ai.provider=openai` | +| `ai.provider` | `FANOUT_AI_PROVIDER` | — | Empty disables chat; exactly `anthropic` or `openai` enables it | +| `ai.anthropic.api_key` | `ANTHROPIC_API_KEY` | — | Required when `ai.provider=anthropic` | +| `ai.openai.api_key` | `OPENAI_API_KEY` | — | Required when `ai.provider=openai` | | `storage.retention_days` | `FANOUT_RETENTION_DAYS` | `30` | Telemetry retention window | | `mcp.enabled` | `FANOUT_MCP_ENABLED` | `true` | Serve the MCP endpoint at `/mcp` | -Only the selected AI provider's credential and endpoint are used. Without its -key, chat is disabled even if the other provider's key is present. Unsupported -provider names fail at startup, including when no key is configured. +AI chat is disabled by default, even with inherited vendor credentials. Select +exactly `anthropic` or `openai` to enable it; the selected provider's key is +required at startup. Only that provider's credential and endpoint are validated +and used. With no provider selected, AI keys and endpoints are ignored. Unsupported +provider names, including different casing or surrounding whitespace, fail startup. `FANOUT_AI_MODEL` remains the application model selector. `ANTHROPIC_BASE_URL` defaults to `https://api.anthropic.com`; Fanout appends diff --git a/cmd/fanout-docgen/main.go b/cmd/fanout-docgen/main.go index 89bad736..aedb0b2e 100644 --- a/cmd/fanout-docgen/main.go +++ b/cmd/fanout-docgen/main.go @@ -220,8 +220,8 @@ func validateGeneratedDefaults(path string, fields []field) error { if line == "" || strings.HasPrefix(line, "#") { continue } - name, _, assignment := strings.Cut(line, "=") - if !assignment || !public[strings.TrimSpace(name)] { + name, value, assignment := strings.Cut(line, "=") + if !assignment || !public[strings.TrimSpace(name)] || strings.ContainsAny(value, "\"'`\\") { return refuse() } } @@ -333,6 +333,7 @@ type field struct { Env string // e.g. "FANOUT_DUCKDB_MEMORY" Type string Default string + Local *string // nil unless the schema declares a local override Secret bool Doc string } @@ -369,12 +370,17 @@ func collect(source string) ([]field, error) { ) } seen[sf.Name] = true + var local *string + if value, ok := sf.Tag.Lookup("local"); ok { + local = &value + } fields = append(fields, field{ GoName: sf.Name, Key: key, Env: env, Type: typeName(sf.Type), Default: sf.Tag.Get("default"), + Local: local, Secret: sf.Tag.Get("secret") == "true", Doc: docs[sf.Name], }) @@ -540,7 +546,7 @@ var groups = map[string]groupPage{ "ai": { Title: "AI provider settings", Description: "The provider, key, model and base URL backing the chat investigator.", - Summary: "Settings for the model behind the chat investigator. Without a key the agent is disabled.", + Summary: "Chat is disabled unless a provider is selected; its API key is then required.", ReadWhen: []string{ "The chat investigator is disabled and you want it on.", "You are pointing Fanout at a self-hosted or proxied model endpoint.", @@ -655,7 +661,7 @@ func renderPage(name string, page groupPage, fields []field) []byte { } // renderEnvironment shares the exact reflected schema used by the site pages. -// Empty secret values are safe placeholders and never enable an integration. +// Empty values never override YAML; defaults are documented in comments. func renderEnvironment(fields []field) []byte { var b strings.Builder b.WriteString("# Generated by cmd/fanout-docgen from internal/config/config.go.\n") @@ -665,7 +671,7 @@ func renderEnvironment(fields []field) []byte { b.WriteString("# All settings require restart. Empty environment values mean no override.\n") b.WriteString("# Secret entries below are empty placeholders; use runtime secret injection.\n") b.WriteString("# Local authentication requires FANOUT_AUTH_CODE_SECRET (32+ characters).\n") - b.WriteString("# Only the AI key/endpoint selected by FANOUT_AI_PROVIDER is used.\n") + b.WriteString("# Chat is disabled unless FANOUT_AI_PROVIDER selects anthropic or openai; its key is required.\n") for _, f := range fields { fmt.Fprintf(&b, "\n# %s; %s", f.Key, f.Type) if f.Secret { @@ -675,18 +681,18 @@ func renderEnvironment(fields []field) []byte { if f.Doc != "" { fmt.Fprintf(&b, "# %s\n", f.Doc) } - value := f.Default - if f.Secret { - value = "" + defaultValue := f.Default + if f.Secret || defaultValue == "" { + defaultValue = "(empty)" } - fmt.Fprintf(&b, "%s=%s\n", f.Env, value) + fmt.Fprintf(&b, "# Default: %s\n", defaultValue) + fmt.Fprintf(&b, "%s=\n", f.Env) } return []byte(b.String()) } -// renderDefaults contains no secret fields, even empty ones. Production replaces -// the local-only listener with the schema's shared-listener default; operators -// supply deployment-specific origins and all credentials at launch time. +// renderDefaults emits only schema-declared local overrides, without secrets. +// Production resets exactly those fields to their built-in defaults. func renderDefaults(fields []field, production bool) []byte { var b strings.Builder b.WriteString("# Generated nonsecret configuration by cmd/fanout-docgen.\n") @@ -694,21 +700,21 @@ func renderDefaults(fields []field, production bool) []byte { if production { b.WriteString("# Production overrides for .env. Deployment supplies its public URL and secrets.\n") } else { - b.WriteString("# Shared local defaults; .env.production overrides them for production.\n") + b.WriteString("# Shared local overrides; .env.production resets them for production.\n") b.WriteString("# Use just run, or inject these values explicitly; Fanout does not load dotenv.\n") b.WriteString("# Personal nonsecret overrides belong in ignored .env.local.\n") } for _, f := range fields { - if f.Secret || production && f.Key != "server.addr" { + if f.Secret || f.Local == nil { continue } fmt.Fprintf(&b, "\n# %s; %s\n", f.Key, f.Type) if f.Doc != "" { fmt.Fprintf(&b, "# %s\n", f.Doc) } - value := f.Default - if !production && f.Key == "server.addr" { - value = "127.0.0.1" + value + value := *f.Local + if production { + value = f.Default } fmt.Fprintf(&b, "%s=%s\n", f.Env, value) } diff --git a/cmd/fanout-docgen/main_test.go b/cmd/fanout-docgen/main_test.go index 9a098e53..ccd8ad02 100644 --- a/cmd/fanout-docgen/main_test.go +++ b/cmd/fanout-docgen/main_test.go @@ -27,9 +27,6 @@ func TestCollectCrossChecksReflectionAgainstSource(t *testing.T) { if f.Key == "" || f.Env == "" { t.Errorf("%s: koanf and env tags are both required", f.GoName) } - if !strings.HasPrefix(f.Env, "FANOUT_") && !strings.HasPrefix(f.Key, "ai.anthropic.") && !strings.HasPrefix(f.Key, "ai.openai.") { - t.Errorf("%s: custom environment variable %q lacks FANOUT_ prefix", f.GoName, f.Env) - } if f.Type == "" { t.Errorf("%s: no rendered type", f.GoName) } @@ -70,8 +67,11 @@ func TestEnvironmentTemplateMatchesSchema(t *testing.T) { } for _, f := range fields { value, exists := seen[f.Env] - if !exists || value != f.Default || f.Secret && value != "" { - t.Errorf("template does not match the safe schema default for %s", f.Env) + if !exists || value != "" { + t.Errorf("template must leave %s empty so it never overrides YAML", f.Env) + } + if f.Default != "" && !strings.Contains(string(body), "# Default: "+f.Default+"\n"+f.Env+"=\n") { + t.Errorf("template must document the default for %s", f.Env) } } env = append(env, "FANOUT_AUTH_CODE_SECRET="+strings.Repeat("x", 32)) @@ -102,8 +102,8 @@ func TestPublicEnvironmentDefaultsExcludeSecrets(t *testing.T) { t.Errorf("%s is stale; run just docs-generate", tc.path) } for _, f := range fields { - if f.Secret && strings.Contains(string(body), f.Env+"=") { - t.Errorf("public defaults must exclude secret setting %s", f.Env) + if present := strings.Contains(string(body), f.Env+"="); present != (!f.Secret && f.Local != nil) { + t.Errorf("public defaults must contain exactly the nonsecret local overrides: %s", f.Env) } } if !strings.Contains(string(body), "FANOUT_ADDR="+tc.addr+"\n") { @@ -125,6 +125,10 @@ func TestPublicDefaultGenerationPreservesPrivateFiles(t *testing.T) { "AI_API_KEY=synthetic-private-value\n", "# Generated nonsecret configuration by cmd/fanout-docgen.\nANTHROPIC_API_KEY=synthetic-private-value\n", "# Generated nonsecret configuration by cmd/fanout-docgen.\nFANOUT_AI_API_KEY=synthetic-private-value\n", + "# Generated nonsecret configuration by cmd/fanout-docgen.\nFANOUT_DATA_DIR=\"safe\n# synthetic-private-value\"\n", + "# Generated nonsecret configuration by cmd/fanout-docgen.\nFANOUT_DATA_DIR='synthetic-private-value'\n", + "# Generated nonsecret configuration by cmd/fanout-docgen.\nFANOUT_DATA_DIR=`synthetic-private-value`\n", + "# Generated nonsecret configuration by cmd/fanout-docgen.\nFANOUT_DATA_DIR=synthetic\\private-value\n", } { if err := os.WriteFile(path, []byte(content), 0o600); err != nil { t.Fatal(err) diff --git a/cmd/fanout/main.go b/cmd/fanout/main.go index 63addc2c..40e2d357 100644 --- a/cmd/fanout/main.go +++ b/cmd/fanout/main.go @@ -373,7 +373,7 @@ func main() { agent.NewRuntime(provider, toolRegistry, agent.NewStore(sqlite.DB)).Register(e.Group("/api/agent", api.RequireCapability(api.RunAgent))) slog.Info("AG-UI agent enabled", "path", "/api/agent/runs", "provider", cfg.AIProvider) } else { - slog.Info("AG-UI agent disabled", "reason", "selected AI provider API key is not configured") + slog.Info("AG-UI agent disabled", "reason", "ai.provider (FANOUT_AI_PROVIDER) is not selected") } // The compiled browser client is an embedded asset, not a second runtime. diff --git a/docs/operations.md b/docs/operations.md index 7cf432cb..308b5a02 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -9,18 +9,23 @@ give ingest credentials only to trusted services and collectors. Runtime configuration uses built-in defaults, then an explicit `--config` YAML document, then recognized nonempty environment values. Every setting requires a restart. Fanout does not load dotenv files itself. The generated root -`.env.example` documents every setting with empty credential placeholders; -`.env` and `.env.production` contain only nonsecret settings. +`.env.example` documents every default in comments and leaves all values empty; +`.env` and `.env.production` contain only nonsecret environment-specific overrides. | File | Use | Public Git policy | | --- | --- | --- | -| `.env` | Shared defaults with a loopback local listener | Committed; generated; no secret fields | +| `.env` | Local loopback listener override | Committed; generated; no secret fields | | `.env.production` | Production listener override | Committed; generated; no deployment-specific credentials or origins | -| `.env.example` | Complete configuration template | Committed; generated; empty secret placeholders | +| `.env.example` | Complete configuration template | Committed; generated; all values empty | | `.env.local` / `.env.production.local` | Personal nonsecret files | Ignored; production never loads them | | `secrets/fanout.env` | Private complete SOPS credential profile, when used | Operator-owned; ignored and excluded from Docker | | `secrets/fanout.production.env` | Private production profile, when credentials differ | Operator-owned; ignored and excluded from Docker | +`.env` is now tracked and generated. Before pulling, move nonsecret personal +values from an older private `.env` to ignored `.env.local` and credentials to +runtime injection, then preserve the old file outside the checkout. Git refuses +to overwrite an untracked `.env`. + Build the binary, then use `just run` locally or `just run-production` for production. The Node launcher uses `node:util.parseEnv`; dotenv is not shell code and values are not interpolated. Local order is `.env`, optional `.env.local`, @@ -30,9 +35,10 @@ Unknown names, secret entries, and unresolved `op://` references in ordinary dotenv files fail before launch. A missing required file never selects another profile. Empty values retain Fanout's no-override semantics. -The launcher injects dotenv settings as environment values, so they override -the native binary's `--config` YAML. Launch the binary directly for YAML-only -ordinary configuration. Supported binary flags are forwarded unchanged. +The launcher injects nonempty dotenv settings as environment overrides of +`--config` YAML. The generated files override only the listener; other YAML +settings survive both launch modes. `.env.production` resets every local override +to its schema default. Supported binary flags are forwarded unchanged. Keep ordinary YAML/dotenv configuration nonsecret where possible. Deliver credentials through deployment environment injection, restricted managed mounts, @@ -79,11 +85,15 @@ names and safe metadata. Update their mappings, verify the real consumers, and verify recovery from the destination before removing the old authority. This repository ships no credential profiles or decryption keys. -For AI, deliver only the selected provider's `ANTHROPIC_API_KEY` or -`OPENAI_API_KEY`. Provider endpoint overrides are `ANTHROPIC_BASE_URL` and +AI chat is disabled by default. Select exactly `anthropic` or `openai` through +`ai.provider` or `FANOUT_AI_PROVIDER`, then deliver its required `ANTHROPIC_API_KEY` +or `OPENAI_API_KEY`; a selected provider without its key fails startup. Inherited +keys do not enable chat. With no provider selected, AI keys and endpoints are +ignored; only the selected provider's endpoint is validated and used. +Provider endpoint overrides are `ANTHROPIC_BASE_URL` and `OPENAI_BASE_URL`; the OpenAI base includes `/v1`, while Anthropic's excludes it. The old generic key/base URL names and YAML keys are rejected; the -[configuration upgrade instructions](../README.md#configuration) describe +[configuration upgrade instructions](../site/src/content/docs/start/install.mdx#ai-configuration-upgrades) describe their exact replacements. Verify authentication and request paths against the configured provider after rollout, without logging credentials. diff --git a/fanout.example.yaml b/fanout.example.yaml index 086b4043..c303a4c2 100644 --- a/fanout.example.yaml +++ b/fanout.example.yaml @@ -41,17 +41,17 @@ alerts: history_days: 7 # FANOUT_ALERTS_HISTORY_DAYS ai: - provider: anthropic # FANOUT_AI_PROVIDER + provider: "" # FANOUT_AI_PROVIDER; empty disables chat; select anthropic or openai model: "" # FANOUT_AI_MODEL anthropic: # Defaults to https://api.anthropic.com; Fanout appends /v1/messages. base_url: "" # ANTHROPIC_BASE_URL - # Optional; enables chat when provider is anthropic: ANTHROPIC_API_KEY + # Required when provider is anthropic: ANTHROPIC_API_KEY # api_key: "" openai: # Defaults to https://api.openai.com/v1; Fanout appends /responses. base_url: "" # OPENAI_BASE_URL - # Optional; enables chat when provider is openai: OPENAI_API_KEY + # Required when provider is openai: OPENAI_API_KEY # api_key: "" smtp: diff --git a/internal/agent/provider_contract_test.go b/internal/agent/provider_contract_test.go index e3edf8d4..9083175c 100644 --- a/internal/agent/provider_contract_test.go +++ b/internal/agent/provider_contract_test.go @@ -149,8 +149,8 @@ func TestOpenAIEncryptedContentConfigurationError(t *testing.T) { } func TestNewProviderNewestDefaults(t *testing.T) { - for _, kind := range []string{"", "anthropic", "openai"} { - p, err := NewProvider(kind, "test", "", "") + for _, kind := range []string{"anthropic", "openai"} { + p, err := NewProvider(kind, "test", "", "https://provider.example/gateway") if err != nil { t.Fatal(err) } @@ -167,6 +167,14 @@ func TestNewProviderNewestDefaults(t *testing.T) { } } +func TestNewProviderRejectsUnsupportedNames(t *testing.T) { + for _, kind := range []string{"", "OpenAI", " OpenAI ", " openai ", "Anthropic", "unsupported"} { + if _, err := NewProvider(kind, "test", "", "https://provider.example"); err == nil { + t.Errorf("provider %q must be rejected", kind) + } + } +} + func TestAnthropicCacheBreakpoints(t *testing.T) { for _, finalRole := range []Role{RoleUser, RoleTool} { t.Run(string(finalRole), func(t *testing.T) { diff --git a/internal/agent/provider_http.go b/internal/agent/provider_http.go index 2180b08b..04f260db 100644 --- a/internal/agent/provider_http.go +++ b/internal/agent/provider_http.go @@ -12,28 +12,20 @@ import ( "sort" "strings" "time" - - "github.com/labstack/fanout/internal/config" ) func NewProvider(kind, apiKey, model, baseURL string) (Provider, error) { - switch strings.ToLower(strings.TrimSpace(kind)) { - case "", "anthropic": + switch kind { + case "anthropic": if model == "" { model = "claude-sonnet-5-5" } - if baseURL == "" { - baseURL = config.DefaultAnthropicBaseURL - } - return &anthropicProvider{apiKey: apiKey, model: model, baseURL: strings.TrimRight(baseURL, "/"), client: modelHTTPClient()}, nil + return &anthropicProvider{apiKey: apiKey, model: model, baseURL: baseURL, client: modelHTTPClient()}, nil case "openai": if model == "" { model = "gpt-6.1-sol" } - if baseURL == "" { - baseURL = config.DefaultOpenAIBaseURL - } - return &openAIProvider{apiKey: apiKey, model: model, baseURL: strings.TrimRight(baseURL, "/"), client: modelHTTPClient()}, nil + return &openAIProvider{apiKey: apiKey, model: model, baseURL: baseURL, client: modelHTTPClient()}, nil default: return nil, fmt.Errorf("unsupported AI provider %q", kind) } diff --git a/internal/agent/provider_http_test.go b/internal/agent/provider_http_test.go index 1648b8d5..04642c45 100644 --- a/internal/agent/provider_http_test.go +++ b/internal/agent/provider_http_test.go @@ -12,6 +12,7 @@ import ( "testing" agtypes "github.com/ag-ui-protocol/ag-ui/sdks/community/go/pkg/core/types" + "github.com/labstack/fanout/internal/config" ) // lines joins SSE lines with \n. @@ -39,7 +40,8 @@ func TestProviderBaseURLPreservesGatewayPath(t *testing.T) { _, _ = io.WriteString(w, tc.response) })) defer server.Close() - p, err := NewProvider(tc.provider, "test-key", "", server.URL+tc.basePath+slash) + cfg := config.Config{AIProvider: tc.provider, AnthropicBaseURL: server.URL + tc.basePath + slash, OpenAIBaseURL: server.URL + tc.basePath + slash} + p, err := NewProvider(tc.provider, "test-key", "", cfg.AIBaseURL()) if err != nil { t.Fatal(err) } diff --git a/internal/config/config.go b/internal/config/config.go index ff0fdd7c..3e40f764 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -25,7 +25,7 @@ const ( // Elapsed-time settings use time.Duration with unit-bearing values. type Config struct { // Addr binds the browser, API, MCP, OTLP/gRPC and OTLP/HTTP listener. - Addr string `koanf:"server.addr" env:"FANOUT_ADDR" default:":7520"` + Addr string `koanf:"server.addr" env:"FANOUT_ADDR" default:":7520" local:"127.0.0.1:7520"` // DataDir stores telemetry, query state, and the control database. DataDir string `koanf:"storage.data_dir" env:"FANOUT_DATA_DIR" default:"./data"` // IngestBatchSize limits telemetry rows per batch; valid values are 1–50000. @@ -94,21 +94,21 @@ type Config struct { AlertEvaluationInterval time.Duration `koanf:"alerts.evaluation_interval" env:"FANOUT_ALERTS_EVALUATION_INTERVAL" default:"30s"` // AlertHistoryDays sets alert history retention in days. AlertHistoryDays int `koanf:"alerts.history_days" env:"FANOUT_ALERTS_HISTORY_DAYS" default:"7"` - // AIProvider selects anthropic or openai independently of which keys are present. - AIProvider string `koanf:"ai.provider" env:"FANOUT_AI_PROVIDER" default:"anthropic"` + // AIProvider enables chat by selecting exactly anthropic or openai. Empty disables chat. + AIProvider string `koanf:"ai.provider" env:"FANOUT_AI_PROVIDER" default:""` // AIModel overrides the provider's default: claude-sonnet-5-5 for Anthropic // and gpt-6.1-sol for OpenAI. The OpenAI provider expects a reasoning model // supporting reasoning.encrypted_content for stateless Responses replay. // A 400 rejecting encrypted content is reported as a server-side configuration // error; choose a compatible reasoning model with ai.model. AIModel string `koanf:"ai.model" env:"FANOUT_AI_MODEL"` - // AnthropicAPIKey enables chat only when ai.provider is anthropic. + // AnthropicAPIKey is required when ai.provider is anthropic. AnthropicAPIKey string `koanf:"ai.anthropic.api_key" env:"ANTHROPIC_API_KEY" secret:"true"` // AnthropicBaseURL defaults to https://api.anthropic.com. Fanout appends // /v1/messages, preserving any gateway path prefix. Use an HTTP(S) base URL // without embedded credentials, a query, or a fragment. AnthropicBaseURL string `koanf:"ai.anthropic.base_url" env:"ANTHROPIC_BASE_URL"` - // OpenAIAPIKey enables chat only when ai.provider is openai. + // OpenAIAPIKey is required when ai.provider is openai. OpenAIAPIKey string `koanf:"ai.openai.api_key" env:"OPENAI_API_KEY" secret:"true"` // OpenAIBaseURL defaults to https://api.openai.com/v1. Fanout appends // /responses, preserving the API version and any gateway path prefix. @@ -329,18 +329,24 @@ func (c Config) Validate() error { return fmt.Errorf("OIDC auto-provisioning requires auth.oidc.allowed_groups or auth.oidc.allowed_domains") } } - switch strings.ToLower(strings.TrimSpace(c.AIProvider)) { + switch c.AIProvider { case "", "anthropic", "openai": default: return fmt.Errorf("ai.provider must be anthropic or openai") } - baseURL, key := c.AIBaseURL(), "ai.anthropic.base_url (ANTHROPIC_BASE_URL)" - if strings.EqualFold(strings.TrimSpace(c.AIProvider), "openai") { - key = "ai.openai.base_url (OPENAI_BASE_URL)" - } - u, err := url.Parse(baseURL) - if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || strings.Contains(baseURL, "#") { - return fmt.Errorf("%s must be an HTTP(S) base URL without credentials, query, or fragment", key) + if c.AgentConfigured() { + key, endpoint := "ai.anthropic.api_key (ANTHROPIC_API_KEY)", "ai.anthropic.base_url (ANTHROPIC_BASE_URL)" + if c.AIProvider == "openai" { + key, endpoint = "ai.openai.api_key (OPENAI_API_KEY)", "ai.openai.base_url (OPENAI_BASE_URL)" + } + if strings.TrimSpace(c.AIAPIKey()) == "" { + return fmt.Errorf("%s is required when ai.provider is %s", key, c.AIProvider) + } + baseURL := c.AIBaseURL() + u, err := url.Parse(baseURL) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || strings.Contains(baseURL, "#") { + return fmt.Errorf("%s must be an HTTP(S) base URL without credentials, query, or fragment", endpoint) + } } for _, raw := range strings.Split(c.TrustedProxyCIDRs, ",") { value := strings.TrimSpace(raw) @@ -387,13 +393,13 @@ func (c Config) SMTPConfigured() bool { // AgentConfigured reports whether the in-process AI agent should be started. func (c Config) AgentConfigured() bool { - return strings.TrimSpace(c.AIAPIKey()) != "" + return c.AIProvider != "" } // AIAPIKey returns only the explicitly selected provider's credential. func (c Config) AIAPIKey() string { - switch strings.ToLower(strings.TrimSpace(c.AIProvider)) { - case "", "anthropic": + switch c.AIProvider { + case "anthropic": return c.AnthropicAPIKey case "openai": return c.OpenAIAPIKey @@ -404,9 +410,14 @@ func (c Config) AIAPIKey() string { // AIBaseURL returns the selected provider's API base, including its default. func (c Config) AIBaseURL() string { - baseURL, fallback := c.AnthropicBaseURL, DefaultAnthropicBaseURL - if strings.EqualFold(strings.TrimSpace(c.AIProvider), "openai") { + var baseURL, fallback string + switch c.AIProvider { + case "anthropic": + baseURL, fallback = c.AnthropicBaseURL, DefaultAnthropicBaseURL + case "openai": baseURL, fallback = c.OpenAIBaseURL, DefaultOpenAIBaseURL + default: + return "" } if strings.TrimSpace(baseURL) == "" { return fallback diff --git a/internal/config/config_test.go b/internal/config/config_test.go index a14bf674..740ff930 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -665,7 +665,7 @@ func TestValidate(t *testing.T) { c := valid c.SMTPHost, c.SMTPUser, c.SMTPPass, c.SMTPFrom = "", "", "", "" c.AnthropicAPIKey = "" - c.AIProvider = "anthropic" + c.AIProvider = "" if err := c.Validate(); err != nil { t.Fatalf("minimal local config should pass: %v", err) } diff --git a/internal/config/launcher_test.go b/internal/config/launcher_test.go new file mode 100644 index 00000000..f87c514a --- /dev/null +++ b/internal/config/launcher_test.go @@ -0,0 +1,71 @@ +package config + +import ( + "context" + "encoding/json" + "os" + "os/exec" + "path/filepath" + "testing" + "time" +) + +func TestGeneratedLauncherEnvironmentPreservesYAML(t *testing.T) { + root := t.TempDir() + for _, name := range []string{".env", ".env.production", ".env.example"} { + body, err := os.ReadFile(filepath.Join("../..", name)) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, name), body, 0o600); err != nil { + t.Fatal(err) + } + } + path := filepath.Join(root, "fanout.yaml") + if err := os.WriteFile(path, []byte("storage:\n data_dir: ./yaml-data\n retention_days: 17\n"), 0o600); err != nil { + t.Fatal(err) + } + launcher, err := filepath.Abs("../../scripts/run-fanout.mjs") + if err != nil { + t.Fatal(err) + } + for _, environment := range []string{"local", "production"} { + t.Run(environment, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "node", "--input-type=module", "-e", ` +const { pathToFileURL } = await import('node:url'); +const { prepareEnvironment } = await import(pathToFileURL(process.argv[3]).href); +process.stdout.write(JSON.stringify(prepareEnvironment(process.argv[1], process.argv[2], { + FANOUT_AUTH_CODE_SECRET: '0123456789abcdef0123456789abcdef' +})));`, root, environment, launcher) + cmd.Env = []string{"PATH=" + os.Getenv("PATH")} + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("Node launcher: %v\n%s", err, output) + } + var values map[string]string + if err := json.Unmarshal(output, &values); err != nil { + t.Fatal(err) + } + var env []string + for name, value := range values { + env = append(env, name+"="+value) + } + cfg, err := Load(LoadOptions{Path: path, Environ: env}) + if err != nil { + t.Fatal(err) + } + if cfg.DataDir != "./yaml-data" || cfg.RetentionDays != 17 { + t.Fatal("generated launcher defaults overrode YAML settings") + } + wantAddr := "127.0.0.1:7520" + if environment == "production" { + wantAddr = ":7520" + } + if cfg.Addr != wantAddr { + t.Fatalf("listener = %q, want %q", cfg.Addr, wantAddr) + } + }) + } +} diff --git a/internal/config/provider_test.go b/internal/config/provider_test.go index 994bd784..c9a49735 100644 --- a/internal/config/provider_test.go +++ b/internal/config/provider_test.go @@ -13,9 +13,10 @@ func TestLoadSelectsProviderCredentials(t *testing.T) { credentials []string }{ {"anthropic", "anthropic", "anthropic-test", "https://anthropic.example/gateway", []string{"ANTHROPIC_API_KEY=anthropic-test", "ANTHROPIC_BASE_URL=https://anthropic.example/gateway/", "OPENAI_API_KEY=openai-test", "OPENAI_BASE_URL=https://openai.example/v1"}}, - {"openai", " OpenAI ", "openai-test", "https://openai.example/gateway/v1", []string{"ANTHROPIC_API_KEY=anthropic-test", "OPENAI_API_KEY=openai-test", "OPENAI_BASE_URL=https://openai.example/gateway/v1/"}}, - {"other key does not enable anthropic", "anthropic", "", DefaultAnthropicBaseURL, []string{"OPENAI_API_KEY=openai-test", "OPENAI_BASE_URL=not-used"}}, - {"other key does not enable openai", "openai", "", DefaultOpenAIBaseURL, []string{"ANTHROPIC_API_KEY=anthropic-test", "ANTHROPIC_BASE_URL=not-used"}}, + {"openai", "openai", "openai-test", "https://openai.example/gateway/v1", []string{"ANTHROPIC_API_KEY=anthropic-test", "ANTHROPIC_BASE_URL=not-used", "OPENAI_API_KEY=openai-test", "OPENAI_BASE_URL=https://openai.example/gateway/v1/"}}, + {"inherited credentials do not enable chat", "", "", "", []string{"ANTHROPIC_API_KEY=anthropic-test", "OPENAI_API_KEY=openai-test", "ANTHROPIC_BASE_URL=not-used", "OPENAI_BASE_URL=not-used"}}, + {"anthropic default endpoint", "anthropic", "anthropic-test", DefaultAnthropicBaseURL, []string{"ANTHROPIC_API_KEY=anthropic-test", "OPENAI_BASE_URL=not-used"}}, + {"openai default endpoint", "openai", "openai-test", DefaultOpenAIBaseURL, []string{"OPENAI_API_KEY=openai-test", "ANTHROPIC_BASE_URL=not-used"}}, } { t.Run(tc.name, func(t *testing.T) { env := []string{"FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), "FANOUT_AI_PROVIDER=" + tc.provider} @@ -23,7 +24,7 @@ func TestLoadSelectsProviderCredentials(t *testing.T) { if err != nil { t.Fatal(err) } - if cfg.AIAPIKey() != tc.key || cfg.AIBaseURL() != tc.baseURL || cfg.AgentConfigured() != (tc.key != "") { + if cfg.AIProvider != tc.provider || cfg.AIAPIKey() != tc.key || cfg.AIBaseURL() != tc.baseURL || cfg.AgentConfigured() != (tc.provider != "") { t.Fatal("selected provider credential, endpoint, or enabled state is incorrect") } }) @@ -55,7 +56,7 @@ func TestLoadRejectsInvalidProviderURLWithoutLeakingValues(t *testing.T) { for _, provider := range []string{"anthropic", "openai"} { name := strings.ToUpper(provider) + "_BASE_URL" for _, value := range []string{"file:///private", "https://user:private-value@example.com", "https://example.com?key=private-value", "https://example.com#private-value", "https://example.com?", "https://example.com#", "https://", "https://example.com/%zz"} { - _, err := Load(LoadOptions{Environ: []string{"FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), "FANOUT_AI_PROVIDER=" + provider, name + "=" + value}}) + _, err := Load(LoadOptions{Environ: []string{"FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), "FANOUT_AI_PROVIDER=" + provider, strings.ToUpper(provider) + "_API_KEY=test-key", name + "=" + value}}) if err == nil || !strings.Contains(err.Error(), name) || strings.Contains(err.Error(), value) || strings.Contains(err.Error(), "private-value") { t.Fatalf("expected redacted validation error naming %s", name) } @@ -64,8 +65,29 @@ func TestLoadRejectsInvalidProviderURLWithoutLeakingValues(t *testing.T) { } func TestLoadRejectsUnsupportedProviderWithoutAKey(t *testing.T) { - _, err := Load(LoadOptions{Environ: []string{"FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), "FANOUT_AI_PROVIDER=unsupported"}}) - if err == nil || !strings.Contains(err.Error(), "ai.provider") { - t.Fatal("unsupported provider must fail at startup, even without a key") + for _, provider := range []string{"unsupported", "OpenAI", " OpenAI ", " openai ", "Anthropic"} { + _, err := Load(LoadOptions{Environ: []string{"FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), "FANOUT_AI_PROVIDER=" + provider}}) + if err == nil || !strings.Contains(err.Error(), "ai.provider") { + t.Fatal("unsupported provider must fail at startup, even without a key") + } + } +} + +func TestLoadRequiresSelectedProviderKey(t *testing.T) { + for _, provider := range []string{"anthropic", "openai"} { + otherKey := "OPENAI_API_KEY" + if provider == "openai" { + otherKey = "ANTHROPIC_API_KEY" + } + for _, value := range []string{"", " "} { + name := strings.ToUpper(provider) + "_API_KEY" + _, err := Load(LoadOptions{Environ: []string{ + "FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), "FANOUT_AI_PROVIDER=" + provider, + name + "=" + value, otherKey + "=other-provider-key", + }}) + if err == nil || !strings.Contains(err.Error(), "ai."+provider+".api_key") || !strings.Contains(err.Error(), name) { + t.Fatalf("expected missing key error naming both YAML and environment keys, got %v", err) + } + } } } diff --git a/scripts/run-fanout.mjs b/scripts/run-fanout.mjs index e931992f..cc0528f3 100644 --- a/scripts/run-fanout.mjs +++ b/scripts/run-fanout.mjs @@ -35,7 +35,7 @@ function inventory(root) { const names = new Set(Object.keys(parseEnv(content))); const secrets = new Set(); for (const block of content.split(/\n\s*\n/)) { - if (/^# [^\n]+; secret$/m.test(block)) { + if (/^# [a-z0-9_.]+; [a-zA-Z0-9_.]+; secret$/.test(block.split(/\r?\n/, 1)[0])) { for (const name of Object.keys(parseEnv(block))) secrets.add(name); } } @@ -82,18 +82,15 @@ export async function main(args) { const environment = /** @type {"local" | "production"} */ (args[1]); const env = prepareEnvironment(root, environment, process.env); const child = spawn(args[3], args.slice(4), { cwd: root, env, stdio: "inherit" }); - const terminate = () => child.kill("SIGTERM"); - const interrupt = () => child.kill("SIGINT"); - process.once("SIGTERM", terminate); - process.once("SIGINT", interrupt); + const handlers = new Map(["SIGINT", "SIGTERM", "SIGHUP"].map(signal => [signal, () => child.kill(signal)])); + for (const [signal, handler] of handlers) process.on(signal, handler); try { return await new Promise((resolveExit, reject) => { child.once("error", reject); - child.once("exit", (code, signal) => resolveExit(code ?? (signal === "SIGINT" ? 130 : signal === "SIGTERM" ? 143 : 1))); + child.once("exit", (code, signal) => resolveExit(code ?? ({ SIGINT: 130, SIGTERM: 143, SIGHUP: 129 }[signal] ?? 1))); }); } finally { - process.off("SIGTERM", terminate); - process.off("SIGINT", interrupt); + for (const [signal, handler] of handlers) process.off(signal, handler); } } diff --git a/scripts/run-fanout.test.ts b/scripts/run-fanout.test.ts index 4dc4ac20..d946cf6d 100644 --- a/scripts/run-fanout.test.ts +++ b/scripts/run-fanout.test.ts @@ -1,5 +1,6 @@ import { afterEach, expect, test } from "bun:test"; -import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { spawn } from "node:child_process"; import { join } from "node:path"; import { tmpdir } from "node:os"; import { pathToFileURL } from "node:url"; @@ -12,13 +13,13 @@ function fixture(): string { const root = mkdtempSync(join(tmpdir(), "fanout-env-")); roots.push(root); writeFileSync(join(root, ".env.example"), `# server.addr; string -FANOUT_ADDR=:7520 +FANOUT_ADDR= # ai.model; string FANOUT_AI_MODEL= # storage.data_dir; string -FANOUT_DATA_DIR=./data +FANOUT_DATA_DIR= # auth.code_secret; string; secret FANOUT_AUTH_CODE_SECRET= @@ -26,7 +27,7 @@ FANOUT_AUTH_CODE_SECRET= # ai.anthropic.api_key; string; secret ANTHROPIC_API_KEY= `); - writeFileSync(join(root, ".env"), "FANOUT_ADDR=127.0.0.1:7520\nFANOUT_AI_MODEL=base\n"); + writeFileSync(join(root, ".env"), "FANOUT_ADDR=127.0.0.1:7520\n"); return root; } @@ -45,14 +46,33 @@ test("production excludes personal files and requires its declared override file writeFileSync(join(root, ".env.local"), "ANTHROPIC_API_KEY=personal-secret\n"); writeFileSync(join(root, ".env.production.local"), "FANOUT_AI_MODEL=personal-production\n"); expect(() => prepareEnvironment(root, "production", {})).toThrow("Cannot read required"); - writeFileSync(join(root, ".env.production"), "FANOUT_ADDR=:7520\nFANOUT_AI_MODEL=production\n"); + writeFileSync(join(root, ".env.production"), "FANOUT_ADDR=:7520\n"); const env = prepareEnvironment(root, "production", {}); expect(env.FANOUT_ADDR).toBe(":7520"); - expect(env.FANOUT_AI_MODEL).toBe("production"); + expect(env.FANOUT_AI_MODEL).toBeUndefined(); expect(env.ANTHROPIC_API_KEY).toBeUndefined(); expect(prepareEnvironment(root, "production", { FANOUT_AI_MODEL: "deployment" }).FANOUT_AI_MODEL).toBe("deployment"); }); +test("CRLF generated inventory preserves secret classification for local and production preparation", () => { + const inventory = readFileSync(join(import.meta.dir, "../.env.example"), "utf8").replace(/\r?\n/g, "\r\n"); + for (const environment of ["local", "production"] as const) { + const root = fixture(); + writeFileSync(join(root, ".env.example"), inventory); + writeFileSync(join(root, ".env.production"), "FANOUT_ADDR=:7520\n"); + const injected = { FANOUT_AUTH_CODE_SECRET: "synthetic-only", ANTHROPIC_API_KEY: "synthetic-only" }; + const env = prepareEnvironment(root, environment, injected); + expect(env.FANOUT_ADDR).toBe(environment === "local" ? "127.0.0.1:7520" : ":7520"); + expect(env.FANOUT_AUTH_CODE_SECRET).toBe(injected.FANOUT_AUTH_CODE_SECRET); + expect(env.ANTHROPIC_API_KEY).toBe(injected.ANTHROPIC_API_KEY); + const file = environment === "local" ? ".env.local" : ".env.production"; + for (const name of Object.keys(injected)) { + writeFileSync(join(root, file), `${name}=synthetic-only\n`); + expect(() => prepareEnvironment(root, environment, {})).toThrow(`Secret setting ${name} belongs in runtime injection, not ${file}`); + } + } +}); + test("dotenv parsing preserves dollars, quotes, whitespace, and multiline values without shell execution", () => { const root = fixture(); writeFileSync(join(root, ".env.local"), `FANOUT_DATA_DIR=' $HOME $(printf unsafe) "quoted" ' @@ -101,3 +121,84 @@ test("launcher requires an explicit environment and command", async () => { await expect(main([])).rejects.toThrow("Usage:"); await expect(main(["--environment", "local", "--"])).rejects.toThrow("Usage:"); }); + +test("secret classification uses only the strict first-line schema header", () => { + const root = fixture(); + writeFileSync(join(root, ".env.example"), `# ai.model; string +# documentation; string; secret +FANOUT_AI_MODEL= + +# Invalid-Key; string; secret +FANOUT_DATA_DIR= + +# auth.code_secret; string; secret +# An ordinary doc line cannot remove the secret marker. +FANOUT_AUTH_CODE_SECRET= +`); + writeFileSync(join(root, ".env"), "FANOUT_AI_MODEL=public\nFANOUT_DATA_DIR=./public\n"); + expect(prepareEnvironment(root, "local", {}).FANOUT_AI_MODEL).toBe("public"); + expect(prepareEnvironment(root, "local", {}).FANOUT_DATA_DIR).toBe("./public"); + writeFileSync(join(root, ".env"), "FANOUT_AUTH_CODE_SECRET=synthetic-only\n"); + expect(() => prepareEnvironment(root, "local", {})).toThrow("Secret setting FANOUT_AUTH_CODE_SECRET"); +}); + +function launchFixture(childCode: string) { + const root = fixture(); + mkdirSync(join(root, "scripts")); + copyFileSync(join(import.meta.dir, "run-fanout.mjs"), join(root, "scripts/run-fanout.mjs")); + const harness = ` +import { main } from './scripts/run-fanout.mjs'; +const signals = ['SIGINT', 'SIGTERM', 'SIGHUP']; +const before = signals.map(signal => process.listenerCount(signal)); +const code = await main(['--environment', 'local', '--', process.execPath, '-e', process.argv[2]]); +console.log('cleanup:' + signals.every((signal, index) => process.listenerCount(signal) === before[index])); +process.exitCode = code; +`; + writeFileSync(join(root, "harness.mjs"), harness); + return spawn("node", [join(root, "harness.mjs"), childCode], { env: { PATH: process.env.PATH }, stdio: ["ignore", "pipe", "pipe"] }); +} + +test("launcher forwards repeated SIGINT, SIGTERM, and SIGHUP until exit and removes handlers", async () => { + const signals = ["SIGINT", "SIGTERM", "SIGHUP", "SIGINT", "SIGTERM", "SIGHUP"] as const; + const launcher = launchFixture(` +let count = 0; +setTimeout(() => process.exit(1), 10000); +for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) process.on(signal, () => { + console.log(signal); + if (++count === 6) process.exit(0); +}); +console.log('ready'); +`); + let output = ""; + let index = 0; + const result = new Promise((resolveExit, reject) => { + launcher.once("error", reject); + launcher.once("close", code => resolveExit(code)); + // Acknowledgement of each delivery avoids timing-dependent signal coalescing. + launcher.stdout.on("data", data => { + output += data.toString(); + const lines = output.trimEnd().split("\n"); + if (lines.length === index + 1 && index < signals.length) launcher.kill(signals[index++]); + }); + }); + try { + expect(await result).toBe(0); + expect(output.trimEnd().split("\n")).toEqual(["ready", ...signals, "cleanup:true"]); + } finally { + if (launcher.exitCode === null) launcher.kill("SIGKILL"); + } +}, 15000); + +test("launcher maps child signal exits to conventional codes and removes handlers", async () => { + for (const [signal, code] of [["SIGINT", 130], ["SIGTERM", 143], ["SIGHUP", 129]] as const) { + const launcher = launchFixture(`process.kill(process.pid, '${signal}');`); + let output = ""; + launcher.stdout.on("data", data => { output += data.toString(); }); + const exitCode = await new Promise((resolveExit, reject) => { + launcher.once("error", reject); + launcher.once("close", code => resolveExit(code)); + }); + expect(exitCode).toBe(code); + expect(output.trim()).toBe("cleanup:true"); + } +}); diff --git a/site/src/content/docs/reference/configuration.mdx b/site/src/content/docs/reference/configuration.mdx index db71fc27..c2029809 100644 --- a/site/src/content/docs/reference/configuration.mdx +++ b/site/src/content/docs/reference/configuration.mdx @@ -25,7 +25,9 @@ A nonempty recognized environment variable wins over the YAML file, and the file wins over the built-in default. Empty environment values mean no override. All settings are read once at startup and require restart. Fanout does not load dotenv files itself. The generated `.env.example` documents types, defaults, -and empty credential placeholders; `.env` and `.env.production` omit secret fields. +and leaves every value empty so copying it never overrides YAML. `.env` contains +only schema-declared local overrides; `.env.production` resets exactly those +fields to their built-in defaults. Both omit secret fields. For a source checkout, build the binary and use `just run` for local development or `just run-production` for an explicit production launch. Local order is @@ -38,8 +40,8 @@ unknown names, and secret entries in ordinary dotenv files fail before launch. Inject credentials from a private selected SOPS profile or the deployment's secret service. This public repository ships no operator secrets or recipients. Dotenv values are injected as environment, so nonempty values override the -corresponding `--config` YAML. Launch the binary directly for YAML-only ordinary -configuration. +corresponding `--config` YAML. The generated files override only the listener; +other YAML settings survive both launch modes. ## Naming @@ -49,9 +51,12 @@ use `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, `OPENAI_API_KEY`, and settings table — `storage.duckdb.memory` maps to `FANOUT_DUCKDB_MEMORY`. Provider settings map to native structured keys under `ai.anthropic` or `ai.openai`. -`FANOUT_AI_PROVIDER` selects the provider and `FANOUT_AI_MODEL` selects its model. -Only the selected provider's key and endpoint are used. Without that key, chat is -disabled. Other vendor/tool settings, including `ANTHROPIC_MODEL` and +`FANOUT_AI_PROVIDER` (YAML `ai.provider`) defaults to empty, disabling chat. +Selecting exactly `anthropic` or `openai` enables chat and requires that provider's +key at startup; other names, casing, or surrounding whitespace fail. Only the +selected provider's key and endpoint are validated and used. With no provider +selected, inherited AI keys and endpoints are ignored. `FANOUT_AI_MODEL` selects +the model. Other vendor/tool settings, including `ANTHROPIC_MODEL` and `OPENAI_MODEL`, are ignored. See [AI settings](/reference/settings/ai) for the different base URL meanings and [upgrade instructions](/start/install#ai-configuration-upgrades) for migrating the retired generic key and base URL settings. diff --git a/site/src/content/docs/reference/settings/ai.mdx b/site/src/content/docs/reference/settings/ai.mdx index 42155dde..300df2e8 100644 --- a/site/src/content/docs/reference/settings/ai.mdx +++ b/site/src/content/docs/reference/settings/ai.mdx @@ -1,7 +1,7 @@ --- title: "AI provider settings" description: "The provider, key, model and base URL backing the chat investigator." -summary: "Settings for the model behind the chat investigator. Without a key the agent is disabled." +summary: "Chat is disabled unless a provider is selected; its API key is then required." read_when: - "The chat investigator is disabled and you want it on." - "You are pointing Fanout at a self-hosted or proxied model endpoint." @@ -28,13 +28,13 @@ ignored; only the explicitly selected AI provider's key and endpoint are used. | `ai.model` | `FANOUT_AI_MODEL` | string | — | | `ai.openai.api_key` | `OPENAI_API_KEY` | string · secret | — | | `ai.openai.base_url` | `OPENAI_BASE_URL` | string | — | -| `ai.provider` | `FANOUT_AI_PROVIDER` | string | `anthropic` | +| `ai.provider` | `FANOUT_AI_PROVIDER` | string | — | ## Notes ### `ai.anthropic.api_key` -Enables chat only when ai.provider is anthropic. +Required when ai.provider is anthropic. ### `ai.anthropic.base_url` @@ -46,7 +46,7 @@ Overrides the provider's default: claude-sonnet-5-5 for Anthropic and gpt-6.1-so ### `ai.openai.api_key` -Enables chat only when ai.provider is openai. +Required when ai.provider is openai. ### `ai.openai.base_url` @@ -54,7 +54,7 @@ Defaults to https://api.openai.com/v1. Fanout appends /responses, preserving the ### `ai.provider` -Selects anthropic or openai independently of which keys are present. +Enables chat by selecting exactly anthropic or openai. Empty disables chat. :::caution[Secrets] Settings marked `secret` are redacted from Fanout's startup configuration diff --git a/site/src/content/docs/start/install.mdx b/site/src/content/docs/start/install.mdx index 8b35596e..1488cc2d 100644 --- a/site/src/content/docs/start/install.mdx +++ b/site/src/content/docs/start/install.mdx @@ -32,10 +32,11 @@ pick whichever registry your hosts already pull from. `FANOUT_AUTH_CODE_SECRET` is required in the default `local` auth mode and must be at least 32 characters — the container refuses to start without it. It is the only setting the command above adds; the image ships listener and data-directory -defaults of its own. Add `ANTHROPIC_API_KEY` (or `OPENAI_API_KEY` with -`FANOUT_AI_PROVIDER=openai`) to enable the chat investigator, -which is optional: without a key the process starts normally and logs that the -agent is disabled. +defaults of its own. Select `FANOUT_AI_PROVIDER=anthropic` with `ANTHROPIC_API_KEY`, +or `FANOUT_AI_PROVIDER=openai` with `OPENAI_API_KEY`, to enable the chat +investigator. Chat is optional and disabled when no provider is selected, even +with inherited vendor credentials or endpoints. Selecting a provider requires +its key at startup and validates only its endpoint. :::caution[Use a named volume, or fix the ownership yourself] The image is distroless and runs unprivileged as UID/GID 65532. A **named @@ -113,11 +114,12 @@ reasoning is in the repository's release contract. ## AI configuration upgrades -Replace `FANOUT_AI_API_KEY` with `ANTHROPIC_API_KEY`, or `OPENAI_API_KEY` when -`FANOUT_AI_PROVIDER=openai`. Move YAML `ai.api_key` to the corresponding -`ai.anthropic.api_key` or `ai.openai.api_key`. The other provider's key does not -enable chat; provider and model choices remain `FANOUT_AI_PROVIDER` and -`FANOUT_AI_MODEL`. +Select exactly `anthropic` or `openai` with `FANOUT_AI_PROVIDER` (YAML +`ai.provider`) to enable chat; empty disables it. Replace `FANOUT_AI_API_KEY` +with the selected provider's required `ANTHROPIC_API_KEY` or `OPENAI_API_KEY`. +Move YAML `ai.api_key` to the corresponding `ai.anthropic.api_key` or +`ai.openai.api_key`. Inherited keys never select a provider. Model selection +remains `FANOUT_AI_MODEL`. Replace `FANOUT_AI_BASE_URL` with the matching `ANTHROPIC_BASE_URL` or `OPENAI_BASE_URL` (YAML `ai.anthropic.base_url` or `ai.openai.base_url`). For From 38a71460aaf5194328ef75ebd54b23e66ee2057b Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Tue, 13 Oct 2026 21:49:30 -0700 Subject: [PATCH 3/6] feat(config)!: configure Fanout from the environment only Fanout's settings are all flat, so a second configuration source added precedence bugs without adding expressiveness. - Remove the YAML file, the --config flag, fanout.example.yaml, fanout.docker.yaml and the koanf dependency. The loader resolves built-in defaults, then the process environment, from the schema tags. Environment names are unchanged; error messages name them directly. - Every secret also accepts _FILE (both set fails, one trailing newline is stripped, an empty file fails, contents are never printed). Vendor variables are read only for the selected AI provider. - The binary listens on 127.0.0.1:7520 by default. The image sets FANOUT_ADDR=:7520 and FANOUT_DATA_DIR and bakes in no configuration file. - fanout healthcheck derives its probe from FANOUT_ADDR, supports TLS on any bind address, and never routes the local probe through a proxy. - Only the generated .env.example is tracked. just run loads an optional, ignored personal .env; inherited values win. - Docs describe environment and file-based secret delivery and carry one upgrade table for every breaking change on this branch. --- .dockerignore | 6 +- .env | 9 - .env.example | 276 ++++---- .env.production | 7 - .gitignore | 8 +- AGENTS.md | 47 +- CONTRIBUTING.md | 32 +- Dockerfile | 4 +- README.md | 178 ++--- SECURITY.md | 5 +- THIRD_PARTY_NOTICES | 95 --- cmd/fanout-docgen/main.go | 152 +--- cmd/fanout-docgen/main_test.go | 83 +-- cmd/fanout/main.go | 77 ++- cmd/fanout/main_test.go | 210 +++++- context7.json | 2 +- docs/authentication.md | 4 +- docs/operations.md | 249 ++++--- fanout.docker.yaml | 7 - fanout.example.yaml | 94 --- go.mod | 10 +- go.sum | 16 - internal/agent/provider_contract_test.go | 2 +- internal/agent/provider_http.go | 2 +- internal/config/config.go | 191 +++-- internal/config/config_test.go | 654 ++++-------------- internal/config/launcher_test.go | 71 -- internal/config/loader.go | 290 +++----- internal/config/pinned_memory_test.go | 2 +- internal/config/provider_test.go | 89 ++- internal/config/sizing.go | 12 +- internal/query/duck.go | 4 +- justfile | 9 +- scripts/run-fanout.mjs | 88 +-- scripts/run-fanout.test.ts | 127 +--- site/src/content/docs/guides/troubleshoot.mdx | 6 +- .../content/docs/reference/configuration.mdx | 126 ++-- .../content/docs/reference/settings/ai.mdx | 50 +- .../docs/reference/settings/alerts.mdx | 24 +- .../content/docs/reference/settings/auth.mdx | 82 +-- .../docs/reference/settings/ingest.mdx | 28 +- .../content/docs/reference/settings/mcp.mdx | 16 +- .../docs/reference/settings/metrics.mdx | 24 +- .../docs/reference/settings/server.mdx | 40 +- .../content/docs/reference/settings/smtp.mdx | 36 +- .../docs/reference/settings/storage.mdx | 44 +- site/src/content/docs/start/install.mdx | 47 +- site/src/pages/llms.txt.ts | 2 +- site/src/styles/fanout.css | 2 +- 49 files changed, 1403 insertions(+), 2236 deletions(-) delete mode 100644 .env delete mode 100644 .env.production delete mode 100644 fanout.docker.yaml delete mode 100644 fanout.example.yaml delete mode 100644 internal/config/launcher_test.go diff --git a/.dockerignore b/.dockerignore index 437e7221..7ed41f3d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -19,10 +19,8 @@ ui/host/node_modules/ ui/host/dist/ # Dev files -.env -.env.* -**/.env -**/.env.* +.env* +**/.env* secrets/ .sops.yaml .tmp/ diff --git a/.env b/.env deleted file mode 100644 index 2dd7367b..00000000 --- a/.env +++ /dev/null @@ -1,9 +0,0 @@ -# Generated nonsecret configuration by cmd/fanout-docgen. -# Regenerate with just docs-generate. Credentials never belong in this file. -# Shared local overrides; .env.production resets them for production. -# Use just run, or inject these values explicitly; Fanout does not load dotenv. -# Personal nonsecret overrides belong in ignored .env.local. - -# server.addr; string -# Binds the browser, API, MCP, OTLP/gRPC and OTLP/HTTP listener. -FANOUT_ADDR=127.0.0.1:7520 diff --git a/.env.example b/.env.example index c2042d6e..30d6311d 100644 --- a/.env.example +++ b/.env.example @@ -1,258 +1,264 @@ # Generated by cmd/fanout-docgen from internal/config/config.go. # Regenerate with just docs-generate; verify with just docs-generate-check. # Fanout does not load dotenv files. Inject through your launcher or deployment. -# Precedence: built-in defaults < --config YAML < nonempty environment values. +# Precedence: built-in defaults < nonempty environment values. # All settings require restart. Empty environment values mean no override. # Secret entries below are empty placeholders; use runtime secret injection. # Local authentication requires FANOUT_AUTH_CODE_SECRET (32+ characters). # Chat is disabled unless FANOUT_AI_PROVIDER selects anthropic or openai; its key is required. -# ai.anthropic.api_key; string; secret -# Required when ai.provider is anthropic. +# ANTHROPIC_API_KEY; string; secret +# Required when FANOUT_AI_PROVIDER is anthropic. # Default: (empty) ANTHROPIC_API_KEY= +# Alternatively set ANTHROPIC_API_KEY_FILE to a secret file path; do not set both. -# ai.anthropic.base_url; string +# ANTHROPIC_BASE_URL; string # Defaults to https://api.anthropic.com. Fanout appends /v1/messages, preserving any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. # Default: (empty) ANTHROPIC_BASE_URL= -# ai.model; string -# Overrides the provider's default: claude-sonnet-5-5 for Anthropic and gpt-6.1-sol for OpenAI. The OpenAI provider expects a reasoning model supporting reasoning.encrypted_content for stateless Responses replay. A 400 rejecting encrypted content is reported as a server-side configuration error; choose a compatible reasoning model with ai.model. -# Default: (empty) -FANOUT_AI_MODEL= - -# ai.openai.api_key; string; secret -# Required when ai.provider is openai. -# Default: (empty) -OPENAI_API_KEY= +# FANOUT_ADDR; string +# Binds the browser, API, MCP, OTLP/gRPC and OTLP/HTTP listener. +# Default: 127.0.0.1:7520 +FANOUT_ADDR= -# ai.openai.base_url; string -# Defaults to https://api.openai.com/v1. Fanout appends /responses, preserving the API version and any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. +# FANOUT_AI_MODEL; string +# Overrides the provider's default: claude-sonnet-5-5 for Anthropic and gpt-6.1-sol for OpenAI. The OpenAI provider expects a reasoning model supporting reasoning.encrypted_content for stateless Responses replay. A 400 rejecting encrypted content is reported as a server-side configuration error; choose a compatible reasoning model with FANOUT_AI_MODEL. # Default: (empty) -OPENAI_BASE_URL= +FANOUT_AI_MODEL= -# ai.provider; string +# FANOUT_AI_PROVIDER; string # Enables chat by selecting exactly anthropic or openai. Empty disables chat. # Default: (empty) FANOUT_AI_PROVIDER= -# alerts.enabled; boolean +# FANOUT_ALERTS_ENABLED; boolean # Runs the alert evaluation engine. # Default: true FANOUT_ALERTS_ENABLED= -# alerts.evaluation_interval; duration +# FANOUT_ALERTS_EVALUATION_INTERVAL; duration # Schedules alert evaluation; use a Go duration of at least 1s. # Default: 30s FANOUT_ALERTS_EVALUATION_INTERVAL= -# alerts.history_days; integer +# FANOUT_ALERTS_HISTORY_DAYS; integer # Sets alert history retention in days. # Default: 7 FANOUT_ALERTS_HISTORY_DAYS= -# auth.code_secret; string; secret +# FANOUT_AUTH_CODE_SECRET; string; secret # Signs local sign-in codes; required in local mode with at least 32 characters. # Default: (empty) FANOUT_AUTH_CODE_SECRET= +# Alternatively set FANOUT_AUTH_CODE_SECRET_FILE to a secret file path; do not set both. -# auth.mode; string +# FANOUT_AUTH_MODE; string # Selects local sign-in or oidc; each mode validates its prerequisites. # Default: local FANOUT_AUTH_MODE= -# auth.oidc.admin_groups; string +# FANOUT_DATA_DIR; string +# Stores telemetry, query state, and the control database. +# Default: ./data +FANOUT_DATA_DIR= + +# FANOUT_DEFAULT_NAMESPACE; string +# Supplies the namespace for telemetry without one. +# Default: default +FANOUT_DEFAULT_NAMESPACE= + +# FANOUT_DUCKDB_MAX_CONNECTIONS; integer +# Caps the DuckDB connection pool. Reads scan immutable Parquet concurrently; rollup-cache writes are serialized by the query write gate. Zero means "size it from the machine" — the same spelling DuckDBThreads uses for deferring to a default. Resolution happens in resolveSizing and is reported in the startup configuration log. +# Default: 0 +FANOUT_DUCKDB_MAX_CONNECTIONS= + +# FANOUT_DUCKDB_MEMORY; string +# Caps DuckDB's memory (e.g. "8GB"). Empty means Fanout sizes it from detected memory, reserving headroom for the Go runtime; see resolveSizing. DuckDB's own default is 80% of detected RAM, which is calculated as though DuckDB owned the machine and has been observed to get the process OOM-killed once the Go heap is added on top. +# Default: (empty) +FANOUT_DUCKDB_MEMORY= + +# FANOUT_DUCKDB_THREADS; integer +# Caps DuckDB's global query worker pool. Zero leaves DuckDB's own default in place (one worker per core). Set it to leave cores free for ingest on a query-heavy co-tenant host. +# Default: (empty) +FANOUT_DUCKDB_THREADS= + +# FANOUT_INGEST_BATCH_SIZE; integer +# Limits telemetry rows per batch; valid values are 1–50000. +# Default: 50000 +FANOUT_INGEST_BATCH_SIZE= + +# FANOUT_INGEST_MAX_IN_FLIGHT_BYTES; integer +# Caps the decoded telemetry the process will hold across concurrent requests. Concurrency is HTTP/2 streams times connections and each handler holds its batch until the commit is durable, so without a ceiling a burst is bounded only by memory. Zero is unbounded, which is the behaviour that predates this setting. +# Default: 268435456 +FANOUT_INGEST_MAX_IN_FLIGHT_BYTES= + +# FANOUT_MAINTENANCE_INTERVAL; duration +# Controls Parquet retention and compaction, and query-cache checkpointing. +# Default: 1h +FANOUT_MAINTENANCE_INTERVAL= + +# FANOUT_MCP_ENABLED; boolean +# Serves the MCP endpoint at /mcp. +# Default: true +FANOUT_MCP_ENABLED= + +# FANOUT_METRICS_PUBLIC; boolean +# Allows unauthenticated Prometheus scrapes. +# Default: false +FANOUT_METRICS_PUBLIC= + +# FANOUT_METRICS_TOKEN; string; secret +# Authorizes Prometheus scrapes with a bearer token; optional when using an admin session. +# Default: (empty) +FANOUT_METRICS_TOKEN= +# Alternatively set FANOUT_METRICS_TOKEN_FILE to a secret file path; do not set both. + +# FANOUT_OIDC_ADMIN_GROUPS; string # Lists identity-provider groups mapped to admin, separated by commas. # Default: (empty) FANOUT_OIDC_ADMIN_GROUPS= -# auth.oidc.allowed_domains; string +# FANOUT_OIDC_ALLOWED_DOMAINS; string # Lists allowed email domains, separated by commas; wildcard * requires verified email. # Default: (empty) FANOUT_OIDC_ALLOWED_DOMAINS= -# auth.oidc.allowed_groups; string +# FANOUT_OIDC_ALLOWED_GROUPS; string # Lists allowed identity-provider groups, separated by commas. # Default: (empty) FANOUT_OIDC_ALLOWED_GROUPS= -# auth.oidc.auto_provision; boolean +# FANOUT_OIDC_AUTO_PROVISION; boolean # Creates accounts on OIDC sign-in; requires an allowed group or domain. # Default: false FANOUT_OIDC_AUTO_PROVISION= -# auth.oidc.client_id; string +# FANOUT_OIDC_CLIENT_ID; string # Identifies Fanout's client registration; required in OIDC mode. # Default: (empty) FANOUT_OIDC_CLIENT_ID= -# auth.oidc.client_secret; string; secret +# FANOUT_OIDC_CLIENT_SECRET; string; secret # Authenticates Fanout's client registration; required in OIDC mode. # Default: (empty) FANOUT_OIDC_CLIENT_SECRET= +# Alternatively set FANOUT_OIDC_CLIENT_SECRET_FILE to a secret file path; do not set both. -# auth.oidc.default_role; string +# FANOUT_OIDC_DEFAULT_ROLE; string # Assigns viewer, operator, or admin to users without a group role mapping. # Default: viewer FANOUT_OIDC_DEFAULT_ROLE= -# auth.oidc.email_claim; string +# FANOUT_OIDC_EMAIL_CLAIM; string # Names the claim containing the user's email address. # Default: email FANOUT_OIDC_EMAIL_CLAIM= -# auth.oidc.email_verification; string +# FANOUT_OIDC_EMAIL_VERIFICATION; string # Selects required verified-email claims or issuer trust; issuer trust requires an allowlist. # Default: required FANOUT_OIDC_EMAIL_VERIFICATION= -# auth.oidc.issuer_url; string +# FANOUT_OIDC_ISSUER_URL; string # Selects the identity provider issuer; required in OIDC mode. # Default: (empty) FANOUT_OIDC_ISSUER_URL= -# auth.oidc.operator_groups; string +# FANOUT_OIDC_OPERATOR_GROUPS; string # Lists identity-provider groups mapped to operator, separated by commas. # Default: (empty) FANOUT_OIDC_OPERATOR_GROUPS= -# auth.self_signup; boolean -# Allows verified email addresses to create viewer accounts; requires local auth and SMTP. -# Default: false -FANOUT_SELF_SIGNUP= - -# auth.session_absolute_ttl; duration -# Limits total session lifetime; it must be at least session_idle_ttl. -# Default: 168h -FANOUT_SESSION_ABSOLUTE_TTL= - -# auth.session_idle_ttl; duration -# Expires inactive sessions; use a Go duration of at least 5m. -# Default: 12h -FANOUT_SESSION_IDLE_TTL= - -# ingest.batch_size; integer -# Limits telemetry rows per batch; valid values are 1–50000. -# Default: 50000 -FANOUT_INGEST_BATCH_SIZE= - -# ingest.default_namespace; string -# Supplies the namespace for telemetry without one. -# Default: default -FANOUT_DEFAULT_NAMESPACE= - -# ingest.max_in_flight_bytes; integer -# Caps the decoded telemetry the process will hold across concurrent requests. Concurrency is HTTP/2 streams times connections and each handler holds its batch until the commit is durable, so without a ceiling a burst is bounded only by memory. Zero is unbounded, which is the behaviour that predates this setting. -# Default: 268435456 -FANOUT_INGEST_MAX_IN_FLIGHT_BYTES= - -# mcp.enabled; boolean -# Serves the MCP endpoint at /mcp. -# Default: true -FANOUT_MCP_ENABLED= - -# metrics.public; boolean -# Allows unauthenticated Prometheus scrapes. -# Default: false -FANOUT_METRICS_PUBLIC= - -# metrics.token; string; secret -# Authorizes Prometheus scrapes with a bearer token; optional when using an admin session. -# Default: (empty) -FANOUT_METRICS_TOKEN= - -# server.addr; string -# Binds the browser, API, MCP, OTLP/gRPC and OTLP/HTTP listener. -# Default: :7520 -FANOUT_ADDR= - -# server.pprof_enabled; boolean +# FANOUT_PPROF_ENABLED; boolean # Exposes Go's net/http/pprof handlers at /debug/pprof/* for CPU/heap/mutex/goroutine profiling under load. Off by default; the routes require an admin browser session with operations:read. # Default: false FANOUT_PPROF_ENABLED= -# server.public_url; string +# FANOUT_PUBLIC_URL; string # Sets the external browser origin. Required for OIDC; HTTPS is required with MCP. # Default: (empty) FANOUT_PUBLIC_URL= -# server.tls.cert_file; string -# Selects the TLS certificate file; requires tls.key_file. -# Default: (empty) -FANOUT_TLS_CERT_FILE= +# FANOUT_RETENTION_DAYS; integer +# Sets telemetry retention in days; zero disables expiry. +# Default: 30 +FANOUT_RETENTION_DAYS= -# server.tls.key_file; string -# Selects the TLS private key file; requires tls.cert_file. -# Default: (empty) -FANOUT_TLS_KEY_FILE= +# FANOUT_ROLLUP_INTERVAL; duration +# Schedules analytical rollups; use a Go duration of at least 1s. +# Default: 1m +FANOUT_ROLLUP_INTERVAL= -# server.trusted_proxy_cidrs; string -# Lists trusted proxy networks, separated by commas; empty trusts no forwarded client addresses. -# Default: (empty) -FANOUT_TRUSTED_PROXY_CIDRS= +# FANOUT_ROLLUP_SKIP_TO_LATEST; boolean +# RollupSkipToLatest, set once at boot, advances the service and edge rollup watermarks to the current max ingested timestamp. Existing history is not aggregated into those analytical rollups. Completed-batch trace caches still rebuild all retained history in bounded passes; uncached files remain immediately queryable. Useful for large pre-seeded datasets (benchmarks, restores); off in normal operation. +# Default: false +FANOUT_ROLLUP_SKIP_TO_LATEST= + +# FANOUT_SELF_SIGNUP; boolean +# Allows verified email addresses to create viewer accounts; requires local auth and SMTP. +# Default: false +FANOUT_SELF_SIGNUP= -# smtp.from; string +# FANOUT_SESSION_ABSOLUTE_TTL; duration +# Limits total session lifetime; it must be at least FANOUT_SESSION_IDLE_TTL. +# Default: 168h +FANOUT_SESSION_ABSOLUTE_TTL= + +# FANOUT_SESSION_IDLE_TTL; duration +# Expires inactive sessions; use a Go duration of at least 5m. +# Default: 12h +FANOUT_SESSION_IDLE_TTL= + +# FANOUT_SMTP_FROM; string # Sets the email sender address; required with SMTP. # Default: (empty) FANOUT_SMTP_FROM= -# smtp.host; string +# FANOUT_SMTP_HOST; string # Selects the mail server. Host, username, password, and from must all be set to enable SMTP. # Default: (empty) FANOUT_SMTP_HOST= -# smtp.password; string; secret +# FANOUT_SMTP_PASSWORD; string; secret # Authenticates to the mail server; required with SMTP. # Default: (empty) FANOUT_SMTP_PASSWORD= +# Alternatively set FANOUT_SMTP_PASSWORD_FILE to a secret file path; do not set both. -# smtp.port; integer +# FANOUT_SMTP_PORT; integer # Sets the mail server's positive TCP port. # Default: 587 FANOUT_SMTP_PORT= -# smtp.username; string +# FANOUT_SMTP_USERNAME; string # Authenticates to the mail server; required with SMTP. # Default: (empty) FANOUT_SMTP_USERNAME= -# storage.data_dir; string -# Stores telemetry, query state, and the control database. -# Default: ./data -FANOUT_DATA_DIR= - -# storage.duckdb.max_connections; integer -# Caps the DuckDB connection pool. Reads scan immutable Parquet concurrently; rollup-cache writes are serialized by the query write gate. Zero means "size it from the machine" — the same spelling DuckDBThreads uses for deferring to a default. Resolution happens in resolveSizing and is reported in the startup configuration log. -# Default: 0 -FANOUT_DUCKDB_MAX_CONNECTIONS= - -# storage.duckdb.memory; string -# Caps DuckDB's memory (e.g. "8GB"). Empty means Fanout sizes it from detected memory, reserving headroom for the Go runtime; see resolveSizing. DuckDB's own default is 80% of detected RAM, which is calculated as though DuckDB owned the machine and has been observed to get the process OOM-killed once the Go heap is added on top. +# FANOUT_TLS_CERT_FILE; string +# Selects the TLS certificate file; requires FANOUT_TLS_KEY_FILE. # Default: (empty) -FANOUT_DUCKDB_MEMORY= +FANOUT_TLS_CERT_FILE= -# storage.duckdb.threads; integer -# Caps DuckDB's global query worker pool. Zero leaves DuckDB's own default in place (one worker per core). Set it to leave cores free for ingest on a query-heavy co-tenant host. +# FANOUT_TLS_KEY_FILE; string +# Selects the TLS private key file; requires FANOUT_TLS_CERT_FILE. # Default: (empty) -FANOUT_DUCKDB_THREADS= - -# storage.maintenance_interval; duration -# Controls Parquet retention and compaction, and query-cache checkpointing. -# Default: 1h -FANOUT_MAINTENANCE_INTERVAL= +FANOUT_TLS_KEY_FILE= -# storage.retention_days; integer -# Sets telemetry retention in days; zero disables expiry. -# Default: 30 -FANOUT_RETENTION_DAYS= +# FANOUT_TRUSTED_PROXY_CIDRS; string +# Lists trusted proxy networks, separated by commas; empty trusts no forwarded client addresses. +# Default: (empty) +FANOUT_TRUSTED_PROXY_CIDRS= -# storage.rollup_interval; duration -# Schedules analytical rollups; use a Go duration of at least 1s. -# Default: 1m -FANOUT_ROLLUP_INTERVAL= +# OPENAI_API_KEY; string; secret +# Required when FANOUT_AI_PROVIDER is openai. +# Default: (empty) +OPENAI_API_KEY= +# Alternatively set OPENAI_API_KEY_FILE to a secret file path; do not set both. -# storage.rollup_skip_to_latest; boolean -# RollupSkipToLatest, set once at boot, advances the service and edge rollup watermarks to the current max ingested timestamp. Existing history is not aggregated into those analytical rollups. Completed-batch trace caches still rebuild all retained history in bounded passes; uncached files remain immediately queryable. Useful for large pre-seeded datasets (benchmarks, restores); off in normal operation. -# Default: false -FANOUT_ROLLUP_SKIP_TO_LATEST= +# OPENAI_BASE_URL; string +# Defaults to https://api.openai.com/v1. Fanout appends /responses, preserving the API version and any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. +# Default: (empty) +OPENAI_BASE_URL= diff --git a/.env.production b/.env.production deleted file mode 100644 index b8636c7b..00000000 --- a/.env.production +++ /dev/null @@ -1,7 +0,0 @@ -# Generated nonsecret configuration by cmd/fanout-docgen. -# Regenerate with just docs-generate. Credentials never belong in this file. -# Production overrides for .env. Deployment supplies its public URL and secrets. - -# server.addr; string -# Binds the browser, API, MCP, OTLP/gRPC and OTLP/HTTP listener. -FANOUT_ADDR=:7520 diff --git a/.gitignore b/.gitignore index 5b142896..1950c6f5 100644 --- a/.gitignore +++ b/.gitignore @@ -39,12 +39,8 @@ coverage.out coverage.html cover.out -# Only generated public defaults/templates are tracked. Personal overrides, -# credential migration sources, and operator secret profiles remain private. -.env -.env.* -!/.env -!/.env.production +# Personal environment files and operator secret profiles remain private. +.env* !/.env.example /secrets/ /.sops.yaml diff --git a/AGENTS.md b/AGENTS.md index 6228e685..ac4ba3fe 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,36 +2,37 @@ ## Configuration conventions -- Use consumer-first environment naming: custom product settings use - `FANOUT_`; AI keys and endpoints use `ANTHROPIC_API_KEY`, +- Configuration is environment-only. `internal/config/config.go` is the + authoritative schema: built-in defaults, then nonempty process environment. + Custom names use `FANOUT_` plus a clear name, adding the area only when it + clarifies; vendor credentials and endpoints keep `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, `OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Provider and model selection remain `FANOUT_AI_PROVIDER` and `FANOUT_AI_MODEL`. - An empty provider disables chat; exactly `anthropic` or `openai` enables it and - requires that provider's API key. Ignore AI credentials and endpoints when chat - is disabled; validate only the selected provider's endpoint. -- `internal/config/config.go` is the authoritative schema. Run - `just docs-generate` after configuration changes and verify - `just docs-generate-check`; the generator owns `.env`, `.env.production`, - `.env.example`, and site references. Public defaults exclude all secret fields. - `.env` contains only schema-declared `local` overrides; `.env.production` resets - those fields to their built-in defaults. `.env.example` keeps all values empty - and documents defaults in comments so copying it never overrides YAML. -- Preserve the explicit YAML/process-environment loader. Fanout does not load - dotenv files; launchers and deployments own file loading and secret injection. - Only Fanout's owned namespace rejects unknown environment names; shared vendor - namespaces recognize the documented names and ignore unrelated consumers. -- The Node launcher used by `just run` loads `.env`, then optional `.env.local`, - then explicitly injected values. `just run-production` loads `.env` and required - `.env.production`, then injected values, and excludes all personal files. - Use Node's native dotenv parser; never source dotenv as shell code. Secret - entries and unknown names are rejected in these nonsecret files. + An empty provider disables chat; exactly `anthropic` or `openai` requires + that provider's key. Ignore AI credentials and endpoints when chat is disabled; + validate only the selected provider's endpoint. +- Every `secret:"true"` field accepts `_FILE`. Both forms nonempty fail; + secret files lose exactly one trailing LF or CRLF and must remain nonempty. + Errors name variables and paths, never secret contents. No file-mode check. +- `group` tags serve docgen only, using ai, alerts, auth, ingest, mcp, metrics, + server, smtp, and storage. Run `just docs-generate` after schema changes and + verify `just docs-generate-check`. The generator owns `.env.example` and the + site reference; every template value is empty, with defaults and file forms + in comments. Secret defaults must never appear in public artifacts. +- The binary does not load dotenv. `just run` uses Node's native `parseEnv` for + optional ignored `./.env`, then inherited environment values win. Personal + development credentials may live there. Never shell-source dotenv. Only + Fanout's namespace rejects unknown names; tolerate platform service variables + and ignore unrelated names in shared vendor namespaces. +- Native listener default is `127.0.0.1:7520`. The image supplies + `FANOUT_ADDR=:7520` and `FANOUT_DATA_DIR=/var/lib/fanout/data`. - Repository-managed secret profiles adopt SOPS authority through a verified migration, with key recovery in 1Password. Existing vault/platform delivery remains authoritative until all consumers and recovery are verified. Follow `docs/operations.md` for delivery; never commit credential profiles in plaintext. - This public product repository ships no operator secrets or recipients. Keep - operator `secrets/`, personal files, and private migration sources ignored and - outside Docker inputs; select private SOPS profiles explicitly at launch. + operator `secrets/`, personal environment files, and private migration sources + ignored and outside Docker inputs; select private SOPS profiles explicitly. ## Public repository diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4d2d73f1..57062e32 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -17,10 +17,11 @@ You need Go 1.27.2 and a C/C++ toolchain on native Linux or macOS (amd64 or arm6 also requires a 32-character authentication code secret. SMTP and AI chat are optional; selecting an AI provider requires its key — see [README](README.md#requirements). -`.env` is now tracked and generated. Before pulling, move nonsecret personal -values from an older private `.env` to ignored `.env.local` and credentials to -runtime injection, then preserve the old file outside the checkout. Git refuses -to overwrite an untracked `.env`. See [runtime secret delivery](docs/operations.md#runtime-secret-delivery). +Copy `.env.example` to ignored `.env` for personal development settings and +credentials. After building, `just run` loads it with Node's native dotenv +parser; inherited environment values win. The binary validates the settings. +Keep this file out of Git and Docker inputs. See +[runtime secret delivery](docs/operations.md#runtime-secret-delivery). The DuckDB wrapper downloads the pinned, checksum-verified engine from [Fanout's dependency artifact mirror](https://github.com/labstack/fanout/releases/tag/duckdb-v2.0.0-alpha43763) @@ -89,18 +90,17 @@ font is missing rather than shipping a card in a substitute face, which typst would otherwise do without complaint. **Runtime configuration is strict.** `internal/config/config.go` owns the -schema, documented in [`fanout.example.yaml`](fanout.example.yaml). New settings -need a YAML key, a consumer-defined environment name (or `FANOUT_` for custom -settings), a purpose comment, a test, and matching documentation. Unknown YAML -keys and owned `FANOUT_` names deliberately fail startup. Recognized provider -names are explicit; unrelated shared-vendor/tool variables are ignored. -Run `just docs-generate` to regenerate the settings reference, `.env`, -`.env.production`, and `.env.example`; `just docs-generate-check` verifies them. -Public defaults exclude secret fields, and templates use empty placeholders. -Private overrides and operator secret profiles stay ignored and outside Docker. -After building, `just run` loads shared/local nonsecret settings with the Node -launcher; `just run-production` excludes personal files. Inject credentials -explicitly. Never introduce aliases for retired names or shell-source dotenv. +schema. New settings need an environment name, a documentation `group` tag, +a purpose comment, a test, and matching documentation. Custom settings use +`FANOUT_` plus a clear name, adding the area only when it clarifies; vendor +credentials and endpoints keep the vendor's documented names. Unknown owned +names fail startup, while unrelated shared-vendor/tool variables are ignored. +Secrets use `secret:"true"` and automatically accept an `_FILE` form; do not +set both forms nonempty. Run `just docs-generate` to regenerate the settings +reference and `.env.example`; `just docs-generate-check` verifies them. The +template leaves all values empty and documents defaults in comments. +Personal environment files and operator profiles stay ignored and outside +Docker. Never shell-source dotenv. **SQLite migrations own the control schema.** Write forward SQL changes in `internal/db/migrations` using Goose annotations. `just db-migrate-create NAME` diff --git a/Dockerfile b/Dockerfile index 34614b4b..3713f60a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,7 +46,6 @@ LABEL org.opencontainers.image.title="Fanout" \ org.opencontainers.image.version="${VERSION}" COPY --from=build /app/fanout /usr/local/bin/fanout -COPY fanout.docker.yaml /etc/fanout/fanout.yaml COPY LICENSE NOTICE THIRD_PARTY_NOTICES TRADEMARK.md /usr/share/licenses/fanout/ COPY --from=build --chown=nonroot:nonroot /runtime/var/lib/fanout /var/lib/fanout @@ -55,6 +54,8 @@ COPY --from=build --chown=nonroot:nonroot /runtime/var/lib/fanout /var/lib/fanou USER nonroot:nonroot WORKDIR /var/lib/fanout +ENV FANOUT_ADDR=:7520 FANOUT_DATA_DIR=/var/lib/fanout/data + EXPOSE 7520 # Startup does DuckDB catalog attachment and maintenance, so the grace period @@ -64,4 +65,3 @@ HEALTHCHECK --interval=30s --timeout=3s --start-period=40s --retries=3 \ CMD ["fanout", "healthcheck"] ENTRYPOINT ["fanout"] -CMD ["--config", "/etc/fanout/fanout.yaml"] diff --git a/README.md b/README.md index b25d9f62..3855f6db 100644 --- a/README.md +++ b/README.md @@ -161,8 +161,7 @@ without SMTP, mint a 15-minute, single-use link against the running container's control database: ```sh -docker exec fanout fanout --config /etc/fanout/fanout.yaml \ - login-link admin@example.com +docker exec fanout fanout login-link admin@example.com ``` Select `FANOUT_AI_PROVIDER=anthropic` and inject `ANTHROPIC_API_KEY`, or select @@ -175,20 +174,10 @@ The distroless image runs unprivileged as UID 65532. A bind-mounted host directo `/var/lib/fanout/data` must be writable by that user; a named volume, as above, needs no such handling. -The image selects `/etc/fanout/fanout.yaml` by default. That file contains only -the container listener and data-directory defaults; it does not contain -credentials. Start a container-specific document from that file so it retains -the shared listener and persistent data-directory settings: - -```sh -cp fanout.docker.yaml fanout.yaml -# Add the remaining settings, then mount it over the image document: -docker run -v ./fanout.yaml:/etc/fanout/fanout.yaml:ro \ - labstack/fanout:latest -``` - -All surfaces use `server.addr` (`:7520` by default). Publish only that -port; OTLP/HTTP and OTLP/gRPC use the same address as the application. +The image sets `FANOUT_ADDR=:7520` and +`FANOUT_DATA_DIR=/var/lib/fanout/data`. Override settings with deployment +environment values or mounted secret files. Publish port `7520`; the UI, API, +MCP, OTLP/HTTP, and OTLP/gRPC share this listener. ### From source @@ -223,102 +212,59 @@ capacity tests. ## Configuration -Configuration is resolved once at startup and validated before Fanout opens -data files or listeners. Sources apply in this order: built-in defaults, an -optional YAML document selected with `--config`, then recognized environment -variables. Fanout does not search for configuration or load `.env` files. -All runtime settings are read once at startup and require a restart to change. - -[`fanout.example.yaml`](fanout.example.yaml) is the complete commented schema: - -```sh -cp fanout.example.yaml fanout.yaml -./bin/fanout --config ./fanout.yaml -``` - -Environment variables override the corresponding YAML values and are useful -for container injection and secrets. An empty environment value means "no -override"; use YAML for an explicit empty string. Unknown YAML keys and unknown -`FANOUT_` variables are startup errors, except for the service-discovery names -Kubernetes and Docker link-style networking inject for a Service named -`fanout`. Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, -`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other environment variables, including -other settings in those shared vendor namespaces, are ignored. In particular, -`ANTHROPIC_MODEL` and `OPENAI_MODEL` do not select Fanout's model. - -[`.env.example`](.env.example) is generated from the same configuration schema -as the settings reference, with types and defaults in comments and every value -empty, so copying the template never overrides YAML. The shared [`.env`](.env) -contains only the nonsecret local listener override; the generated -[`.env.production`](.env.production) resets it to the schema default for production. -Personal nonsecret overrides belong in ignored `.env.local`. This public -repository excludes operator `secrets/` profiles and private recipient policy. - -After building, use `just run` locally or `just run-production` for an explicit -production launch. The Node launcher parses dotenv without shell execution: -local loading is `.env`, then optional `.env.local`, then injected values; -production loading is `.env`, then required `.env.production`, then injected -values. Production never reads `.local` files. A missing required file fails; -no other environment is substituted. Credentials must be injected separately, -and secret entries or unknown names in nonsecret dotenv files are rejected. - -For example, after selecting a private, complete credential profile: - -```sh -sops exec-env secrets/fanout.env 'just run' -sops exec-env secrets/fanout.production.env 'just run-production' -``` - -Select only a profile that actually exists for the intended resources. An -ordinary dotenv loader does not resolve 1Password references or decrypt SOPS. -The binary retains its YAML/environment precedence: every nonempty dotenv value -injected by the launcher overrides the corresponding `--config` YAML value. -Generated dotenv files override only the listener, so other YAML settings survive -both launch modes. -See [runtime secret delivery](docs/operations.md#runtime-secret-delivery) for -SOPS injection and migration precautions. Run `just docs-generate` after changing -the schema; `just docs-generate-check` verifies all three dotenv artifacts and -the reference. The generator refuses to overwrite private or unmanaged defaults. - -YAML null values are rejected. Boolean values must use YAML 1.2 `true` or -`false` (not `yes`, `no`, `on`, or `off`). If a YAML document contains a -credential, Fanout requires that the file not be accessible by group or others -(for example, mode `0600`). The definitions live in -[`internal/config/config.go`](internal/config/config.go); the settings most -operators touch are: - -| YAML key | Environment override | Default | Purpose | -| --- | --- | --- | --- | -| `server.addr` | `FANOUT_ADDR` | `:7520` | UI, API, MCP, and both OTLP transports | -| `storage.data_dir` | `FANOUT_DATA_DIR` | `./data` | Parquet, query state, and control SQLite | -| `auth.mode` | `FANOUT_AUTH_MODE` | `local` | `local` (login link or SMTP) or `oidc` | -| `auth.code_secret` | `FANOUT_AUTH_CODE_SECRET` | — | Required in local mode, 32+ characters | -| `ai.provider` | `FANOUT_AI_PROVIDER` | — | Empty disables chat; exactly `anthropic` or `openai` enables it | -| `ai.anthropic.api_key` | `ANTHROPIC_API_KEY` | — | Required when `ai.provider=anthropic` | -| `ai.openai.api_key` | `OPENAI_API_KEY` | — | Required when `ai.provider=openai` | -| `storage.retention_days` | `FANOUT_RETENTION_DAYS` | `30` | Telemetry retention window | -| `mcp.enabled` | `FANOUT_MCP_ENABLED` | `true` | Serve the MCP endpoint at `/mcp` | - -AI chat is disabled by default, even with inherited vendor credentials. Select -exactly `anthropic` or `openai` to enable it; the selected provider's key is -required at startup. Only that provider's credential and endpoint are validated -and used. With no provider selected, AI keys and endpoints are ignored. Unsupported -provider names, including different casing or surrounding whitespace, fail startup. -`FANOUT_AI_MODEL` remains the application model selector. - -`ANTHROPIC_BASE_URL` defaults to `https://api.anthropic.com`; Fanout appends -`/v1/messages`. `OPENAI_BASE_URL` defaults to `https://api.openai.com/v1`; -Fanout appends `/responses`. Both retain gateway path prefixes and accept a -trailing slash. The selected base URL must use HTTP(S) without embedded -credentials, a query, or a fragment. - -When upgrading, replace `FANOUT_AI_API_KEY` with the selected vendor's key and -move YAML `ai.api_key` to `ai.anthropic.api_key` or `ai.openai.api_key`. -Replace `FANOUT_AI_BASE_URL` and YAML `ai.base_url` with the matching vendor -setting. For OpenAI, append `/v1` to the old base URL to preserve the request -destination; Anthropic's base URL keeps its previous meaning. Retired settings -are rejected. Coordinate the new names with launchers, deployment inputs, and -secret-store fields before restarting; they have no aliases. +Configuration is resolved once at startup from built-in defaults followed by +nonempty process environment values, and validated before data files or +listeners open. Every change requires a restart. Native installs bind +`127.0.0.1:7520` by default; set `FANOUT_ADDR=:7520` for other hosts to reach it. + +Custom settings use `FANOUT_` plus a clear name, adding the area only when it +clarifies. Vendor credentials and endpoints keep their documented names: +`ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, `OPENAI_API_KEY`, and +`OPENAI_BASE_URL`. Unknown `FANOUT_` variables fail startup, except for +platform-injected service-discovery names. Unrelated vendor/tool variables are +ignored. Empty values mean unset. Bools, integers, and Go durations are parsed +as their declared types. + +[`.env.example`](.env.example) is generated from +[`internal/config/config.go`](internal/config/config.go), with every value +empty and defaults in comments. Copy it to ignored `.env` for personal +settings and development credentials. After building, `just run` uses Node's +`parseEnv` to load optional `./.env`, then lets inherited environment values +win, including empty values. The binary itself reads only process environment; +production launchers and deployments supply their own values. + +Every secret also accepts a file path through `_FILE`: +`FANOUT_AUTH_CODE_SECRET_FILE`, `FANOUT_OIDC_CLIENT_SECRET_FILE`, +`FANOUT_SMTP_PASSWORD_FILE`, `FANOUT_METRICS_TOKEN_FILE`, +`ANTHROPIC_API_KEY_FILE`, and `OPENAI_API_KEY_FILE`. Set either the value or +its file path. Setting both nonempty fails startup. Fanout strips exactly one +trailing LF or CRLF from a secret file and rejects an empty result. + +| Environment variable | Default | Purpose | +| --- | --- | --- | +| `FANOUT_ADDR` | `127.0.0.1:7520` | UI, API, MCP, and both OTLP transports | +| `FANOUT_DATA_DIR` | `./data` | Parquet, query state, and control SQLite | +| `FANOUT_AUTH_MODE` | `local` | Login link or SMTP, or `oidc` | +| `FANOUT_AUTH_CODE_SECRET` | — | Required in local mode, 32+ characters | +| `FANOUT_AI_PROVIDER` | — | Empty disables chat; `anthropic` or `openai` enables it | +| `ANTHROPIC_API_KEY` | — | Required for Anthropic chat | +| `OPENAI_API_KEY` | — | Required for OpenAI chat | +| `FANOUT_RETENTION_DAYS` | `30` | Telemetry retention window | +| `FANOUT_MCP_ENABLED` | `true` | Serve the MCP endpoint at `/mcp` | + +Chat requires an explicit provider and its credential. Inherited keys never +enable it. Only the selected provider's endpoint is validated and used. +`FANOUT_AI_MODEL` selects the model. `ANTHROPIC_BASE_URL` defaults to +`https://api.anthropic.com`; Fanout appends `/v1/messages`. `OPENAI_BASE_URL` +defaults to `https://api.openai.com/v1`; Fanout appends `/responses`. +Both preserve gateway prefixes and accept a trailing slash. The selected URL +must use HTTP(S) without credentials, a query, or a fragment. + +See the [configuration upgrade table](https://fanout.run/start/install/#configuration-upgrades) +and [runtime secret delivery](docs/operations.md#runtime-secret-delivery) for +deployment examples. This public repository excludes operator secret profiles +and private recipient policy. Run `just docs-generate` after changing the +schema; `just docs-generate-check` verifies `.env.example` and the reference. ### Advanced DuckDB sizing @@ -332,14 +278,14 @@ These variables are escape hatches for measured, unusual workloads: | Variable | Automatic behavior | When to override | | --- | --- | --- | -| `storage.duckdb.memory` / `FANOUT_DUCKDB_MEMORY` | 60% of the container or host memory available to Fanout | A measured co-tenant workload needs a different Go/DuckDB split | -| `storage.duckdb.max_connections` / `FANOUT_DUCKDB_MAX_CONNECTIONS` | Available Go CPUs, bounded to 2–16 connections | Query concurrency has been benchmarked for this machine | -| `storage.duckdb.threads` / `FANOUT_DUCKDB_THREADS` | DuckDB chooses its own query worker count | Query-heavy work must leave specific cores free for ingest | +| `FANOUT_DUCKDB_MEMORY` | 60% of the container or host memory available to Fanout | A measured co-tenant workload needs a different Go/DuckDB split | +| `FANOUT_DUCKDB_MAX_CONNECTIONS` | Available Go CPUs, bounded to 2–16 connections | Query concurrency has been benchmarked for this machine | +| `FANOUT_DUCKDB_THREADS` | DuckDB chooses its own query worker count | Query-heavy work must leave specific cores free for ingest | An explicit value always wins. Startup logs and `/readyz` report the detected host and cgroup limits, the selected source, and whether detection was incomplete. If Fanout cannot conclusively inspect a container limit, it warns; -set `storage.duckdb.memory` or `FANOUT_DUCKDB_MEMORY` for a guaranteed bound. +set `FANOUT_DUCKDB_MEMORY` for a guaranteed bound. For TLS and reverse proxies, health checks, backups and restores, upgrades, retention, and recovery, see the [operator runbook](docs/operations.md). diff --git a/SECURITY.md b/SECURITY.md index d3a1bfd4..7eb0bcfa 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -37,9 +37,8 @@ release is supported. There are no backports to earlier tags. ## Operating Fanout safely -Fanout serves plaintext by default. Configure `server.tls.cert_file` and -`server.tls.key_file` in YAML (or `FANOUT_TLS_CERT_FILE` and -`FANOUT_TLS_KEY_FILE`), or place it behind a reverse proxy that terminates TLS. +Fanout serves plaintext by default. Configure `FANOUT_TLS_CERT_FILE` and +`FANOUT_TLS_KEY_FILE`, or place it behind a reverse proxy that terminates TLS. Browser and API reads require an authenticated account, and every OTLP request requires the separately managed ingest token. Demo instances use the same credential paths as production instances. diff --git a/THIRD_PARTY_NOTICES b/THIRD_PARTY_NOTICES index 55c75338..0230def8 100644 --- a/THIRD_PARTY_NOTICES +++ b/THIRD_PARTY_NOTICES @@ -23,7 +23,6 @@ COMPONENT INVENTORY - Go: github.com/duckdb/duckdb-go-bindings v0.10506.0 - Go: github.com/duckdb/duckdb-go/v2 v2.10506.0 - Go: github.com/dustin/go-humanize v1.1.0 -- Go: github.com/fsnotify/fsnotify v1.10.1 - Go: github.com/go-jose/go-jose/v4 v4.1.5 - Go: github.com/go-viper/mapstructure/v2 v2.5.0 - Go: github.com/google/jsonschema-go v0.4.3 @@ -31,16 +30,9 @@ COMPONENT INVENTORY - Go: github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 - Go: github.com/klauspost/compress v1.20.1 - Go: github.com/klauspost/cpuid/v2 v2.4.0 -- Go: github.com/knadh/koanf/maps v0.1.3 -- Go: github.com/knadh/koanf/parsers/yaml v1.1.1 -- Go: github.com/knadh/koanf/providers/confmap v1.0.1 -- Go: github.com/knadh/koanf/providers/file v1.2.1 -- Go: github.com/knadh/koanf/v2 v2.3.7 - Go: github.com/labstack/echo/v5 v5.4.0 - Go: github.com/mattn/go-isatty v0.0.24 - Go: github.com/mfridman/interpolate v0.0.2 -- Go: github.com/mitchellh/copystructure v1.2.0 -- Go: github.com/mitchellh/reflectwalk v1.0.2 - Go: github.com/modelcontextprotocol/go-sdk v1.8.0 - Go: github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 - Go: github.com/ncruces/go-strftime v1.1.0 @@ -1177,39 +1169,8 @@ SOFTWARE. ---- Applies to ------------------------------------------------------------- -- Go: github.com/fsnotify/fsnotify v1.10.1 / LICENSE ----------------------------------------------------------------------------- - -Copyright © 2012 The Go Authors. All rights reserved. -Copyright © fsnotify Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright notice, this - list of conditions and the following disclaimer in the documentation and/or - other materials provided with the distribution. -* Neither the name of Google Inc. nor the names of its contributors may be used - to endorse or promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - --- Applies to ------------------------------------------------------------- - Go: github.com/go-viper/mapstructure/v2 v2.5.0 / LICENSE -- Go: github.com/mitchellh/reflectwalk v1.0.2 / LICENSE ---------------------------------------------------------------------------- The MIT License (MIT) @@ -1659,36 +1620,6 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ---- Applies to ------------------------------------------------------------- -- Go: github.com/knadh/koanf/maps v0.1.3 / LICENSE -- Go: github.com/knadh/koanf/parsers/yaml v1.1.1 / LICENSE -- Go: github.com/knadh/koanf/providers/confmap v1.0.1 / LICENSE -- Go: github.com/knadh/koanf/providers/file v1.2.1 / LICENSE -- Go: github.com/knadh/koanf/v2 v2.3.7 / LICENSE ----------------------------------------------------------------------------- - -The MIT License - -Copyright (c) 2019, Kailash Nadh. https://github.com/knadh - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - --- Applies to ------------------------------------------------------------- - Go: github.com/labstack/echo/v5 v5.4.0 / LICENSE ---------------------------------------------------------------------------- @@ -1758,32 +1689,6 @@ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ---- Applies to ------------------------------------------------------------- -- Go: github.com/mitchellh/copystructure v1.2.0 / LICENSE ----------------------------------------------------------------------------- - -The MIT License (MIT) - -Copyright (c) 2014 Mitchell Hashimoto - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. - --- Applies to ------------------------------------------------------------- - Go: github.com/modelcontextprotocol/go-sdk v1.8.0 / LICENSE ---------------------------------------------------------------------------- diff --git a/cmd/fanout-docgen/main.go b/cmd/fanout-docgen/main.go index aedb0b2e..ebb93909 100644 --- a/cmd/fanout-docgen/main.go +++ b/cmd/fanout-docgen/main.go @@ -52,21 +52,19 @@ func main() { // list going stale is loud rather than a quietly shorter table. routeDirs = flag.String("route-dirs", "internal/api,internal/agent,cmd/fanout", "comma-separated directories whose files register HTTP routes") - outDir = flag.String("out", "site/src/content/docs/reference", "reference root to write generated pages into") - envExample = flag.String("env-example", ".env.example", "path to the generated environment template") - envDefaults = flag.String("env-defaults", ".env", "path to the generated nonsecret local defaults") - envProduction = flag.String("env-production", ".env.production", "path to the generated nonsecret production overrides") - check = flag.Bool("check", false, "exit non-zero when a written page differs from the one on disk") + outDir = flag.String("out", "site/src/content/docs/reference", "reference root to write generated pages into") + envExample = flag.String("env-example", ".env.example", "path to the generated environment template") + check = flag.Bool("check", false, "exit non-zero when a written page differs from the one on disk") ) flag.Parse() - if err := run(*source, *alerts, *routeDirs, *outDir, *envExample, *envDefaults, *envProduction, *check); err != nil { + if err := run(*source, *alerts, *routeDirs, *outDir, *envExample, *check); err != nil { fmt.Fprintf(os.Stderr, "fanout-docgen: %v\n", err) os.Exit(1) } } -func run(source, alertSource, routeDirs, outDir, envExample, envDefaults, envProduction string, check bool) error { +func run(source, alertSource, routeDirs, outDir, envExample string, check bool) error { fields, err := collect(source) if err != nil { return err @@ -120,20 +118,7 @@ func run(source, alertSource, routeDirs, outDir, envExample, envDefaults, envPro } pages["mcp-tools.mdx"] = toolsPage - outputs := map[string][]byte{ - envExample: renderEnvironment(fields), - envDefaults: renderDefaults(fields, false), - envProduction: renderDefaults(fields, true), - } - // Never replace an operator's dotenv file or a managed secret mount with - // generated public defaults. Check before writing any artifacts. - if !check { - for _, path := range []string{envDefaults, envProduction} { - if err := validateGeneratedDefaults(path, fields); err != nil { - return err - } - } - } + outputs := map[string][]byte{envExample: renderEnvironment(fields)} for name, body := range pages { outputs[filepath.Join(outDir, name)] = body } @@ -190,44 +175,6 @@ func run(source, alertSource, routeDirs, outDir, envExample, envDefaults, envPro return nil } -func validateGeneratedDefaults(path string, fields []field) error { - info, err := os.Lstat(path) - if os.IsNotExist(err) { - return nil - } - if err != nil { - return err - } - refuse := func() error { - return fmt.Errorf("refuse to overwrite private or unmanaged environment file %s; preserve it outside the public defaults before adopting the layout", path) - } - if !info.Mode().IsRegular() { - return refuse() - } - existing, err := os.ReadFile(path) - if err != nil { - return err - } - if !bytes.HasPrefix(existing, []byte("# Generated nonsecret configuration by cmd/fanout-docgen.\n")) { - return refuse() - } - public := make(map[string]bool, len(fields)) - for _, f := range fields { - public[f.Env] = !f.Secret - } - for _, line := range strings.Split(string(existing), "\n") { - line = strings.TrimSpace(line) - if line == "" || strings.HasPrefix(line, "#") { - continue - } - name, value, assignment := strings.Cut(line, "=") - if !assignment || !public[strings.TrimSpace(name)] || strings.ContainsAny(value, "\"'`\\") { - return refuse() - } - } - return nil -} - // alertEnvCount is set by collectAlertEnv so the summary line can report it // without threading the slice back through run's return. var alertEnvCount []field @@ -329,18 +276,17 @@ func renderAlertEnv(vars []field) []byte { // field is one setting, assembled from both views of the same declaration. type field struct { GoName string - Key string // koanf key, e.g. "storage.duckdb.memory" + Key string // alert expression variable + Group string // documentation area, e.g. "storage" Env string // e.g. "FANOUT_DUCKDB_MEMORY" Type string Default string - Local *string // nil unless the schema declares a local override Secret bool Doc string } func (f field) group() string { - key, _, _ := strings.Cut(f.Key, ".") - return key + return f.Group } // collect reflects over config.Config for the contract, parses source for the @@ -360,27 +306,22 @@ func collect(source string) ([]field, error) { if !sf.IsExported() { continue } - key := sf.Tag.Get("koanf") + group := sf.Tag.Get("group") env := sf.Tag.Get("env") - if key == "" || env == "" { + if group == "" || env == "" { return nil, fmt.Errorf( - "config.Config field %s is exported but carries no koanf/env tag; "+ + "config.Config field %s is exported but carries no group/env tag; "+ "either tag it so it reaches the reference, or unexport it", sf.Name, ) } seen[sf.Name] = true - var local *string - if value, ok := sf.Tag.Lookup("local"); ok { - local = &value - } fields = append(fields, field{ GoName: sf.Name, - Key: key, + Group: group, Env: env, Type: typeName(sf.Type), Default: sf.Tag.Get("default"), - Local: local, Secret: sf.Tag.Get("secret") == "true", Doc: docs[sf.Name], }) @@ -399,7 +340,7 @@ func collect(source string) ([]field, error) { } } - sort.Slice(fields, func(i, j int) bool { return fields[i].Key < fields[j].Key }) + sort.Slice(fields, func(i, j int) bool { return fields[i].Env < fields[j].Env }) return fields, nil } @@ -496,7 +437,7 @@ type groupPage struct { ReadWhen []string } -// Registering a group is deliberate. A new koanf prefix appearing in the type +// Registering a group is deliberate. A new group appearing in the type // is a new section of the product, and it should be named by a person rather // than published under a bare key — so an unregistered group fails the run // instead of shipping a page called "metrics" with no explanation on it. @@ -614,26 +555,30 @@ func renderPage(name string, page groupPage, fields []field) []byte { b.WriteString("{/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */}\n\n") b.WriteString("Every setting below is read once at startup and requires a restart to change.\n") - b.WriteString("It is read from the environment variable in the second\n") - b.WriteString("column, or from the same key in a YAML configuration file. An unrecognised\n") - b.WriteString("`FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces\n") - b.WriteString("as a refusal to start rather than as a default nobody chose.\n\n") + b.WriteString("Built-in defaults are overridden by nonempty environment values.\n") + b.WriteString("Unknown `FANOUT_` names fail startup; platform-injected service names are tolerated.\n") + b.WriteString("Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies.\n") + b.WriteString("Vendor credentials and endpoints keep the vendor's documented names.\n\n") b.WriteString("Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`,\n") b.WriteString("`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are\n") b.WriteString("ignored; only the explicitly selected AI provider's key and endpoint are used.\n\n") - b.WriteString("| Setting | Environment variable | Type | Default |\n") - b.WriteString("|---|---|---|---|\n") + b.WriteString("| Environment variable | Type | Default |\n") + b.WriteString("|---|---|---|\n") for _, f := range fields { typ := f.Type if f.Secret { typ += " · secret" } def := "—" - if f.Default != "" { + if f.Default != "" && !f.Secret { def = "`" + f.Default + "`" } - fmt.Fprintf(&b, "| `%s` | `%s` | %s | %s |\n", f.Key, f.Env, typ, def) + names := "`" + f.Env + "`" + if f.Secret { + names += " or `" + f.Env + "_FILE`" + } + fmt.Fprintf(&b, "| %s | %s | %s |\n", names, typ, def) } var noted []field @@ -645,15 +590,15 @@ func renderPage(name string, page groupPage, fields []field) []byte { if len(noted) > 0 { b.WriteString("\n## Notes\n") for _, f := range noted { - fmt.Fprintf(&b, "\n### `%s`\n\n%s\n", f.Key, f.Doc) + fmt.Fprintf(&b, "\n### `%s`\n\n%s\n", f.Env, f.Doc) } } if hasSecret(fields) { b.WriteString("\n:::caution[Secrets]\n") b.WriteString("Settings marked `secret` are redacted from Fanout's startup configuration\n") - b.WriteString("log. They are still plain environment variables: anything that can read the\n") - b.WriteString("process environment can read them.\n") + b.WriteString("log. Supply either the value or its `_FILE` path, never both. File contents\n") + b.WriteString("lose exactly one trailing newline (LF or CRLF); empty secrets fail startup.\n") b.WriteString(":::\n") } @@ -661,19 +606,19 @@ func renderPage(name string, page groupPage, fields []field) []byte { } // renderEnvironment shares the exact reflected schema used by the site pages. -// Empty values never override YAML; defaults are documented in comments. +// Values stay empty; defaults and secret file forms are documented in comments. func renderEnvironment(fields []field) []byte { var b strings.Builder b.WriteString("# Generated by cmd/fanout-docgen from internal/config/config.go.\n") b.WriteString("# Regenerate with just docs-generate; verify with just docs-generate-check.\n") b.WriteString("# Fanout does not load dotenv files. Inject through your launcher or deployment.\n") - b.WriteString("# Precedence: built-in defaults < --config YAML < nonempty environment values.\n") + b.WriteString("# Precedence: built-in defaults < nonempty environment values.\n") b.WriteString("# All settings require restart. Empty environment values mean no override.\n") b.WriteString("# Secret entries below are empty placeholders; use runtime secret injection.\n") b.WriteString("# Local authentication requires FANOUT_AUTH_CODE_SECRET (32+ characters).\n") b.WriteString("# Chat is disabled unless FANOUT_AI_PROVIDER selects anthropic or openai; its key is required.\n") for _, f := range fields { - fmt.Fprintf(&b, "\n# %s; %s", f.Key, f.Type) + fmt.Fprintf(&b, "\n# %s; %s", f.Env, f.Type) if f.Secret { b.WriteString("; secret") } @@ -687,36 +632,9 @@ func renderEnvironment(fields []field) []byte { } fmt.Fprintf(&b, "# Default: %s\n", defaultValue) fmt.Fprintf(&b, "%s=\n", f.Env) - } - return []byte(b.String()) -} - -// renderDefaults emits only schema-declared local overrides, without secrets. -// Production resets exactly those fields to their built-in defaults. -func renderDefaults(fields []field, production bool) []byte { - var b strings.Builder - b.WriteString("# Generated nonsecret configuration by cmd/fanout-docgen.\n") - b.WriteString("# Regenerate with just docs-generate. Credentials never belong in this file.\n") - if production { - b.WriteString("# Production overrides for .env. Deployment supplies its public URL and secrets.\n") - } else { - b.WriteString("# Shared local overrides; .env.production resets them for production.\n") - b.WriteString("# Use just run, or inject these values explicitly; Fanout does not load dotenv.\n") - b.WriteString("# Personal nonsecret overrides belong in ignored .env.local.\n") - } - for _, f := range fields { - if f.Secret || f.Local == nil { - continue - } - fmt.Fprintf(&b, "\n# %s; %s\n", f.Key, f.Type) - if f.Doc != "" { - fmt.Fprintf(&b, "# %s\n", f.Doc) - } - value := *f.Local - if production { - value = f.Default + if f.Secret { + fmt.Fprintf(&b, "# Alternatively set %s_FILE to a secret file path; do not set both.\n", f.Env) } - fmt.Fprintf(&b, "%s=%s\n", f.Env, value) } return []byte(b.String()) } diff --git a/cmd/fanout-docgen/main_test.go b/cmd/fanout-docgen/main_test.go index ccd8ad02..af9cd618 100644 --- a/cmd/fanout-docgen/main_test.go +++ b/cmd/fanout-docgen/main_test.go @@ -24,8 +24,8 @@ func TestCollectCrossChecksReflectionAgainstSource(t *testing.T) { } for _, f := range fields { - if f.Key == "" || f.Env == "" { - t.Errorf("%s: koanf and env tags are both required", f.GoName) + if f.Group == "" || f.Env == "" { + t.Errorf("%s: group and env tags are both required", f.GoName) } if f.Type == "" { t.Errorf("%s: no rendered type", f.GoName) @@ -66,81 +66,24 @@ func TestEnvironmentTemplateMatchesSchema(t *testing.T) { t.Fatalf("template has %d settings, schema has %d", len(seen), len(fields)) } for _, f := range fields { + if f.Secret && !strings.Contains(string(body), f.Env+"_FILE") { + t.Errorf("missing file form for %s", f.Env) + } value, exists := seen[f.Env] if !exists || value != "" { - t.Errorf("template must leave %s empty so it never overrides YAML", f.Env) + t.Errorf("template must leave %s empty so it preserves defaults", f.Env) } if f.Default != "" && !strings.Contains(string(body), "# Default: "+f.Default+"\n"+f.Env+"=\n") { t.Errorf("template must document the default for %s", f.Env) } } - env = append(env, "FANOUT_AUTH_CODE_SECRET="+strings.Repeat("x", 32)) - if _, err := config.Load(config.LoadOptions{Environ: env}); err != nil { - t.Fatalf("template rejected by the runtime loader: %v", err) - } -} - -func TestPublicEnvironmentDefaultsExcludeSecrets(t *testing.T) { - fields, err := collect(configSource) - if err != nil { - t.Fatal(err) - } - for _, tc := range []struct { - path string - production bool - addr string - }{ - {"../../.env", false, "127.0.0.1:7520"}, - {"../../.env.production", true, ":7520"}, - } { - body := renderDefaults(fields, tc.production) - written, err := os.ReadFile(tc.path) - if err != nil { - t.Fatal(err) - } - if string(written) != string(body) { - t.Errorf("%s is stale; run just docs-generate", tc.path) - } - for _, f := range fields { - if present := strings.Contains(string(body), f.Env+"="); present != (!f.Secret && f.Local != nil) { - t.Errorf("public defaults must contain exactly the nonsecret local overrides: %s", f.Env) - } - } - if !strings.Contains(string(body), "FANOUT_ADDR="+tc.addr+"\n") { - t.Errorf("%s has the wrong listener scope", tc.path) + for i, entry := range env { + if entry == "FANOUT_AUTH_CODE_SECRET=" { + env[i] += strings.Repeat("x", 32) } - if err := validateGeneratedDefaults(tc.path, fields); err != nil { - t.Fatal(err) - } - } -} - -func TestPublicDefaultGenerationPreservesPrivateFiles(t *testing.T) { - fields, err := collect(configSource) - if err != nil { - t.Fatal(err) } - path := filepath.Join(t.TempDir(), ".env") - for _, content := range []string{ - "AI_API_KEY=synthetic-private-value\n", - "# Generated nonsecret configuration by cmd/fanout-docgen.\nANTHROPIC_API_KEY=synthetic-private-value\n", - "# Generated nonsecret configuration by cmd/fanout-docgen.\nFANOUT_AI_API_KEY=synthetic-private-value\n", - "# Generated nonsecret configuration by cmd/fanout-docgen.\nFANOUT_DATA_DIR=\"safe\n# synthetic-private-value\"\n", - "# Generated nonsecret configuration by cmd/fanout-docgen.\nFANOUT_DATA_DIR='synthetic-private-value'\n", - "# Generated nonsecret configuration by cmd/fanout-docgen.\nFANOUT_DATA_DIR=`synthetic-private-value`\n", - "# Generated nonsecret configuration by cmd/fanout-docgen.\nFANOUT_DATA_DIR=synthetic\\private-value\n", - } { - if err := os.WriteFile(path, []byte(content), 0o600); err != nil { - t.Fatal(err) - } - err := validateGeneratedDefaults(path, fields) - if err == nil || strings.Contains(err.Error(), "synthetic-private-value") { - t.Fatal("expected redacted refusal to replace a private file") - } - unchanged, err := os.ReadFile(path) - if err != nil || string(unchanged) != content { - t.Fatal("private dotenv file was modified") - } + if _, err := config.Load(config.LoadOptions{Environ: env}); err != nil { + t.Fatalf("template rejected by the runtime loader: %v", err) } } @@ -163,7 +106,7 @@ func TestEverySettingGroupIsRegistered(t *testing.T) { t.Errorf( "setting %q is in group %q, which has no entry in the groups registry; "+ "add a title and description for it", - f.Key, f.group(), + f.Env, f.group(), ) } } @@ -190,7 +133,7 @@ func TestRenderProducesOnePageWithFrontmatterPerGroup(t *testing.T) { if !strings.Contains(text, "generated: true") { t.Errorf("%s: missing the generated marker, so the site cannot tell it apart from an authored page", name) } - if !strings.Contains(text, "| Setting | Environment variable | Type | Default |") { + if !strings.Contains(text, "| Environment variable | Type | Default |") { t.Errorf("%s: missing the settings table", name) } // Every page is indexed in llms.txt, which fails the site build when a diff --git a/cmd/fanout/main.go b/cmd/fanout/main.go index 40e2d357..c7bb293a 100644 --- a/cmd/fanout/main.go +++ b/cmd/fanout/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "crypto/tls" "errors" "flag" "fmt" @@ -13,6 +14,7 @@ import ( "net/url" "os" "os/signal" + "reflect" "regexp" "runtime" "strings" @@ -58,7 +60,7 @@ type repairCommand struct { } func main() { - configPath, showVersion, loginEmail, healthURL, repair, err := parseCommandLine(os.Args[1:], os.Stderr) + showVersion, loginEmail, healthURL, repair, err := parseCommandLine(os.Args[1:], os.Stderr) if errors.Is(err, flag.ErrHelp) { return } @@ -70,7 +72,8 @@ func main() { return } if healthURL != "" { - if err := checkHealth(healthURL); err != nil { + derivedProbe := len(os.Args) == 2 && os.Args[1] == "healthcheck" + if err := checkHealth(healthURL, derivedProbe); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } @@ -79,7 +82,7 @@ func main() { slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, nil))) - cfg, err := config.Load(config.LoadOptions{Path: configPath}) + cfg, err := config.Load(config.LoadOptions{}) if err != nil { slog.Error("invalid configuration", "err", err) os.Exit(1) @@ -373,7 +376,7 @@ func main() { agent.NewRuntime(provider, toolRegistry, agent.NewStore(sqlite.DB)).Register(e.Group("/api/agent", api.RequireCapability(api.RunAgent))) slog.Info("AG-UI agent enabled", "path", "/api/agent/runs", "provider", cfg.AIProvider) } else { - slog.Info("AG-UI agent disabled", "reason", "ai.provider (FANOUT_AI_PROVIDER) is not selected") + slog.Info("AG-UI agent disabled", "reason", "FANOUT_AI_PROVIDER is not selected") } // The compiled browser client is an embedded asset, not a second runtime. @@ -423,15 +426,19 @@ func main() { } } -func parseCommandLine(args []string, output io.Writer) (configPath string, showVersion bool, loginEmail, healthURL string, repair *repairCommand, err error) { +func parseCommandLine(args []string, output io.Writer) (showVersion bool, loginEmail, healthURL string, repair *repairCommand, err error) { if len(args) == 1 && args[0] == "version" { - return "", true, "", "", nil, nil + return true, "", "", nil, nil } if len(args) >= 1 && len(args) <= 2 && args[0] == "healthcheck" { if len(args) == 2 { - return "", false, "", args[1], nil, nil + return false, "", args[1], nil, nil } - return "", false, "", "http://127.0.0.1:7520/healthz", nil, nil + healthURL, err := healthcheckURL(os.Getenv("FANOUT_ADDR"), os.Getenv("FANOUT_TLS_CERT_FILE")) + if err != nil { + fmt.Fprintln(output, err) + } + return false, "", healthURL, nil, err } flags := flag.NewFlagSet("fanout", flag.ContinueOnError) @@ -439,34 +446,33 @@ func parseCommandLine(args []string, output io.Writer) (configPath string, showV flags.Usage = func() { fmt.Fprintln(output, "Usage of fanout:") fmt.Fprintln(output, " fanout [flags]") - fmt.Fprintln(output, " fanout [--config path] login-link ") - fmt.Fprintln(output, " fanout [--config path] repair verify") - fmt.Fprintln(output, " fanout [--config path] repair quarantine --batch ") + fmt.Fprintln(output, " fanout login-link ") + fmt.Fprintln(output, " fanout repair verify") + fmt.Fprintln(output, " fanout repair quarantine --batch ") fmt.Fprintln(output, " fanout healthcheck [url]") fmt.Fprintln(output, " fanout version") fmt.Fprintln(output, "Flags:") flags.PrintDefaults() } - flags.StringVar(&configPath, "config", "", "path to a Fanout YAML configuration file") flags.BoolVar(&showVersion, "version", false, "print the Fanout version") flags.BoolVar(&showVersion, "v", false, "print the Fanout version") if err := flags.Parse(args); err != nil { - return "", false, "", "", nil, err + return false, "", "", nil, err } if flags.NArg() == 2 && flags.Arg(0) == "login-link" { - return configPath, false, flags.Arg(1), "", nil, nil + return false, flags.Arg(1), "", nil, nil } if flags.NArg() > 0 && flags.Arg(0) == "repair" { repair, err := parseRepairCommand(flags.Args()[1:], output) - return configPath, false, "", "", repair, err + return false, "", "", repair, err } if flags.NArg() != 0 { err := fmt.Errorf("unexpected arguments: %s", strings.Join(flags.Args(), " ")) fmt.Fprintln(output, err) flags.Usage() - return "", false, "", "", nil, err + return false, "", "", nil, err } - return configPath, showVersion, "", "", nil, nil + return showVersion, "", "", nil, nil } func parseRepairCommand(args []string, output io.Writer) (*repairCommand, error) { @@ -518,8 +524,43 @@ func runRepair(root string, command repairCommand, output io.Writer) error { } } -func checkHealth(healthURL string) error { +func healthcheckURL(addr, certFile string) (string, error) { + if addr == "" { + field, _ := reflect.TypeOf(config.Config{}).FieldByName("Addr") + addr = field.Tag.Get("default") + } + host, port, err := net.SplitHostPort(addr) + if err != nil { + return "", fmt.Errorf("FANOUT_ADDR must be a host:port address: %w", err) + } + if host == "" { + host = "127.0.0.1" + } + if ip := net.ParseIP(host); ip != nil && ip.IsUnspecified() { + host = "127.0.0.1" + if ip.To4() == nil { + host = "::1" + } + } + scheme := "http" + if certFile != "" { + scheme = "https" + } + return (&url.URL{Scheme: scheme, Host: net.JoinHostPort(host, port), Path: "/healthz"}).String(), nil +} + +func checkHealth(healthURL string, derivedProbe bool) error { client := &http.Client{Timeout: 3 * time.Second} + if derivedProbe { + // A derived probe targets this process's own listener directly, whose + // certificate may name its public hostname rather than any bind IP. + // Explicit URLs retain environment proxies and certificate verification. + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + defer transport.CloseIdleConnections() + client.Transport = transport + } response, err := client.Get(healthURL) if err != nil { return fmt.Errorf("healthcheck: %w", err) diff --git a/cmd/fanout/main_test.go b/cmd/fanout/main_test.go index 3fa98739..f460be73 100644 --- a/cmd/fanout/main_test.go +++ b/cmd/fanout/main_test.go @@ -3,16 +3,25 @@ package main import ( "bytes" "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" "errors" "flag" "io" + "math/big" + "net" "net/http" "net/http/httptest" "net/url" "os" + "os/exec" "path/filepath" "strings" "testing" + "time" "github.com/labstack/fanout/internal/auth" "github.com/labstack/fanout/internal/config" @@ -22,10 +31,12 @@ import ( ) func TestParseCommandLine(t *testing.T) { + t.Setenv("FANOUT_ADDR", "") + t.Setenv("FANOUT_TLS_CERT_FILE", "") + t.Setenv("FANOUT_AUTH_CODE_SECRET", "") tests := []struct { name string args []string - wantPath string wantVersion bool wantEmail string wantHealth string @@ -33,16 +44,14 @@ func TestParseCommandLine(t *testing.T) { wantErr bool }{ {name: "server defaults", args: nil}, - {name: "config", args: []string{"--config", "/etc/fanout.yaml"}, wantPath: "/etc/fanout.yaml"}, {name: "version command", args: []string{"version"}, wantVersion: true}, {name: "version flag", args: []string{"--version"}, wantVersion: true}, {name: "short version flag", args: []string{"-v"}, wantVersion: true}, {name: "login link", args: []string{"login-link", "admin@example.com"}, wantEmail: "admin@example.com"}, - {name: "login link with config", args: []string{"--config", "/etc/fanout.yaml", "login-link", "admin@example.com"}, wantPath: "/etc/fanout.yaml", wantEmail: "admin@example.com"}, {name: "default healthcheck", args: []string{"healthcheck"}, wantHealth: "http://127.0.0.1:7520/healthz"}, {name: "custom healthcheck", args: []string{"healthcheck", "http://fanout:8080/healthz"}, wantHealth: "http://fanout:8080/healthz"}, {name: "repair verify", args: []string{"repair", "verify"}, wantRepair: &repairCommand{action: "verify"}}, - {name: "repair quarantine with config", args: []string{"--config", "/etc/fanout.yaml", "repair", "quarantine", "--batch", "broken"}, wantPath: "/etc/fanout.yaml", wantRepair: &repairCommand{action: "quarantine", batch: "broken"}}, + {name: "repair quarantine", args: []string{"repair", "quarantine", "--batch", "broken"}, wantRepair: &repairCommand{action: "quarantine", batch: "broken"}}, {name: "repair quarantine missing batch", args: []string{"repair", "quarantine"}, wantErr: true}, {name: "unknown repair action", args: []string{"repair", "delete"}, wantErr: true}, {name: "login link missing email", args: []string{"login-link"}, wantErr: true}, @@ -51,12 +60,12 @@ func TestParseCommandLine(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - path, showVersion, email, healthURL, repair, err := parseCommandLine(test.args, io.Discard) + showVersion, email, healthURL, repair, err := parseCommandLine(test.args, io.Discard) if (err != nil) != test.wantErr { t.Fatalf("error = %v, wantErr %v", err, test.wantErr) } - if path != test.wantPath || showVersion != test.wantVersion || email != test.wantEmail || healthURL != test.wantHealth { - t.Fatalf("result = (%q, %v, %q, %q), want (%q, %v, %q, %q)", path, showVersion, email, healthURL, test.wantPath, test.wantVersion, test.wantEmail, test.wantHealth) + if showVersion != test.wantVersion || email != test.wantEmail || healthURL != test.wantHealth { + t.Fatalf("result = (%v, %q, %q), want (%v, %q, %q)", showVersion, email, healthURL, test.wantVersion, test.wantEmail, test.wantHealth) } if repair == nil != (test.wantRepair == nil) || repair != nil && *repair != *test.wantRepair { t.Fatalf("repair = %#v, want %#v", repair, test.wantRepair) @@ -66,7 +75,7 @@ func TestParseCommandLine(t *testing.T) { } func TestParseCommandLineHelp(t *testing.T) { - _, _, _, _, _, err := parseCommandLine([]string{"--help"}, io.Discard) + _, _, _, _, err := parseCommandLine([]string{"--help"}, io.Discard) if !errors.Is(err, flag.ErrHelp) { t.Fatalf("error = %v, want flag.ErrHelp", err) } @@ -74,7 +83,7 @@ func TestParseCommandLineHelp(t *testing.T) { func TestParseCommandLinePrintsOneErrorAndUsage(t *testing.T) { var output bytes.Buffer - _, _, _, _, _, err := parseCommandLine([]string{"serve"}, &output) + _, _, _, _, err := parseCommandLine([]string{"serve"}, &output) if err == nil { t.Fatal("expected unexpected-argument error") } @@ -110,7 +119,7 @@ func TestCheckHealth(t *testing.T) { writer.WriteHeader(test.statusCode) })) defer server.Close() - if err := checkHealth(server.URL); (err != nil) != test.wantErr { + if err := checkHealth(server.URL, false); (err != nil) != test.wantErr { t.Fatalf("checkHealth() error = %v, wantErr %v", err, test.wantErr) } }) @@ -205,3 +214,184 @@ func TestCreateLoginLink(t *testing.T) { t.Fatalf("issued audit events = %d, want 1", events) } } + +func TestHealthcheckURL(t *testing.T) { + for _, tc := range []struct{ addr, cert, want string }{ + {"", "", "http://127.0.0.1:7520/healthz"}, + {":8080", "", "http://127.0.0.1:8080/healthz"}, + {"0.0.0.0:8080", "", "http://127.0.0.1:8080/healthz"}, + {"[::]:8080", "", "http://[::1]:8080/healthz"}, + {"[::1]:8080", "", "http://[::1]:8080/healthz"}, + {"[2001:db8::1]:8080", "", "http://[2001:db8::1]:8080/healthz"}, + {"127.0.0.1:9000", "cert.pem", "https://127.0.0.1:9000/healthz"}, + {"localhost:9000", "", "http://localhost:9000/healthz"}, + {"example.com:9000", "", "http://example.com:9000/healthz"}, + } { + got, err := healthcheckURL(tc.addr, tc.cert) + if err != nil || got != tc.want { + t.Fatalf("healthcheckURL(%q) = %q, %v; want %q", tc.addr, got, err, tc.want) + } + } + if _, err := healthcheckURL("invalid", ""); err == nil { + t.Fatal("expected invalid address error") + } +} + +func TestHealthcheckUsesEnvironmentWithoutValidatingSecrets(t *testing.T) { + t.Setenv("FANOUT_ADDR", "[::]:9090") + t.Setenv("FANOUT_TLS_CERT_FILE", "cert.pem") + t.Setenv("FANOUT_AUTH_CODE_SECRET_FILE", "/missing") + _, _, probe, _, err := parseCommandLine([]string{"healthcheck"}, io.Discard) + if err != nil || probe != "https://[::1]:9090/healthz" { + t.Fatalf("probe = %s, %v", probe, err) + } +} + +func TestCheckHealthWithLocalTLS(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) + defer server.Close() + if err := checkHealth(server.URL+"/healthz", true); err != nil { + t.Fatal(err) + } + if err := checkHealth(server.URL+"/healthz", false); err == nil { + t.Fatal("explicit URL must verify the certificate") + } +} + +func TestCheckHealthWithInterfaceTLS(t *testing.T) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + certificate := &x509.Certificate{ + SerialNumber: big.NewInt(1), DNSNames: []string{"fanout.example.com"}, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, certificate, certificate, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/healthz" { + t.Errorf("probe path = %q, want /healthz", request.URL.Path) + } + w.WriteHeader(http.StatusOK) + })) + server.TLS = &tls.Config{Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: key}}} + server.StartTLS() + defer server.Close() + roots := x509.NewCertPool() + roots.AddCert(server.Certificate()) + for _, addr := range []string{"192.0.2.10:7520", "[2001:db8::10]:7520"} { + t.Run(addr, func(t *testing.T) { + original := http.DefaultTransport + transport := original.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{RootCAs: roots} + // Route the interface bind to a local test listener so both IP families + // exercise real TLS without requiring those addresses on the test host. + transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { + if address != addr { + t.Errorf("probe address = %q, want %q", address, addr) + } + var dialer net.Dialer + return dialer.DialContext(ctx, network, server.Listener.Addr().String()) + } + http.DefaultTransport = transport + t.Cleanup(func() { + http.DefaultTransport = original + transport.CloseIdleConnections() + }) + t.Setenv("FANOUT_ADDR", addr) + t.Setenv("FANOUT_TLS_CERT_FILE", "cert.pem") + _, _, probe, _, err := parseCommandLine([]string{"healthcheck"}, io.Discard) + if err != nil { + t.Fatal(err) + } + if err := checkHealth(probe, true); err != nil { + t.Fatalf("derived TLS probe: %v", err) + } + _, _, explicitProbe, _, err := parseCommandLine([]string{"healthcheck", probe}, io.Discard) + if err != nil { + t.Fatal(err) + } + var hostnameError x509.HostnameError + if err := checkHealth(explicitProbe, false); !errors.As(err, &hostnameError) { + t.Fatalf("explicit TLS probe must verify the hostname, got %v", err) + } + }) + } +} + +func TestCheckHealthDerivedIgnoresEnvironmentProxy(t *testing.T) { + // ProxyFromEnvironment caches settings on first use, so use a fresh process + // to ensure earlier tests cannot hide the configured proxies. + if os.Getenv("FANOUT_TEST_PROXY_CHILD") != "1" { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + proxy := "http://" + listener.Addr().String() + if err := listener.Close(); err != nil { + t.Fatal(err) + } + t.Setenv("HTTP_PROXY", proxy) + t.Setenv("HTTPS_PROXY", proxy) + t.Setenv("NO_PROXY", "") + t.Setenv("no_proxy", "") + t.Setenv("REQUEST_METHOD", "") + t.Setenv("FANOUT_TEST_PROXY_CHILD", "1") + command := exec.Command(os.Args[0], "-test.run=^TestCheckHealthDerivedIgnoresEnvironmentProxy$", "-test.v") + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("proxy regression subprocess: %v\n%s", err, output) + } + return + } + for _, scheme := range []string{"http", "https"} { + t.Run(scheme, func(t *testing.T) { + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if request.URL.Path != "/healthz" { + t.Errorf("probe path = %q, want /healthz", request.URL.Path) + } + w.WriteHeader(http.StatusOK) + })) + certFile := "" + if scheme == "https" { + server.StartTLS() + certFile = "cert.pem" + } else { + server.Start() + } + defer server.Close() + const addr = "192.0.2.10:7520" + original := http.DefaultTransport + transport := original.(*http.Transport).Clone() + transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { + if address == addr { + address = server.Listener.Addr().String() + } else if "http://"+address != os.Getenv("HTTP_PROXY") { + return nil, errors.New("unexpected dial address: " + address) + } + var dialer net.Dialer + return dialer.DialContext(ctx, network, address) + } + http.DefaultTransport = transport + t.Cleanup(func() { + http.DefaultTransport = original + transport.CloseIdleConnections() + }) + t.Setenv("FANOUT_ADDR", addr) + t.Setenv("FANOUT_TLS_CERT_FILE", certFile) + _, _, probe, _, err := parseCommandLine([]string{"healthcheck"}, io.Discard) + if err != nil { + t.Fatal(err) + } + if err := checkHealth(probe, true); err != nil { + t.Fatalf("derived %s probe with unreachable proxy: %v", scheme, err) + } + if err := checkHealth(probe, false); err == nil || !strings.Contains(err.Error(), strings.TrimPrefix(os.Getenv("HTTP_PROXY"), "http://")) { + t.Fatalf("explicit %s probe must use the unreachable proxy, got %v", scheme, err) + } + }) + } +} diff --git a/context7.json b/context7.json index fe906ee4..69a99477 100644 --- a/context7.json +++ b/context7.json @@ -15,6 +15,6 @@ "Telemetry arrives over OTLP/gRPC or OTLP/HTTP on port 7520. Point an OpenTelemetry SDK or Collector at it; do not invent a bespoke ingest protocol.", "Parquet is the authoritative telemetry store. DuckDB query state and rollups are rebuildable from it, and SQLite is reserved for transactional product state.", "Fanout exposes an MCP server, and the browser client, the in-process agent and any external MCP host all reach the same typed observability contract rather than issuing raw SQL.", - "Configuration is by `FANOUT_*` environment variables or a YAML file; see fanout.example.yaml. The Go module path is `github.com/labstack/fanout`." + "Configuration uses process environment over built-in defaults; see .env.example. Custom names use `FANOUT_`; AI keys and endpoints use documented vendor names. Secret variables also accept `_FILE` paths. The Go module path is `github.com/labstack/fanout`." ] } diff --git a/docs/authentication.md b/docs/authentication.md index 4dd4ea40..126315c6 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -72,13 +72,13 @@ If you are locked out in local mode, run the command with the same configuration and data directory as the server: ```sh -fanout --config /etc/fanout/fanout.yaml login-link admin@example.com +fanout login-link admin@example.com ``` For a container using the bundled configuration: ```sh -docker exec fanout --config /etc/fanout/fanout.yaml \ +docker exec fanout \ login-link admin@example.com ``` diff --git a/docs/operations.md b/docs/operations.md index 308b5a02..cc9c56c7 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -6,105 +6,140 @@ give ingest credentials only to trusted services and collectors. ## Runtime secret delivery -Runtime configuration uses built-in defaults, then an explicit `--config` YAML -document, then recognized nonempty environment values. Every setting requires -a restart. Fanout does not load dotenv files itself. The generated root -`.env.example` documents every default in comments and leaves all values empty; -`.env` and `.env.production` contain only nonsecret environment-specific overrides. - -| File | Use | Public Git policy | -| --- | --- | --- | -| `.env` | Local loopback listener override | Committed; generated; no secret fields | -| `.env.production` | Production listener override | Committed; generated; no deployment-specific credentials or origins | -| `.env.example` | Complete configuration template | Committed; generated; all values empty | -| `.env.local` / `.env.production.local` | Personal nonsecret files | Ignored; production never loads them | -| `secrets/fanout.env` | Private complete SOPS credential profile, when used | Operator-owned; ignored and excluded from Docker | -| `secrets/fanout.production.env` | Private production profile, when credentials differ | Operator-owned; ignored and excluded from Docker | - -`.env` is now tracked and generated. Before pulling, move nonsecret personal -values from an older private `.env` to ignored `.env.local` and credentials to -runtime injection, then preserve the old file outside the checkout. Git refuses -to overwrite an untracked `.env`. - -Build the binary, then use `just run` locally or `just run-production` for -production. The Node launcher uses `node:util.parseEnv`; dotenv is not shell code -and values are not interpolated. Local order is `.env`, optional `.env.local`, -then inherited/injected values. Production order is `.env`, required -`.env.production`, then inherited/injected values; no personal files are loaded. -Unknown names, secret entries, and unresolved `op://` references in ordinary -dotenv files fail before launch. A missing required file never selects another -profile. Empty values retain Fanout's no-override semantics. - -The launcher injects nonempty dotenv settings as environment overrides of -`--config` YAML. The generated files override only the listener; other YAML -settings survive both launch modes. `.env.production` resets every local override -to its schema default. Supported binary flags are forwarded unchanged. - -Keep ordinary YAML/dotenv configuration nonsecret where possible. Deliver -credentials through deployment environment injection, restricted managed mounts, -or a platform secret service. Do not bake credentials into images or frontend -bundles. Local dotenv scratch files stay ignored and outside Docker inputs. - -For repository-managed application secrets adopting Canon, use one authoritative -SOPS-encrypted profile per workload/credential set, such as -`secrets/fanout.env`, or `secrets/fanout.production.env` when production actually -uses different credentials. Select one complete profile explicitly; do not layer -production credentials over development credentials or fall back on a missing -profile. Platform-owned credentials remain platform-owned. - -Operator profiles and recipient policy are excluded from this public product -repository. Create them only for actual credential sets; do not ship working -development credentials or invent environment profiles. Existing private -credential migration sources remain ignored and are never loaded automatically. - -With a provisioned SOPS key and an existing encrypted profile, use +Runtime configuration uses built-in defaults followed by nonempty process +environment values. Every setting requires a restart. Fanout reads no +configuration document or dotenv file. Generated `.env.example` lists the +settings with empty values and defaults in comments. For development, `just run` +loads optional ignored `./.env` with Node's native `parseEnv`; inherited values +win, including empty values. Personal development credentials may live there. +Personal environment files and operator profiles are excluded from Git and Docker. + +Custom names use `FANOUT_` plus a clear name, adding the area only when it +clarifies. Vendor credentials and endpoints keep their documented names. +Unknown `FANOUT_` names fail startup, except platform service-discovery names; +unrelated names in shared vendor namespaces are ignored. + +### Environment and mounted secret files + +Deliver credentials through deployment environment injection or managed mounts. +Every secret accepts either its value or its file path: + +| Value variable | File path variable | +| --- | --- | +| `FANOUT_AUTH_CODE_SECRET` | `FANOUT_AUTH_CODE_SECRET_FILE` | +| `FANOUT_OIDC_CLIENT_SECRET` | `FANOUT_OIDC_CLIENT_SECRET_FILE` | +| `FANOUT_SMTP_PASSWORD` | `FANOUT_SMTP_PASSWORD_FILE` | +| `FANOUT_METRICS_TOKEN` | `FANOUT_METRICS_TOKEN_FILE` | +| `ANTHROPIC_API_KEY` | `ANTHROPIC_API_KEY_FILE` | +| `OPENAI_API_KEY` | `OPENAI_API_KEY_FILE` | + +Both forms nonempty fail startup. Fanout strips exactly one trailing LF or CRLF +from file contents, then rejects an empty result. Errors identify the variable +and path, never the contents. Fanout performs no file-mode check; the deployment +controls mount access. Do not bake credentials into images or frontend bundles. + +For Docker, mount a provisioned secret and inject its path: + +```sh +docker run --name fanout -p 7520:7520 \ + -v fanout-data:/var/lib/fanout/data \ + --mount type=bind,src="$AUTH_SECRET_PATH",dst=/run/secrets/auth_code,readonly \ + -e FANOUT_AUTH_CODE_SECRET_FILE=/run/secrets/auth_code \ + labstack/fanout:latest +``` + +The source secret must be readable by the image's UID 65532. In Compose, grant +only the required secret to the service: + +```yaml +services: + fanout: + image: labstack/fanout:latest + ports: ["7520:7520"] + volumes: ["fanout-data:/var/lib/fanout/data"] + environment: + FANOUT_AUTH_CODE_SECRET_FILE: /run/secrets/auth_code + secrets: [auth_code] +secrets: + auth_code: + file: ${AUTH_SECRET_PATH} +volumes: + fanout-data: +``` + +For Kubernetes, put ordinary settings in a ConfigMap and credentials in a +Secret provisioned through your platform's secret manager. This Pod fragment +injects both without putting credential values in the manifest: + +```yaml +containers: + - name: fanout + image: labstack/fanout:latest + envFrom: + - configMapRef: + name: fanout-settings + env: + - name: FANOUT_AUTH_CODE_SECRET + valueFrom: + secretKeyRef: + name: fanout-credentials + key: auth_code_secret +``` + +Alternatively mount the Secret as files and set the corresponding `_FILE` +paths. Use the existing persistent-volume and access policy for telemetry data. + +### SOPS runtime injection + +Use one authoritative SOPS-encrypted profile per actual workload/credential +set, such as ignored `secrets/fanout.env`. Select a complete profile explicitly; +never layer different credential sets or fall back on a missing profile. +Platform-owned credentials remain platform-owned. This public repository ships +no operator secrets, recipient policy, or decryption keys. + +With a provisioned SOPS key and existing encrypted profile, launch through [SOPS runtime injection](https://getsops.io/docs/usage/advanced/#passing-secrets-to-other-processes) -to launch without writing decrypted credentials to disk: +without writing decrypted credentials to disk: ```sh -sops exec-env secrets/fanout.env './bin/fanout --config fanout.yaml' -# Or use the shared nonsecret dotenv defaults: +sops exec-env secrets/fanout.env './bin/fanout' +# For local development with optional personal settings: sops exec-env secrets/fanout.env 'just run' -# Select a complete profile for actual production resources: -sops exec-env secrets/fanout.production.env 'just run-production' ``` -This example injects only secrets; ordinary settings remain in `fanout.yaml` -or deployment inputs. SOPS supplies the profile's environment to the child; -avoid conflicting secret values in inherited shell variables. Verify synthetic -values containing dollars, quotes, spaces, and newlines through the actual -deployment parser before migrating a profile. Keep private age identities out -of Git, restrict their local files to mode `0600`, and document recovery in the -product's existing 1Password item. 1Password holds key recovery rather than a -second independently maintained copy of SOPS-owned application secrets. - -Existing vault-owned delivery remains valid until its migration is verified. -Before retiring a vault field or old variable name, inventory local, CI, -deployment, shared-project, supported-version, and recovery consumers using -names and safe metadata. Update their mappings, verify the real consumers, and -verify recovery from the destination before removing the old authority. This -repository ships no credential profiles or decryption keys. - -AI chat is disabled by default. Select exactly `anthropic` or `openai` through -`ai.provider` or `FANOUT_AI_PROVIDER`, then deliver its required `ANTHROPIC_API_KEY` -or `OPENAI_API_KEY`; a selected provider without its key fails startup. Inherited -keys do not enable chat. With no provider selected, AI keys and endpoints are -ignored; only the selected provider's endpoint is validated and used. -Provider endpoint overrides are `ANTHROPIC_BASE_URL` and -`OPENAI_BASE_URL`; the OpenAI base includes `/v1`, while Anthropic's excludes it. -The old generic key/base URL names and YAML keys are rejected; the -[configuration upgrade instructions](../site/src/content/docs/start/install.mdx#ai-configuration-upgrades) describe -their exact replacements. Verify authentication and request paths against the -configured provider after rollout, without logging credentials. +The profile supplies environment values to the child; ordinary settings may +also come from deployment inputs. A dotenv parser neither decrypts SOPS nor +resolves 1Password references. Verify synthetic values containing dollars, +quotes, spaces, and newlines through the actual deployment parser before +migrating a profile. Keep private age identities out of Git, restrict their +local files to mode `0600`, and document key recovery in the existing 1Password +item. 1Password holds recovery rather than a second maintained copy of +SOPS-owned application secrets. + +Existing vault/platform delivery remains authoritative until consumers and +recovery are verified. Before retiring a vault field or old name, inventory +local, CI, deployment, shared-project, supported-version, and recovery consumers +using names and safe metadata. Update mappings, verify real consumers, and +verify recovery from the destination before removing the old authority. Private +migration sources stay ignored and are never loaded automatically. + +AI chat is disabled by default. Select exactly `anthropic` or `openai` with +`FANOUT_AI_PROVIDER`, then deliver its required `ANTHROPIC_API_KEY` or +`OPENAI_API_KEY`. A selected provider without its key fails startup. Inherited +keys never enable chat. Only the selected endpoint is validated and used. +Overrides are `ANTHROPIC_BASE_URL` and `OPENAI_BASE_URL`; the OpenAI base +includes `/v1`, while Anthropic's excludes it. See the +[configuration upgrade table](../site/src/content/docs/start/install.mdx#configuration-upgrades). ## Network surface | Default | Purpose | Exposure | |---|---|---| -| `:7520` | Browser, API, MCP, health, metrics, OTLP/gRPC and OTLP/HTTP | Users, trusted services or a reverse proxy | +| `127.0.0.1:7520` (native), `:7520` (image) | Browser, API, MCP, health, metrics, OTLP/gRPC and OTLP/HTTP | Users, trusted services or a reverse proxy | -Both native and container deployments use one listener. Set -`FANOUT_ADDR=127.0.0.1:7520` for loopback-only access. Every OTLP request +Both native and container deployments use one listener. Native installs default +to loopback; set `FANOUT_ADDR=:7520` for remote access. The image supplies +`FANOUT_ADDR=:7520` and `FANOUT_DATA_DIR=/var/lib/fanout/data`. Every OTLP request requires the ingest token created during first-admin setup; browser sessions and MCP tokens do not authorize ingest. Do not embed the ingest token in a browser or shipped mobile application. @@ -144,6 +179,14 @@ original host. ## Health and monitoring +`fanout healthcheck` derives `/healthz` from `FANOUT_ADDR` or its built-in +default. Empty and unspecified bind hosts become loopback, including IPv6. +`FANOUT_TLS_CERT_FILE` selects HTTPS. The local loopback probe skips certificate +verification because the certificate may be issued for the public hostname. +Healthcheck reads only these two settings and does not require valid secrets. +An explicit `fanout healthcheck ` overrides URL derivation. + + - `/healthz` is the liveness check. - `/readyz` includes storage readiness and resolved runtime sizing. - `/metrics` exposes Prometheus metrics. Keep it private or configure @@ -207,22 +250,15 @@ cascades to sessions, OAuth grants, and owned dashboards. Use `fanout login-link ## Single-port migration -This is a breaking configuration change: - -1. Remove `FANOUT_OTLP_GRPC_ADDR` and `FANOUT_OTLP_HTTP_ADDR`, or YAML - `ingest.otlp_grpc_addr` and `ingest.otlp_http_addr`. They are rejected at startup. -2. Rename `FANOUT_HTTP_ADDR` to `FANOUT_ADDR`, or YAML `server.http_addr` - to `server.addr` (default `:7520`). Publish only that container port. -3. Change exporters from ports `4317`/`4318` to `7520`, or to the public HTTPS - origin and port. The ingest token and OTLP signal paths do not change. -4. Enable HTTP/2 backend forwarding for gRPC at any proxy. Remove obsolete - ingest port mappings and proxy upstreams. A separate ingest hostname may - still route to the same Fanout port if required by the edge configuration. -5. Set `FANOUT_PUBLIC_URL` to the external origin so setup instructions start - with the reachable endpoint. If collectors use a different private hostname, - edit the endpoint on **Connect telemetry** before copying the configuration. - Remove `FANOUT_INGEST_ADVERTISED_ENDPOINT` or YAML - `ingest.advertised_endpoint`; these guidance-only settings are rejected. +All transports use `FANOUT_ADDR`. Exporters use port `7520` or the public +HTTPS origin and port; the ingest token and signal paths remain the same. +Enable HTTP/2 backend forwarding for gRPC at proxies and remove obsolete ingest +port mappings and upstreams. A separate ingest hostname may still route to the +same Fanout port if required by the edge configuration. + +Set `FANOUT_PUBLIC_URL` to the external origin. If collectors use a different +private hostname, edit the endpoint on **Connect telemetry** before copying the +configuration. Local comparison results are in [the single-port benchmark report](benchmarks/single-port-2026-09-08.md). @@ -238,10 +274,7 @@ ingest token is still required on the private path. Single-port removes the need for a proxy solely to combine listeners. Splitting public application access from private ingest still requires network-aware -routing. The mandatory default is deliberate for this pre-release product: -there is one existing operator, so keeping two listener topologies would add -configuration and testing costs without an installed compatibility base. The -address rename is a separate breaking change, bundled into this migration. +routing. The shared listener keeps one network topology for this pre-release product. ## Upgrade and rollback @@ -262,7 +295,7 @@ SMTP is optional. Creating a local user without it succeeds and returns operator with access to the same configuration and data directory then runs: ```sh -fanout --config /etc/fanout/fanout.yaml login-link user@example.com +fanout login-link user@example.com ``` The resulting link expires after 15 minutes, works once, and is recorded in the diff --git a/fanout.docker.yaml b/fanout.docker.yaml deleted file mode 100644 index ad49d6e6..00000000 --- a/fanout.docker.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# One listener for the browser, API, MCP and both OTLP transports. -# Runtime environment variables override these container defaults. -server: - addr: ":7520" - -storage: - data_dir: /var/lib/fanout/data diff --git a/fanout.example.yaml b/fanout.example.yaml deleted file mode 100644 index c303a4c2..00000000 --- a/fanout.example.yaml +++ /dev/null @@ -1,94 +0,0 @@ -# Fanout runtime configuration. -# -# Start with: fanout --config /path/to/fanout.yaml -# Every value can be overridden by its documented environment variable. -# Secrets are intentionally omitted below; inject them through the environment -# or add them to a suitably protected copy of this file. -# Use YAML 1.2 true/false for booleans; yes/no/on/off are strings and rejected. -# Null values (a key with nothing after the colon) are rejected. - -server: - addr: ":7520" # FANOUT_ADDR - public_url: "" # FANOUT_PUBLIC_URL - trusted_proxy_cidrs: "" # FANOUT_TRUSTED_PROXY_CIDRS - pprof_enabled: false # FANOUT_PPROF_ENABLED - tls: - cert_file: "" # FANOUT_TLS_CERT_FILE - key_file: "" # FANOUT_TLS_KEY_FILE - -ingest: - default_namespace: default # FANOUT_DEFAULT_NAMESPACE - batch_size: 50000 # FANOUT_INGEST_BATCH_SIZE - # Decoded telemetry held across concurrent requests before new ones are - # refused with a retryable status. 0 disables the ceiling. - max_in_flight_bytes: 268435456 # FANOUT_INGEST_MAX_IN_FLIGHT_BYTES - -storage: - data_dir: ./data # FANOUT_DATA_DIR - retention_days: 30 # FANOUT_RETENTION_DAYS - rollup_interval: 1m # FANOUT_ROLLUP_INTERVAL - rollup_skip_to_latest: false # FANOUT_ROLLUP_SKIP_TO_LATEST - maintenance_interval: 1h # FANOUT_MAINTENANCE_INTERVAL - duckdb: - # Empty/zero values let Fanout size these from the machine. - memory: "" # FANOUT_DUCKDB_MEMORY - threads: 0 # FANOUT_DUCKDB_THREADS - max_connections: 0 # FANOUT_DUCKDB_MAX_CONNECTIONS - -alerts: - enabled: true # FANOUT_ALERTS_ENABLED - evaluation_interval: 30s # FANOUT_ALERTS_EVALUATION_INTERVAL - history_days: 7 # FANOUT_ALERTS_HISTORY_DAYS - -ai: - provider: "" # FANOUT_AI_PROVIDER; empty disables chat; select anthropic or openai - model: "" # FANOUT_AI_MODEL - anthropic: - # Defaults to https://api.anthropic.com; Fanout appends /v1/messages. - base_url: "" # ANTHROPIC_BASE_URL - # Required when provider is anthropic: ANTHROPIC_API_KEY - # api_key: "" - openai: - # Defaults to https://api.openai.com/v1; Fanout appends /responses. - base_url: "" # OPENAI_BASE_URL - # Required when provider is openai: OPENAI_API_KEY - # api_key: "" - -smtp: - host: "" # FANOUT_SMTP_HOST - port: 587 # FANOUT_SMTP_PORT - username: "" # FANOUT_SMTP_USERNAME - from: "" # FANOUT_SMTP_FROM - # Optional with the other SMTP fields; enables email-code login: FANOUT_SMTP_PASSWORD - # password: "" - -auth: - mode: local # FANOUT_AUTH_MODE - session_idle_ttl: 12h # FANOUT_SESSION_IDLE_TTL - session_absolute_ttl: 168h # FANOUT_SESSION_ABSOLUTE_TTL - # Required in local auth mode: FANOUT_AUTH_CODE_SECRET (32+ characters) - # code_secret: "" - # Allow verified email addresses to create viewer accounts. - self_signup: false # FANOUT_SELF_SIGNUP - oidc: - issuer_url: "" # FANOUT_OIDC_ISSUER_URL - client_id: "" # FANOUT_OIDC_CLIENT_ID - email_claim: email # FANOUT_OIDC_EMAIL_CLAIM - email_verification: required # FANOUT_OIDC_EMAIL_VERIFICATION - auto_provision: false # FANOUT_OIDC_AUTO_PROVISION - allowed_groups: "" # FANOUT_OIDC_ALLOWED_GROUPS - # Use "*" only with email_verification: required to admit any verified identity. - allowed_domains: "" # FANOUT_OIDC_ALLOWED_DOMAINS - default_role: viewer # FANOUT_OIDC_DEFAULT_ROLE - operator_groups: "" # FANOUT_OIDC_OPERATOR_GROUPS - admin_groups: "" # FANOUT_OIDC_ADMIN_GROUPS - # Required in OIDC mode: FANOUT_OIDC_CLIENT_SECRET - # client_secret: "" - -metrics: - public: false # FANOUT_METRICS_PUBLIC - # Optional bearer token: FANOUT_METRICS_TOKEN - # token: "" - -mcp: - enabled: true # FANOUT_MCP_ENABLED diff --git a/go.mod b/go.mod index e8fa1808..1a85b188 100644 --- a/go.mod +++ b/go.mod @@ -9,14 +9,9 @@ require ( github.com/alexedwards/scs/v2 v2.9.0 github.com/coreos/go-oidc/v3 v3.21.0 github.com/duckdb/duckdb-go/v2 v2.10506.0 - github.com/go-viper/mapstructure/v2 v2.5.0 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/jsonschema-go v0.4.3 github.com/google/uuid v1.6.0 - github.com/knadh/koanf/parsers/yaml v1.1.1 - github.com/knadh/koanf/providers/confmap v1.0.1 - github.com/knadh/koanf/providers/file v1.2.1 - github.com/knadh/koanf/v2 v2.3.7 github.com/labstack/echo/v5 v5.4.0 github.com/modelcontextprotocol/go-sdk v1.8.0 github.com/parquet-go/parquet-go v0.32.0 @@ -50,19 +45,16 @@ require ( github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10506.0 // indirect github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10506.0 // indirect github.com/dustin/go-humanize v1.1.0 // indirect - github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/go-jose/go-jose/v4 v4.1.5 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/goccy/go-json v0.11.2 // indirect github.com/google/flatbuffers v25.12.19+incompatible // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.31.0 // indirect github.com/klauspost/compress v1.20.1 // indirect github.com/klauspost/cpuid/v2 v2.4.0 // indirect - github.com/knadh/koanf/maps v0.1.3 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/mattn/go-isatty v0.0.24 // indirect github.com/mfridman/interpolate v0.0.2 // indirect - github.com/mitchellh/copystructure v1.2.0 // indirect - github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/ncruces/go-strftime v1.1.0 // indirect github.com/parquet-go/bitpack v1.1.0 // indirect diff --git a/go.sum b/go.sum index 742ca040..f1edba6e 100644 --- a/go.sum +++ b/go.sum @@ -42,8 +42,6 @@ github.com/duckdb/duckdb-go/v2 v2.10506.0 h1:mcZjUQ/kSNeZdtGCqyj5/JEDUdj7/kvILfI github.com/duckdb/duckdb-go/v2 v2.10506.0/go.mod h1:6pLO+4GMcb0q36+z+RlQ7h+MMxum1lL0eAVf4GsPAxs= github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg= github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI= -github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= -github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/go-jose/go-jose/v4 v4.1.5 h1:RjgjO2LOtWOJKUC5wpwY9LR3B3vwVAz6JS2YHfYU6eA= github.com/go-jose/go-jose/v4 v4.1.5/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= @@ -75,16 +73,6 @@ github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLM github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI= github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= -github.com/knadh/koanf/maps v0.1.3 h1:P1z7EvTqdFBrPYbzSvorvrpib+sjkUMxf0FVvA5NKK4= -github.com/knadh/koanf/maps v0.1.3/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= -github.com/knadh/koanf/parsers/yaml v1.1.1 h1:u70vV5IyaM0HvONh8HoqBC97oTgO33KcpZbTLiKVinU= -github.com/knadh/koanf/parsers/yaml v1.1.1/go.mod h1:HHmcHXUrp9cOPcuC+2wrr44GTUB0EC+PyfN3HZD9tFg= -github.com/knadh/koanf/providers/confmap v1.0.1 h1:L15hbvMqlvhwUuCtL9BkL+rqiMAjk6cZc8O9XoDtE3A= -github.com/knadh/koanf/providers/confmap v1.0.1/go.mod h1:txHYHiI2hAtF0/0sCmcuol4IDcuQbKTybiB1nOcUo1A= -github.com/knadh/koanf/providers/file v1.2.1 h1:bEWbtQwYrA+W2DtdBrQWyXqJaJSG3KrP3AESOJYp9wM= -github.com/knadh/koanf/providers/file v1.2.1/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA= -github.com/knadh/koanf/v2 v2.3.7 h1:amceufOeoQcq6VFKjm7/ggJ3t0Dkqaxy5fza4j3YgTA= -github.com/knadh/koanf/v2 v2.3.7/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/labstack/echo/v5 v5.4.0 h1:iY674460IvSmUcj7MziL3YrgyTClghz5BpdZoKiGxR4= @@ -93,10 +81,6 @@ github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsRe github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mfridman/interpolate v0.0.2 h1:pnuTK7MQIxxFz1Gr+rjSIx9u7qVjf5VOoM/u6BbAxPY= github.com/mfridman/interpolate v0.0.2/go.mod h1:p+7uk6oE07mpE/Ik1b8EckO0O4ZXiGAfshKBWLUM9Xg= -github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= -github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= -github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= -github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/modelcontextprotocol/go-sdk v1.8.0 h1:KIvahhYqwtbeniWVPs3TcXEA7b8jEtwfBpOTAI+Urx4= github.com/modelcontextprotocol/go-sdk v1.8.0/go.mod h1:dL7u98E/zjJTGzEq+j30jQ8K2k1mb6LeAH4inEcSGts= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= diff --git a/internal/agent/provider_contract_test.go b/internal/agent/provider_contract_test.go index 9083175c..0cac59e2 100644 --- a/internal/agent/provider_contract_test.go +++ b/internal/agent/provider_contract_test.go @@ -140,7 +140,7 @@ func TestOpenAIEncryptedContentConfigurationError(t *testing.T) { } err = p.Stream(context.Background(), StreamParams{}, func(StreamEvent) error { return nil }) var apiErr *APIError - if err == nil || !strings.Contains(err.Error(), "reasoning model") || !strings.Contains(err.Error(), "ai.model") || !errors.As(err, &apiErr) || apiErr.StatusCode != 400 { + if err == nil || !strings.Contains(err.Error(), "reasoning model") || !strings.Contains(err.Error(), "FANOUT_AI_MODEL") || !errors.As(err, &apiErr) || apiErr.StatusCode != 400 { t.Errorf("configuration error = %v", err) } if got := clientErrorMessage(err); got != "Fanout could not reach the model provider. Please try again." { diff --git a/internal/agent/provider_http.go b/internal/agent/provider_http.go index 04f260db..d634db5e 100644 --- a/internal/agent/provider_http.go +++ b/internal/agent/provider_http.go @@ -104,7 +104,7 @@ func (p *openAIProvider) Stream(ctx context.Context, params StreamParams, cb fun if err != nil { var apiErr *APIError if errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusBadRequest && strings.Contains(strings.ToLower(apiErr.Body), "encrypted") { - return fmt.Errorf("OpenAI configuration error: ai.model must select a reasoning model supporting reasoning.encrypted_content (default gpt-6.1-sol): %w", err) + return fmt.Errorf("OpenAI configuration error: FANOUT_AI_MODEL must select a reasoning model supporting reasoning.encrypted_content (default gpt-6.1-sol): %w", err) } return err } diff --git a/internal/config/config.go b/internal/config/config.go index 3e40f764..4534d083 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -21,43 +21,42 @@ const ( // Config is Fanout's canonical configuration schema. Custom environment names // use FANOUT_; provider credentials and endpoints use their vendor's names. -// YAML retains native structured keys, mapped explicitly by the env tags. // Elapsed-time settings use time.Duration with unit-bearing values. type Config struct { // Addr binds the browser, API, MCP, OTLP/gRPC and OTLP/HTTP listener. - Addr string `koanf:"server.addr" env:"FANOUT_ADDR" default:":7520" local:"127.0.0.1:7520"` + Addr string `group:"server" env:"FANOUT_ADDR" default:"127.0.0.1:7520"` // DataDir stores telemetry, query state, and the control database. - DataDir string `koanf:"storage.data_dir" env:"FANOUT_DATA_DIR" default:"./data"` + DataDir string `group:"storage" env:"FANOUT_DATA_DIR" default:"./data"` // IngestBatchSize limits telemetry rows per batch; valid values are 1–50000. - IngestBatchSize int `koanf:"ingest.batch_size" env:"FANOUT_INGEST_BATCH_SIZE" default:"50000"` + IngestBatchSize int `group:"ingest" env:"FANOUT_INGEST_BATCH_SIZE" default:"50000"` // IngestMaxInFlightBytes caps the decoded telemetry the process will hold // across concurrent requests. Concurrency is HTTP/2 streams times // connections and each handler holds its batch until the commit is // durable, so without a ceiling a burst is bounded only by memory. Zero is // unbounded, which is the behaviour that predates this setting. - IngestMaxInFlightBytes int `koanf:"ingest.max_in_flight_bytes" env:"FANOUT_INGEST_MAX_IN_FLIGHT_BYTES" default:"268435456"` + IngestMaxInFlightBytes int `group:"ingest" env:"FANOUT_INGEST_MAX_IN_FLIGHT_BYTES" default:"268435456"` // RollupInterval schedules analytical rollups; use a Go duration of at least 1s. - RollupInterval time.Duration `koanf:"storage.rollup_interval" env:"FANOUT_ROLLUP_INTERVAL" default:"1m"` + RollupInterval time.Duration `group:"storage" env:"FANOUT_ROLLUP_INTERVAL" default:"1m"` // MCPEnabled serves the MCP endpoint at /mcp. - MCPEnabled bool `koanf:"mcp.enabled" env:"FANOUT_MCP_ENABLED" default:"true"` + MCPEnabled bool `group:"mcp" env:"FANOUT_MCP_ENABLED" default:"true"` // RetentionDays sets telemetry retention in days; zero disables expiry. - RetentionDays int `koanf:"storage.retention_days" env:"FANOUT_RETENTION_DAYS" default:"30"` + RetentionDays int `group:"storage" env:"FANOUT_RETENTION_DAYS" default:"30"` // MaintenanceInterval controls Parquet retention and compaction, and // query-cache checkpointing. - MaintenanceInterval time.Duration `koanf:"storage.maintenance_interval" env:"FANOUT_MAINTENANCE_INTERVAL" default:"1h"` + MaintenanceInterval time.Duration `group:"storage" env:"FANOUT_MAINTENANCE_INTERVAL" default:"1h"` // RollupSkipToLatest, set once at boot, advances the service and edge rollup // watermarks to the current max ingested timestamp. Existing history is not // aggregated into those analytical rollups. Completed-batch trace caches // still rebuild all retained history in bounded passes; // uncached files remain immediately queryable. Useful for large pre-seeded // datasets (benchmarks, restores); off in normal operation. - RollupSkipToLatest bool `koanf:"storage.rollup_skip_to_latest" env:"FANOUT_ROLLUP_SKIP_TO_LATEST" default:"false"` + RollupSkipToLatest bool `group:"storage" env:"FANOUT_ROLLUP_SKIP_TO_LATEST" default:"false"` // DefaultNamespace supplies the namespace for telemetry without one. - DefaultNamespace string `koanf:"ingest.default_namespace" env:"FANOUT_DEFAULT_NAMESPACE" default:"default"` + DefaultNamespace string `group:"ingest" env:"FANOUT_DEFAULT_NAMESPACE" default:"default"` // PprofEnabled exposes Go's net/http/pprof handlers at /debug/pprof/* for // CPU/heap/mutex/goroutine profiling under load. Off by default; the routes // require an admin browser session with operations:read. - PprofEnabled bool `koanf:"server.pprof_enabled" env:"FANOUT_PPROF_ENABLED" default:"false"` + PprofEnabled bool `group:"server" env:"FANOUT_PPROF_ENABLED" default:"false"` // The DuckDB knobs below self-size where possible and otherwise default to // values validated on the reference deployment target, a small shared VM // (Hetzner CPX32: 4 vCPU, 8 GB RAM, 160 GB disk). There the self-sizing @@ -77,97 +76,97 @@ type Config struct { // resolveSizing. DuckDB's own default is 80% of detected RAM, which is // calculated as though DuckDB owned the machine and has been observed to // get the process OOM-killed once the Go heap is added on top. - DuckDBMemory string `koanf:"storage.duckdb.memory" env:"FANOUT_DUCKDB_MEMORY"` + DuckDBMemory string `group:"storage" env:"FANOUT_DUCKDB_MEMORY"` // DuckDBThreads caps DuckDB's global query worker pool. Zero leaves // DuckDB's own default in place (one worker per core). Set it to leave // cores free for ingest on a query-heavy co-tenant host. - DuckDBThreads int `koanf:"storage.duckdb.threads" env:"FANOUT_DUCKDB_THREADS"` + DuckDBThreads int `group:"storage" env:"FANOUT_DUCKDB_THREADS"` // DuckDBMaxConns caps the DuckDB connection pool. Reads scan immutable Parquet // concurrently; rollup-cache writes are serialized by the query write gate. // Zero means "size it from the machine" — the same spelling DuckDBThreads // uses for deferring to a default. Resolution happens in resolveSizing and // is reported in the startup configuration log. - DuckDBMaxConns int `koanf:"storage.duckdb.max_connections" env:"FANOUT_DUCKDB_MAX_CONNECTIONS" default:"0"` + DuckDBMaxConns int `group:"storage" env:"FANOUT_DUCKDB_MAX_CONNECTIONS" default:"0"` // AlertEnabled runs the alert evaluation engine. - AlertEnabled bool `koanf:"alerts.enabled" env:"FANOUT_ALERTS_ENABLED" default:"true"` + AlertEnabled bool `group:"alerts" env:"FANOUT_ALERTS_ENABLED" default:"true"` // AlertEvaluationInterval schedules alert evaluation; use a Go duration of at least 1s. - AlertEvaluationInterval time.Duration `koanf:"alerts.evaluation_interval" env:"FANOUT_ALERTS_EVALUATION_INTERVAL" default:"30s"` + AlertEvaluationInterval time.Duration `group:"alerts" env:"FANOUT_ALERTS_EVALUATION_INTERVAL" default:"30s"` // AlertHistoryDays sets alert history retention in days. - AlertHistoryDays int `koanf:"alerts.history_days" env:"FANOUT_ALERTS_HISTORY_DAYS" default:"7"` + AlertHistoryDays int `group:"alerts" env:"FANOUT_ALERTS_HISTORY_DAYS" default:"7"` // AIProvider enables chat by selecting exactly anthropic or openai. Empty disables chat. - AIProvider string `koanf:"ai.provider" env:"FANOUT_AI_PROVIDER" default:""` + AIProvider string `group:"ai" env:"FANOUT_AI_PROVIDER" default:""` // AIModel overrides the provider's default: claude-sonnet-5-5 for Anthropic // and gpt-6.1-sol for OpenAI. The OpenAI provider expects a reasoning model // supporting reasoning.encrypted_content for stateless Responses replay. // A 400 rejecting encrypted content is reported as a server-side configuration - // error; choose a compatible reasoning model with ai.model. - AIModel string `koanf:"ai.model" env:"FANOUT_AI_MODEL"` - // AnthropicAPIKey is required when ai.provider is anthropic. - AnthropicAPIKey string `koanf:"ai.anthropic.api_key" env:"ANTHROPIC_API_KEY" secret:"true"` + // error; choose a compatible reasoning model with FANOUT_AI_MODEL. + AIModel string `group:"ai" env:"FANOUT_AI_MODEL"` + // AnthropicAPIKey is required when FANOUT_AI_PROVIDER is anthropic. + AnthropicAPIKey string `group:"ai" env:"ANTHROPIC_API_KEY" secret:"true"` // AnthropicBaseURL defaults to https://api.anthropic.com. Fanout appends // /v1/messages, preserving any gateway path prefix. Use an HTTP(S) base URL // without embedded credentials, a query, or a fragment. - AnthropicBaseURL string `koanf:"ai.anthropic.base_url" env:"ANTHROPIC_BASE_URL"` - // OpenAIAPIKey is required when ai.provider is openai. - OpenAIAPIKey string `koanf:"ai.openai.api_key" env:"OPENAI_API_KEY" secret:"true"` + AnthropicBaseURL string `group:"ai" env:"ANTHROPIC_BASE_URL"` + // OpenAIAPIKey is required when FANOUT_AI_PROVIDER is openai. + OpenAIAPIKey string `group:"ai" env:"OPENAI_API_KEY" secret:"true"` // OpenAIBaseURL defaults to https://api.openai.com/v1. Fanout appends // /responses, preserving the API version and any gateway path prefix. // Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. - OpenAIBaseURL string `koanf:"ai.openai.base_url" env:"OPENAI_BASE_URL"` + OpenAIBaseURL string `group:"ai" env:"OPENAI_BASE_URL"` // SMTPHost selects the mail server. Host, username, password, and from must all be set to enable SMTP. - SMTPHost string `koanf:"smtp.host" env:"FANOUT_SMTP_HOST"` + SMTPHost string `group:"smtp" env:"FANOUT_SMTP_HOST"` // SMTPPort sets the mail server's positive TCP port. - SMTPPort int `koanf:"smtp.port" env:"FANOUT_SMTP_PORT" default:"587"` + SMTPPort int `group:"smtp" env:"FANOUT_SMTP_PORT" default:"587"` // SMTPUser authenticates to the mail server; required with SMTP. - SMTPUser string `koanf:"smtp.username" env:"FANOUT_SMTP_USERNAME"` + SMTPUser string `group:"smtp" env:"FANOUT_SMTP_USERNAME"` // SMTPPass authenticates to the mail server; required with SMTP. - SMTPPass string `koanf:"smtp.password" env:"FANOUT_SMTP_PASSWORD" secret:"true"` + SMTPPass string `group:"smtp" env:"FANOUT_SMTP_PASSWORD" secret:"true"` // SMTPFrom sets the email sender address; required with SMTP. - SMTPFrom string `koanf:"smtp.from" env:"FANOUT_SMTP_FROM"` + SMTPFrom string `group:"smtp" env:"FANOUT_SMTP_FROM"` // AuthMode selects local sign-in or oidc; each mode validates its prerequisites. - AuthMode string `koanf:"auth.mode" env:"FANOUT_AUTH_MODE" default:"local"` + AuthMode string `group:"auth" env:"FANOUT_AUTH_MODE" default:"local"` // PublicURL sets the external browser origin. Required for OIDC; HTTPS is required with MCP. - PublicURL string `koanf:"server.public_url" env:"FANOUT_PUBLIC_URL"` + PublicURL string `group:"server" env:"FANOUT_PUBLIC_URL"` // AuthCodeSecret signs local sign-in codes; required in local mode with at least 32 characters. - AuthCodeSecret string `koanf:"auth.code_secret" env:"FANOUT_AUTH_CODE_SECRET" secret:"true"` + AuthCodeSecret string `group:"auth" env:"FANOUT_AUTH_CODE_SECRET" secret:"true"` // SelfSignup allows verified email addresses to create viewer accounts; requires local auth and SMTP. - SelfSignup bool `koanf:"auth.self_signup" env:"FANOUT_SELF_SIGNUP" default:"false"` + SelfSignup bool `group:"auth" env:"FANOUT_SELF_SIGNUP" default:"false"` // SessionIdleTTL expires inactive sessions; use a Go duration of at least 5m. - SessionIdleTTL time.Duration `koanf:"auth.session_idle_ttl" env:"FANOUT_SESSION_IDLE_TTL" default:"12h"` - // SessionAbsoluteTTL limits total session lifetime; it must be at least session_idle_ttl. - SessionAbsoluteTTL time.Duration `koanf:"auth.session_absolute_ttl" env:"FANOUT_SESSION_ABSOLUTE_TTL" default:"168h"` + SessionIdleTTL time.Duration `group:"auth" env:"FANOUT_SESSION_IDLE_TTL" default:"12h"` + // SessionAbsoluteTTL limits total session lifetime; it must be at least FANOUT_SESSION_IDLE_TTL. + SessionAbsoluteTTL time.Duration `group:"auth" env:"FANOUT_SESSION_ABSOLUTE_TTL" default:"168h"` // OIDCIssuerURL selects the identity provider issuer; required in OIDC mode. - OIDCIssuerURL string `koanf:"auth.oidc.issuer_url" env:"FANOUT_OIDC_ISSUER_URL"` + OIDCIssuerURL string `group:"auth" env:"FANOUT_OIDC_ISSUER_URL"` // OIDCClientID identifies Fanout's client registration; required in OIDC mode. - OIDCClientID string `koanf:"auth.oidc.client_id" env:"FANOUT_OIDC_CLIENT_ID"` + OIDCClientID string `group:"auth" env:"FANOUT_OIDC_CLIENT_ID"` // OIDCClientSecret authenticates Fanout's client registration; required in OIDC mode. - OIDCClientSecret string `koanf:"auth.oidc.client_secret" env:"FANOUT_OIDC_CLIENT_SECRET" secret:"true"` + OIDCClientSecret string `group:"auth" env:"FANOUT_OIDC_CLIENT_SECRET" secret:"true"` // OIDCEmailClaim names the claim containing the user's email address. - OIDCEmailClaim string `koanf:"auth.oidc.email_claim" env:"FANOUT_OIDC_EMAIL_CLAIM" default:"email"` + OIDCEmailClaim string `group:"auth" env:"FANOUT_OIDC_EMAIL_CLAIM" default:"email"` // OIDCEmailVerification selects required verified-email claims or issuer trust; issuer trust requires an allowlist. - OIDCEmailVerification string `koanf:"auth.oidc.email_verification" env:"FANOUT_OIDC_EMAIL_VERIFICATION" default:"required"` + OIDCEmailVerification string `group:"auth" env:"FANOUT_OIDC_EMAIL_VERIFICATION" default:"required"` // OIDCAutoProvision creates accounts on OIDC sign-in; requires an allowed group or domain. - OIDCAutoProvision bool `koanf:"auth.oidc.auto_provision" env:"FANOUT_OIDC_AUTO_PROVISION" default:"false"` + OIDCAutoProvision bool `group:"auth" env:"FANOUT_OIDC_AUTO_PROVISION" default:"false"` // OIDCAllowedGroups lists allowed identity-provider groups, separated by commas. - OIDCAllowedGroups string `koanf:"auth.oidc.allowed_groups" env:"FANOUT_OIDC_ALLOWED_GROUPS"` + OIDCAllowedGroups string `group:"auth" env:"FANOUT_OIDC_ALLOWED_GROUPS"` // OIDCAllowedDomains lists allowed email domains, separated by commas; wildcard * requires verified email. - OIDCAllowedDomains string `koanf:"auth.oidc.allowed_domains" env:"FANOUT_OIDC_ALLOWED_DOMAINS"` + OIDCAllowedDomains string `group:"auth" env:"FANOUT_OIDC_ALLOWED_DOMAINS"` // OIDCDefaultRole assigns viewer, operator, or admin to users without a group role mapping. - OIDCDefaultRole string `koanf:"auth.oidc.default_role" env:"FANOUT_OIDC_DEFAULT_ROLE" default:"viewer"` + OIDCDefaultRole string `group:"auth" env:"FANOUT_OIDC_DEFAULT_ROLE" default:"viewer"` // OIDCOperatorGroups lists identity-provider groups mapped to operator, separated by commas. - OIDCOperatorGroups string `koanf:"auth.oidc.operator_groups" env:"FANOUT_OIDC_OPERATOR_GROUPS"` + OIDCOperatorGroups string `group:"auth" env:"FANOUT_OIDC_OPERATOR_GROUPS"` // OIDCAdminGroups lists identity-provider groups mapped to admin, separated by commas. - OIDCAdminGroups string `koanf:"auth.oidc.admin_groups" env:"FANOUT_OIDC_ADMIN_GROUPS"` + OIDCAdminGroups string `group:"auth" env:"FANOUT_OIDC_ADMIN_GROUPS"` // MetricsToken authorizes Prometheus scrapes with a bearer token; optional when using an admin session. - MetricsToken string `koanf:"metrics.token" env:"FANOUT_METRICS_TOKEN" secret:"true"` + MetricsToken string `group:"metrics" env:"FANOUT_METRICS_TOKEN" secret:"true"` // MetricsPublic allows unauthenticated Prometheus scrapes. - MetricsPublic bool `koanf:"metrics.public" env:"FANOUT_METRICS_PUBLIC" default:"false"` + MetricsPublic bool `group:"metrics" env:"FANOUT_METRICS_PUBLIC" default:"false"` // TrustedProxyCIDRs lists trusted proxy networks, separated by commas; empty trusts no forwarded client addresses. - TrustedProxyCIDRs string `koanf:"server.trusted_proxy_cidrs" env:"FANOUT_TRUSTED_PROXY_CIDRS"` - // TLSCertFile selects the TLS certificate file; requires tls.key_file. - TLSCertFile string `koanf:"server.tls.cert_file" env:"FANOUT_TLS_CERT_FILE"` - // TLSKeyFile selects the TLS private key file; requires tls.cert_file. - TLSKeyFile string `koanf:"server.tls.key_file" env:"FANOUT_TLS_KEY_FILE"` + TrustedProxyCIDRs string `group:"server" env:"FANOUT_TRUSTED_PROXY_CIDRS"` + // TLSCertFile selects the TLS certificate file; requires FANOUT_TLS_KEY_FILE. + TLSCertFile string `group:"server" env:"FANOUT_TLS_CERT_FILE"` + // TLSKeyFile selects the TLS private key file; requires FANOUT_TLS_CERT_FILE. + TLSKeyFile string `group:"server" env:"FANOUT_TLS_KEY_FILE"` resolvedSizing sizingSource } @@ -176,10 +175,10 @@ type Config struct { func (c Config) LogStartup() { c.logResolvedSizing(c.resolvedSizing) if !c.SecureCookies() { - slog.Warn("browser session cookies are not Secure; configure server.public_url or local TLS before exposing Fanout") + slog.Warn("browser session cookies are not Secure; configure FANOUT_PUBLIC_URL or local TLS before exposing Fanout") } if strings.TrimSpace(c.PublicURL) != "" && !c.TLSEnabled() && strings.TrimSpace(c.TrustedProxyCIDRs) == "" { - slog.Warn("reverse-proxy client IPs are not trusted; configure server.trusted_proxy_cidrs so audit and rate limits use end-client addresses") + slog.Warn("reverse-proxy client IPs are not trusted; configure FANOUT_TRUSTED_PROXY_CIDRS so audit and rate limits use end-client addresses") } if c.MetricsPublic { slog.Warn("Prometheus metrics are publicly accessible", "path", "/metrics") @@ -218,7 +217,7 @@ func (c Config) ControlSQLitePath() string { } // TLSEnabled reports whether a cert/key pair is configured. When true, HTTP -// serves HTTPS and OTLP gRPC over TLS on server.addr. +// serves HTTPS and OTLP gRPC over TLS on FANOUT_ADDR. func (c Config) TLSEnabled() bool { return strings.TrimSpace(c.TLSCertFile) != "" && strings.TrimSpace(c.TLSKeyFile) != "" @@ -227,120 +226,120 @@ func (c Config) TLSEnabled() bool { // Validate checks that config values are sane. func (c Config) Validate() error { if strings.TrimSpace(c.Addr) == "" { - return fmt.Errorf("server.addr must not be empty") + return fmt.Errorf("FANOUT_ADDR must not be empty") } if strings.TrimSpace(c.DataDir) == "" { - return fmt.Errorf("storage.data_dir must not be empty") + return fmt.Errorf("FANOUT_DATA_DIR must not be empty") } if c.IngestBatchSize <= 0 || c.IngestBatchSize > maxIngestBatchSize { - return fmt.Errorf("ingest.batch_size must be between 1 and %d, got %d", maxIngestBatchSize, c.IngestBatchSize) + return fmt.Errorf("FANOUT_INGEST_BATCH_SIZE must be between 1 and %d, got %d", maxIngestBatchSize, c.IngestBatchSize) } if c.RollupInterval < time.Second { - return fmt.Errorf("storage.rollup_interval must be at least 1s, got %s", c.RollupInterval) + return fmt.Errorf("FANOUT_ROLLUP_INTERVAL must be at least 1s, got %s", c.RollupInterval) } if c.RetentionDays < 0 { - return fmt.Errorf("storage.retention_days must be >= 0, got %d", c.RetentionDays) + return fmt.Errorf("FANOUT_RETENTION_DAYS must be >= 0, got %d", c.RetentionDays) } if c.MaintenanceInterval < time.Second { - return fmt.Errorf("storage.maintenance_interval must be at least 1s, got %s", c.MaintenanceInterval) + return fmt.Errorf("FANOUT_MAINTENANCE_INTERVAL must be at least 1s, got %s", c.MaintenanceInterval) } if c.DuckDBThreads < 0 { - return fmt.Errorf("storage.duckdb.threads must be >= 0, got %d", c.DuckDBThreads) + return fmt.Errorf("FANOUT_DUCKDB_THREADS must be >= 0, got %d", c.DuckDBThreads) } if c.DuckDBMaxConns <= 0 { - return fmt.Errorf("storage.duckdb.max_connections must resolve to > 0, got %d", c.DuckDBMaxConns) + return fmt.Errorf("FANOUT_DUCKDB_MAX_CONNECTIONS must resolve to > 0, got %d", c.DuckDBMaxConns) } if c.AlertEvaluationInterval < time.Second { - return fmt.Errorf("alerts.evaluation_interval must be at least 1s, got %s", c.AlertEvaluationInterval) + return fmt.Errorf("FANOUT_ALERTS_EVALUATION_INTERVAL must be at least 1s, got %s", c.AlertEvaluationInterval) } if c.AlertHistoryDays < 0 { - return fmt.Errorf("alerts.history_days must be >= 0, got %d", c.AlertHistoryDays) + return fmt.Errorf("FANOUT_ALERTS_HISTORY_DAYS must be >= 0, got %d", c.AlertHistoryDays) } if c.MCPEnabled { u, err := url.Parse(c.MCPResourceURL()) if err != nil || u.Scheme != "https" || u.Host == "" || u.User != nil || u.Path != "/mcp" || u.RawQuery != "" || u.Fragment != "" { - return fmt.Errorf("server.public_url must be an HTTPS origin when MCP is enabled") + return fmt.Errorf("FANOUT_PUBLIC_URL must be an HTTPS origin when MCP is enabled") } } authMode := strings.ToLower(strings.TrimSpace(c.AuthMode)) if authMode == "" { - return fmt.Errorf("auth.mode must not be empty") + return fmt.Errorf("FANOUT_AUTH_MODE must not be empty") } if authMode != "local" && authMode != "oidc" { - return fmt.Errorf("auth.mode must be local or oidc") + return fmt.Errorf("FANOUT_AUTH_MODE must be local or oidc") } idleTTL := c.SessionIdleTTL absoluteTTL := c.SessionAbsoluteTTL if idleTTL < 5*time.Minute { - return fmt.Errorf("auth.session_idle_ttl must be at least 5m") + return fmt.Errorf("FANOUT_SESSION_IDLE_TTL must be at least 5m") } if absoluteTTL <= 0 || idleTTL > absoluteTTL { - return fmt.Errorf("auth.session_absolute_ttl must be positive and at least auth.session_idle_ttl") + return fmt.Errorf("FANOUT_SESSION_ABSOLUTE_TTL must be positive and at least FANOUT_SESSION_IDLE_TTL") } if authMode == "local" { if len(strings.TrimSpace(c.AuthCodeSecret)) < 32 { - return fmt.Errorf("auth.code_secret must be at least 32 characters") + return fmt.Errorf("FANOUT_AUTH_CODE_SECRET must be at least 32 characters") } } if anySet(c.SMTPHost, c.SMTPUser, c.SMTPPass, c.SMTPFrom) { if !c.SMTPConfigured() { - return fmt.Errorf("smtp.host, smtp.username, smtp.password, and smtp.from must all be set when SMTP is configured") + return fmt.Errorf("FANOUT_SMTP_HOST, FANOUT_SMTP_USERNAME, FANOUT_SMTP_PASSWORD, and FANOUT_SMTP_FROM must all be set when SMTP is configured") } if c.SMTPPort <= 0 { - return fmt.Errorf("smtp.port must be > 0") + return fmt.Errorf("FANOUT_SMTP_PORT must be > 0") } } if c.SelfSignup { if authMode != "local" { - return fmt.Errorf("auth.self_signup requires auth.mode=local") + return fmt.Errorf("FANOUT_SELF_SIGNUP requires FANOUT_AUTH_MODE=local") } if !c.SMTPConfigured() { - return fmt.Errorf("auth.self_signup requires SMTP") + return fmt.Errorf("FANOUT_SELF_SIGNUP requires SMTP") } } if authMode == "oidc" { issuer, err := url.Parse(strings.TrimSpace(c.OIDCIssuerURL)) if err != nil || issuer.Scheme != "https" || issuer.Host == "" { - return fmt.Errorf("auth.oidc.issuer_url must be an HTTPS URL") + return fmt.Errorf("FANOUT_OIDC_ISSUER_URL must be an HTTPS URL") } if strings.TrimSpace(c.OIDCClientID) == "" || strings.TrimSpace(c.OIDCClientSecret) == "" { - return fmt.Errorf("auth.oidc.client_id and auth.oidc.client_secret are required in oidc mode") + return fmt.Errorf("FANOUT_OIDC_CLIENT_ID and FANOUT_OIDC_CLIENT_SECRET are required in oidc mode") } publicURL, err := url.Parse(strings.TrimSpace(c.PublicURL)) if err != nil || publicURL.Scheme != "https" || publicURL.Host == "" { - return fmt.Errorf("server.public_url must be an HTTPS URL in oidc mode") + return fmt.Errorf("FANOUT_PUBLIC_URL must be an HTTPS URL in oidc mode") } if c.OIDCEmailVerification != "required" && c.OIDCEmailVerification != "issuer" { - return fmt.Errorf("auth.oidc.email_verification must be required or issuer") + return fmt.Errorf("FANOUT_OIDC_EMAIL_VERIFICATION must be required or issuer") } if c.OIDCEmailVerification != "required" && csvContains(c.OIDCAllowedDomains, "*") { - return fmt.Errorf("auth.oidc.allowed_domains wildcard requires auth.oidc.email_verification=required") + return fmt.Errorf("FANOUT_OIDC_ALLOWED_DOMAINS wildcard requires FANOUT_OIDC_EMAIL_VERIFICATION=required") } if strings.TrimSpace(c.OIDCEmailClaim) == "" { - return fmt.Errorf("auth.oidc.email_claim must not be empty") + return fmt.Errorf("FANOUT_OIDC_EMAIL_CLAIM must not be empty") } if c.OIDCEmailVerification == "issuer" && strings.TrimSpace(c.OIDCAllowedGroups) == "" && strings.TrimSpace(c.OIDCAllowedDomains) == "" { - return fmt.Errorf("OIDC issuer email verification requires auth.oidc.allowed_groups or auth.oidc.allowed_domains") + return fmt.Errorf("OIDC issuer email verification requires FANOUT_OIDC_ALLOWED_GROUPS or FANOUT_OIDC_ALLOWED_DOMAINS") } if !appauth.ValidRole(c.OIDCDefaultRole) { - return fmt.Errorf("auth.oidc.default_role must be viewer, operator, or admin") + return fmt.Errorf("FANOUT_OIDC_DEFAULT_ROLE must be viewer, operator, or admin") } if c.OIDCAutoProvision && strings.TrimSpace(c.OIDCAllowedGroups) == "" && strings.TrimSpace(c.OIDCAllowedDomains) == "" { - return fmt.Errorf("OIDC auto-provisioning requires auth.oidc.allowed_groups or auth.oidc.allowed_domains") + return fmt.Errorf("OIDC auto-provisioning requires FANOUT_OIDC_ALLOWED_GROUPS or FANOUT_OIDC_ALLOWED_DOMAINS") } } switch c.AIProvider { case "", "anthropic", "openai": default: - return fmt.Errorf("ai.provider must be anthropic or openai") + return fmt.Errorf("FANOUT_AI_PROVIDER must be anthropic or openai") } if c.AgentConfigured() { - key, endpoint := "ai.anthropic.api_key (ANTHROPIC_API_KEY)", "ai.anthropic.base_url (ANTHROPIC_BASE_URL)" + key, endpoint := "ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL" if c.AIProvider == "openai" { - key, endpoint = "ai.openai.api_key (OPENAI_API_KEY)", "ai.openai.base_url (OPENAI_BASE_URL)" + key, endpoint = "OPENAI_API_KEY", "OPENAI_BASE_URL" } if strings.TrimSpace(c.AIAPIKey()) == "" { - return fmt.Errorf("%s is required when ai.provider is %s", key, c.AIProvider) + return fmt.Errorf("%s is required when FANOUT_AI_PROVIDER is %s", key, c.AIProvider) } baseURL := c.AIBaseURL() u, err := url.Parse(baseURL) @@ -354,11 +353,11 @@ func (c Config) Validate() error { continue } if _, _, err := net.ParseCIDR(value); err != nil { - return fmt.Errorf("server.trusted_proxy_cidrs contains invalid CIDR %q", value) + return fmt.Errorf("FANOUT_TRUSTED_PROXY_CIDRS contains invalid CIDR %q", value) } } if anySet(c.TLSCertFile, c.TLSKeyFile) && !c.TLSEnabled() { - return fmt.Errorf("server.tls requires cert_file and key_file") + return fmt.Errorf("FANOUT_TLS_CERT_FILE and FANOUT_TLS_KEY_FILE must both be set") } return nil } @@ -372,7 +371,7 @@ func (c Config) SecureCookies() bool { } // MCPResourceURL returns the canonical OAuth resource URI for the MCP route. -// MCP is served by the main HTTP server, so it always shares server.public_url. +// MCP is served by the main HTTP server, so it always shares FANOUT_PUBLIC_URL. // The localhost value preserves zero-configuration development when no public // origin is configured. func (c Config) MCPResourceURL() string { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 740ff930..e9ddc477 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -7,10 +7,6 @@ import ( "strings" "testing" "time" - - "github.com/knadh/koanf/parsers/yaml" - "github.com/knadh/koanf/providers/file" - "github.com/knadh/koanf/v2" ) func validEnvironment() []string { @@ -27,546 +23,198 @@ func validEnvironment() []string { func TestLoadReturnsDefaults(t *testing.T) { cfg, err := Load(LoadOptions{Environ: validEnvironment()}) if err != nil { - t.Fatalf("Load: %v", err) - } - - if cfg.Addr != ":7520" { - t.Errorf("Addr = %q, want %q", cfg.Addr, ":7520") - } - - if cfg.DataDir != "./data" { - t.Errorf("DataDir = %q, want %q", cfg.DataDir, "./data") - } - if cfg.IngestBatchSize != 50000 { - t.Errorf("IngestBatchSize = %d, want %d", cfg.IngestBatchSize, 50000) - } - if cfg.RollupInterval != time.Minute { - t.Errorf("RollupInterval = %s, want %s", cfg.RollupInterval, time.Minute) - } - if cfg.RetentionDays != 30 { - t.Errorf("RetentionDays = %d, want %d", cfg.RetentionDays, 30) - } - if cfg.DefaultNamespace != "default" { - t.Errorf("DefaultNamespace = %q, want %q", cfg.DefaultNamespace, "default") - } - // DuckDBMemory is no longer asserted to be empty: Load resolves it from - // detected memory, and what it resolves to depends on the machine running - // the test. On a host where detection declines it stays empty, which is the - // documented fallback rather than a failure. The resolution rules - // themselves are covered in sizing_test.go. - if cfg.DuckDBThreads != 0 { - t.Errorf("DuckDBThreads = %d, want 0 (DuckDB self-sizes to core count)", cfg.DuckDBThreads) - } - // Sized from the machine rather than fixed at 4, so assert the contract: - // above the write-gate invariant and within the auto-sizing ceiling. - if cfg.DuckDBMaxConns < minDuckDBConns || cfg.DuckDBMaxConns > maxAutoDuckDBConns { - t.Errorf("DuckDBMaxConns = %d, want between %d and %d", - cfg.DuckDBMaxConns, minDuckDBConns, maxAutoDuckDBConns) - } -} - -func TestLoadLayering(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte(`server: - addr: ":1111" -mcp: - enabled: false -metrics: - public: true -auth: - session_idle_ttl: 10h - self_signup: true -`), 0o600); err != nil { - t.Fatalf("write config: %v", err) - } - - cfg, err := Load(LoadOptions{ - Path: path, - Environ: append(validEnvironment(), "FANOUT_ADDR=:2222"), - }) - if err != nil { - t.Fatalf("Load: %v", err) + t.Fatal(err) } - if cfg.Addr != ":2222" { - t.Fatalf("Addr = %q, want environment override", cfg.Addr) + if cfg.Addr != "127.0.0.1:7520" || cfg.DataDir != "./data" || cfg.IngestBatchSize != 50000 || cfg.RollupInterval != time.Minute || cfg.RetentionDays != 30 || cfg.DefaultNamespace != "default" || !cfg.MCPEnabled || !cfg.AlertEnabled || cfg.AIProvider != "" { + t.Fatal("built-in defaults were not resolved") } - if cfg.MCPEnabled || !cfg.MetricsPublic || cfg.SessionIdleTTL != 10*time.Hour || !cfg.SelfSignup { - t.Fatalf("YAML values were not merged: %+v", cfg) + if cfg.DuckDBThreads != 0 || cfg.DuckDBMaxConns < minDuckDBConns || cfg.DuckDBMaxConns > maxAutoDuckDBConns { + t.Fatal("machine sizing did not preserve the connection and thread defaults") } } func TestLoadTypedEnvironmentValues(t *testing.T) { cfg, err := Load(LoadOptions{Environ: append(validEnvironment(), - "FANOUT_MCP_ENABLED=false", - "FANOUT_METRICS_PUBLIC=true", - "FANOUT_SESSION_IDLE_TTL=10h", - "FANOUT_SELF_SIGNUP=true", + "FANOUT_ADDR=:9000", "FANOUT_MCP_ENABLED=false", "FANOUT_METRICS_PUBLIC=true", + "FANOUT_SESSION_IDLE_TTL=10h", "FANOUT_SELF_SIGNUP=true", "FANOUT_RETENTION_DAYS=0", + "FANOUT_ROLLUP_INTERVAL=5m", "FANOUT_MAINTENANCE_INTERVAL=2h", "FANOUT_ALERTS_EVALUATION_INTERVAL=45s", )}) if err != nil { - t.Fatalf("Load: %v", err) + t.Fatal(err) } - if cfg.MCPEnabled || !cfg.MetricsPublic || cfg.SessionIdleTTL != 10*time.Hour || !cfg.SelfSignup { - t.Fatalf("environment values were not decoded: %+v", cfg) + if cfg.Addr != ":9000" || cfg.MCPEnabled || !cfg.MetricsPublic || cfg.SessionIdleTTL != 10*time.Hour || !cfg.SelfSignup || cfg.RetentionDays != 0 || cfg.RollupInterval != 5*time.Minute || cfg.MaintenanceInterval != 2*time.Hour || cfg.AlertEvaluationInterval != 45*time.Second { + t.Fatal("environment values were not decoded") } } func TestLoadTreatsEmptyEnvironmentValuesAsAbsent(t *testing.T) { - cfg, err := Load(LoadOptions{Environ: append(validEnvironment(), - "FANOUT_ADDR=", - "FANOUT_MCP_ENABLED=", - "FANOUT_RETENTION_DAYS=", - )}) - if err != nil { - t.Fatalf("Load: %v", err) - } - if cfg.Addr != ":7520" || !cfg.MCPEnabled || cfg.RetentionDays != 30 { - t.Fatalf("empty environment values erased defaults: %+v", cfg) - } -} - -func TestMCPResourceURL(t *testing.T) { - tests := []struct { - name string - publicURL string - want string - }{ - {name: "local default", want: "https://localhost:7520/mcp"}, - {name: "public origin", publicURL: "https://fanout.example.com", want: "https://fanout.example.com/mcp"}, - {name: "trimmed origin", publicURL: " https://fanout.example.com/ ", want: "https://fanout.example.com/mcp"}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - cfg := Config{PublicURL: tc.publicURL} - if got := cfg.MCPResourceURL(); got != tc.want { - t.Fatalf("MCPResourceURL() = %q, want %q", got, tc.want) - } - }) - } -} - -func TestExampleConfigurationMatchesSchema(t *testing.T) { - path := filepath.Join("..", "..", "fanout.example.yaml") - cfg, err := Load(LoadOptions{ - Path: path, - Environ: validEnvironment(), - }) - if err != nil { - t.Fatalf("Load example: %v", err) - } - if cfg.Addr != ":7520" || cfg.DataDir != "./data" { - t.Fatalf("example defaults = HTTP %q data %q", cfg.Addr, cfg.DataDir) - } - defaults, err := Load(LoadOptions{Environ: validEnvironment()}) + cfg, err := Load(LoadOptions{Environ: append(validEnvironment(), "FANOUT_ADDR=", "FANOUT_MCP_ENABLED=", "FANOUT_RETENTION_DAYS=", "FANOUT_AUTH_CODE_SECRET_FILE=")}) if err != nil { - t.Fatalf("Load defaults: %v", err) - } - if !reflect.DeepEqual(cfg, defaults) { - t.Error("example values do not match built-in defaults") - } - - raw, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read example: %v", err) - } - example := koanf.New(".") - if err := example.Load(file.Provider(path), yaml.Parser()); err != nil { - t.Fatalf("parse example: %v", err) - } - active := make(map[string]bool, len(example.Keys())) - for _, key := range example.Keys() { - active[key] = true - } - - specs, err := configFieldSpecs() - if err != nil { - t.Fatalf("field specs: %v", err) - } - wantSecrets := map[string]bool{ - "ai.anthropic.api_key": true, - "ai.openai.api_key": true, - "smtp.password": true, - "auth.code_secret": true, - "auth.oidc.client_secret": true, - "metrics.token": true, - } - foundSecrets := make(map[string]bool, len(wantSecrets)) - rawText := string(raw) - for _, spec := range specs { - if !strings.Contains(rawText, spec.env) { - t.Errorf("example does not document %s", spec.env) - } - if spec.secret != wantSecrets[spec.key] { - t.Errorf("secret classification for %s = %t, want %t", spec.key, spec.secret, wantSecrets[spec.key]) - } - if spec.secret { - foundSecrets[spec.key] = true - } - if spec.secret && active[spec.key] { - t.Errorf("secret key %s must be documented but inactive in the example", spec.key) - } - if spec.secret { - leaf := spec.key[strings.LastIndex(spec.key, ".")+1:] - if !strings.Contains(rawText, "# "+leaf+":") { - t.Errorf("secret key %s is not documented as a commented YAML key", spec.key) - } - } - if !spec.secret && !active[spec.key] { - t.Errorf("non-secret key %s must be active in the example", spec.key) - } + t.Fatal(err) } - if !reflect.DeepEqual(foundSecrets, wantSecrets) { - t.Errorf("secret settings = %v, want exactly %v", foundSecrets, wantSecrets) + if cfg.Addr != "127.0.0.1:7520" || !cfg.MCPEnabled || cfg.RetentionDays != 30 { + t.Fatal("empty values erased defaults") } } func TestConfigurationSchemaUsesUnitBearingDurations(t *testing.T) { specs, err := configFieldSpecs() if err != nil { - t.Fatalf("field specs: %v", err) + t.Fatal(err) } - durationType := reflect.TypeOf(time.Duration(0)) for _, spec := range specs { - key := strings.ToLower(spec.key) - env := strings.ToLower(spec.env) - if strings.Contains(key, "_seconds") || strings.Contains(key, "_ms") || - strings.Contains(env, "_seconds") || strings.Contains(env, "_ms") { - t.Errorf("elapsed-time setting uses a unit suffix: %s / %s", spec.key, spec.env) + if strings.HasSuffix(spec.env, "_SECONDS") || strings.HasSuffix(spec.env, "_MS") { + t.Errorf("unit suffix in %s", spec.env) } - if spec.typ == durationType && !strings.HasSuffix(key, "_interval") && !strings.HasSuffix(key, "_ttl") && !strings.HasSuffix(key, "_retention") { - t.Errorf("duration setting %s must end in _interval or _ttl", spec.key) + if spec.typ == reflect.TypeOf(time.Duration(0)) && !strings.HasSuffix(spec.env, "_INTERVAL") && !strings.HasSuffix(spec.env, "_TTL") { + t.Errorf("duration name %s must end in _INTERVAL or _TTL", spec.env) } } } -func TestLoadDurationIntervals(t *testing.T) { - t.Run("YAML", func(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - document := `storage: - rollup_interval: 5m - maintenance_interval: 1h -alerts: - evaluation_interval: 45s -` - if err := os.WriteFile(path, []byte(document), 0o600); err != nil { - t.Fatalf("write config: %v", err) - } - cfg, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}) - if err != nil { - t.Fatalf("Load: %v", err) - } - if cfg.RollupInterval != 5*time.Minute || - cfg.MaintenanceInterval != time.Hour || - cfg.AlertEvaluationInterval != 45*time.Second { - t.Fatalf("duration values were not decoded: %+v", cfg) - } - }) - - t.Run("environment", func(t *testing.T) { - cfg, err := Load(LoadOptions{Environ: append(validEnvironment(), - "FANOUT_ROLLUP_INTERVAL=5m", - "FANOUT_MAINTENANCE_INTERVAL=1h", - "FANOUT_ALERTS_EVALUATION_INTERVAL=45s", - )}) - if err != nil { - t.Fatalf("Load: %v", err) - } - if cfg.RollupInterval != 5*time.Minute || - cfg.MaintenanceInterval != time.Hour || - cfg.AlertEvaluationInterval != 45*time.Second { - t.Fatalf("duration values were not decoded: %+v", cfg) - } - }) -} - -func TestDockerConfigurationMatchesSchema(t *testing.T) { - path := filepath.Join("..", "..", "fanout.docker.yaml") - cfg, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}) - if err != nil { - t.Fatalf("Load Docker config: %v", err) - } - if cfg.Addr != ":7520" || cfg.DataDir != "/var/lib/fanout/data" { - t.Fatalf("Docker config values = HTTP %q data %q", cfg.Addr, cfg.DataDir) - } -} - -func TestLoadRejectsUnknownInputs(t *testing.T) { - t.Run("YAML key", func(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte("server:\n htpp_addr: ':1111'\n"), 0o600); err != nil { - t.Fatalf("write config: %v", err) - } - _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}) - if err == nil || !strings.Contains(err.Error(), "server.htpp_addr") { - t.Fatalf("error = %v, want unknown YAML key", err) - } - }) - - t.Run("environment variable", func(t *testing.T) { - _, err := Load(LoadOptions{Environ: append(validEnvironment(), "FANOUT_HTPP_ADDR=:1111")}) - if err == nil || !strings.Contains(err.Error(), "FANOUT_HTPP_ADDR") { - t.Fatalf("error = %v, want unknown environment variable", err) - } - }) - - t.Run("removed ingest YAML key", func(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte("ingest:\n public_endpoint: https://ingest.example.com\n"), 0o600); err != nil { - t.Fatalf("write config: %v", err) - } - _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}) - if err == nil || !strings.Contains(err.Error(), "ingest.public_endpoint") { - t.Fatalf("error = %v, want removed YAML key rejected", err) - } - }) - - t.Run("removed ingest environment variable", func(t *testing.T) { - _, err := Load(LoadOptions{Environ: append(validEnvironment(), "FANOUT_INGEST_ENDPOINT=https://ingest.example.com")}) - if err == nil || !strings.Contains(err.Error(), "FANOUT_INGEST_ENDPOINT") { - t.Fatalf("error = %v, want removed environment variable rejected", err) - } - }) - - t.Run("removed YAML keys", func(t *testing.T) { - for _, test := range []struct { - key, document string - }{ - {"ai.api_key", "ai:\n api_key: secret\n"}, - {"ai.base_url", "ai:\n base_url: https://ai.example.com\n"}, - {"agent.provider", "agent:\n provider: anthropic\n"}, - {"agent.api_key", "agent:\n api_key: secret\n"}, - {"agent.model", "agent:\n model: test\n"}, - {"agent.base_url", "agent:\n base_url: https://ai.example.com\n"}, - {"ingest.flush_seconds", "ingest:\n flush_seconds: 15\n"}, - {"storage.rollup_every_seconds", "storage:\n rollup_every_seconds: 60\n"}, - {"storage.merge_every_seconds", "storage:\n merge_every_seconds: 60\n"}, - {"storage.maintenance_every_seconds", "storage:\n maintenance_every_seconds: 3600\n"}, - {"alerts.evaluation_interval_seconds", "alerts:\n evaluation_interval_seconds: 30\n"}, - {"mcp.public_url", "mcp:\n public_url: https://fanout.example.com/mcp\n"}, - } { - t.Run(test.key, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte(test.document), 0o600); err != nil { - t.Fatalf("write config: %v", err) - } - _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}) - if err == nil || !strings.Contains(err.Error(), test.key) { - t.Fatalf("error = %v, want removed key %s rejected", err, test.key) - } - }) - } - }) - - t.Run("removed environment variables", func(t *testing.T) { - for _, name := range []string{ - "FANOUT_AI_API_KEY", - "FANOUT_AI_BASE_URL", - "FANOUT_ROLLUP_EVERY_SECONDS", - "FANOUT_MERGE_EVERY_SECONDS", - "FANOUT_MERGE_INTERVAL", - "FANOUT_MAINTENANCE_EVERY_SECONDS", - "FANOUT_ALERTS_EVALUATION_INTERVAL_SECONDS", - "FANOUT_MCP_PUBLIC_URL", - } { - t.Run(name, func(t *testing.T) { - _, err := Load(LoadOptions{Environ: append(validEnvironment(), name+"=1")}) - if err == nil || !strings.Contains(err.Error(), name) { - t.Fatalf("error = %v, want removed environment variable %s rejected", err, name) - } - }) - } - }) - - t.Run("near platform variable", func(t *testing.T) { - _, err := Load(LoadOptions{Environ: append(validEnvironment(), "FANOUT_SERVICE_PORTAL=7520")}) - if err == nil || !strings.Contains(err.Error(), "FANOUT_SERVICE_PORTAL") { - t.Fatalf("error = %v, want unknown environment variable", err) - } - }) - - t.Run("malformed environment entry", func(t *testing.T) { - _, err := Load(LoadOptions{Environ: append(validEnvironment(), "FANOUT_ADDR")}) - if err == nil || !strings.Contains(err.Error(), "NAME=value") { - t.Fatalf("error = %v, want malformed environment error", err) +func TestLoadRejectsUnknownAndMalformedEnvironment(t *testing.T) { + for _, entry := range []string{"FANOUT_HTPP_ADDR=:1111", "FANOUT_SERVICE_PORTAL=7520", "FANOUT_ADDR", "FANOUT_ADDR_FILE=/unused", "FANOUT_AI_API_KEY=synthetic-private-value"} { + _, err := Load(LoadOptions{Environ: append(validEnvironment(), entry)}) + name, _, _ := strings.Cut(entry, "=") + if err == nil || !strings.Contains(err.Error(), name) || strings.Contains(err.Error(), "synthetic-private-value") { + t.Fatalf("expected redacted error naming %s", name) } - }) -} - -func TestLoadIgnoresKubernetesServiceEnvironment(t *testing.T) { - environ := append(validEnvironment(), - "FANOUT_SERVICE_HOST=10.0.0.1", - "FANOUT_SERVICE_PORT=7520", - "FANOUT_SERVICE_PORT_HTTP=7520", - "FANOUT_PORT=tcp://10.0.0.1:7520", - "FANOUT_PORT_7520_TCP_ADDR=10.0.0.1", - ) - if _, err := Load(LoadOptions{Environ: environ}); err != nil { - t.Fatalf("Load with Kubernetes service variables: %v", err) - } -} - -func TestLoadIgnoresEnvironmentOutsideNamespace(t *testing.T) { - if _, err := Load(LoadOptions{Environ: append(validEnvironment(), - "OTHER_APP_SETTING=value", - )}); err != nil { - t.Fatalf("Load with unrelated environment variable: %v", err) } } -func TestLoadAllowsEmptyYAMLSections(t *testing.T) { - for _, document := range []string{ - "metrics:\n", - "auth:\n oidc:\n # client_id: fanout\n", - "server: {}\n", - } { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte(document), 0o600); err != nil { - t.Fatal(err) - } - if _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}); err != nil { - t.Fatalf("Load %q: %v", document, err) +func TestLoadRejectsDuplicateEnvironmentNames(t *testing.T) { + for _, entry := range []string{"FANOUT_ADDR=", "ANTHROPIC_API_KEY=test", "FANOUT_METRICS_TOKEN_FILE="} { + name, _, _ := strings.Cut(entry, "=") + _, err := Load(LoadOptions{Environ: []string{entry, entry}}) + if err == nil || !strings.Contains(err.Error(), "duplicate") || !strings.Contains(err.Error(), name) { + t.Fatalf("expected duplicate error for %s", name) } } } -func TestLoadRejectsYAMLNullValues(t *testing.T) { - for _, key := range []struct { - name, document string - }{ - {"HTTP address", "server:\n addr:\n"}, - {"MCP enabled", "mcp:\n enabled:\n"}, - {"retention", "storage:\n retention_days:\n"}, - } { - t.Run(key.name, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte(key.document), 0o600); err != nil { - t.Fatal(err) - } - _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}) - if err == nil || !strings.Contains(err.Error(), "must not be null") { - t.Fatalf("error = %v, want null rejection", err) - } - }) +func TestLoadIgnoresPlatformAndUnrelatedEnvironment(t *testing.T) { + env := append(validEnvironment(), "FANOUT_SERVICE_HOST=10.0.0.1", "FANOUT_SERVICE_PORT=7520", "FANOUT_SERVICE_PORT_HTTP=7520", "FANOUT_PORT=tcp://10.0.0.1:7520", "FANOUT_PORT_7520_TCP_ADDR=10.0.0.1", "OTHER_APP_SETTING=value", "OPENAI_MODEL=other-model") + if _, err := Load(LoadOptions{Environ: env}); err != nil { + t.Fatal(err) } } -func TestLoadRejectsInvalidFilesAndValues(t *testing.T) { - t.Run("missing file", func(t *testing.T) { - _, err := Load(LoadOptions{Path: filepath.Join(t.TempDir(), "missing.yaml"), Environ: validEnvironment()}) - if err == nil { - t.Fatal("expected missing file error") - } - }) - - for _, test := range []struct { - name, document string - }{ - {"invalid duration", "storage:\n rollup_interval: never\n"}, - {"fractional integer", "smtp:\n port: 25.9\n"}, - {"integer as boolean", "mcp:\n enabled: 2\n"}, - {"YAML keyword as boolean", "alerts:\n enabled: off\n"}, +func TestLoadRejectsInvalidTypedValuesWithoutLeakingValues(t *testing.T) { + for _, entry := range []string{ + "FANOUT_ROLLUP_INTERVAL=private-value", "FANOUT_SMTP_PORT=25.9", "FANOUT_MCP_ENABLED=off", "FANOUT_RETENTION_DAYS=999999999999999999999999999", + "FANOUT_INGEST_BATCH_SIZE=0", "FANOUT_DUCKDB_MAX_CONNECTIONS=-3", "FANOUT_DUCKDB_THREADS=-4", + "FANOUT_ROLLUP_INTERVAL=999ms", "FANOUT_MAINTENANCE_INTERVAL=-5s", "FANOUT_ALERTS_EVALUATION_INTERVAL=0s", } { - t.Run(test.name, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte(test.document), 0o600); err != nil { - t.Fatalf("write config: %v", err) - } - _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}) - if err == nil { - t.Fatal("expected type error") + env := validEnvironment() + name, _, _ := strings.Cut(entry, "=") + for i, existing := range env { + if strings.HasPrefix(existing, name+"=") { + env = append(env[:i], env[i+1:]...) + break } - }) - } - - t.Run("invalid merged config", func(t *testing.T) { - _, err := Load(LoadOptions{Environ: append(validEnvironment(), "FANOUT_INGEST_BATCH_SIZE=0")}) - if err == nil || !strings.Contains(err.Error(), "batch_size") { - t.Fatalf("error = %v, want validation error", err) - } - }) - - t.Run("explicit empty auth mode", func(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte("auth:\n mode: \"\"\n"), 0o600); err != nil { - t.Fatal(err) } - _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}) - if err == nil || !strings.Contains(err.Error(), "auth.mode must not be empty") { - t.Fatalf("error = %v, want explicit empty auth-mode rejection", err) + _, err := Load(LoadOptions{Environ: append(env, entry)}) + if err == nil || !strings.Contains(err.Error(), name) || strings.Contains(err.Error(), "private-value") { + t.Fatalf("expected redacted validation error for %s", name) } - }) - - for _, test := range []struct { - name, assignment, key string - }{ - {"negative max connections is not auto", "FANOUT_DUCKDB_MAX_CONNECTIONS=-3", "max_connections"}, - {"zero alert interval", "FANOUT_ALERTS_EVALUATION_INTERVAL=0s", "evaluation_interval"}, - {"negative maintenance interval", "FANOUT_MAINTENANCE_INTERVAL=-5s", "maintenance_interval"}, - {"negative DuckDB threads", "FANOUT_DUCKDB_THREADS=-4", "duckdb.threads"}, - } { - t.Run(test.name, func(t *testing.T) { - _, err := Load(LoadOptions{Environ: append(validEnvironment(), test.assignment)}) - if err == nil || !strings.Contains(err.Error(), test.key) { - t.Fatalf("error = %v, want validation error for %s", err, test.key) - } - }) - } - - for _, test := range []struct { - name, assignment, key string - }{ - {"subsecond rollup interval", "FANOUT_ROLLUP_INTERVAL=999ms", "rollup_interval"}, - {"subsecond maintenance interval", "FANOUT_MAINTENANCE_INTERVAL=500ms", "maintenance_interval"}, - {"subsecond alert interval", "FANOUT_ALERTS_EVALUATION_INTERVAL=999ms", "evaluation_interval"}, - } { - t.Run(test.name, func(t *testing.T) { - _, err := Load(LoadOptions{Environ: append(validEnvironment(), test.assignment)}) - if err == nil || !strings.Contains(err.Error(), test.key) { - t.Fatalf("error = %v, want lower-bound validation for %s", err, test.key) - } - }) } } func TestLoadIgnoresDotenvInputs(t *testing.T) { dir := t.TempDir() if err := os.WriteFile(filepath.Join(dir, ".env"), []byte("FANOUT_ADDR=:9999\n"), 0o600); err != nil { - t.Fatalf("write .env: %v", err) + t.Fatal(err) } t.Chdir(dir) - cfg, err := Load(LoadOptions{Environ: validEnvironment()}) if err != nil { - t.Fatalf("Load: %v", err) + t.Fatal(err) } - if cfg.Addr != ":7520" { - t.Fatalf("Addr = %q, want default", cfg.Addr) + if cfg.Addr != "127.0.0.1:7520" { + t.Fatal("binary loaded dotenv") } } -func TestLoadErrorsDoNotContainSecrets(t *testing.T) { - const secret = "do-not-print-this-password" - path := filepath.Join(t.TempDir(), "fanout.yaml") - document := "smtp:\n password: " + secret + "\n port: 25.9\n" - if err := os.WriteFile(path, []byte(document), 0o600); err != nil { +func TestLoadSecretFiles(t *testing.T) { + names := []string{"FANOUT_AUTH_CODE_SECRET", "FANOUT_OIDC_CLIENT_SECRET", "FANOUT_SMTP_PASSWORD", "FANOUT_METRICS_TOKEN", "ANTHROPIC_API_KEY", "OPENAI_API_KEY"} + specs, err := configFieldSpecs() + if err != nil { t.Fatal(err) } - _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}) - if err == nil { - t.Fatal("expected decode error") - } - if strings.Contains(err.Error(), secret) { - t.Fatalf("error leaked secret: %v", err) + for _, name := range names { + for _, tc := range []struct{ name, suffix, retained string }{ + {"no newline", "", ""}, + {"LF", "\n", ""}, + {"CRLF", "\r\n", ""}, + {"two LF", "\n\n", "\n"}, + {"two CRLF", "\r\n\r\n", "\r\n"}, + {"bare CR", "\r", "\r"}, + {"spaces", " \n", " "}, + } { + t.Run(name+"/"+tc.name, func(t *testing.T) { + secret := strings.Repeat("x", 32) + path := filepath.Join(t.TempDir(), "secret") + if err := os.WriteFile(path, []byte(secret+tc.suffix), 0o644); err != nil { + t.Fatal(err) + } + env := []string{} + for _, entry := range validEnvironment() { + if !strings.HasPrefix(entry, name+"=") { + env = append(env, entry) + } + } + env = append(env, name+"=", name+"_FILE="+path) + if isProviderEnvironment(name) { + provider, _, _ := strings.Cut(strings.ToLower(name), "_") + env = append(env, "FANOUT_AI_PROVIDER="+provider) + } + cfg, err := Load(LoadOptions{Environ: env}) + if err != nil { + t.Fatal(err) + } + want := secret + tc.retained + for _, spec := range specs { + if spec.env == name && reflect.ValueOf(cfg).Field(spec.index).String() != want { + t.Fatalf("%s did not preserve exact file contents", name) + } + } + }) + } } } -func TestLoadRequiresPrivatePermissionsForConfigSecrets(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte("ai:\n anthropic:\n api_key: secret-from-file\n"), 0o644); err != nil { - t.Fatal(err) +func TestLoadRejectsConflictingOrInvalidSecretFiles(t *testing.T) { + const name = "FANOUT_METRICS_TOKEN" + const secret = "synthetic-private-value" + path := filepath.Join(t.TempDir(), "secret") + for _, content := range []string{"", "\n", "\r\n", secret} { + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + env := append(validEnvironment(), name+"_FILE="+path) + if content == secret { + env = append(env, name+"="+secret) + } + _, err := Load(LoadOptions{Environ: env}) + if err == nil || !strings.Contains(err.Error(), name+"_FILE") || !strings.Contains(err.Error(), path) || strings.Contains(err.Error(), secret) { + t.Fatal("expected redacted file error naming variable and path") + } } - _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}) - if err == nil || !strings.Contains(err.Error(), "must not be accessible by group or others") { - t.Fatalf("error = %v, want file permission rejection", err) + for _, invalidPath := range []string{path + "-missing", filepath.Dir(path)} { + _, err := Load(LoadOptions{Environ: append(validEnvironment(), name+"_FILE="+invalidPath)}) + if err == nil || !strings.Contains(err.Error(), name+"_FILE") || !strings.Contains(err.Error(), invalidPath) { + t.Fatal("expected file read error naming variable and path") + } + } +} + +func TestMCPResourceURL(t *testing.T) { + for _, tc := range []struct{ publicURL, want string }{ + {"", "https://localhost:7520/mcp"}, {"https://fanout.example.com", "https://fanout.example.com/mcp"}, {" https://fanout.example.com/ ", "https://fanout.example.com/mcp"}, + } { + if got := (Config{PublicURL: tc.publicURL}).MCPResourceURL(); got != tc.want { + t.Fatalf("MCPResourceURL = %s, want %s", got, tc.want) + } } } @@ -733,7 +381,7 @@ func TestValidate(t *testing.T) { } { c.PublicURL = invalid if err := c.Validate(); err == nil { - t.Errorf("server.public_url %q should be rejected with MCP enabled", invalid) + t.Errorf("FANOUT_PUBLIC_URL %q should be rejected with MCP enabled", invalid) } } }) @@ -758,32 +406,32 @@ func TestFieldSpecsRejectInvalidSchemas(t *testing.T) { want string }{ { - name: "duplicate key", + name: "secret file name collision", typ: reflect.TypeOf(struct { - A string `koanf:"same" env:"FANOUT_A"` - B string `koanf:"same" env:"FANOUT_B"` + A string `env:"FANOUT_SECRET" secret:"true"` + B string `env:"FANOUT_SECRET_FILE"` }{}), - want: "share key", + want: "share environment variable", }, { name: "duplicate environment", typ: reflect.TypeOf(struct { - A string `koanf:"a" env:"FANOUT_SAME"` - B string `koanf:"b" env:"FANOUT_SAME"` + A string `env:"FANOUT_SAME"` + B string `env:"FANOUT_SAME"` }{}), want: "share environment variable", }, { name: "missing tag", typ: reflect.TypeOf(struct { - A string `koanf:"a"` + A string }{}), want: "must declare", }, { name: "missing prefix", typ: reflect.TypeOf(struct { - A string `koanf:"a" env:"A"` + A string `env:"A"` }{}), want: "outside the supported configuration contract", }, @@ -817,31 +465,3 @@ func TestSecureCookies(t *testing.T) { }) } } - -func TestRemovedListenerAndEndpointSettingsRejected(t *testing.T) { - t.Run("old server address", func(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte("server:\n http_addr: ':7520'\n"), 0o600); err != nil { - t.Fatal(err) - } - if _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}); err == nil || !strings.Contains(err.Error(), "server.http_addr") { - t.Fatalf("removed key: %v", err) - } - }) - for _, key := range []string{"otlp_grpc_addr", "otlp_http_addr", "advertised_endpoint"} { - t.Run(key, func(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - if err := os.WriteFile(path, []byte("ingest:\n "+key+": ':4317'\n"), 0o600); err != nil { - t.Fatal(err) - } - if _, err := Load(LoadOptions{Path: path, Environ: validEnvironment()}); err == nil || !strings.Contains(err.Error(), key) { - t.Fatalf("removed key: %v", err) - } - }) - } - for _, name := range []string{"FANOUT_HTTP_ADDR", "FANOUT_OTLP_GRPC_ADDR", "FANOUT_OTLP_HTTP_ADDR", "FANOUT_INGEST_ADVERTISED_ENDPOINT"} { - if _, err := Load(LoadOptions{Environ: append(validEnvironment(), name+"=:4317")}); err == nil || !strings.Contains(err.Error(), name) { - t.Fatalf("removed variable: %v", err) - } - } -} diff --git a/internal/config/launcher_test.go b/internal/config/launcher_test.go deleted file mode 100644 index f87c514a..00000000 --- a/internal/config/launcher_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package config - -import ( - "context" - "encoding/json" - "os" - "os/exec" - "path/filepath" - "testing" - "time" -) - -func TestGeneratedLauncherEnvironmentPreservesYAML(t *testing.T) { - root := t.TempDir() - for _, name := range []string{".env", ".env.production", ".env.example"} { - body, err := os.ReadFile(filepath.Join("../..", name)) - if err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(root, name), body, 0o600); err != nil { - t.Fatal(err) - } - } - path := filepath.Join(root, "fanout.yaml") - if err := os.WriteFile(path, []byte("storage:\n data_dir: ./yaml-data\n retention_days: 17\n"), 0o600); err != nil { - t.Fatal(err) - } - launcher, err := filepath.Abs("../../scripts/run-fanout.mjs") - if err != nil { - t.Fatal(err) - } - for _, environment := range []string{"local", "production"} { - t.Run(environment, func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - cmd := exec.CommandContext(ctx, "node", "--input-type=module", "-e", ` -const { pathToFileURL } = await import('node:url'); -const { prepareEnvironment } = await import(pathToFileURL(process.argv[3]).href); -process.stdout.write(JSON.stringify(prepareEnvironment(process.argv[1], process.argv[2], { - FANOUT_AUTH_CODE_SECRET: '0123456789abcdef0123456789abcdef' -})));`, root, environment, launcher) - cmd.Env = []string{"PATH=" + os.Getenv("PATH")} - output, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("Node launcher: %v\n%s", err, output) - } - var values map[string]string - if err := json.Unmarshal(output, &values); err != nil { - t.Fatal(err) - } - var env []string - for name, value := range values { - env = append(env, name+"="+value) - } - cfg, err := Load(LoadOptions{Path: path, Environ: env}) - if err != nil { - t.Fatal(err) - } - if cfg.DataDir != "./yaml-data" || cfg.RetentionDays != 17 { - t.Fatal("generated launcher defaults overrode YAML settings") - } - wantAddr := "127.0.0.1:7520" - if environment == "production" { - wantAddr = ":7520" - } - if cfg.Addr != wantAddr { - t.Fatalf("listener = %q, want %q", cfg.Addr, wantAddr) - } - }) - } -} diff --git a/internal/config/loader.go b/internal/config/loader.go index 659eafc5..44a25e4b 100644 --- a/internal/config/loader.go +++ b/internal/config/loader.go @@ -8,23 +8,16 @@ import ( "strconv" "strings" "time" - - "github.com/go-viper/mapstructure/v2" - "github.com/knadh/koanf/parsers/yaml" - "github.com/knadh/koanf/providers/confmap" - "github.com/knadh/koanf/providers/file" - "github.com/knadh/koanf/v2" ) -// LoadOptions selects the explicit configuration inputs. A nil Environ uses -// os.Environ; an empty slice deliberately supplies no environment variables. +// LoadOptions selects the environment. A nil Environ uses os.Environ; +// an empty slice deliberately supplies no environment variables. type LoadOptions struct { - Path string Environ []string } type fieldSpec struct { - key string + index int env string typ reflect.Type defaultValue any @@ -32,79 +25,72 @@ type fieldSpec struct { secret bool } -// Load resolves defaults, an optional YAML document, and process environment -// in that order, then sizes and validates the effective configuration. +// Load resolves built-in defaults and process environment, then sizes and +// validates the effective configuration. func Load(options LoadOptions) (Config, error) { specs, err := configFieldSpecs() if err != nil { return Config{}, err } - - specByKey := make(map[string]fieldSpec, len(specs)) - specByEnv := make(map[string]fieldSpec, len(specs)) - defaults := make(map[string]any) + known := make(map[string]bool, len(specs)) + var cfg Config + value := reflect.ValueOf(&cfg).Elem() for _, spec := range specs { - specByKey[spec.key] = spec - specByEnv[spec.env] = spec - if spec.hasDefault { - defaults[spec.key] = spec.defaultValue - } - } - - k := koanf.New(".") - if err := k.Load(confmap.Provider(defaults, "."), nil); err != nil { - return Config{}, fmt.Errorf("load configuration defaults: %w", err) - } - - if options.Path != "" { - info, statErr := os.Stat(options.Path) - if statErr != nil { - return Config{}, fmt.Errorf("load config %q: %w", options.Path, statErr) - } - if !info.Mode().IsRegular() { - return Config{}, fmt.Errorf("load config %q: not a regular file", options.Path) - } - fromFile := koanf.New(".") - if err := fromFile.Load(file.Provider(options.Path), yaml.Parser()); err != nil { - return Config{}, fmt.Errorf("load config %q: %w", options.Path, err) + known[spec.env] = true + if spec.secret { + known[spec.env+"_FILE"] = true } - if err := validateYAMLKeys(fromFile, specByKey); err != nil { - return Config{}, fmt.Errorf("config %q: %w", options.Path, err) - } - if containsSecretValues(fromFile, specs) && info.Mode().Perm()&0o077 != 0 { - return Config{}, fmt.Errorf("config %q contains secrets and must not be accessible by group or others (mode is %04o)", options.Path, info.Mode().Perm()) - } - if err := k.Merge(fromFile); err != nil { - return Config{}, fmt.Errorf("merge config %q: %w", options.Path, err) + if spec.hasDefault { + value.Field(spec.index).Set(reflect.ValueOf(spec.defaultValue)) } } - environ := options.Environ if environ == nil { environ = os.Environ() } - overrides, err := environmentOverrides(environ, specByEnv) + values, err := environmentValues(environ, known) if err != nil { return Config{}, err } - if err := k.Load(confmap.Provider(overrides, "."), nil); err != nil { - return Config{}, fmt.Errorf("load environment configuration: %w", err) - } - - var cfg Config - if err := k.UnmarshalWithConf("", &cfg, koanf.UnmarshalConf{ - Tag: "koanf", - FlatPaths: true, - DecoderConfig: &mapstructure.DecoderConfig{ - DecodeHook: mapstructure.StringToTimeDurationHookFunc(), - ErrorUnused: true, - WeaklyTypedInput: false, - }, - }); err != nil { - return Config{}, fmt.Errorf("decode configuration: %w", err) + for _, spec := range specs { + // Shared vendor settings, including secret mounts, belong only to the + // explicitly selected provider. Fanout-owned secrets always resolve. + if strings.HasPrefix(spec.env, "ANTHROPIC_") && values["FANOUT_AI_PROVIDER"] != "anthropic" || + strings.HasPrefix(spec.env, "OPENAI_") && values["FANOUT_AI_PROVIDER"] != "openai" { + continue + } + text := values[spec.env] + if spec.secret { + name := spec.env + "_FILE" + if path := values[name]; path != "" { + if text != "" { + return Config{}, fmt.Errorf("%s and %s (%q) must not both be set", spec.env, name, path) + } + content, err := os.ReadFile(path) + if err != nil { + return Config{}, fmt.Errorf("read %s file %q: %w", name, path, err) + } + text = string(content) + if strings.HasSuffix(text, "\r\n") { + text = strings.TrimSuffix(text, "\r\n") + } else { + text = strings.TrimSuffix(text, "\n") + } + if text == "" { + return Config{}, fmt.Errorf("%s file %q is empty", name, path) + } + } + } + if text == "" { + continue + } + parsed, err := parseTextValue(text, spec.typ) + if err != nil { + return Config{}, fmt.Errorf("environment variable %s %w", spec.env, err) + } + value.Field(spec.index).Set(reflect.ValueOf(parsed)) } - // Size before validating: validation should judge the configuration the - // process will actually run with, not the holes left for it to fill. + // Validation judges the sizing the process will actually use. cfg.resolveSizing() if err := cfg.Validate(); err != nil { return Config{}, fmt.Errorf("validate configuration: %w", err) @@ -121,26 +107,29 @@ func fieldSpecs(typ reflect.Type) ([]fieldSpec, error) { return nil, fmt.Errorf("configuration schema must be a struct, got %s", typ) } specs := make([]fieldSpec, 0, typ.NumField()) - keys := make(map[string]string, typ.NumField()) envs := make(map[string]string, typ.NumField()) for index := 0; index < typ.NumField(); index++ { field := typ.Field(index) if !field.IsExported() { continue } - key := field.Tag.Get("koanf") - envName := field.Tag.Get("env") - if key == "" || envName == "" { - return nil, fmt.Errorf("configuration field %s must declare koanf and env tags", field.Name) + name := field.Tag.Get("env") + if name == "" { + return nil, fmt.Errorf("configuration field %s must declare an env tag", field.Name) } - if previous, exists := keys[key]; exists { - return nil, fmt.Errorf("configuration fields %s and %s share key %q", previous, field.Name, key) + if !strings.HasPrefix(name, "FANOUT_") && !isProviderEnvironment(name) { + return nil, fmt.Errorf("environment variable %s is outside the supported configuration contract", name) } - if previous, exists := envs[envName]; exists { - return nil, fmt.Errorf("configuration fields %s and %s share environment variable %q", previous, field.Name, envName) + secret := field.Tag.Get("secret") == "true" + names := []string{name} + if secret { + names = append(names, name+"_FILE") } - if !strings.HasPrefix(envName, "FANOUT_") && !isProviderEnvironment(envName) { - return nil, fmt.Errorf("configuration field %s environment variable %q is outside the supported configuration contract", field.Name, envName) + for _, env := range names { + if previous, exists := envs[env]; exists { + return nil, fmt.Errorf("configuration fields %s and %s share environment variable %s", previous, field.Name, env) + } + envs[env] = field.Name } defaultText, hasDefault := field.Tag.Lookup("default") var defaultValue any @@ -148,158 +137,47 @@ func fieldSpecs(typ reflect.Type) ([]fieldSpec, error) { var err error defaultValue, err = parseTextValue(defaultText, field.Type) if err != nil { - return nil, fmt.Errorf("configuration field %s has invalid default: %w", field.Name, err) + return nil, fmt.Errorf("environment variable %s has invalid default: %w", name, err) } } specs = append(specs, fieldSpec{ - key: key, - env: envName, + index: index, + env: name, typ: field.Type, defaultValue: defaultValue, hasDefault: hasDefault, - secret: field.Tag.Get("secret") == "true", + secret: secret, }) - keys[key] = field.Name - envs[envName] = field.Name } return specs, nil } -func validateYAMLKeys(k *koanf.Koanf, known map[string]fieldSpec) error { - var unknown []string - var emptyContainers []string - for _, key := range k.Keys() { - value := k.Get(key) - if spec, exists := known[key]; exists { - if value == nil { - return fmt.Errorf("key %s must not be null", key) - } - if err := validateYAMLValue(value, spec.typ); err != nil { - return fmt.Errorf("key %s %w", key, err) - } - continue - } - if isSchemaContainer(key, known) && isEmptyContainer(value) { - emptyContainers = append(emptyContainers, key) - continue - } - unknown = append(unknown, key) - } - if len(unknown) == 0 { - for _, key := range emptyContainers { - k.Delete(key) - } - return nil - } - sort.Strings(unknown) - return fmt.Errorf("contains unknown keys: %s", strings.Join(unknown, ", ")) -} - -func isSchemaContainer(key string, known map[string]fieldSpec) bool { - prefix := key + "." - for candidate := range known { - if strings.HasPrefix(candidate, prefix) { - return true - } - } - return false -} - -func validateYAMLValue(value any, typ reflect.Type) error { - if typ == reflect.TypeOf(time.Duration(0)) { - text, ok := value.(string) - if !ok { - return fmt.Errorf("must be a duration string") - } - if _, err := time.ParseDuration(text); err != nil { - return fmt.Errorf("must be a Go duration: %w", err) - } - return nil - } - kind := reflect.TypeOf(value).Kind() - switch typ.Kind() { - case reflect.String: - if kind != reflect.String { - return fmt.Errorf("must be a string") - } - case reflect.Bool: - if kind != reflect.Bool { - return fmt.Errorf("must be true or false") - } - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - if kind < reflect.Int || kind > reflect.Int64 { - return fmt.Errorf("must be an integer") - } - default: - return fmt.Errorf("uses unsupported configuration type %s", typ) - } - return nil -} - -func isEmptyContainer(value any) bool { - if value == nil { - return true - } - rv := reflect.ValueOf(value) - return (rv.Kind() == reflect.Map || rv.Kind() == reflect.Slice) && rv.Len() == 0 -} - -func containsSecretValues(k *koanf.Koanf, specs []fieldSpec) bool { - for _, spec := range specs { - if !spec.secret { - continue - } - value := k.Get(spec.key) - if value == nil { - continue - } - if text, ok := value.(string); ok && strings.TrimSpace(text) == "" { - continue - } - return true - } - return false -} - -func environmentOverrides(environ []string, known map[string]fieldSpec) (map[string]any, error) { - overrides := make(map[string]any) +func environmentValues(environ []string, known map[string]bool) (map[string]string, error) { + values := make(map[string]string) var unknown []string for _, assignment := range environ { name, value, found := strings.Cut(assignment, "=") - _, isKnown := known[name] - if !found { - if isKnown || strings.HasPrefix(name, "FANOUT_") { - return nil, fmt.Errorf("environment entry %s must use NAME=value form", name) - } + isKnown := known[name] + if !isKnown && (!strings.HasPrefix(name, "FANOUT_") || isPlatformInjectedEnvironment(name)) { continue } - if !isKnown && !strings.HasPrefix(name, "FANOUT_") { - continue + if !found { + return nil, fmt.Errorf("environment entry %s must use NAME=value form", name) } - spec, exists := known[name] - if !exists { - if isPlatformInjectedEnvironment(name) { - continue - } + if !isKnown { unknown = append(unknown, name) continue } - // An explicitly empty environment value means "no override", so it cannot - // erase a safe default or silently disable a feature. - if value == "" { - continue - } - parsed, err := parseTextValue(value, spec.typ) - if err != nil { - return nil, fmt.Errorf("environment variable %s has an invalid value for %s: %w", name, spec.key, err) + if _, exists := values[name]; exists { + return nil, fmt.Errorf("duplicate environment variable %s", name) } - overrides[spec.key] = parsed + values[name] = value } if len(unknown) > 0 { sort.Strings(unknown) return nil, fmt.Errorf("unknown Fanout environment variables: %s", strings.Join(unknown, ", ")) } - return overrides, nil + return values, nil } // Vendor namespaces are shared with other consumers. Accept these exact names; @@ -325,7 +203,7 @@ func parseTextValue(value string, typ reflect.Type) (any, error) { if typ == reflect.TypeOf(time.Duration(0)) { parsed, err := time.ParseDuration(value) if err != nil { - return nil, fmt.Errorf("must be a Go duration: %w", err) + return nil, fmt.Errorf("must be a Go duration") } return parsed, nil } @@ -335,13 +213,13 @@ func parseTextValue(value string, typ reflect.Type) (any, error) { case reflect.Bool: parsed, err := strconv.ParseBool(value) if err != nil { - return nil, fmt.Errorf("must be true or false: %w", err) + return nil, fmt.Errorf("must be true or false") } return parsed, nil case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: parsed, err := strconv.ParseInt(value, 10, typ.Bits()) if err != nil { - return nil, fmt.Errorf("must be an integer: %w", err) + return nil, fmt.Errorf("must be an integer") } result := reflect.New(typ).Elem() result.SetInt(parsed) diff --git a/internal/config/pinned_memory_test.go b/internal/config/pinned_memory_test.go index 74de5a80..29826aa4 100644 --- a/internal/config/pinned_memory_test.go +++ b/internal/config/pinned_memory_test.go @@ -2,7 +2,7 @@ package config import "testing" -// Pinning storage.duckdb.memory skips detection entirely, so nothing compares +// Pinning FANOUT_DUCKDB_MEMORY skips detection entirely, so nothing compares // the value to the machine. Both production deployments pinned 4GB on 8 GiB // hosts and were OOM-killed repeatedly: DuckDB's budget is not the process's // budget, and the Go runtime, the ingest appender and compaction's merges all diff --git a/internal/config/provider_test.go b/internal/config/provider_test.go index c9a49735..bdcb4865 100644 --- a/internal/config/provider_test.go +++ b/internal/config/provider_test.go @@ -1,12 +1,77 @@ package config import ( + "fmt" "os" "path/filepath" "strings" "testing" ) +func TestLoadIgnoresInactiveProviderSecretFiles(t *testing.T) { + emptyPath := filepath.Join(t.TempDir(), "empty") + if err := os.WriteFile(emptyPath, nil, 0o644); err != nil { + t.Fatal(err) + } + for _, provider := range []string{"", "anthropic", "openai"} { + for _, vendor := range []string{"anthropic", "openai"} { + if vendor == provider { + continue + } + for _, file := range []struct{ name, path string }{ + {"missing", "/missing"}, {"empty", emptyPath}, + } { + for _, bothSet := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/%s/%s/both_set=%t", provider, vendor, file.name, bothSet), func(t *testing.T) { + env := []string{"FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), "FANOUT_AI_PROVIDER=" + provider} + if provider != "" { + env = append(env, strings.ToUpper(provider)+"_API_KEY=selected-key") + } + name := strings.ToUpper(vendor) + env = append(env, name+"_API_KEY_FILE="+file.path, name+"_BASE_URL=unused-invalid-url") + if bothSet { + env = append(env, name+"_API_KEY=unused-key") + } + cfg, err := Load(LoadOptions{Environ: env}) + if err != nil { + t.Fatal(err) + } + if vendor == "anthropic" && (cfg.AnthropicAPIKey != "" || cfg.AnthropicBaseURL != "") || + vendor == "openai" && (cfg.OpenAIAPIKey != "" || cfg.OpenAIBaseURL != "") { + t.Fatal("inactive provider settings must be ignored") + } + }) + } + } + } + } +} + +func TestLoadRejectsSelectedProviderSecretFileErrors(t *testing.T) { + for _, provider := range []string{"anthropic", "openai"} { + for _, bothSet := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/both_set=%t", provider, bothSet), func(t *testing.T) { + name := strings.ToUpper(provider) + "_API_KEY" + path := filepath.Join(t.TempDir(), "missing") + env := []string{ + "FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), "FANOUT_AI_PROVIDER=" + provider, + name + "_FILE=" + path, + } + if bothSet { + env = append(env, name+"=synthetic-private-value") + } + _, err := Load(LoadOptions{Environ: env}) + if err == nil || !strings.Contains(err.Error(), name+"_FILE") || !strings.Contains(err.Error(), path) || strings.Contains(err.Error(), "synthetic-private-value") { + t.Fatalf("expected redacted file error naming variable and path, got %v", err) + } + if bothSet && !strings.Contains(err.Error(), "must not both be set") { + t.Fatalf("expected selected provider conflict, got %v", err) + } + }) + } + } +} + func TestLoadSelectsProviderCredentials(t *testing.T) { for _, tc := range []struct { name, provider, key, baseURL string @@ -31,24 +96,20 @@ func TestLoadSelectsProviderCredentials(t *testing.T) { } } -func TestLoadProviderEnvironmentOverridesYAML(t *testing.T) { - path := filepath.Join(t.TempDir(), "fanout.yaml") - document := "ai:\n provider: openai\n model: configured-model\n anthropic:\n api_key: anthropic-file\n base_url: https://anthropic.example\n openai:\n api_key: openai-file\n base_url: https://openai.example/v1\n" - if err := os.WriteFile(path, []byte(document), 0o600); err != nil { - t.Fatal(err) - } - cfg, err := Load(LoadOptions{Path: path, Environ: []string{ +func TestLoadIgnoresOtherVendorEnvironment(t *testing.T) { + cfg, err := Load(LoadOptions{Environ: []string{ "FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), - "OPENAI_API_KEY=openai-env", "OPENAI_BASE_URL=https://gateway.example/api/v1/", - "ANTHROPIC_API_KEY=", "ANTHROPIC_BASE_URL=", "FANOUT_AI_MODEL=", + "FANOUT_AI_PROVIDER=openai", "OPENAI_API_KEY=openai-env", + "OPENAI_BASE_URL=https://gateway.example/api/v1/", "FANOUT_AI_MODEL=configured-model", + "ANTHROPIC_API_KEY=", "ANTHROPIC_BASE_URL=", "OPENAI_MODEL=unrelated-tool", "ANTHROPIC_MODEL=unrelated-tool", "OPENAI_PROJECT_ID=another-consumer", "ANTHROPIC_AUTH_TOKEN=another-consumer", }}) if err != nil { t.Fatal(err) } - if cfg.AIAPIKey() != "openai-env" || cfg.AIBaseURL() != "https://gateway.example/api/v1" || cfg.AIModel != "configured-model" || cfg.AnthropicAPIKey != "anthropic-file" || cfg.AnthropicBaseURL != "https://anthropic.example" { - t.Fatal("provider environment precedence or shared consumer isolation is incorrect") + if cfg.AIAPIKey() != "openai-env" || cfg.AIBaseURL() != "https://gateway.example/api/v1" || cfg.AIModel != "configured-model" || cfg.AnthropicAPIKey != "" || cfg.AnthropicBaseURL != "" { + t.Fatal("provider environment or shared consumer isolation is incorrect") } } @@ -67,7 +128,7 @@ func TestLoadRejectsInvalidProviderURLWithoutLeakingValues(t *testing.T) { func TestLoadRejectsUnsupportedProviderWithoutAKey(t *testing.T) { for _, provider := range []string{"unsupported", "OpenAI", " OpenAI ", " openai ", "Anthropic"} { _, err := Load(LoadOptions{Environ: []string{"FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), "FANOUT_AI_PROVIDER=" + provider}}) - if err == nil || !strings.Contains(err.Error(), "ai.provider") { + if err == nil || !strings.Contains(err.Error(), "FANOUT_AI_PROVIDER") { t.Fatal("unsupported provider must fail at startup, even without a key") } } @@ -85,8 +146,8 @@ func TestLoadRequiresSelectedProviderKey(t *testing.T) { "FANOUT_AUTH_CODE_SECRET=" + strings.Repeat("x", 32), "FANOUT_AI_PROVIDER=" + provider, name + "=" + value, otherKey + "=other-provider-key", }}) - if err == nil || !strings.Contains(err.Error(), "ai."+provider+".api_key") || !strings.Contains(err.Error(), name) { - t.Fatalf("expected missing key error naming both YAML and environment keys, got %v", err) + if err == nil || !strings.Contains(err.Error(), name) { + t.Fatalf("expected missing key error naming the environment variable, got %v", err) } } } diff --git a/internal/config/sizing.go b/internal/config/sizing.go index 5aa1f490..b8fa0312 100644 --- a/internal/config/sizing.go +++ b/internal/config/sizing.go @@ -134,13 +134,13 @@ func (c Config) logResolvedSizing(src sizingSource) { "gomaxprocs", runtime.GOMAXPROCS(0), ) if src.MemoryAuto && src.MemoryDetectionIncomplete { - slog.Warn("memory limit detection was incomplete; automatic DuckDB sizing may exceed a container limit — configure storage.duckdb.memory explicitly", + slog.Warn("memory limit detection was incomplete; automatic DuckDB sizing may exceed a container limit — configure FANOUT_DUCKDB_MEMORY explicitly", "memory_source", src.MemorySource, "detected_memory_bytes", src.DetectedRAM, ) } if src.MemoryAuto && c.DuckDBMemory == "" { - slog.Warn("could not detect available memory; DuckDB will size itself to 80% of RAM, which can exceed the machine once the Go runtime is added — configure storage.duckdb.memory explicitly") + slog.Warn("could not detect available memory; DuckDB will size itself to 80% of RAM, which can exceed the machine once the Go runtime is added — configure FANOUT_DUCKDB_MEMORY explicitly") } // A pinned budget skipped detection, so this is the only place it is ever // compared to the machine it will run on. @@ -174,7 +174,7 @@ func resolveDuckDBMemory(available uint64) string { // pinnedMemoryConcern reports why an operator-pinned DuckDB budget looks wrong // for this machine, or "" when it looks fine or cannot be judged. // -// Pinning storage.duckdb.memory skips detection entirely (resolveSizing only +// Pinning FANOUT_DUCKDB_MEMORY skips detection entirely (resolveSizing only // fills a value that is empty), so without this nothing ever compares the // figure to the machine it will run on. DuckDB's budget is not the process's // budget: the Go heap, the ingest appender and compaction's merges all sit on @@ -192,14 +192,14 @@ func pinnedMemoryConcern(pinned string, detectedRAM uint64) string { } budget, ok := parseByteSize(pinned) if !ok { - return fmt.Sprintf("storage.duckdb.memory %q could not be parsed as a size; DuckDB may reject it or fall back to its own default", pinned) + return fmt.Sprintf("FANOUT_DUCKDB_MEMORY %q could not be parsed as a size; DuckDB may reject it or fall back to its own default", pinned) } if budget >= detectedRAM { - return fmt.Sprintf("storage.duckdb.memory %q is at or above the %d bytes detected for this machine; the Go runtime and compaction allocate on top of it", pinned, detectedRAM) + return fmt.Sprintf("FANOUT_DUCKDB_MEMORY %q is at or above the %d bytes detected for this machine; the Go runtime and compaction allocate on top of it", pinned, detectedRAM) } share := detectedRAM / 100 * duckDBMemoryPercent if budget > share { - return fmt.Sprintf("storage.duckdb.memory %q takes more than the %d%% of %d bytes automatic sizing would leave DuckDB; the remainder has to cover the Go heap, the ingest appender and compaction merges", pinned, duckDBMemoryPercent, detectedRAM) + return fmt.Sprintf("FANOUT_DUCKDB_MEMORY %q takes more than the %d%% of %d bytes automatic sizing would leave DuckDB; the remainder has to cover the Go heap, the ingest appender and compaction merges", pinned, duckDBMemoryPercent, detectedRAM) } return "" } diff --git a/internal/query/duck.go b/internal/query/duck.go index 1d29989b..7e8458ce 100644 --- a/internal/query/duck.go +++ b/internal/query/duck.go @@ -153,7 +153,7 @@ func duckDBPoolSize(cfg config.Config) int { // of a constant that's wrong everywhere but the box it was tuned for. func duckDSN(dbPath, mem string, threads int) (string, error) { if strings.ContainsAny(mem, "&?'\"\\; ") { - return "", fmt.Errorf("invalid storage.duckdb.memory value: %q", mem) + return "", fmt.Errorf("invalid FANOUT_DUCKDB_MEMORY value: %q", mem) } params := make([]string, 0, 2) if threads > 0 { @@ -292,7 +292,7 @@ func NewDuck(ctx context.Context, cfg config.Config, repository *telemetrystore. return tags }) if cfg.DuckDBMemory == "" { - // Only when the operator hasn't pinned storage.duckdb.memory: keep DuckDB's + // Only when the operator hasn't pinned FANOUT_DUCKDB_MEMORY: keep DuckDB's // cgroup-aware auto limit on big boxes but leave absolute RAM headroom on // small ones, so the kernel doesn't OOM-kill the whole process. if err := applyMemoryHeadroom(ctx, db); err != nil { diff --git a/justfile b/justfile index 658e2ddf..25c9f315 100644 --- a/justfile +++ b/justfile @@ -32,14 +32,9 @@ embedded := "internal/ui/dist internal/mcp/apps" default: @just --list -# Launch with shared nonsecret defaults, personal overrides, then injected values. -# Build the binary first with `just build`; secrets must be injected explicitly. +# Load optional ignored .env, then inherited values. Build with `just build` first. run *ARGS='': - node scripts/run-fanout.mjs --environment local -- ./bin/fanout {{ARGS}} - -# Production explicitly excludes every developer-local dotenv file. -run-production *ARGS='': - node scripts/run-fanout.mjs --environment production -- ./bin/fanout {{ARGS}} + node scripts/run-fanout.mjs -- ./bin/fanout {{ARGS}} # Split out so CI installs exactly what a contributor does, without also # writing git hooks into the runner. diff --git a/scripts/run-fanout.mjs b/scripts/run-fanout.mjs index cc0528f3..12da133a 100644 --- a/scripts/run-fanout.mjs +++ b/scripts/run-fanout.mjs @@ -1,87 +1,35 @@ import { spawn } from "node:child_process"; -import { readFileSync, statSync } from "node:fs"; +import { readFileSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { parseEnv } from "node:util"; -/** @param {string} path @param {boolean} required */ -function readDotenv(path, required) { - let content; +/** @param {string} root @param {Record} inherited */ +export function prepareEnvironment(root, inherited) { + let content = ""; try { - if (!statSync(path).isFile()) throw new Error("not a regular file"); - content = readFileSync(path, "utf8"); + content = readFileSync(resolve(root, ".env"), "utf8"); } catch (error) { - if (!required && error && typeof error === "object" && "code" in error && error.code === "ENOENT") return {}; - throw new Error(`Cannot read required or invalid dotenv file: ${path}`); - } - try { - // Node's dotenv parser preserves literal dollars and never executes shell code. - return parseEnv(content); - } catch { - throw new Error(`Invalid dotenv syntax: ${path}`); - } -} - -// The generated template is the schema-derived inventory, including secret -// classification. There is no independently maintained configuration allowlist. -/** @param {string} root */ -function inventory(root) { - let content; - try { - content = readFileSync(resolve(root, ".env.example"), "utf8"); - } catch { - throw new Error("Missing generated environment inventory; run just docs-generate"); - } - const names = new Set(Object.keys(parseEnv(content))); - const secrets = new Set(); - for (const block of content.split(/\n\s*\n/)) { - if (/^# [a-z0-9_.]+; [a-zA-Z0-9_.]+; secret$/.test(block.split(/\r?\n/, 1)[0])) { - for (const name of Object.keys(parseEnv(block))) secrets.add(name); - } - } - if (names.size === 0 || secrets.size === 0) throw new Error("Missing generated environment inventory; run just docs-generate"); - return { names, secrets }; -} - -/** - * @param {string} root - * @param {"local" | "production"} environment - * @param {Record} inherited - * @returns {Record} - */ -export function prepareEnvironment(root, environment, inherited) { - if (environment !== "local" && environment !== "production") throw new Error("Environment must be local or production"); - const { names, secrets } = inventory(root); - const files = [".env", ...(environment === "production" ? [".env.production"] : [".env.local"])]; - /** @type {Record} */ - const values = {}; - for (const file of files) { - const layer = readDotenv(resolve(root, file), file !== ".env.local"); - for (const [name, value] of Object.entries(layer)) { - if (value === undefined) throw new Error(`Invalid dotenv syntax: ${file}`); - if (!names.has(name)) throw new Error(`Unknown dotenv setting ${name} in ${file}`); - if (secrets.has(name)) throw new Error(`Secret setting ${name} belongs in runtime injection, not ${file}`); - if (value.includes("op://")) throw new Error(`Unresolved secret reference for ${name} in ${file}; use runtime injection`); - if (value !== "") values[name] = value; + if (!error || typeof error !== "object" || !("code" in error) || error.code !== "ENOENT") { + throw new Error("Cannot read dotenv file: .env"); } } - // Explicitly injected values win. An empty recognized value retains Fanout's - // existing "no override" semantics instead of erasing a file setting. + // Node's parser preserves literal dollars and never executes shell code. + const values = parseEnv(content); for (const [name, value] of Object.entries(inherited)) { - if (value !== undefined && (value !== "" || !names.has(name))) values[name] = value; + if (value !== undefined) values[name] = value; } return values; } /** @param {string[]} args @returns {Promise} */ export async function main(args) { - if (args.length < 4 || args[0] !== "--environment" || args[2] !== "--") { - throw new Error("Usage: node scripts/run-fanout.mjs --environment local|production -- command [args]"); + if (args.length < 2 || args[0] !== "--") { + throw new Error("Usage: node scripts/run-fanout.mjs -- command [args]"); } const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); - const environment = /** @type {"local" | "production"} */ (args[1]); - const env = prepareEnvironment(root, environment, process.env); - const child = spawn(args[3], args.slice(4), { cwd: root, env, stdio: "inherit" }); + const env = prepareEnvironment(root, process.env); + const child = spawn(args[1], args.slice(2), { cwd: root, env, stdio: "inherit" }); const handlers = new Map(["SIGINT", "SIGTERM", "SIGHUP"].map(signal => [signal, () => child.kill(signal)])); for (const [signal, handler] of handlers) process.on(signal, handler); try { @@ -98,13 +46,9 @@ if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.m try { process.exitCode = await main(process.argv.slice(2)); } catch (error) { - // Launch errors can contain command arguments. Only preparation errors - // below carry safe setting/file names; never print resolved environment values. const message = error instanceof Error ? error.message : "Environment launch failed"; - console.error(message.startsWith("Unknown dotenv") || message.startsWith("Secret setting") || - message.startsWith("Unresolved secret") || message.startsWith("Cannot read") || - message.startsWith("Invalid dotenv") || message.startsWith("Usage:") || - message.startsWith("Environment must") || message.startsWith("Missing generated") ? message : "Cannot launch command; ensure it is executable (for Fanout, run just build)"); + console.error(message.startsWith("Cannot read dotenv") || message.startsWith("Usage:") + ? message : "Cannot launch command; ensure it is executable (for Fanout, run just build)"); process.exitCode = 1; } } diff --git a/scripts/run-fanout.test.ts b/scripts/run-fanout.test.ts index d946cf6d..73aa8990 100644 --- a/scripts/run-fanout.test.ts +++ b/scripts/run-fanout.test.ts @@ -1,5 +1,5 @@ import { afterEach, expect, test } from "bun:test"; -import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { copyFileSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { spawn } from "node:child_process"; import { join } from "node:path"; import { tmpdir } from "node:os"; @@ -12,134 +12,43 @@ afterEach(() => { for (const root of roots.splice(0)) rmSync(root, { recursive: function fixture(): string { const root = mkdtempSync(join(tmpdir(), "fanout-env-")); roots.push(root); - writeFileSync(join(root, ".env.example"), `# server.addr; string -FANOUT_ADDR= - -# ai.model; string -FANOUT_AI_MODEL= - -# storage.data_dir; string -FANOUT_DATA_DIR= - -# auth.code_secret; string; secret -FANOUT_AUTH_CODE_SECRET= - -# ai.anthropic.api_key; string; secret -ANTHROPIC_API_KEY= -`); - writeFileSync(join(root, ".env"), "FANOUT_ADDR=127.0.0.1:7520\n"); return root; } -test("local dotenv defaults, personal overrides, and explicit injection have ordered precedence", () => { +test("dotenv is optional and inherited values win, including empty values", () => { const root = fixture(); - writeFileSync(join(root, ".env.local"), "FANOUT_ADDR=127.0.0.1:9000\nFANOUT_AI_MODEL=personal\n"); - const env = prepareEnvironment(root, "local", { FANOUT_ADDR: "", FANOUT_AI_MODEL: "injected", FANOUT_AUTH_CODE_SECRET: "synthetic-only", PATH: "/synthetic/path" }); - expect(env.FANOUT_ADDR).toBe("127.0.0.1:9000"); + expect(prepareEnvironment(root, { PATH: "/synthetic/path" })).toEqual({ PATH: "/synthetic/path" }); + writeFileSync(join(root, ".env"), "FANOUT_ADDR=:9000\nFANOUT_AI_MODEL=personal\nANTHROPIC_API_KEY=synthetic-only\nFANOUT_UNKNOWN=value\n"); + const env = prepareEnvironment(root, { FANOUT_ADDR: "", FANOUT_AI_MODEL: "injected", PATH: "/synthetic/path" }); + expect(env.FANOUT_ADDR).toBe(""); expect(env.FANOUT_AI_MODEL).toBe("injected"); - expect(env.FANOUT_AUTH_CODE_SECRET).toBe("synthetic-only"); + expect(env.ANTHROPIC_API_KEY).toBe("synthetic-only"); + expect(env.FANOUT_UNKNOWN).toBe("value"); expect(env.PATH).toBe("/synthetic/path"); }); -test("production excludes personal files and requires its declared override file", () => { - const root = fixture(); - writeFileSync(join(root, ".env.local"), "ANTHROPIC_API_KEY=personal-secret\n"); - writeFileSync(join(root, ".env.production.local"), "FANOUT_AI_MODEL=personal-production\n"); - expect(() => prepareEnvironment(root, "production", {})).toThrow("Cannot read required"); - writeFileSync(join(root, ".env.production"), "FANOUT_ADDR=:7520\n"); - const env = prepareEnvironment(root, "production", {}); - expect(env.FANOUT_ADDR).toBe(":7520"); - expect(env.FANOUT_AI_MODEL).toBeUndefined(); - expect(env.ANTHROPIC_API_KEY).toBeUndefined(); - expect(prepareEnvironment(root, "production", { FANOUT_AI_MODEL: "deployment" }).FANOUT_AI_MODEL).toBe("deployment"); -}); - -test("CRLF generated inventory preserves secret classification for local and production preparation", () => { - const inventory = readFileSync(join(import.meta.dir, "../.env.example"), "utf8").replace(/\r?\n/g, "\r\n"); - for (const environment of ["local", "production"] as const) { - const root = fixture(); - writeFileSync(join(root, ".env.example"), inventory); - writeFileSync(join(root, ".env.production"), "FANOUT_ADDR=:7520\n"); - const injected = { FANOUT_AUTH_CODE_SECRET: "synthetic-only", ANTHROPIC_API_KEY: "synthetic-only" }; - const env = prepareEnvironment(root, environment, injected); - expect(env.FANOUT_ADDR).toBe(environment === "local" ? "127.0.0.1:7520" : ":7520"); - expect(env.FANOUT_AUTH_CODE_SECRET).toBe(injected.FANOUT_AUTH_CODE_SECRET); - expect(env.ANTHROPIC_API_KEY).toBe(injected.ANTHROPIC_API_KEY); - const file = environment === "local" ? ".env.local" : ".env.production"; - for (const name of Object.keys(injected)) { - writeFileSync(join(root, file), `${name}=synthetic-only\n`); - expect(() => prepareEnvironment(root, environment, {})).toThrow(`Secret setting ${name} belongs in runtime injection, not ${file}`); - } - } -}); - -test("dotenv parsing preserves dollars, quotes, whitespace, and multiline values without shell execution", () => { +test("dotenv parsing preserves literal shell text, CRLF, and multiline values", () => { const root = fixture(); - writeFileSync(join(root, ".env.local"), `FANOUT_DATA_DIR=' $HOME $(printf unsafe) "quoted" ' + writeFileSync(join(root, ".env"), `FANOUT_DATA_DIR=' $HOME $(printf unsafe) "quoted" ' FANOUT_AI_MODEL="line one line two" -`); - const env = prepareEnvironment(root, "local", {}); +`.replace(/\n/g, "\r\n")); + const env = prepareEnvironment(root, {}); expect(env.FANOUT_DATA_DIR).toBe(' $HOME $(printf unsafe) "quoted" '); expect(env.FANOUT_AI_MODEL).toBe("line one\nline two"); - // Verify the real Node runtime as well as Bun's test-runner implementation. const result = Bun.spawnSync(["node", "--input-type=module", "-e", - "const {prepareEnvironment}=await import(process.argv[2]); process.stdout.write(JSON.stringify(prepareEnvironment(process.argv[1], 'local', {})));", + "const {prepareEnvironment}=await import(process.argv[2]); process.stdout.write(JSON.stringify(prepareEnvironment(process.argv[1], {})));", root, pathToFileURL(join(import.meta.dir, "run-fanout.mjs")).href], { cwd: root }); expect(result.exitCode).toBe(0); expect(JSON.parse(result.stdout.toString())).toEqual(env); }); -test("public and personal dotenv files reject secrets, retired names, and unresolved references without leaking values", () => { - for (const assignment of [ - "ANTHROPIC_API_KEY=synthetic-private-value", - "FANOUT_AUTH_CODE_SECRET=", - "FANOUT_AI_API_KEY=synthetic-private-value", - "FANOUT_AI_BASE_URL=https://example.com/synthetic-private-value", - "AI_API_KEY=synthetic-private-value", - "FANOUT_AI_MODEL=op://synthetic-private-value", - ]) { - for (const file of [".env", ".env.local"]) { - const root = fixture(); - writeFileSync(join(root, file), assignment+"\n"); - let message = ""; - try { prepareEnvironment(root, "local", {}); } catch (error) { message = (error as Error).message; } - expect(message).not.toBe(""); - expect(message).not.toContain("synthetic-private-value"); - } - } -}); - -test("missing defaults and unsupported environments fail rather than fall back", () => { +test("unreadable dotenv fails and launcher requires a command", async () => { const root = fixture(); - rmSync(join(root, ".env")); - expect(() => prepareEnvironment(root, "local", {})).toThrow("Cannot read required"); - expect(() => prepareEnvironment(root, "staging" as "local", {})).toThrow("Environment must"); -}); - -test("launcher requires an explicit environment and command", async () => { + mkdirSync(join(root, ".env")); + expect(() => prepareEnvironment(root, {})).toThrow("Cannot read dotenv"); await expect(main([])).rejects.toThrow("Usage:"); - await expect(main(["--environment", "local", "--"])).rejects.toThrow("Usage:"); -}); - -test("secret classification uses only the strict first-line schema header", () => { - const root = fixture(); - writeFileSync(join(root, ".env.example"), `# ai.model; string -# documentation; string; secret -FANOUT_AI_MODEL= - -# Invalid-Key; string; secret -FANOUT_DATA_DIR= - -# auth.code_secret; string; secret -# An ordinary doc line cannot remove the secret marker. -FANOUT_AUTH_CODE_SECRET= -`); - writeFileSync(join(root, ".env"), "FANOUT_AI_MODEL=public\nFANOUT_DATA_DIR=./public\n"); - expect(prepareEnvironment(root, "local", {}).FANOUT_AI_MODEL).toBe("public"); - expect(prepareEnvironment(root, "local", {}).FANOUT_DATA_DIR).toBe("./public"); - writeFileSync(join(root, ".env"), "FANOUT_AUTH_CODE_SECRET=synthetic-only\n"); - expect(() => prepareEnvironment(root, "local", {})).toThrow("Secret setting FANOUT_AUTH_CODE_SECRET"); + await expect(main(["--"])).rejects.toThrow("Usage:"); }); function launchFixture(childCode: string) { @@ -150,7 +59,7 @@ function launchFixture(childCode: string) { import { main } from './scripts/run-fanout.mjs'; const signals = ['SIGINT', 'SIGTERM', 'SIGHUP']; const before = signals.map(signal => process.listenerCount(signal)); -const code = await main(['--environment', 'local', '--', process.execPath, '-e', process.argv[2]]); +const code = await main(['--', process.execPath, '-e', process.argv[2]]); console.log('cleanup:' + signals.every((signal, index) => process.listenerCount(signal) === before[index])); process.exitCode = code; `; diff --git a/site/src/content/docs/guides/troubleshoot.mdx b/site/src/content/docs/guides/troubleshoot.mdx index b813a75b..3dd0405d 100644 --- a/site/src/content/docs/guides/troubleshoot.mdx +++ b/site/src/content/docs/guides/troubleshoot.mdx @@ -83,7 +83,7 @@ trace index. First stop Fanout and back up `telemetry/`, then verify the full authoritative set: ```sh -fanout --config /etc/fanout/fanout.yaml repair verify +fanout repair verify ``` The command is read-only and names every unreadable batch. Restore each named @@ -92,7 +92,7 @@ only its telemetry is preferable to leaving the instance offline, set it aside explicitly: ```sh -fanout --config /etc/fanout/fanout.yaml repair quarantine --batch +fanout repair quarantine --batch ``` Quarantine is an atomic rename beside the authoritative batch set, not a @@ -137,7 +137,7 @@ operator with shell access to the same configuration and data directory then mints a link directly: ```sh -fanout --config /etc/fanout/fanout.yaml login-link user@example.com +fanout login-link user@example.com ``` The link expires after 15 minutes, works once, and is recorded in the diff --git a/site/src/content/docs/reference/configuration.mdx b/site/src/content/docs/reference/configuration.mdx index c2029809..d747f006 100644 --- a/site/src/content/docs/reference/configuration.mdx +++ b/site/src/content/docs/reference/configuration.mdx @@ -1,87 +1,85 @@ --- title: Configuration description: How Fanout is configured, where settings come from, and how the reference stays true to the binary. -summary: The rules that govern every setting — precedence, naming, secrets, and what happens to a name that changes. +summary: The rules that govern every setting — precedence, naming, secret files, and validation. read_when: - You are configuring an instance and want the rules before the list. - A setting you set appears to have been ignored. status: preview --- -Fanout is configured from the environment, or from a YAML file using the same -keys. Every setting is listed under [Settings](/reference/settings/server), +Fanout resolves built-in defaults followed by nonempty process environment +values. Every setting is listed under [Settings](/reference/settings/server), grouped by the part of the product it belongs to. -Those pages are generated from `internal/config` by `cmd/fanout-docgen`, and -the root `.env`, `.env.production`, and `.env.example` are generated from the -same schema. `just check` fails -when a committed page or template no longer matches the type the loader -binds. That is the whole point of generating them: the reference cannot describe -a setting the binary does not accept, and cannot omit one it does. +Those pages and `.env.example` are generated from `internal/config` by +`cmd/fanout-docgen`. `just docs-generate-check` verifies that they match the +schema the binary uses. ## Where a value comes from -A nonempty recognized environment variable wins over the YAML file, and the file -wins over the built-in default. Empty environment values mean no override. -All settings are read once at startup and require restart. Fanout does not load -dotenv files itself. The generated `.env.example` documents types, defaults, -and leaves every value empty so copying it never overrides YAML. `.env` contains -only schema-declared local overrides; `.env.production` resets exactly those -fields to their built-in defaults. Both omit secret fields. - -For a source checkout, build the binary and use `just run` for local development -or `just run-production` for an explicit production launch. Local order is -`.env`, optional ignored `.env.local`, then injected environment values. -Production order is `.env`, required `.env.production`, then injected values; -personal files are excluded. The Node launcher uses the native dotenv parser -without shell execution or variable interpolation. Missing required files, -unknown names, and secret entries in ordinary dotenv files fail before launch. - -Inject credentials from a private selected SOPS profile or the deployment's -secret service. This public repository ships no operator secrets or recipients. -Dotenv values are injected as environment, so nonempty values override the -corresponding `--config` YAML. The generated files override only the listener; -other YAML settings survive both launch modes. +All settings are read once at startup and require a restart. An empty +environment value means unset and preserves the built-in default. Native +installs listen on `127.0.0.1:7520`; the image supplies `FANOUT_ADDR=:7520` +and `FANOUT_DATA_DIR=/var/lib/fanout/data`. + +The binary reads only process environment. `.env.example` leaves every value +empty and documents types, defaults, and secret file forms in comments. For a +source checkout, copy it to ignored `.env` for personal settings and development +credentials. Build the binary, then use `just run`: Node's native `parseEnv` +loads optional `./.env`, and inherited environment values win, including empty +values. Values are never executed as shell code or interpolated. The binary +validates unknown names and types. Production deployments supply their own +environment values or managed secret mounts. ## Naming -Custom environment settings use the `FANOUT_` prefix. Provider keys and endpoints -use `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, `OPENAI_API_KEY`, and -`OPENAI_BASE_URL`. The YAML key is the dotted path in the first column of each -settings table — `storage.duckdb.memory` maps to `FANOUT_DUCKDB_MEMORY`. -Provider settings map to native structured keys under `ai.anthropic` or `ai.openai`. - -`FANOUT_AI_PROVIDER` (YAML `ai.provider`) defaults to empty, disabling chat. -Selecting exactly `anthropic` or `openai` enables chat and requires that provider's -key at startup; other names, casing, or surrounding whitespace fail. Only the -selected provider's key and endpoint are validated and used. With no provider -selected, inherited AI keys and endpoints are ignored. `FANOUT_AI_MODEL` selects -the model. Other vendor/tool settings, including `ANTHROPIC_MODEL` and -`OPENAI_MODEL`, are ignored. See [AI settings](/reference/settings/ai) for -the different base URL meanings and [upgrade instructions](/start/install#ai-configuration-upgrades) -for migrating the retired generic key and base URL settings. - -## An unknown variable is an error - -Fanout rejects an unrecognised `FANOUT_`-prefixed variable at startup rather -than ignoring it. This matters more than it looks: Fanout is pre-release and -settings get renamed without aliases, so the alternative is a variable you -believe is in effect, silently replaced by a default nobody chose, discovered -months later. A refusal to start is the version of that failure you can act on. - -The consequence is that upgrading can require a configuration change, and the -process tells you which one at the first boot after the upgrade. +Custom names use `FANOUT_` plus a clear name, adding the area only when it +clarifies. Vendor credentials and endpoints keep `ANTHROPIC_API_KEY`, +`ANTHROPIC_BASE_URL`, `OPENAI_API_KEY`, and `OPENAI_BASE_URL`. + +`FANOUT_AI_PROVIDER` defaults to empty, disabling chat. Exactly `anthropic` +or `openai` enables chat and requires that provider's key. Only the selected +provider's endpoint is validated and used. Inherited credentials never select +a provider. `FANOUT_AI_MODEL` selects the model; unrelated vendor/tool settings +such as `ANTHROPIC_MODEL` and `OPENAI_MODEL` are ignored. See +[AI settings](/reference/settings/ai) for base URL meanings and the +[configuration upgrade table](/start/install#configuration-upgrades). + +## Unknown names + +Unknown `FANOUT_` names fail startup. Platform-injected service-discovery names +are tolerated, and unrelated names in shared vendor namespaces are ignored. +Errors name the relevant environment variable. Boolean, integer, and Go +duration values must parse as their declared types. ## Secrets -Settings marked `secret` are redacted from the startup configuration log. They -are ordinary environment variables otherwise: anything that can read the process -environment can read them. Fanout does not encrypt them at rest and does not -integrate with a secret manager — mount them the way you would for any other -process. +Every secret accepts either its value or its file path: + +| Value variable | File path variable | +|---|---| +| `FANOUT_AUTH_CODE_SECRET` | `FANOUT_AUTH_CODE_SECRET_FILE` | +| `FANOUT_OIDC_CLIENT_SECRET` | `FANOUT_OIDC_CLIENT_SECRET_FILE` | +| `FANOUT_SMTP_PASSWORD` | `FANOUT_SMTP_PASSWORD_FILE` | +| `FANOUT_METRICS_TOKEN` | `FANOUT_METRICS_TOKEN_FILE` | +| `ANTHROPIC_API_KEY` | `ANTHROPIC_API_KEY_FILE` | +| `OPENAI_API_KEY` | `OPENAI_API_KEY_FILE` | + +Setting both forms nonempty fails startup. File contents lose exactly one +trailing LF or CRLF; an empty result fails. Errors name variables and paths, +never contents. The deployment controls file access; Fanout performs no +file-mode check. Secrets are redacted from startup logs. + +Use deployment environment injection, Docker/Compose secrets, Kubernetes +Secrets, or a private selected SOPS profile. This public repository ships no +operator profiles or recipients. See +[runtime secret delivery](https://github.com/labstack/fanout/blob/main/docs/operations.md#runtime-secret-delivery) +for examples. ## Validation -The whole configuration is validated before any listener opens, so a value the -binary will not accept stops it at startup. [First boot](/start/first-boot) -lists what is checked. +The whole configuration is validated before any listener opens. A value the +binary cannot accept stops startup. [First boot](/start/first-boot) lists what +is checked. The healthcheck command derives its probe from the listener and +TLS environment settings without validating unrelated settings or secrets. diff --git a/site/src/content/docs/reference/settings/ai.mdx b/site/src/content/docs/reference/settings/ai.mdx index 300df2e8..5ed983de 100644 --- a/site/src/content/docs/reference/settings/ai.mdx +++ b/site/src/content/docs/reference/settings/ai.mdx @@ -12,52 +12,52 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} Every setting below is read once at startup and requires a restart to change. -It is read from the environment variable in the second -column, or from the same key in a YAML configuration file. An unrecognised -`FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces -as a refusal to start rather than as a default nobody chose. +Built-in defaults are overridden by nonempty environment values. +Unknown `FANOUT_` names fail startup; platform-injected service names are tolerated. +Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. +Vendor credentials and endpoints keep the vendor's documented names. Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, `OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are ignored; only the explicitly selected AI provider's key and endpoint are used. -| Setting | Environment variable | Type | Default | -|---|---|---|---| -| `ai.anthropic.api_key` | `ANTHROPIC_API_KEY` | string · secret | — | -| `ai.anthropic.base_url` | `ANTHROPIC_BASE_URL` | string | — | -| `ai.model` | `FANOUT_AI_MODEL` | string | — | -| `ai.openai.api_key` | `OPENAI_API_KEY` | string · secret | — | -| `ai.openai.base_url` | `OPENAI_BASE_URL` | string | — | -| `ai.provider` | `FANOUT_AI_PROVIDER` | string | — | +| Environment variable | Type | Default | +|---|---|---| +| `ANTHROPIC_API_KEY` or `ANTHROPIC_API_KEY_FILE` | string · secret | — | +| `ANTHROPIC_BASE_URL` | string | — | +| `FANOUT_AI_MODEL` | string | — | +| `FANOUT_AI_PROVIDER` | string | — | +| `OPENAI_API_KEY` or `OPENAI_API_KEY_FILE` | string · secret | — | +| `OPENAI_BASE_URL` | string | — | ## Notes -### `ai.anthropic.api_key` +### `ANTHROPIC_API_KEY` -Required when ai.provider is anthropic. +Required when FANOUT_AI_PROVIDER is anthropic. -### `ai.anthropic.base_url` +### `ANTHROPIC_BASE_URL` Defaults to https://api.anthropic.com. Fanout appends /v1/messages, preserving any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. -### `ai.model` +### `FANOUT_AI_MODEL` -Overrides the provider's default: claude-sonnet-5-5 for Anthropic and gpt-6.1-sol for OpenAI. The OpenAI provider expects a reasoning model supporting reasoning.encrypted_content for stateless Responses replay. A 400 rejecting encrypted content is reported as a server-side configuration error; choose a compatible reasoning model with ai.model. +Overrides the provider's default: claude-sonnet-5-5 for Anthropic and gpt-6.1-sol for OpenAI. The OpenAI provider expects a reasoning model supporting reasoning.encrypted_content for stateless Responses replay. A 400 rejecting encrypted content is reported as a server-side configuration error; choose a compatible reasoning model with FANOUT_AI_MODEL. -### `ai.openai.api_key` +### `FANOUT_AI_PROVIDER` -Required when ai.provider is openai. +Enables chat by selecting exactly anthropic or openai. Empty disables chat. -### `ai.openai.base_url` +### `OPENAI_API_KEY` -Defaults to https://api.openai.com/v1. Fanout appends /responses, preserving the API version and any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. +Required when FANOUT_AI_PROVIDER is openai. -### `ai.provider` +### `OPENAI_BASE_URL` -Enables chat by selecting exactly anthropic or openai. Empty disables chat. +Defaults to https://api.openai.com/v1. Fanout appends /responses, preserving the API version and any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. :::caution[Secrets] Settings marked `secret` are redacted from Fanout's startup configuration -log. They are still plain environment variables: anything that can read the -process environment can read them. +log. Supply either the value or its `_FILE` path, never both. File contents +lose exactly one trailing newline (LF or CRLF); empty secrets fail startup. ::: diff --git a/site/src/content/docs/reference/settings/alerts.mdx b/site/src/content/docs/reference/settings/alerts.mdx index 86dec738..bc9437c4 100644 --- a/site/src/content/docs/reference/settings/alerts.mdx +++ b/site/src/content/docs/reference/settings/alerts.mdx @@ -11,31 +11,31 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} Every setting below is read once at startup and requires a restart to change. -It is read from the environment variable in the second -column, or from the same key in a YAML configuration file. An unrecognised -`FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces -as a refusal to start rather than as a default nobody chose. +Built-in defaults are overridden by nonempty environment values. +Unknown `FANOUT_` names fail startup; platform-injected service names are tolerated. +Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. +Vendor credentials and endpoints keep the vendor's documented names. Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, `OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are ignored; only the explicitly selected AI provider's key and endpoint are used. -| Setting | Environment variable | Type | Default | -|---|---|---|---| -| `alerts.enabled` | `FANOUT_ALERTS_ENABLED` | boolean | `true` | -| `alerts.evaluation_interval` | `FANOUT_ALERTS_EVALUATION_INTERVAL` | duration | `30s` | -| `alerts.history_days` | `FANOUT_ALERTS_HISTORY_DAYS` | integer | `7` | +| Environment variable | Type | Default | +|---|---|---| +| `FANOUT_ALERTS_ENABLED` | boolean | `true` | +| `FANOUT_ALERTS_EVALUATION_INTERVAL` | duration | `30s` | +| `FANOUT_ALERTS_HISTORY_DAYS` | integer | `7` | ## Notes -### `alerts.enabled` +### `FANOUT_ALERTS_ENABLED` Runs the alert evaluation engine. -### `alerts.evaluation_interval` +### `FANOUT_ALERTS_EVALUATION_INTERVAL` Schedules alert evaluation; use a Go duration of at least 1s. -### `alerts.history_days` +### `FANOUT_ALERTS_HISTORY_DAYS` Sets alert history retention in days. diff --git a/site/src/content/docs/reference/settings/auth.mdx b/site/src/content/docs/reference/settings/auth.mdx index 923ae2a1..6595389b 100644 --- a/site/src/content/docs/reference/settings/auth.mdx +++ b/site/src/content/docs/reference/settings/auth.mdx @@ -12,102 +12,102 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} Every setting below is read once at startup and requires a restart to change. -It is read from the environment variable in the second -column, or from the same key in a YAML configuration file. An unrecognised -`FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces -as a refusal to start rather than as a default nobody chose. +Built-in defaults are overridden by nonempty environment values. +Unknown `FANOUT_` names fail startup; platform-injected service names are tolerated. +Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. +Vendor credentials and endpoints keep the vendor's documented names. Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, `OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are ignored; only the explicitly selected AI provider's key and endpoint are used. -| Setting | Environment variable | Type | Default | -|---|---|---|---| -| `auth.code_secret` | `FANOUT_AUTH_CODE_SECRET` | string · secret | — | -| `auth.mode` | `FANOUT_AUTH_MODE` | string | `local` | -| `auth.oidc.admin_groups` | `FANOUT_OIDC_ADMIN_GROUPS` | string | — | -| `auth.oidc.allowed_domains` | `FANOUT_OIDC_ALLOWED_DOMAINS` | string | — | -| `auth.oidc.allowed_groups` | `FANOUT_OIDC_ALLOWED_GROUPS` | string | — | -| `auth.oidc.auto_provision` | `FANOUT_OIDC_AUTO_PROVISION` | boolean | `false` | -| `auth.oidc.client_id` | `FANOUT_OIDC_CLIENT_ID` | string | — | -| `auth.oidc.client_secret` | `FANOUT_OIDC_CLIENT_SECRET` | string · secret | — | -| `auth.oidc.default_role` | `FANOUT_OIDC_DEFAULT_ROLE` | string | `viewer` | -| `auth.oidc.email_claim` | `FANOUT_OIDC_EMAIL_CLAIM` | string | `email` | -| `auth.oidc.email_verification` | `FANOUT_OIDC_EMAIL_VERIFICATION` | string | `required` | -| `auth.oidc.issuer_url` | `FANOUT_OIDC_ISSUER_URL` | string | — | -| `auth.oidc.operator_groups` | `FANOUT_OIDC_OPERATOR_GROUPS` | string | — | -| `auth.self_signup` | `FANOUT_SELF_SIGNUP` | boolean | `false` | -| `auth.session_absolute_ttl` | `FANOUT_SESSION_ABSOLUTE_TTL` | duration | `168h` | -| `auth.session_idle_ttl` | `FANOUT_SESSION_IDLE_TTL` | duration | `12h` | +| Environment variable | Type | Default | +|---|---|---| +| `FANOUT_AUTH_CODE_SECRET` or `FANOUT_AUTH_CODE_SECRET_FILE` | string · secret | — | +| `FANOUT_AUTH_MODE` | string | `local` | +| `FANOUT_OIDC_ADMIN_GROUPS` | string | — | +| `FANOUT_OIDC_ALLOWED_DOMAINS` | string | — | +| `FANOUT_OIDC_ALLOWED_GROUPS` | string | — | +| `FANOUT_OIDC_AUTO_PROVISION` | boolean | `false` | +| `FANOUT_OIDC_CLIENT_ID` | string | — | +| `FANOUT_OIDC_CLIENT_SECRET` or `FANOUT_OIDC_CLIENT_SECRET_FILE` | string · secret | — | +| `FANOUT_OIDC_DEFAULT_ROLE` | string | `viewer` | +| `FANOUT_OIDC_EMAIL_CLAIM` | string | `email` | +| `FANOUT_OIDC_EMAIL_VERIFICATION` | string | `required` | +| `FANOUT_OIDC_ISSUER_URL` | string | — | +| `FANOUT_OIDC_OPERATOR_GROUPS` | string | — | +| `FANOUT_SELF_SIGNUP` | boolean | `false` | +| `FANOUT_SESSION_ABSOLUTE_TTL` | duration | `168h` | +| `FANOUT_SESSION_IDLE_TTL` | duration | `12h` | ## Notes -### `auth.code_secret` +### `FANOUT_AUTH_CODE_SECRET` Signs local sign-in codes; required in local mode with at least 32 characters. -### `auth.mode` +### `FANOUT_AUTH_MODE` Selects local sign-in or oidc; each mode validates its prerequisites. -### `auth.oidc.admin_groups` +### `FANOUT_OIDC_ADMIN_GROUPS` Lists identity-provider groups mapped to admin, separated by commas. -### `auth.oidc.allowed_domains` +### `FANOUT_OIDC_ALLOWED_DOMAINS` Lists allowed email domains, separated by commas; wildcard * requires verified email. -### `auth.oidc.allowed_groups` +### `FANOUT_OIDC_ALLOWED_GROUPS` Lists allowed identity-provider groups, separated by commas. -### `auth.oidc.auto_provision` +### `FANOUT_OIDC_AUTO_PROVISION` Creates accounts on OIDC sign-in; requires an allowed group or domain. -### `auth.oidc.client_id` +### `FANOUT_OIDC_CLIENT_ID` Identifies Fanout's client registration; required in OIDC mode. -### `auth.oidc.client_secret` +### `FANOUT_OIDC_CLIENT_SECRET` Authenticates Fanout's client registration; required in OIDC mode. -### `auth.oidc.default_role` +### `FANOUT_OIDC_DEFAULT_ROLE` Assigns viewer, operator, or admin to users without a group role mapping. -### `auth.oidc.email_claim` +### `FANOUT_OIDC_EMAIL_CLAIM` Names the claim containing the user's email address. -### `auth.oidc.email_verification` +### `FANOUT_OIDC_EMAIL_VERIFICATION` Selects required verified-email claims or issuer trust; issuer trust requires an allowlist. -### `auth.oidc.issuer_url` +### `FANOUT_OIDC_ISSUER_URL` Selects the identity provider issuer; required in OIDC mode. -### `auth.oidc.operator_groups` +### `FANOUT_OIDC_OPERATOR_GROUPS` Lists identity-provider groups mapped to operator, separated by commas. -### `auth.self_signup` +### `FANOUT_SELF_SIGNUP` Allows verified email addresses to create viewer accounts; requires local auth and SMTP. -### `auth.session_absolute_ttl` +### `FANOUT_SESSION_ABSOLUTE_TTL` -Limits total session lifetime; it must be at least session_idle_ttl. +Limits total session lifetime; it must be at least FANOUT_SESSION_IDLE_TTL. -### `auth.session_idle_ttl` +### `FANOUT_SESSION_IDLE_TTL` Expires inactive sessions; use a Go duration of at least 5m. :::caution[Secrets] Settings marked `secret` are redacted from Fanout's startup configuration -log. They are still plain environment variables: anything that can read the -process environment can read them. +log. Supply either the value or its `_FILE` path, never both. File contents +lose exactly one trailing newline (LF or CRLF); empty secrets fail startup. ::: diff --git a/site/src/content/docs/reference/settings/ingest.mdx b/site/src/content/docs/reference/settings/ingest.mdx index 50c75255..c82cd6ae 100644 --- a/site/src/content/docs/reference/settings/ingest.mdx +++ b/site/src/content/docs/reference/settings/ingest.mdx @@ -12,31 +12,31 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} Every setting below is read once at startup and requires a restart to change. -It is read from the environment variable in the second -column, or from the same key in a YAML configuration file. An unrecognised -`FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces -as a refusal to start rather than as a default nobody chose. +Built-in defaults are overridden by nonempty environment values. +Unknown `FANOUT_` names fail startup; platform-injected service names are tolerated. +Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. +Vendor credentials and endpoints keep the vendor's documented names. Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, `OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are ignored; only the explicitly selected AI provider's key and endpoint are used. -| Setting | Environment variable | Type | Default | -|---|---|---|---| -| `ingest.batch_size` | `FANOUT_INGEST_BATCH_SIZE` | integer | `50000` | -| `ingest.default_namespace` | `FANOUT_DEFAULT_NAMESPACE` | string | `default` | -| `ingest.max_in_flight_bytes` | `FANOUT_INGEST_MAX_IN_FLIGHT_BYTES` | integer | `268435456` | +| Environment variable | Type | Default | +|---|---|---| +| `FANOUT_DEFAULT_NAMESPACE` | string | `default` | +| `FANOUT_INGEST_BATCH_SIZE` | integer | `50000` | +| `FANOUT_INGEST_MAX_IN_FLIGHT_BYTES` | integer | `268435456` | ## Notes -### `ingest.batch_size` +### `FANOUT_DEFAULT_NAMESPACE` -Limits telemetry rows per batch; valid values are 1–50000. +Supplies the namespace for telemetry without one. -### `ingest.default_namespace` +### `FANOUT_INGEST_BATCH_SIZE` -Supplies the namespace for telemetry without one. +Limits telemetry rows per batch; valid values are 1–50000. -### `ingest.max_in_flight_bytes` +### `FANOUT_INGEST_MAX_IN_FLIGHT_BYTES` Caps the decoded telemetry the process will hold across concurrent requests. Concurrency is HTTP/2 streams times connections and each handler holds its batch until the commit is durable, so without a ceiling a burst is bounded only by memory. Zero is unbounded, which is the behaviour that predates this setting. diff --git a/site/src/content/docs/reference/settings/mcp.mdx b/site/src/content/docs/reference/settings/mcp.mdx index 2803d1df..fb3c3c37 100644 --- a/site/src/content/docs/reference/settings/mcp.mdx +++ b/site/src/content/docs/reference/settings/mcp.mdx @@ -11,21 +11,21 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} Every setting below is read once at startup and requires a restart to change. -It is read from the environment variable in the second -column, or from the same key in a YAML configuration file. An unrecognised -`FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces -as a refusal to start rather than as a default nobody chose. +Built-in defaults are overridden by nonempty environment values. +Unknown `FANOUT_` names fail startup; platform-injected service names are tolerated. +Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. +Vendor credentials and endpoints keep the vendor's documented names. Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, `OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are ignored; only the explicitly selected AI provider's key and endpoint are used. -| Setting | Environment variable | Type | Default | -|---|---|---|---| -| `mcp.enabled` | `FANOUT_MCP_ENABLED` | boolean | `true` | +| Environment variable | Type | Default | +|---|---|---| +| `FANOUT_MCP_ENABLED` | boolean | `true` | ## Notes -### `mcp.enabled` +### `FANOUT_MCP_ENABLED` Serves the MCP endpoint at /mcp. diff --git a/site/src/content/docs/reference/settings/metrics.mdx b/site/src/content/docs/reference/settings/metrics.mdx index f51484c8..76f4f101 100644 --- a/site/src/content/docs/reference/settings/metrics.mdx +++ b/site/src/content/docs/reference/settings/metrics.mdx @@ -11,32 +11,32 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} Every setting below is read once at startup and requires a restart to change. -It is read from the environment variable in the second -column, or from the same key in a YAML configuration file. An unrecognised -`FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces -as a refusal to start rather than as a default nobody chose. +Built-in defaults are overridden by nonempty environment values. +Unknown `FANOUT_` names fail startup; platform-injected service names are tolerated. +Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. +Vendor credentials and endpoints keep the vendor's documented names. Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, `OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are ignored; only the explicitly selected AI provider's key and endpoint are used. -| Setting | Environment variable | Type | Default | -|---|---|---|---| -| `metrics.public` | `FANOUT_METRICS_PUBLIC` | boolean | `false` | -| `metrics.token` | `FANOUT_METRICS_TOKEN` | string · secret | — | +| Environment variable | Type | Default | +|---|---|---| +| `FANOUT_METRICS_PUBLIC` | boolean | `false` | +| `FANOUT_METRICS_TOKEN` or `FANOUT_METRICS_TOKEN_FILE` | string · secret | — | ## Notes -### `metrics.public` +### `FANOUT_METRICS_PUBLIC` Allows unauthenticated Prometheus scrapes. -### `metrics.token` +### `FANOUT_METRICS_TOKEN` Authorizes Prometheus scrapes with a bearer token; optional when using an admin session. :::caution[Secrets] Settings marked `secret` are redacted from Fanout's startup configuration -log. They are still plain environment variables: anything that can read the -process environment can read them. +log. Supply either the value or its `_FILE` path, never both. File contents +lose exactly one trailing newline (LF or CRLF); empty secrets fail startup. ::: diff --git a/site/src/content/docs/reference/settings/server.mdx b/site/src/content/docs/reference/settings/server.mdx index 0c8674bc..cf695023 100644 --- a/site/src/content/docs/reference/settings/server.mdx +++ b/site/src/content/docs/reference/settings/server.mdx @@ -12,46 +12,46 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} Every setting below is read once at startup and requires a restart to change. -It is read from the environment variable in the second -column, or from the same key in a YAML configuration file. An unrecognised -`FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces -as a refusal to start rather than as a default nobody chose. +Built-in defaults are overridden by nonempty environment values. +Unknown `FANOUT_` names fail startup; platform-injected service names are tolerated. +Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. +Vendor credentials and endpoints keep the vendor's documented names. Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, `OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are ignored; only the explicitly selected AI provider's key and endpoint are used. -| Setting | Environment variable | Type | Default | -|---|---|---|---| -| `server.addr` | `FANOUT_ADDR` | string | `:7520` | -| `server.pprof_enabled` | `FANOUT_PPROF_ENABLED` | boolean | `false` | -| `server.public_url` | `FANOUT_PUBLIC_URL` | string | — | -| `server.tls.cert_file` | `FANOUT_TLS_CERT_FILE` | string | — | -| `server.tls.key_file` | `FANOUT_TLS_KEY_FILE` | string | — | -| `server.trusted_proxy_cidrs` | `FANOUT_TRUSTED_PROXY_CIDRS` | string | — | +| Environment variable | Type | Default | +|---|---|---| +| `FANOUT_ADDR` | string | `127.0.0.1:7520` | +| `FANOUT_PPROF_ENABLED` | boolean | `false` | +| `FANOUT_PUBLIC_URL` | string | — | +| `FANOUT_TLS_CERT_FILE` | string | — | +| `FANOUT_TLS_KEY_FILE` | string | — | +| `FANOUT_TRUSTED_PROXY_CIDRS` | string | — | ## Notes -### `server.addr` +### `FANOUT_ADDR` Binds the browser, API, MCP, OTLP/gRPC and OTLP/HTTP listener. -### `server.pprof_enabled` +### `FANOUT_PPROF_ENABLED` Exposes Go's net/http/pprof handlers at /debug/pprof/* for CPU/heap/mutex/goroutine profiling under load. Off by default; the routes require an admin browser session with operations:read. -### `server.public_url` +### `FANOUT_PUBLIC_URL` Sets the external browser origin. Required for OIDC; HTTPS is required with MCP. -### `server.tls.cert_file` +### `FANOUT_TLS_CERT_FILE` -Selects the TLS certificate file; requires tls.key_file. +Selects the TLS certificate file; requires FANOUT_TLS_KEY_FILE. -### `server.tls.key_file` +### `FANOUT_TLS_KEY_FILE` -Selects the TLS private key file; requires tls.cert_file. +Selects the TLS private key file; requires FANOUT_TLS_CERT_FILE. -### `server.trusted_proxy_cidrs` +### `FANOUT_TRUSTED_PROXY_CIDRS` Lists trusted proxy networks, separated by commas; empty trusts no forwarded client addresses. diff --git a/site/src/content/docs/reference/settings/smtp.mdx b/site/src/content/docs/reference/settings/smtp.mdx index 3cd00537..248a49f9 100644 --- a/site/src/content/docs/reference/settings/smtp.mdx +++ b/site/src/content/docs/reference/settings/smtp.mdx @@ -11,47 +11,47 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} Every setting below is read once at startup and requires a restart to change. -It is read from the environment variable in the second -column, or from the same key in a YAML configuration file. An unrecognised -`FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces -as a refusal to start rather than as a default nobody chose. +Built-in defaults are overridden by nonempty environment values. +Unknown `FANOUT_` names fail startup; platform-injected service names are tolerated. +Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. +Vendor credentials and endpoints keep the vendor's documented names. Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, `OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are ignored; only the explicitly selected AI provider's key and endpoint are used. -| Setting | Environment variable | Type | Default | -|---|---|---|---| -| `smtp.from` | `FANOUT_SMTP_FROM` | string | — | -| `smtp.host` | `FANOUT_SMTP_HOST` | string | — | -| `smtp.password` | `FANOUT_SMTP_PASSWORD` | string · secret | — | -| `smtp.port` | `FANOUT_SMTP_PORT` | integer | `587` | -| `smtp.username` | `FANOUT_SMTP_USERNAME` | string | — | +| Environment variable | Type | Default | +|---|---|---| +| `FANOUT_SMTP_FROM` | string | — | +| `FANOUT_SMTP_HOST` | string | — | +| `FANOUT_SMTP_PASSWORD` or `FANOUT_SMTP_PASSWORD_FILE` | string · secret | — | +| `FANOUT_SMTP_PORT` | integer | `587` | +| `FANOUT_SMTP_USERNAME` | string | — | ## Notes -### `smtp.from` +### `FANOUT_SMTP_FROM` Sets the email sender address; required with SMTP. -### `smtp.host` +### `FANOUT_SMTP_HOST` Selects the mail server. Host, username, password, and from must all be set to enable SMTP. -### `smtp.password` +### `FANOUT_SMTP_PASSWORD` Authenticates to the mail server; required with SMTP. -### `smtp.port` +### `FANOUT_SMTP_PORT` Sets the mail server's positive TCP port. -### `smtp.username` +### `FANOUT_SMTP_USERNAME` Authenticates to the mail server; required with SMTP. :::caution[Secrets] Settings marked `secret` are redacted from Fanout's startup configuration -log. They are still plain environment variables: anything that can read the -process environment can read them. +log. Supply either the value or its `_FILE` path, never both. File contents +lose exactly one trailing newline (LF or CRLF); empty secrets fail startup. ::: diff --git a/site/src/content/docs/reference/settings/storage.mdx b/site/src/content/docs/reference/settings/storage.mdx index 68c6428c..c84bfabf 100644 --- a/site/src/content/docs/reference/settings/storage.mdx +++ b/site/src/content/docs/reference/settings/storage.mdx @@ -12,56 +12,56 @@ generated: true {/* Generated by cmd/fanout-docgen from internal/config. Edit the generator, not this page. */} Every setting below is read once at startup and requires a restart to change. -It is read from the environment variable in the second -column, or from the same key in a YAML configuration file. An unrecognised -`FANOUT_`-prefixed variable is a startup error, so a renamed setting surfaces -as a refusal to start rather than as a default nobody chose. +Built-in defaults are overridden by nonempty environment values. +Unknown `FANOUT_` names fail startup; platform-injected service names are tolerated. +Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. +Vendor credentials and endpoints keep the vendor's documented names. Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, `OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are ignored; only the explicitly selected AI provider's key and endpoint are used. -| Setting | Environment variable | Type | Default | -|---|---|---|---| -| `storage.data_dir` | `FANOUT_DATA_DIR` | string | `./data` | -| `storage.duckdb.max_connections` | `FANOUT_DUCKDB_MAX_CONNECTIONS` | integer | `0` | -| `storage.duckdb.memory` | `FANOUT_DUCKDB_MEMORY` | string | — | -| `storage.duckdb.threads` | `FANOUT_DUCKDB_THREADS` | integer | — | -| `storage.maintenance_interval` | `FANOUT_MAINTENANCE_INTERVAL` | duration | `1h` | -| `storage.retention_days` | `FANOUT_RETENTION_DAYS` | integer | `30` | -| `storage.rollup_interval` | `FANOUT_ROLLUP_INTERVAL` | duration | `1m` | -| `storage.rollup_skip_to_latest` | `FANOUT_ROLLUP_SKIP_TO_LATEST` | boolean | `false` | +| Environment variable | Type | Default | +|---|---|---| +| `FANOUT_DATA_DIR` | string | `./data` | +| `FANOUT_DUCKDB_MAX_CONNECTIONS` | integer | `0` | +| `FANOUT_DUCKDB_MEMORY` | string | — | +| `FANOUT_DUCKDB_THREADS` | integer | — | +| `FANOUT_MAINTENANCE_INTERVAL` | duration | `1h` | +| `FANOUT_RETENTION_DAYS` | integer | `30` | +| `FANOUT_ROLLUP_INTERVAL` | duration | `1m` | +| `FANOUT_ROLLUP_SKIP_TO_LATEST` | boolean | `false` | ## Notes -### `storage.data_dir` +### `FANOUT_DATA_DIR` Stores telemetry, query state, and the control database. -### `storage.duckdb.max_connections` +### `FANOUT_DUCKDB_MAX_CONNECTIONS` Caps the DuckDB connection pool. Reads scan immutable Parquet concurrently; rollup-cache writes are serialized by the query write gate. Zero means "size it from the machine" — the same spelling DuckDBThreads uses for deferring to a default. Resolution happens in resolveSizing and is reported in the startup configuration log. -### `storage.duckdb.memory` +### `FANOUT_DUCKDB_MEMORY` Caps DuckDB's memory (e.g. "8GB"). Empty means Fanout sizes it from detected memory, reserving headroom for the Go runtime; see resolveSizing. DuckDB's own default is 80% of detected RAM, which is calculated as though DuckDB owned the machine and has been observed to get the process OOM-killed once the Go heap is added on top. -### `storage.duckdb.threads` +### `FANOUT_DUCKDB_THREADS` Caps DuckDB's global query worker pool. Zero leaves DuckDB's own default in place (one worker per core). Set it to leave cores free for ingest on a query-heavy co-tenant host. -### `storage.maintenance_interval` +### `FANOUT_MAINTENANCE_INTERVAL` Controls Parquet retention and compaction, and query-cache checkpointing. -### `storage.retention_days` +### `FANOUT_RETENTION_DAYS` Sets telemetry retention in days; zero disables expiry. -### `storage.rollup_interval` +### `FANOUT_ROLLUP_INTERVAL` Schedules analytical rollups; use a Go duration of at least 1s. -### `storage.rollup_skip_to_latest` +### `FANOUT_ROLLUP_SKIP_TO_LATEST` RollupSkipToLatest, set once at boot, advances the service and edge rollup watermarks to the current max ingested timestamp. Existing history is not aggregated into those analytical rollups. Completed-batch trace caches still rebuild all retained history in bounded passes; uncached files remain immediately queryable. Useful for large pre-seeded datasets (benchmarks, restores); off in normal operation. diff --git a/site/src/content/docs/start/install.mdx b/site/src/content/docs/start/install.mdx index 1488cc2d..2720ff39 100644 --- a/site/src/content/docs/start/install.mdx +++ b/site/src/content/docs/start/install.mdx @@ -45,8 +45,9 @@ is not: `-v "$PWD/data:/var/lib/fanout/data"` fails to open its data directory unless that directory is already writable by 65532. ::: -Both the image and native binary bind the shared listener on all interfaces. -Set `FANOUT_ADDR=127.0.0.1:7520` for a local-only instance. +The image sets `FANOUT_ADDR=:7520` and `FANOUT_DATA_DIR=/var/lib/fanout/data`. +Native installs default to `127.0.0.1:7520`; set `FANOUT_ADDR=:7520` to accept +connections from other hosts. @@ -72,6 +73,7 @@ run it: ```sh chmod +x fanout +export FANOUT_AUTH_CODE_SECRET=$(openssl rand -hex 32) ./fanout ``` @@ -112,41 +114,38 @@ Every tag above also exists under `ghcr.io/labstack/fanout`. Standalone signatures and SBOMs are deliberately not published yet — the reasoning is in the repository's release contract. -## AI configuration upgrades +## Configuration upgrades -Select exactly `anthropic` or `openai` with `FANOUT_AI_PROVIDER` (YAML -`ai.provider`) to enable chat; empty disables it. Replace `FANOUT_AI_API_KEY` -with the selected provider's required `ANTHROPIC_API_KEY` or `OPENAI_API_KEY`. -Move YAML `ai.api_key` to the corresponding `ai.anthropic.api_key` or -`ai.openai.api_key`. Inherited keys never select a provider. Model selection -remains `FANOUT_AI_MODEL`. +Apply these deployment changes before restarting with the new binary or image. +Every setting requires a restart. -Replace `FANOUT_AI_BASE_URL` with the matching `ANTHROPIC_BASE_URL` or -`OPENAI_BASE_URL` (YAML `ai.anthropic.base_url` or `ai.openai.base_url`). For -OpenAI, append `/v1` to the old origin/path: Fanout now appends `/responses` -instead of `/v1/responses`. Anthropic still appends `/v1/messages`. Both preserve -gateway prefixes and accept a trailing slash. Retired names are startup errors; -update deployment/secret inputs with the binary. All settings require restart. +| Previous setup | Current setup | +|---|---| +| YAML configuration and `--config` | Configuration comes from built-in defaults and nonempty environment values. Set deployment environment variables for your settings. | +| A mount at `/etc/fanout/fanout.yaml` | The new image silently ignores this file. Transfer its settings to environment variables before upgrading. | +| `FANOUT_AI_API_KEY` and `FANOUT_AI_BASE_URL` | Use the selected provider's `ANTHROPIC_API_KEY` / `ANTHROPIC_BASE_URL` or `OPENAI_API_KEY` / `OPENAI_BASE_URL`. | +| Chat enabled by an inherited key | Set `FANOUT_AI_PROVIDER` to exactly `anthropic` or `openai` and supply its key. Empty disables chat. Model selection remains `FANOUT_AI_MODEL`. | +| OpenAI base without an API version | Include `/v1` in `OPENAI_BASE_URL`; Fanout appends `/responses`. Anthropic still appends `/v1/messages`. Both preserve gateway prefixes. | +| Native listener on all interfaces | Native default is `127.0.0.1:7520`. Set `FANOUT_ADDR=:7520` for remote access. The container image supplies `:7520`. | +| Tracked `.env` defaults | Only generated `.env.example` is tracked, with empty values. Personal `.env` is ignored; `just run` loads it optionally and inherited values win. | + +Secrets also accept `_FILE` paths for managed mounts. Set either the value +or the path, never both nonempty. See [runtime secret delivery](https://github.com/labstack/fanout/blob/main/docs/operations.md#runtime-secret-delivery). ## What an instance opens | Port or path | Purpose | Native default | |---|---|---| -| `7520` | Browser, API, chat, MCP, OTLP/gRPC and OTLP/HTTP | `:7520`, all interfaces | +| `7520` | Browser, API, chat, MCP, OTLP/gRPC and OTLP/HTTP | `127.0.0.1:7520`, loopback | | `./data` | Telemetry, query state, users, sessions, dashboards, alerts and agent history | `./data` | Every OTLP export requires the ingest token, including on the shared port. Ingest rejects requests before setup completes. Read [authenticate exporters](/start/send-telemetry) before pointing a service at the instance. -Upgrading from separate ports? Remove `FANOUT_OTLP_GRPC_ADDR` and -`FANOUT_OTLP_HTTP_ADDR` (or YAML `ingest.otlp_grpc_addr` and -`ingest.otlp_http_addr`), update exporters to port `7520` or your public HTTPS -origin, and remove the old port mappings. Rename `FANOUT_HTTP_ADDR` to -`FANOUT_ADDR` (YAML `server.http_addr` to `server.addr`). Removed settings are -startup errors. Also remove `FANOUT_INGEST_ADVERTISED_ENDPOINT` or YAML -`ingest.advertised_endpoint`; change a private collector hostname directly on -**Connect telemetry** before copying its generated configuration. +Exporters use the shared port `7520` or the public HTTPS origin. Every OTLP +export requires the ingest token. A private collector hostname can be entered +on **Connect telemetry** before copying its generated configuration. Separate interface bindings are also removed. If the dashboard is public but ingest must remain LAN-only, keep Fanout's backend port private and deny diff --git a/site/src/pages/llms.txt.ts b/site/src/pages/llms.txt.ts index 96174879..3e0a5d8f 100644 --- a/site/src/pages/llms.txt.ts +++ b/site/src/pages/llms.txt.ts @@ -63,7 +63,7 @@ export const GET: APIRoute = async ({ site }) => { "", "## Operating Fanout from an agent", "", - "- Fanout is a server. Its whole command line is three forms: `fanout [flags]` to run it, `fanout version`, and `fanout [--config path] login-link ` — a local-auth-mode recovery path that needs shell access on the host. Everything else is done over HTTP or MCP; there are no operational subcommands to script.", + "- Fanout reads built-in defaults and process environment. Native installs listen on 127.0.0.1:7520; the image supplies FANOUT_ADDR=:7520. Secret variables accept _FILE paths. Commands: `fanout [flags]`, `fanout version`, `fanout login-link ` (local recovery with host access), `fanout healthcheck [url]`, `fanout repair verify`, and `fanout repair quarantine --batch `. Product operations use HTTP or MCP.", "- MCP is the intended agent interface. The server exposes its tools at `/mcp`.", "- MCP and the HTTP API authenticate with an API key, which is not the same credential as the ingest token. Both are `fo_`-prefixed; the routes tell them apart by prefix.", "- Telemetry goes in over OTLP only: gRPC and HTTP on the same port as the application (7520 by default). There is no bespoke ingest API to script against.", diff --git a/site/src/styles/fanout.css b/site/src/styles/fanout.css index cb9070e4..9704db7e 100644 --- a/site/src/styles/fanout.css +++ b/site/src/styles/fanout.css @@ -262,7 +262,7 @@ h1#_top { /* Code inside a heading is set smaller than the heading's own size. The generated settings reference titles every note with a setting key, and a key is one long unbreakable token: at Starlight's h3 size, - `storage.duckdb.max_connections` is wider than the content column and breaks + `FANOUT_DUCKDB_MAX_CONNECTIONS` is wider than the content column and breaks mid-identifier, rendering as "max_connection" above a lone "s". */ .sl-markdown-content :is(h1, h2, h3, h4, h5, h6) code { font-size: 0.78em; From b0ace86dbfd5a711304782abd30b2f0b5fc5b581 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Tue, 13 Oct 2026 22:11:31 -0700 Subject: [PATCH 4/6] fix(config): address review of environment-only configuration - .env.example comments out every entry, so an env file copied from it sets nothing and cannot replace the image's FANOUT_ADDR with an empty value. - fanout healthcheck probes 127.0.0.1 for every wildcard bind, skips certificate verification only for derived IP-literal probes (zoned IPv6 included), verifies hostnames, and works out whether its URL is derived once, in argument parsing. It uses the exported DefaultAddr. - One provider table drives key, base URL and file resolution, validation, and error names. - The AI vendor-variable note appears only on the AI settings page. - Delete migration and upgrade guidance and tests that named removed settings; no legacy paths are supported. - Provider tests pass resolved base URLs and assert request paths. --- .env.example | 103 +++++++++--------- CONTRIBUTING.md | 3 +- README.md | 11 +- cmd/fanout-docgen/main.go | 15 ++- cmd/fanout-docgen/main_test.go | 30 +++-- cmd/fanout/main.go | 56 +++++----- cmd/fanout/main_test.go | 72 ++++++++---- docs/operations.md | 66 ++--------- internal/agent/provider_contract_test.go | 4 +- internal/agent/provider_http_test.go | 11 +- internal/config/config.go | 61 +++++++---- internal/config/config_test.go | 5 +- internal/config/loader.go | 16 +-- internal/config/provider_test.go | 8 ++ .../docs/explanation/storage-model.mdx | 7 +- .../docs/guides/back-up-and-restore.mdx | 9 -- .../content/docs/reference/configuration.mdx | 14 +-- .../docs/reference/settings/alerts.mdx | 4 - .../content/docs/reference/settings/auth.mdx | 4 - .../docs/reference/settings/ingest.mdx | 4 - .../content/docs/reference/settings/mcp.mdx | 4 - .../docs/reference/settings/metrics.mdx | 4 - .../docs/reference/settings/server.mdx | 4 - .../content/docs/reference/settings/smtp.mdx | 4 - .../docs/reference/settings/storage.mdx | 4 - site/src/content/docs/start/install.mdx | 22 +--- 26 files changed, 247 insertions(+), 298 deletions(-) diff --git a/.env.example b/.env.example index 30d6311d..f3c44e18 100644 --- a/.env.example +++ b/.env.example @@ -3,262 +3,263 @@ # Fanout does not load dotenv files. Inject through your launcher or deployment. # Precedence: built-in defaults < nonempty environment values. # All settings require restart. Empty environment values mean no override. -# Secret entries below are empty placeholders; use runtime secret injection. +# Every entry is commented out so copying this template sets nothing, preserving image ENV. +# Uncomment only settings you want to supply; use runtime secret injection for secrets. # Local authentication requires FANOUT_AUTH_CODE_SECRET (32+ characters). # Chat is disabled unless FANOUT_AI_PROVIDER selects anthropic or openai; its key is required. # ANTHROPIC_API_KEY; string; secret # Required when FANOUT_AI_PROVIDER is anthropic. # Default: (empty) -ANTHROPIC_API_KEY= +# ANTHROPIC_API_KEY= # Alternatively set ANTHROPIC_API_KEY_FILE to a secret file path; do not set both. # ANTHROPIC_BASE_URL; string # Defaults to https://api.anthropic.com. Fanout appends /v1/messages, preserving any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. # Default: (empty) -ANTHROPIC_BASE_URL= +# ANTHROPIC_BASE_URL= # FANOUT_ADDR; string # Binds the browser, API, MCP, OTLP/gRPC and OTLP/HTTP listener. # Default: 127.0.0.1:7520 -FANOUT_ADDR= +# FANOUT_ADDR= # FANOUT_AI_MODEL; string # Overrides the provider's default: claude-sonnet-5-5 for Anthropic and gpt-6.1-sol for OpenAI. The OpenAI provider expects a reasoning model supporting reasoning.encrypted_content for stateless Responses replay. A 400 rejecting encrypted content is reported as a server-side configuration error; choose a compatible reasoning model with FANOUT_AI_MODEL. # Default: (empty) -FANOUT_AI_MODEL= +# FANOUT_AI_MODEL= # FANOUT_AI_PROVIDER; string # Enables chat by selecting exactly anthropic or openai. Empty disables chat. # Default: (empty) -FANOUT_AI_PROVIDER= +# FANOUT_AI_PROVIDER= # FANOUT_ALERTS_ENABLED; boolean # Runs the alert evaluation engine. # Default: true -FANOUT_ALERTS_ENABLED= +# FANOUT_ALERTS_ENABLED= # FANOUT_ALERTS_EVALUATION_INTERVAL; duration # Schedules alert evaluation; use a Go duration of at least 1s. # Default: 30s -FANOUT_ALERTS_EVALUATION_INTERVAL= +# FANOUT_ALERTS_EVALUATION_INTERVAL= # FANOUT_ALERTS_HISTORY_DAYS; integer # Sets alert history retention in days. # Default: 7 -FANOUT_ALERTS_HISTORY_DAYS= +# FANOUT_ALERTS_HISTORY_DAYS= # FANOUT_AUTH_CODE_SECRET; string; secret # Signs local sign-in codes; required in local mode with at least 32 characters. # Default: (empty) -FANOUT_AUTH_CODE_SECRET= +# FANOUT_AUTH_CODE_SECRET= # Alternatively set FANOUT_AUTH_CODE_SECRET_FILE to a secret file path; do not set both. # FANOUT_AUTH_MODE; string # Selects local sign-in or oidc; each mode validates its prerequisites. # Default: local -FANOUT_AUTH_MODE= +# FANOUT_AUTH_MODE= # FANOUT_DATA_DIR; string # Stores telemetry, query state, and the control database. # Default: ./data -FANOUT_DATA_DIR= +# FANOUT_DATA_DIR= # FANOUT_DEFAULT_NAMESPACE; string # Supplies the namespace for telemetry without one. # Default: default -FANOUT_DEFAULT_NAMESPACE= +# FANOUT_DEFAULT_NAMESPACE= # FANOUT_DUCKDB_MAX_CONNECTIONS; integer # Caps the DuckDB connection pool. Reads scan immutable Parquet concurrently; rollup-cache writes are serialized by the query write gate. Zero means "size it from the machine" — the same spelling DuckDBThreads uses for deferring to a default. Resolution happens in resolveSizing and is reported in the startup configuration log. # Default: 0 -FANOUT_DUCKDB_MAX_CONNECTIONS= +# FANOUT_DUCKDB_MAX_CONNECTIONS= # FANOUT_DUCKDB_MEMORY; string # Caps DuckDB's memory (e.g. "8GB"). Empty means Fanout sizes it from detected memory, reserving headroom for the Go runtime; see resolveSizing. DuckDB's own default is 80% of detected RAM, which is calculated as though DuckDB owned the machine and has been observed to get the process OOM-killed once the Go heap is added on top. # Default: (empty) -FANOUT_DUCKDB_MEMORY= +# FANOUT_DUCKDB_MEMORY= # FANOUT_DUCKDB_THREADS; integer # Caps DuckDB's global query worker pool. Zero leaves DuckDB's own default in place (one worker per core). Set it to leave cores free for ingest on a query-heavy co-tenant host. # Default: (empty) -FANOUT_DUCKDB_THREADS= +# FANOUT_DUCKDB_THREADS= # FANOUT_INGEST_BATCH_SIZE; integer # Limits telemetry rows per batch; valid values are 1–50000. # Default: 50000 -FANOUT_INGEST_BATCH_SIZE= +# FANOUT_INGEST_BATCH_SIZE= # FANOUT_INGEST_MAX_IN_FLIGHT_BYTES; integer # Caps the decoded telemetry the process will hold across concurrent requests. Concurrency is HTTP/2 streams times connections and each handler holds its batch until the commit is durable, so without a ceiling a burst is bounded only by memory. Zero is unbounded, which is the behaviour that predates this setting. # Default: 268435456 -FANOUT_INGEST_MAX_IN_FLIGHT_BYTES= +# FANOUT_INGEST_MAX_IN_FLIGHT_BYTES= # FANOUT_MAINTENANCE_INTERVAL; duration # Controls Parquet retention and compaction, and query-cache checkpointing. # Default: 1h -FANOUT_MAINTENANCE_INTERVAL= +# FANOUT_MAINTENANCE_INTERVAL= # FANOUT_MCP_ENABLED; boolean # Serves the MCP endpoint at /mcp. # Default: true -FANOUT_MCP_ENABLED= +# FANOUT_MCP_ENABLED= # FANOUT_METRICS_PUBLIC; boolean # Allows unauthenticated Prometheus scrapes. # Default: false -FANOUT_METRICS_PUBLIC= +# FANOUT_METRICS_PUBLIC= # FANOUT_METRICS_TOKEN; string; secret # Authorizes Prometheus scrapes with a bearer token; optional when using an admin session. # Default: (empty) -FANOUT_METRICS_TOKEN= +# FANOUT_METRICS_TOKEN= # Alternatively set FANOUT_METRICS_TOKEN_FILE to a secret file path; do not set both. # FANOUT_OIDC_ADMIN_GROUPS; string # Lists identity-provider groups mapped to admin, separated by commas. # Default: (empty) -FANOUT_OIDC_ADMIN_GROUPS= +# FANOUT_OIDC_ADMIN_GROUPS= # FANOUT_OIDC_ALLOWED_DOMAINS; string # Lists allowed email domains, separated by commas; wildcard * requires verified email. # Default: (empty) -FANOUT_OIDC_ALLOWED_DOMAINS= +# FANOUT_OIDC_ALLOWED_DOMAINS= # FANOUT_OIDC_ALLOWED_GROUPS; string # Lists allowed identity-provider groups, separated by commas. # Default: (empty) -FANOUT_OIDC_ALLOWED_GROUPS= +# FANOUT_OIDC_ALLOWED_GROUPS= # FANOUT_OIDC_AUTO_PROVISION; boolean # Creates accounts on OIDC sign-in; requires an allowed group or domain. # Default: false -FANOUT_OIDC_AUTO_PROVISION= +# FANOUT_OIDC_AUTO_PROVISION= # FANOUT_OIDC_CLIENT_ID; string # Identifies Fanout's client registration; required in OIDC mode. # Default: (empty) -FANOUT_OIDC_CLIENT_ID= +# FANOUT_OIDC_CLIENT_ID= # FANOUT_OIDC_CLIENT_SECRET; string; secret # Authenticates Fanout's client registration; required in OIDC mode. # Default: (empty) -FANOUT_OIDC_CLIENT_SECRET= +# FANOUT_OIDC_CLIENT_SECRET= # Alternatively set FANOUT_OIDC_CLIENT_SECRET_FILE to a secret file path; do not set both. # FANOUT_OIDC_DEFAULT_ROLE; string # Assigns viewer, operator, or admin to users without a group role mapping. # Default: viewer -FANOUT_OIDC_DEFAULT_ROLE= +# FANOUT_OIDC_DEFAULT_ROLE= # FANOUT_OIDC_EMAIL_CLAIM; string # Names the claim containing the user's email address. # Default: email -FANOUT_OIDC_EMAIL_CLAIM= +# FANOUT_OIDC_EMAIL_CLAIM= # FANOUT_OIDC_EMAIL_VERIFICATION; string # Selects required verified-email claims or issuer trust; issuer trust requires an allowlist. # Default: required -FANOUT_OIDC_EMAIL_VERIFICATION= +# FANOUT_OIDC_EMAIL_VERIFICATION= # FANOUT_OIDC_ISSUER_URL; string # Selects the identity provider issuer; required in OIDC mode. # Default: (empty) -FANOUT_OIDC_ISSUER_URL= +# FANOUT_OIDC_ISSUER_URL= # FANOUT_OIDC_OPERATOR_GROUPS; string # Lists identity-provider groups mapped to operator, separated by commas. # Default: (empty) -FANOUT_OIDC_OPERATOR_GROUPS= +# FANOUT_OIDC_OPERATOR_GROUPS= # FANOUT_PPROF_ENABLED; boolean # Exposes Go's net/http/pprof handlers at /debug/pprof/* for CPU/heap/mutex/goroutine profiling under load. Off by default; the routes require an admin browser session with operations:read. # Default: false -FANOUT_PPROF_ENABLED= +# FANOUT_PPROF_ENABLED= # FANOUT_PUBLIC_URL; string # Sets the external browser origin. Required for OIDC; HTTPS is required with MCP. # Default: (empty) -FANOUT_PUBLIC_URL= +# FANOUT_PUBLIC_URL= # FANOUT_RETENTION_DAYS; integer # Sets telemetry retention in days; zero disables expiry. # Default: 30 -FANOUT_RETENTION_DAYS= +# FANOUT_RETENTION_DAYS= # FANOUT_ROLLUP_INTERVAL; duration # Schedules analytical rollups; use a Go duration of at least 1s. # Default: 1m -FANOUT_ROLLUP_INTERVAL= +# FANOUT_ROLLUP_INTERVAL= # FANOUT_ROLLUP_SKIP_TO_LATEST; boolean # RollupSkipToLatest, set once at boot, advances the service and edge rollup watermarks to the current max ingested timestamp. Existing history is not aggregated into those analytical rollups. Completed-batch trace caches still rebuild all retained history in bounded passes; uncached files remain immediately queryable. Useful for large pre-seeded datasets (benchmarks, restores); off in normal operation. # Default: false -FANOUT_ROLLUP_SKIP_TO_LATEST= +# FANOUT_ROLLUP_SKIP_TO_LATEST= # FANOUT_SELF_SIGNUP; boolean # Allows verified email addresses to create viewer accounts; requires local auth and SMTP. # Default: false -FANOUT_SELF_SIGNUP= +# FANOUT_SELF_SIGNUP= # FANOUT_SESSION_ABSOLUTE_TTL; duration # Limits total session lifetime; it must be at least FANOUT_SESSION_IDLE_TTL. # Default: 168h -FANOUT_SESSION_ABSOLUTE_TTL= +# FANOUT_SESSION_ABSOLUTE_TTL= # FANOUT_SESSION_IDLE_TTL; duration # Expires inactive sessions; use a Go duration of at least 5m. # Default: 12h -FANOUT_SESSION_IDLE_TTL= +# FANOUT_SESSION_IDLE_TTL= # FANOUT_SMTP_FROM; string # Sets the email sender address; required with SMTP. # Default: (empty) -FANOUT_SMTP_FROM= +# FANOUT_SMTP_FROM= # FANOUT_SMTP_HOST; string # Selects the mail server. Host, username, password, and from must all be set to enable SMTP. # Default: (empty) -FANOUT_SMTP_HOST= +# FANOUT_SMTP_HOST= # FANOUT_SMTP_PASSWORD; string; secret # Authenticates to the mail server; required with SMTP. # Default: (empty) -FANOUT_SMTP_PASSWORD= +# FANOUT_SMTP_PASSWORD= # Alternatively set FANOUT_SMTP_PASSWORD_FILE to a secret file path; do not set both. # FANOUT_SMTP_PORT; integer # Sets the mail server's positive TCP port. # Default: 587 -FANOUT_SMTP_PORT= +# FANOUT_SMTP_PORT= # FANOUT_SMTP_USERNAME; string # Authenticates to the mail server; required with SMTP. # Default: (empty) -FANOUT_SMTP_USERNAME= +# FANOUT_SMTP_USERNAME= # FANOUT_TLS_CERT_FILE; string # Selects the TLS certificate file; requires FANOUT_TLS_KEY_FILE. # Default: (empty) -FANOUT_TLS_CERT_FILE= +# FANOUT_TLS_CERT_FILE= # FANOUT_TLS_KEY_FILE; string # Selects the TLS private key file; requires FANOUT_TLS_CERT_FILE. # Default: (empty) -FANOUT_TLS_KEY_FILE= +# FANOUT_TLS_KEY_FILE= # FANOUT_TRUSTED_PROXY_CIDRS; string # Lists trusted proxy networks, separated by commas; empty trusts no forwarded client addresses. # Default: (empty) -FANOUT_TRUSTED_PROXY_CIDRS= +# FANOUT_TRUSTED_PROXY_CIDRS= # OPENAI_API_KEY; string; secret # Required when FANOUT_AI_PROVIDER is openai. # Default: (empty) -OPENAI_API_KEY= +# OPENAI_API_KEY= # Alternatively set OPENAI_API_KEY_FILE to a secret file path; do not set both. # OPENAI_BASE_URL; string # Defaults to https://api.openai.com/v1. Fanout appends /responses, preserving the API version and any gateway path prefix. Use an HTTP(S) base URL without embedded credentials, a query, or a fragment. # Default: (empty) -OPENAI_BASE_URL= +# OPENAI_BASE_URL= diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 57062e32..e8bbb304 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -18,7 +18,8 @@ also requires a 32-character authentication code secret. SMTP and AI chat are optional; selecting an AI provider requires its key — see [README](README.md#requirements). Copy `.env.example` to ignored `.env` for personal development settings and -credentials. After building, `just run` loads it with Node's native dotenv +credentials. Every entry is commented out; uncomment only values you want to +supply. Copying the template sets nothing. After building, `just run` loads it with Node's native dotenv parser; inherited environment values win. The binary validates the settings. Keep this file out of Git and Docker inputs. See [runtime secret delivery](docs/operations.md#runtime-secret-delivery). diff --git a/README.md b/README.md index 3855f6db..9bfefa7b 100644 --- a/README.md +++ b/README.md @@ -155,7 +155,7 @@ hostname reachable from that Collector. The equivalent OTLP/gRPC endpoint is `localhost:7520` (plaintext locally). Behind a TLS proxy, both transports use the public origin; gRPC requires -HTTP/2 to Fanout. See the [single-port migration notes](docs/operations.md#single-port-migration). +HTTP/2 to Fanout. For later sign-in without SMTP, mint a 15-minute, single-use link against the running container's control database: @@ -227,8 +227,10 @@ as their declared types. [`.env.example`](.env.example) is generated from [`internal/config/config.go`](internal/config/config.go), with every value -empty and defaults in comments. Copy it to ignored `.env` for personal -settings and development credentials. After building, `just run` uses Node's +empty and every entry commented out. Types, defaults, and secret file forms +are documented in comments. Copying it sets nothing and preserves image ENV. +Copy it to ignored `.env` and uncomment personal settings and development +credentials as needed. After building, `just run` uses Node's `parseEnv` to load optional `./.env`, then lets inherited environment values win, including empty values. The binary itself reads only process environment; production launchers and deployments supply their own values. @@ -260,8 +262,7 @@ defaults to `https://api.openai.com/v1`; Fanout appends `/responses`. Both preserve gateway prefixes and accept a trailing slash. The selected URL must use HTTP(S) without credentials, a query, or a fragment. -See the [configuration upgrade table](https://fanout.run/start/install/#configuration-upgrades) -and [runtime secret delivery](docs/operations.md#runtime-secret-delivery) for +See [runtime secret delivery](docs/operations.md#runtime-secret-delivery) for deployment examples. This public repository excludes operator secret profiles and private recipient policy. Run `just docs-generate` after changing the schema; `just docs-generate-check` verifies `.env.example` and the reference. diff --git a/cmd/fanout-docgen/main.go b/cmd/fanout-docgen/main.go index ebb93909..5ece1e13 100644 --- a/cmd/fanout-docgen/main.go +++ b/cmd/fanout-docgen/main.go @@ -559,9 +559,11 @@ func renderPage(name string, page groupPage, fields []field) []byte { b.WriteString("Unknown `FANOUT_` names fail startup; platform-injected service names are tolerated.\n") b.WriteString("Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies.\n") b.WriteString("Vendor credentials and endpoints keep the vendor's documented names.\n\n") - b.WriteString("Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`,\n") - b.WriteString("`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are\n") - b.WriteString("ignored; only the explicitly selected AI provider's key and endpoint are used.\n\n") + if name == "ai" { + b.WriteString("Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`,\n") + b.WriteString("`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are\n") + b.WriteString("ignored; only the explicitly selected AI provider's key and endpoint are used.\n\n") + } b.WriteString("| Environment variable | Type | Default |\n") b.WriteString("|---|---|---|\n") @@ -606,7 +608,7 @@ func renderPage(name string, page groupPage, fields []field) []byte { } // renderEnvironment shares the exact reflected schema used by the site pages. -// Values stay empty; defaults and secret file forms are documented in comments. +// Entries are commented out; defaults and secret file forms stay in comments. func renderEnvironment(fields []field) []byte { var b strings.Builder b.WriteString("# Generated by cmd/fanout-docgen from internal/config/config.go.\n") @@ -614,7 +616,8 @@ func renderEnvironment(fields []field) []byte { b.WriteString("# Fanout does not load dotenv files. Inject through your launcher or deployment.\n") b.WriteString("# Precedence: built-in defaults < nonempty environment values.\n") b.WriteString("# All settings require restart. Empty environment values mean no override.\n") - b.WriteString("# Secret entries below are empty placeholders; use runtime secret injection.\n") + b.WriteString("# Every entry is commented out so copying this template sets nothing, preserving image ENV.\n") + b.WriteString("# Uncomment only settings you want to supply; use runtime secret injection for secrets.\n") b.WriteString("# Local authentication requires FANOUT_AUTH_CODE_SECRET (32+ characters).\n") b.WriteString("# Chat is disabled unless FANOUT_AI_PROVIDER selects anthropic or openai; its key is required.\n") for _, f := range fields { @@ -631,7 +634,7 @@ func renderEnvironment(fields []field) []byte { defaultValue = "(empty)" } fmt.Fprintf(&b, "# Default: %s\n", defaultValue) - fmt.Fprintf(&b, "%s=\n", f.Env) + fmt.Fprintf(&b, "# %s=\n", f.Env) if f.Secret { fmt.Fprintf(&b, "# Alternatively set %s_FILE to a secret file path; do not set both.\n", f.Env) } diff --git a/cmd/fanout-docgen/main_test.go b/cmd/fanout-docgen/main_test.go index af9cd618..a3430930 100644 --- a/cmd/fanout-docgen/main_test.go +++ b/cmd/fanout-docgen/main_test.go @@ -52,15 +52,21 @@ func TestEnvironmentTemplateMatchesSchema(t *testing.T) { var env []string seen := map[string]string{} for _, line := range strings.Split(string(body), "\n") { - if line == "" || strings.HasPrefix(line, "#") { + if line == "" { continue } - name, value, ok := strings.Cut(line, "=") + if !strings.HasPrefix(line, "#") { + t.Fatalf("template must contain only comments: %q", line) + } + entry := strings.TrimPrefix(line, "# ") + if !strings.HasSuffix(entry, "=") { + continue + } + name, value, ok := strings.Cut(entry, "=") if _, duplicate := seen[name]; !ok || duplicate { t.Fatal("invalid or duplicate template entry") } seen[name] = value - env = append(env, line) } if len(seen) != len(fields) { t.Fatalf("template has %d settings, schema has %d", len(seen), len(fields)) @@ -73,18 +79,19 @@ func TestEnvironmentTemplateMatchesSchema(t *testing.T) { if !exists || value != "" { t.Errorf("template must leave %s empty so it preserves defaults", f.Env) } - if f.Default != "" && !strings.Contains(string(body), "# Default: "+f.Default+"\n"+f.Env+"=\n") { + if f.Default != "" && !f.Secret && !strings.Contains(string(body), "# Default: "+f.Default+"\n# "+f.Env+"=\n") { t.Errorf("template must document the default for %s", f.Env) } } - for i, entry := range env { - if entry == "FANOUT_AUTH_CODE_SECRET=" { - env[i] += strings.Repeat("x", 32) - } - } - if _, err := config.Load(config.LoadOptions{Environ: env}); err != nil { + // A copied template contributes no overrides to the image environment. + env = append(env, "FANOUT_ADDR=:7520", "FANOUT_DATA_DIR=/var/lib/fanout/data", "FANOUT_AUTH_CODE_SECRET="+strings.Repeat("x", 32)) + cfg, err := config.Load(config.LoadOptions{Environ: env}) + if err != nil { t.Fatalf("template rejected by the runtime loader: %v", err) } + if cfg.Addr != ":7520" || cfg.DataDir != "/var/lib/fanout/data" { + t.Fatal("template must preserve image environment values") + } } func TestCollectRejectsASourceThatIsNotTheConfigType(t *testing.T) { @@ -127,6 +134,9 @@ func TestRenderProducesOnePageWithFrontmatterPerGroup(t *testing.T) { for name, body := range pages { text := string(body) + if strings.Contains(text, "Fanout also recognizes") != (name == "settings/ai.mdx") { + t.Errorf("%s: vendor-variable paragraph belongs only on the AI settings page", name) + } if !strings.HasPrefix(text, "---\n") { t.Errorf("%s: does not open with frontmatter", name) } diff --git a/cmd/fanout/main.go b/cmd/fanout/main.go index c7bb293a..fa6257bf 100644 --- a/cmd/fanout/main.go +++ b/cmd/fanout/main.go @@ -11,10 +11,10 @@ import ( "net" "net/http" "net/http/pprof" + "net/netip" "net/url" "os" "os/signal" - "reflect" "regexp" "runtime" "strings" @@ -60,7 +60,7 @@ type repairCommand struct { } func main() { - showVersion, loginEmail, healthURL, repair, err := parseCommandLine(os.Args[1:], os.Stderr) + showVersion, loginEmail, healthURL, derivedProbe, repair, err := parseCommandLine(os.Args[1:], os.Stderr) if errors.Is(err, flag.ErrHelp) { return } @@ -71,8 +71,14 @@ func main() { fmt.Println(version) return } - if healthURL != "" { - derivedProbe := len(os.Args) == 2 && os.Args[1] == "healthcheck" + if derivedProbe { + healthURL, err = healthcheckURL(os.Getenv("FANOUT_ADDR"), os.Getenv("FANOUT_TLS_CERT_FILE")) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(2) + } + } + if healthURL != "" || derivedProbe { if err := checkHealth(healthURL, derivedProbe); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) @@ -426,19 +432,15 @@ func main() { } } -func parseCommandLine(args []string, output io.Writer) (showVersion bool, loginEmail, healthURL string, repair *repairCommand, err error) { +func parseCommandLine(args []string, output io.Writer) (showVersion bool, loginEmail, healthURL string, derivedProbe bool, repair *repairCommand, err error) { if len(args) == 1 && args[0] == "version" { - return true, "", "", nil, nil + return true, "", "", false, nil, nil } if len(args) >= 1 && len(args) <= 2 && args[0] == "healthcheck" { if len(args) == 2 { - return false, "", args[1], nil, nil + return false, "", args[1], false, nil, nil } - healthURL, err := healthcheckURL(os.Getenv("FANOUT_ADDR"), os.Getenv("FANOUT_TLS_CERT_FILE")) - if err != nil { - fmt.Fprintln(output, err) - } - return false, "", healthURL, nil, err + return false, "", "", true, nil, nil } flags := flag.NewFlagSet("fanout", flag.ContinueOnError) @@ -457,22 +459,22 @@ func parseCommandLine(args []string, output io.Writer) (showVersion bool, loginE flags.BoolVar(&showVersion, "version", false, "print the Fanout version") flags.BoolVar(&showVersion, "v", false, "print the Fanout version") if err := flags.Parse(args); err != nil { - return false, "", "", nil, err + return false, "", "", false, nil, err } if flags.NArg() == 2 && flags.Arg(0) == "login-link" { - return false, flags.Arg(1), "", nil, nil + return false, flags.Arg(1), "", false, nil, nil } if flags.NArg() > 0 && flags.Arg(0) == "repair" { repair, err := parseRepairCommand(flags.Args()[1:], output) - return false, "", "", repair, err + return false, "", "", false, repair, err } if flags.NArg() != 0 { err := fmt.Errorf("unexpected arguments: %s", strings.Join(flags.Args(), " ")) fmt.Fprintln(output, err) flags.Usage() - return false, "", "", nil, err + return false, "", "", false, nil, err } - return showVersion, "", "", nil, nil + return showVersion, "", "", false, nil, nil } func parseRepairCommand(args []string, output io.Writer) (*repairCommand, error) { @@ -526,8 +528,7 @@ func runRepair(root string, command repairCommand, output io.Writer) error { func healthcheckURL(addr, certFile string) (string, error) { if addr == "" { - field, _ := reflect.TypeOf(config.Config{}).FieldByName("Addr") - addr = field.Tag.Get("default") + addr = config.DefaultAddr } host, port, err := net.SplitHostPort(addr) if err != nil { @@ -538,9 +539,6 @@ func healthcheckURL(addr, certFile string) (string, error) { } if ip := net.ParseIP(host); ip != nil && ip.IsUnspecified() { host = "127.0.0.1" - if ip.To4() == nil { - host = "::1" - } } scheme := "http" if certFile != "" { @@ -552,12 +550,18 @@ func healthcheckURL(addr, certFile string) (string, error) { func checkHealth(healthURL string, derivedProbe bool) error { client := &http.Client{Timeout: 3 * time.Second} if derivedProbe { - // A derived probe targets this process's own listener directly, whose - // certificate may name its public hostname rather than any bind IP. - // Explicit URLs retain environment proxies and certificate verification. + // Derived probes bypass proxies. IP literals skip certificate verification + // because the certificate may name the public hostname instead of a bind IP. + // Hostnames verify normally; explicit URLs retain all default behaviour. + probe, err := url.Parse(healthURL) + if err != nil { + return fmt.Errorf("healthcheck: %w", err) + } transport := http.DefaultTransport.(*http.Transport).Clone() transport.Proxy = nil - transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + if _, err := netip.ParseAddr(probe.Hostname()); err == nil { + transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + } defer transport.CloseIdleConnections() client.Transport = transport } diff --git a/cmd/fanout/main_test.go b/cmd/fanout/main_test.go index f460be73..c3a3ed88 100644 --- a/cmd/fanout/main_test.go +++ b/cmd/fanout/main_test.go @@ -31,15 +31,16 @@ import ( ) func TestParseCommandLine(t *testing.T) { - t.Setenv("FANOUT_ADDR", "") - t.Setenv("FANOUT_TLS_CERT_FILE", "") - t.Setenv("FANOUT_AUTH_CODE_SECRET", "") + t.Setenv("FANOUT_ADDR", "invalid") + t.Setenv("FANOUT_TLS_CERT_FILE", "cert.pem") + t.Setenv("FANOUT_AUTH_CODE_SECRET_FILE", "/missing") tests := []struct { name string args []string wantVersion bool wantEmail string wantHealth string + wantDerived bool wantRepair *repairCommand wantErr bool }{ @@ -48,7 +49,7 @@ func TestParseCommandLine(t *testing.T) { {name: "version flag", args: []string{"--version"}, wantVersion: true}, {name: "short version flag", args: []string{"-v"}, wantVersion: true}, {name: "login link", args: []string{"login-link", "admin@example.com"}, wantEmail: "admin@example.com"}, - {name: "default healthcheck", args: []string{"healthcheck"}, wantHealth: "http://127.0.0.1:7520/healthz"}, + {name: "default healthcheck", args: []string{"healthcheck"}, wantDerived: true}, {name: "custom healthcheck", args: []string{"healthcheck", "http://fanout:8080/healthz"}, wantHealth: "http://fanout:8080/healthz"}, {name: "repair verify", args: []string{"repair", "verify"}, wantRepair: &repairCommand{action: "verify"}}, {name: "repair quarantine", args: []string{"repair", "quarantine", "--batch", "broken"}, wantRepair: &repairCommand{action: "quarantine", batch: "broken"}}, @@ -60,12 +61,12 @@ func TestParseCommandLine(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { - showVersion, email, healthURL, repair, err := parseCommandLine(test.args, io.Discard) + showVersion, email, healthURL, derivedProbe, repair, err := parseCommandLine(test.args, io.Discard) if (err != nil) != test.wantErr { t.Fatalf("error = %v, wantErr %v", err, test.wantErr) } - if showVersion != test.wantVersion || email != test.wantEmail || healthURL != test.wantHealth { - t.Fatalf("result = (%v, %q, %q), want (%v, %q, %q)", showVersion, email, healthURL, test.wantVersion, test.wantEmail, test.wantHealth) + if showVersion != test.wantVersion || email != test.wantEmail || healthURL != test.wantHealth || derivedProbe != test.wantDerived { + t.Fatalf("result = (%v, %q, %q, %v), want (%v, %q, %q, %v)", showVersion, email, healthURL, derivedProbe, test.wantVersion, test.wantEmail, test.wantHealth, test.wantDerived) } if repair == nil != (test.wantRepair == nil) || repair != nil && *repair != *test.wantRepair { t.Fatalf("repair = %#v, want %#v", repair, test.wantRepair) @@ -75,7 +76,7 @@ func TestParseCommandLine(t *testing.T) { } func TestParseCommandLineHelp(t *testing.T) { - _, _, _, _, err := parseCommandLine([]string{"--help"}, io.Discard) + _, _, _, _, _, err := parseCommandLine([]string{"--help"}, io.Discard) if !errors.Is(err, flag.ErrHelp) { t.Fatalf("error = %v, want flag.ErrHelp", err) } @@ -83,7 +84,7 @@ func TestParseCommandLineHelp(t *testing.T) { func TestParseCommandLinePrintsOneErrorAndUsage(t *testing.T) { var output bytes.Buffer - _, _, _, _, err := parseCommandLine([]string{"serve"}, &output) + _, _, _, _, _, err := parseCommandLine([]string{"serve"}, &output) if err == nil { t.Fatal("expected unexpected-argument error") } @@ -220,7 +221,7 @@ func TestHealthcheckURL(t *testing.T) { {"", "", "http://127.0.0.1:7520/healthz"}, {":8080", "", "http://127.0.0.1:8080/healthz"}, {"0.0.0.0:8080", "", "http://127.0.0.1:8080/healthz"}, - {"[::]:8080", "", "http://[::1]:8080/healthz"}, + {"[::]:8080", "", "http://127.0.0.1:8080/healthz"}, {"[::1]:8080", "", "http://[::1]:8080/healthz"}, {"[2001:db8::1]:8080", "", "http://[2001:db8::1]:8080/healthz"}, {"127.0.0.1:9000", "cert.pem", "https://127.0.0.1:9000/healthz"}, @@ -241,8 +242,8 @@ func TestHealthcheckUsesEnvironmentWithoutValidatingSecrets(t *testing.T) { t.Setenv("FANOUT_ADDR", "[::]:9090") t.Setenv("FANOUT_TLS_CERT_FILE", "cert.pem") t.Setenv("FANOUT_AUTH_CODE_SECRET_FILE", "/missing") - _, _, probe, _, err := parseCommandLine([]string{"healthcheck"}, io.Discard) - if err != nil || probe != "https://[::1]:9090/healthz" { + probe, err := healthcheckURL(os.Getenv("FANOUT_ADDR"), os.Getenv("FANOUT_TLS_CERT_FILE")) + if err != nil || probe != "https://127.0.0.1:9090/healthz" { t.Fatalf("probe = %s, %v", probe, err) } } @@ -256,6 +257,11 @@ func TestCheckHealthWithLocalTLS(t *testing.T) { if err := checkHealth(server.URL+"/healthz", false); err == nil { t.Fatal("explicit URL must verify the certificate") } + probe := strings.Replace(server.URL, "127.0.0.1", "localhost", 1) + "/healthz" + var authorityError x509.UnknownAuthorityError + if err := checkHealth(probe, true); !errors.As(err, &authorityError) { + t.Fatalf("derived hostname probe must verify certificate trust, got %v", err) + } } func TestCheckHealthWithInterfaceTLS(t *testing.T) { @@ -283,13 +289,13 @@ func TestCheckHealthWithInterfaceTLS(t *testing.T) { defer server.Close() roots := x509.NewCertPool() roots.AddCert(server.Certificate()) - for _, addr := range []string{"192.0.2.10:7520", "[2001:db8::10]:7520"} { + for _, addr := range []string{"192.0.2.10:7520", "[2001:db8::10]:7520", "[fe80::1%eth0]:7520", "fanout.example.com:7520", "wrong.example.com:7520"} { t.Run(addr, func(t *testing.T) { original := http.DefaultTransport transport := original.(*http.Transport).Clone() transport.TLSClientConfig = &tls.Config{RootCAs: roots} - // Route the interface bind to a local test listener so both IP families - // exercise real TLS without requiring those addresses on the test host. + // Route binds to a local test listener to exercise real TLS without + // requiring those addresses or DNS entries on the test host. transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { if address != addr { t.Errorf("probe address = %q, want %q", address, addr) @@ -304,19 +310,29 @@ func TestCheckHealthWithInterfaceTLS(t *testing.T) { }) t.Setenv("FANOUT_ADDR", addr) t.Setenv("FANOUT_TLS_CERT_FILE", "cert.pem") - _, _, probe, _, err := parseCommandLine([]string{"healthcheck"}, io.Discard) + probe, err := healthcheckURL(os.Getenv("FANOUT_ADDR"), os.Getenv("FANOUT_TLS_CERT_FILE")) if err != nil { t.Fatal(err) } - if err := checkHealth(probe, true); err != nil { + err = checkHealth(probe, true) + var hostnameError x509.HostnameError + if addr == "wrong.example.com:7520" { + if !errors.As(err, &hostnameError) { + t.Fatalf("derived hostname probe must verify the hostname, got %v", err) + } + } else if err != nil { t.Fatalf("derived TLS probe: %v", err) } - _, _, explicitProbe, _, err := parseCommandLine([]string{"healthcheck", probe}, io.Discard) + _, _, explicitProbe, derivedProbe, _, err := parseCommandLine([]string{"healthcheck", probe}, io.Discard) if err != nil { t.Fatal(err) } - var hostnameError x509.HostnameError - if err := checkHealth(explicitProbe, false); !errors.As(err, &hostnameError) { + err = checkHealth(explicitProbe, derivedProbe) + if addr == "fanout.example.com:7520" { + if err != nil { + t.Fatalf("explicit hostname probe: %v", err) + } + } else if !errors.As(err, &hostnameError) { t.Fatalf("explicit TLS probe must verify the hostname, got %v", err) } }) @@ -347,8 +363,12 @@ func TestCheckHealthDerivedIgnoresEnvironmentProxy(t *testing.T) { } return } - for _, scheme := range []string{"http", "https"} { - t.Run(scheme, func(t *testing.T) { + for _, tc := range []struct{ scheme, addr string }{ + {"http", "192.0.2.10:7520"}, {"https", "192.0.2.10:7520"}, + {"http", "fanout.example.com:7520"}, {"https", "example.com:7520"}, + } { + t.Run(tc.scheme+"/"+tc.addr, func(t *testing.T) { + scheme, addr := tc.scheme, tc.addr server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { if request.URL.Path != "/healthz" { t.Errorf("probe path = %q, want /healthz", request.URL.Path) @@ -363,9 +383,13 @@ func TestCheckHealthDerivedIgnoresEnvironmentProxy(t *testing.T) { server.Start() } defer server.Close() - const addr = "192.0.2.10:7520" original := http.DefaultTransport transport := original.(*http.Transport).Clone() + if scheme == "https" { + roots := x509.NewCertPool() + roots.AddCert(server.Certificate()) + transport.TLSClientConfig = &tls.Config{RootCAs: roots} + } transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { if address == addr { address = server.Listener.Addr().String() @@ -382,7 +406,7 @@ func TestCheckHealthDerivedIgnoresEnvironmentProxy(t *testing.T) { }) t.Setenv("FANOUT_ADDR", addr) t.Setenv("FANOUT_TLS_CERT_FILE", certFile) - _, _, probe, _, err := parseCommandLine([]string{"healthcheck"}, io.Discard) + probe, err := healthcheckURL(os.Getenv("FANOUT_ADDR"), os.Getenv("FANOUT_TLS_CERT_FILE")) if err != nil { t.Fatal(err) } diff --git a/docs/operations.md b/docs/operations.md index cc9c56c7..813cfbdf 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -9,7 +9,9 @@ give ingest credentials only to trusted services and collectors. Runtime configuration uses built-in defaults followed by nonempty process environment values. Every setting requires a restart. Fanout reads no configuration document or dotenv file. Generated `.env.example` lists the -settings with empty values and defaults in comments. For development, `just run` +settings as commented-out empty entries, with types, defaults, and secret file +forms in comments. Copying the template sets nothing and preserves image ENV; +uncomment only settings you want to supply. For development, `just run` loads optional ignored `./.env` with Node's native `parseEnv`; inherited values win, including empty values. Personal development credentials may live there. Personal environment files and operator profiles are excluded from Git and Docker. @@ -111,25 +113,17 @@ The profile supplies environment values to the child; ordinary settings may also come from deployment inputs. A dotenv parser neither decrypts SOPS nor resolves 1Password references. Verify synthetic values containing dollars, quotes, spaces, and newlines through the actual deployment parser before -migrating a profile. Keep private age identities out of Git, restrict their +using a profile. Keep private age identities out of Git, restrict their local files to mode `0600`, and document key recovery in the existing 1Password item. 1Password holds recovery rather than a second maintained copy of SOPS-owned application secrets. -Existing vault/platform delivery remains authoritative until consumers and -recovery are verified. Before retiring a vault field or old name, inventory -local, CI, deployment, shared-project, supported-version, and recovery consumers -using names and safe metadata. Update mappings, verify real consumers, and -verify recovery from the destination before removing the old authority. Private -migration sources stay ignored and are never loaded automatically. - AI chat is disabled by default. Select exactly `anthropic` or `openai` with `FANOUT_AI_PROVIDER`, then deliver its required `ANTHROPIC_API_KEY` or `OPENAI_API_KEY`. A selected provider without its key fails startup. Inherited keys never enable chat. Only the selected endpoint is validated and used. Overrides are `ANTHROPIC_BASE_URL` and `OPENAI_BASE_URL`; the OpenAI base -includes `/v1`, while Anthropic's excludes it. See the -[configuration upgrade table](../site/src/content/docs/start/install.mdx#configuration-upgrades). +includes `/v1`, while Anthropic's excludes it. ## Network surface @@ -180,12 +174,12 @@ original host. ## Health and monitoring `fanout healthcheck` derives `/healthz` from `FANOUT_ADDR` or its built-in -default. Empty and unspecified bind hosts become loopback, including IPv6. -`FANOUT_TLS_CERT_FILE` selects HTTPS. The local loopback probe skips certificate -verification because the certificate may be issued for the public hostname. -Healthcheck reads only these two settings and does not require valid secrets. -An explicit `fanout healthcheck ` overrides URL derivation. - +default. Empty, `0.0.0.0`, and `::` bind hosts become `127.0.0.1`. +`FANOUT_TLS_CERT_FILE` selects HTTPS. Derived probes bypass environment proxies. +IP-literal probes skip certificate verification because the certificate may be +issued for the public hostname; hostname probes verify normally. Healthcheck +reads only these two settings and does not require valid secrets. An explicit +`fanout healthcheck ` uses default proxy and certificate verification behaviour. - `/healthz` is the liveness check. - `/readyz` includes storage readiness and resolved runtime sizing. @@ -203,16 +197,6 @@ Fanout uses SQLite with the modernc driver, sqlc query bindings, and embedded Goose SQL migrations. Pending migrations run forward at startup. PostgreSQL and pgx are not required. -This preview replaces the Atlas migration history with a fresh Goose baseline. -Existing Atlas-managed control databases are unsupported; startup rejects them -without converting or resetting their data. To start fresh, stop Fanout, back -up the complete data directory, and move `data/control` aside (under your -configured data directory). Startup creates a new control database. Complete -first-admin setup again and reconfigure the ingest token, users, alert rules, -and dashboards; prior sessions, OAuth grants, and agent history remain only in -the saved control database. Telemetry and query files stay in place. Restore -the saved directory with the previous binary if you need the previous state. - ## Backup The supported portable baseline is a **cold backup** of the complete @@ -248,34 +232,6 @@ Never recover an account by deleting its row from the control database; that cascades to sessions, OAuth grants, and owned dashboards. Use `fanout login-link ` as documented in [authentication.md](authentication.md). -## Single-port migration - -All transports use `FANOUT_ADDR`. Exporters use port `7520` or the public -HTTPS origin and port; the ingest token and signal paths remain the same. -Enable HTTP/2 backend forwarding for gRPC at proxies and remove obsolete ingest -port mappings and upstreams. A separate ingest hostname may still route to the -same Fanout port if required by the edge configuration. - -Set `FANOUT_PUBLIC_URL` to the external origin. If collectors use a different -private hostname, edit the endpoint on **Connect telemetry** before copying the -configuration. - -Local comparison results are in [the single-port benchmark report](benchmarks/single-port-2026-09-08.md). - -### Network isolation changes - -Ingest can no longer bind to a different interface from the application. -For an internet-facing dashboard with LAN-only ingest, keep the backend port -private and configure the public proxy to deny `/v1`, `/v1/*`, and the OTLP -gRPC service paths under `/opentelemetry.proto.collector.*`. Allow collectors -through the private network or an explicitly restricted proxy route. Verify -that both OTLP transports are inaccessible through the public origin; the -ingest token is still required on the private path. - -Single-port removes the need for a proxy solely to combine listeners. Splitting -public application access from private ingest still requires network-aware -routing. The shared listener keeps one network topology for this pre-release product. - ## Upgrade and rollback 1. Read the release notes and pin the target image tag or binary version. diff --git a/internal/agent/provider_contract_test.go b/internal/agent/provider_contract_test.go index 0cac59e2..25306982 100644 --- a/internal/agent/provider_contract_test.go +++ b/internal/agent/provider_contract_test.go @@ -55,7 +55,7 @@ func TestOpenAIOutputItemsReplayVerbatim(t *testing.T) { // Test request replay independently of parser failures: retained output must // replace rebuilt text/calls and calls without executed results must vanish. unsafe := json.RawMessage(`{"type":"function_call","call_id":"not_executed","name":"status","arguments":"{}"}`) - body := captureRequestBody(t, lines(`data: {"type":"response.completed","response":{}}`), func(url string) error { + body := captureRequestBody(t, lines(`data: {"type":"response.completed","response":{}}`), "/v1/responses", func(url string) error { p, err := NewProvider("openai", "test", "", url+"/v1") if err != nil { return err @@ -184,7 +184,7 @@ func TestAnthropicCacheBreakpoints(t *testing.T) { } else { messages = append(messages, ProviderMessage{Role: RoleTool, ToolResult: &ToolResult{ToolCallID: "b", Content: "last"}}) } - body := captureRequestBody(t, lines(`data: {"type":"message_delta","delta":{"stop_reason":"end_turn"}}`), func(url string) error { + body := captureRequestBody(t, lines(`data: {"type":"message_delta","delta":{"stop_reason":"end_turn"}}`), "/v1/messages", func(url string) error { p, err := NewProvider("anthropic", "test", "", url) if err != nil { return err diff --git a/internal/agent/provider_http_test.go b/internal/agent/provider_http_test.go index 04642c45..04feac58 100644 --- a/internal/agent/provider_http_test.go +++ b/internal/agent/provider_http_test.go @@ -351,10 +351,13 @@ func TestParseAnthropicSkipsNonDataLines(t *testing.T) { // captureRequestBody serves one canned SSE response and returns the decoded // request body the provider sent. -func captureRequestBody(t *testing.T, sseBody string, run func(baseURL string) error) map[string]any { +func captureRequestBody(t *testing.T, sseBody, requestPath string, run func(baseURL string) error) map[string]any { t.Helper() var body map[string]any server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != requestPath { + t.Errorf("request path = %q, want %q", r.URL.Path, requestPath) + } if err := json.NewDecoder(r.Body).Decode(&body); err != nil { t.Errorf("decode request body: %v", err) } @@ -369,8 +372,8 @@ func captureRequestBody(t *testing.T, sseBody string, run func(baseURL string) e } func TestOpenAIResponsesRequestBodyShape(t *testing.T) { - body := captureRequestBody(t, lines(`data: {"type":"response.completed","response":{}}`), func(baseURL string) error { - provider, err := NewProvider("openai", "sk-test", "", baseURL+"/v1/") + body := captureRequestBody(t, lines(`data: {"type":"response.completed","response":{}}`), "/v1/responses", func(baseURL string) error { + provider, err := NewProvider("openai", "sk-test", "", baseURL+"/v1") if err != nil { return err } @@ -493,7 +496,7 @@ func TestAnthropicRequestBodyShape(t *testing.T) { `data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ok"}}`, `data: {"type":"message_delta","delta":{"stop_reason":"end_turn"}}`, ) - body := captureRequestBody(t, sseBody, func(baseURL string) error { + body := captureRequestBody(t, sseBody, "/v1/messages", func(baseURL string) error { provider := &anthropicProvider{apiKey: "sk-test", model: "claude-sonnet-5-5", baseURL: baseURL, client: http.DefaultClient} params := StreamParams{ System: "be helpful", diff --git a/internal/config/config.go b/internal/config/config.go index 4534d083..818cd965 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -15,10 +15,36 @@ import ( const maxIngestBatchSize = 50_000 const ( + DefaultAddr = "127.0.0.1:7520" DefaultAnthropicBaseURL = "https://api.anthropic.com" DefaultOpenAIBaseURL = "https://api.openai.com/v1" ) +type providerSettings struct { + keyEnv, baseURLEnv, defaultBaseURL string + key, baseURL func(Config) string +} + +var providers = map[string]providerSettings{ + "anthropic": { + keyEnv: "ANTHROPIC_API_KEY", baseURLEnv: "ANTHROPIC_BASE_URL", defaultBaseURL: DefaultAnthropicBaseURL, + key: func(c Config) string { return c.AnthropicAPIKey }, baseURL: func(c Config) string { return c.AnthropicBaseURL }, + }, + "openai": { + keyEnv: "OPENAI_API_KEY", baseURLEnv: "OPENAI_BASE_URL", defaultBaseURL: DefaultOpenAIBaseURL, + key: func(c Config) string { return c.OpenAIAPIKey }, baseURL: func(c Config) string { return c.OpenAIBaseURL }, + }, +} + +func providerForEnvironment(name string) string { + for provider, settings := range providers { + if name == settings.keyEnv || name == settings.baseURLEnv { + return provider + } + } + return "" +} + // Config is Fanout's canonical configuration schema. Custom environment names // use FANOUT_; provider credentials and endpoints use their vendor's names. // Elapsed-time settings use time.Duration with unit-bearing values. @@ -328,23 +354,18 @@ func (c Config) Validate() error { return fmt.Errorf("OIDC auto-provisioning requires FANOUT_OIDC_ALLOWED_GROUPS or FANOUT_OIDC_ALLOWED_DOMAINS") } } - switch c.AIProvider { - case "", "anthropic", "openai": - default: + provider, supported := providers[c.AIProvider] + if c.AIProvider != "" && !supported { return fmt.Errorf("FANOUT_AI_PROVIDER must be anthropic or openai") } if c.AgentConfigured() { - key, endpoint := "ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL" - if c.AIProvider == "openai" { - key, endpoint = "OPENAI_API_KEY", "OPENAI_BASE_URL" - } if strings.TrimSpace(c.AIAPIKey()) == "" { - return fmt.Errorf("%s is required when FANOUT_AI_PROVIDER is %s", key, c.AIProvider) + return fmt.Errorf("%s is required when FANOUT_AI_PROVIDER is %s", provider.keyEnv, c.AIProvider) } baseURL := c.AIBaseURL() u, err := url.Parse(baseURL) if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || strings.Contains(baseURL, "#") { - return fmt.Errorf("%s must be an HTTP(S) base URL without credentials, query, or fragment", endpoint) + return fmt.Errorf("%s must be an HTTP(S) base URL without credentials, query, or fragment", provider.baseURLEnv) } } for _, raw := range strings.Split(c.TrustedProxyCIDRs, ",") { @@ -397,29 +418,21 @@ func (c Config) AgentConfigured() bool { // AIAPIKey returns only the explicitly selected provider's credential. func (c Config) AIAPIKey() string { - switch c.AIProvider { - case "anthropic": - return c.AnthropicAPIKey - case "openai": - return c.OpenAIAPIKey - default: - return "" + if provider, ok := providers[c.AIProvider]; ok { + return provider.key(c) } + return "" } // AIBaseURL returns the selected provider's API base, including its default. func (c Config) AIBaseURL() string { - var baseURL, fallback string - switch c.AIProvider { - case "anthropic": - baseURL, fallback = c.AnthropicBaseURL, DefaultAnthropicBaseURL - case "openai": - baseURL, fallback = c.OpenAIBaseURL, DefaultOpenAIBaseURL - default: + provider, ok := providers[c.AIProvider] + if !ok { return "" } + baseURL := provider.baseURL(c) if strings.TrimSpace(baseURL) == "" { - return fallback + return provider.defaultBaseURL } return strings.TrimRight(strings.TrimSpace(baseURL), "/") } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index e9ddc477..42a14c84 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -73,7 +73,7 @@ func TestConfigurationSchemaUsesUnitBearingDurations(t *testing.T) { } func TestLoadRejectsUnknownAndMalformedEnvironment(t *testing.T) { - for _, entry := range []string{"FANOUT_HTPP_ADDR=:1111", "FANOUT_SERVICE_PORTAL=7520", "FANOUT_ADDR", "FANOUT_ADDR_FILE=/unused", "FANOUT_AI_API_KEY=synthetic-private-value"} { + for _, entry := range []string{"FANOUT_HTPP_ADDR=:1111", "FANOUT_SERVICE_PORTAL=7520", "FANOUT_ADDR", "FANOUT_ADDR_FILE=/unused", "FANOUT_UNKNOWN_SETTING=synthetic-private-value"} { _, err := Load(LoadOptions{Environ: append(validEnvironment(), entry)}) name, _, _ := strings.Cut(entry, "=") if err == nil || !strings.Contains(err.Error(), name) || strings.Contains(err.Error(), "synthetic-private-value") { @@ -164,8 +164,7 @@ func TestLoadSecretFiles(t *testing.T) { } } env = append(env, name+"=", name+"_FILE="+path) - if isProviderEnvironment(name) { - provider, _, _ := strings.Cut(strings.ToLower(name), "_") + if provider := providerForEnvironment(name); provider != "" { env = append(env, "FANOUT_AI_PROVIDER="+provider) } cfg, err := Load(LoadOptions{Environ: env}) diff --git a/internal/config/loader.go b/internal/config/loader.go index 44a25e4b..1398f19a 100644 --- a/internal/config/loader.go +++ b/internal/config/loader.go @@ -55,8 +55,7 @@ func Load(options LoadOptions) (Config, error) { for _, spec := range specs { // Shared vendor settings, including secret mounts, belong only to the // explicitly selected provider. Fanout-owned secrets always resolve. - if strings.HasPrefix(spec.env, "ANTHROPIC_") && values["FANOUT_AI_PROVIDER"] != "anthropic" || - strings.HasPrefix(spec.env, "OPENAI_") && values["FANOUT_AI_PROVIDER"] != "openai" { + if provider := providerForEnvironment(spec.env); provider != "" && provider != values["FANOUT_AI_PROVIDER"] { continue } text := values[spec.env] @@ -117,7 +116,7 @@ func fieldSpecs(typ reflect.Type) ([]fieldSpec, error) { if name == "" { return nil, fmt.Errorf("configuration field %s must declare an env tag", field.Name) } - if !strings.HasPrefix(name, "FANOUT_") && !isProviderEnvironment(name) { + if !strings.HasPrefix(name, "FANOUT_") && providerForEnvironment(name) == "" { return nil, fmt.Errorf("environment variable %s is outside the supported configuration contract", name) } secret := field.Tag.Get("secret") == "true" @@ -180,17 +179,6 @@ func environmentValues(environ []string, known map[string]bool) (map[string]stri return values, nil } -// Vendor namespaces are shared with other consumers. Accept these exact names; -// only Fanout's own namespace rejects unknown names. -func isProviderEnvironment(name string) bool { - switch name { - case "ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "OPENAI_API_KEY", "OPENAI_BASE_URL": - return true - default: - return false - } -} - func isPlatformInjectedEnvironment(name string) bool { return name == "FANOUT_SERVICE_HOST" || name == "FANOUT_SERVICE_PORT" || diff --git a/internal/config/provider_test.go b/internal/config/provider_test.go index bdcb4865..5096a330 100644 --- a/internal/config/provider_test.go +++ b/internal/config/provider_test.go @@ -4,10 +4,18 @@ import ( "fmt" "os" "path/filepath" + "reflect" "strings" "testing" ) +func TestDefaultAddrMatchesSchema(t *testing.T) { + field, ok := reflect.TypeOf(Config{}).FieldByName("Addr") + if !ok || field.Tag.Get("default") != DefaultAddr { + t.Fatalf("Addr default tag = %q, want %q", field.Tag.Get("default"), DefaultAddr) + } +} + func TestLoadIgnoresInactiveProviderSecretFiles(t *testing.T) { emptyPath := filepath.Join(t.TempDir(), "empty") if err := os.WriteFile(emptyPath, nil, 0o644); err != nil { diff --git a/site/src/content/docs/explanation/storage-model.mdx b/site/src/content/docs/explanation/storage-model.mdx index 8a7d6b51..92bcafb5 100644 --- a/site/src/content/docs/explanation/storage-model.mdx +++ b/site/src/content/docs/explanation/storage-model.mdx @@ -107,11 +107,8 @@ Discarded hashes force decoding; they do not establish a stored checksum. Verification failures such as memory exhaustion or cancellation never authorize quarantine. -Telemetry batches use format 4. An older or unknown batch format makes startup -fail before cleanup or schema rewriting, leaving the files intact. This build -has no reader or automatic conversion for previous formats. Start with an -empty telemetry directory; retain previous data separately for its original -reader. SQLite control state is independent of this telemetry format change. +Telemetry batches use format 4. An unsupported batch format makes startup +fail before cleanup or schema rewriting, leaving the files intact. ## Small files are bounded by compaction diff --git a/site/src/content/docs/guides/back-up-and-restore.mdx b/site/src/content/docs/guides/back-up-and-restore.mdx index 12e88954..ef257adf 100644 --- a/site/src/content/docs/guides/back-up-and-restore.mdx +++ b/site/src/content/docs/guides/back-up-and-restore.mdx @@ -56,15 +56,6 @@ Migrations run forward at startup. A rollback to the previous binary after they have run is not assumed safe — restore the pre-upgrade backup with the previous version instead. -The Goose migration baseline requires a fresh control database. Existing -Atlas-managed control databases are rejected without conversion or reset. For -this transition, stop Fanout, back up the complete directory, and move its -`control` subdirectory aside before starting the new version. Telemetry and -query files remain in place. Complete first-admin setup again, then reconfigure -the ingest token, users, alert rules, and dashboards. Prior sessions, OAuth -grants, and agent history remain in the saved control database; recover that -state using the previous version and its backup. - ## Never delete an account from the control database Removing a user row cascades to sessions, OAuth grants and owned dashboards. To diff --git a/site/src/content/docs/reference/configuration.mdx b/site/src/content/docs/reference/configuration.mdx index d747f006..9804d1f1 100644 --- a/site/src/content/docs/reference/configuration.mdx +++ b/site/src/content/docs/reference/configuration.mdx @@ -23,11 +23,12 @@ environment value means unset and preserves the built-in default. Native installs listen on `127.0.0.1:7520`; the image supplies `FANOUT_ADDR=:7520` and `FANOUT_DATA_DIR=/var/lib/fanout/data`. -The binary reads only process environment. `.env.example` leaves every value -empty and documents types, defaults, and secret file forms in comments. For a -source checkout, copy it to ignored `.env` for personal settings and development -credentials. Build the binary, then use `just run`: Node's native `parseEnv` -loads optional `./.env`, and inherited environment values win, including empty +The binary reads only process environment. `.env.example` comments out every +empty entry and documents types, defaults, and secret file forms in comments. +Copying the template sets nothing and preserves image ENV. For a source +checkout, copy it to ignored `.env` and uncomment personal settings and +development credentials as needed. Build the binary, then use `just run`: +Node's native `parseEnv` loads optional `./.env`, and inherited environment values win, including empty values. Values are never executed as shell code or interpolated. The binary validates unknown names and types. Production deployments supply their own environment values or managed secret mounts. @@ -43,8 +44,7 @@ or `openai` enables chat and requires that provider's key. Only the selected provider's endpoint is validated and used. Inherited credentials never select a provider. `FANOUT_AI_MODEL` selects the model; unrelated vendor/tool settings such as `ANTHROPIC_MODEL` and `OPENAI_MODEL` are ignored. See -[AI settings](/reference/settings/ai) for base URL meanings and the -[configuration upgrade table](/start/install#configuration-upgrades). +[AI settings](/reference/settings/ai) for base URL meanings. ## Unknown names diff --git a/site/src/content/docs/reference/settings/alerts.mdx b/site/src/content/docs/reference/settings/alerts.mdx index bc9437c4..8836fe5c 100644 --- a/site/src/content/docs/reference/settings/alerts.mdx +++ b/site/src/content/docs/reference/settings/alerts.mdx @@ -16,10 +16,6 @@ Unknown `FANOUT_` names fail startup; platform-injected service names are tolera Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. Vendor credentials and endpoints keep the vendor's documented names. -Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, -`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are -ignored; only the explicitly selected AI provider's key and endpoint are used. - | Environment variable | Type | Default | |---|---|---| | `FANOUT_ALERTS_ENABLED` | boolean | `true` | diff --git a/site/src/content/docs/reference/settings/auth.mdx b/site/src/content/docs/reference/settings/auth.mdx index 6595389b..40086be4 100644 --- a/site/src/content/docs/reference/settings/auth.mdx +++ b/site/src/content/docs/reference/settings/auth.mdx @@ -17,10 +17,6 @@ Unknown `FANOUT_` names fail startup; platform-injected service names are tolera Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. Vendor credentials and endpoints keep the vendor's documented names. -Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, -`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are -ignored; only the explicitly selected AI provider's key and endpoint are used. - | Environment variable | Type | Default | |---|---|---| | `FANOUT_AUTH_CODE_SECRET` or `FANOUT_AUTH_CODE_SECRET_FILE` | string · secret | — | diff --git a/site/src/content/docs/reference/settings/ingest.mdx b/site/src/content/docs/reference/settings/ingest.mdx index c82cd6ae..02271e66 100644 --- a/site/src/content/docs/reference/settings/ingest.mdx +++ b/site/src/content/docs/reference/settings/ingest.mdx @@ -17,10 +17,6 @@ Unknown `FANOUT_` names fail startup; platform-injected service names are tolera Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. Vendor credentials and endpoints keep the vendor's documented names. -Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, -`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are -ignored; only the explicitly selected AI provider's key and endpoint are used. - | Environment variable | Type | Default | |---|---|---| | `FANOUT_DEFAULT_NAMESPACE` | string | `default` | diff --git a/site/src/content/docs/reference/settings/mcp.mdx b/site/src/content/docs/reference/settings/mcp.mdx index fb3c3c37..799c2e83 100644 --- a/site/src/content/docs/reference/settings/mcp.mdx +++ b/site/src/content/docs/reference/settings/mcp.mdx @@ -16,10 +16,6 @@ Unknown `FANOUT_` names fail startup; platform-injected service names are tolera Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. Vendor credentials and endpoints keep the vendor's documented names. -Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, -`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are -ignored; only the explicitly selected AI provider's key and endpoint are used. - | Environment variable | Type | Default | |---|---|---| | `FANOUT_MCP_ENABLED` | boolean | `true` | diff --git a/site/src/content/docs/reference/settings/metrics.mdx b/site/src/content/docs/reference/settings/metrics.mdx index 76f4f101..10cf1d1c 100644 --- a/site/src/content/docs/reference/settings/metrics.mdx +++ b/site/src/content/docs/reference/settings/metrics.mdx @@ -16,10 +16,6 @@ Unknown `FANOUT_` names fail startup; platform-injected service names are tolera Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. Vendor credentials and endpoints keep the vendor's documented names. -Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, -`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are -ignored; only the explicitly selected AI provider's key and endpoint are used. - | Environment variable | Type | Default | |---|---|---| | `FANOUT_METRICS_PUBLIC` | boolean | `false` | diff --git a/site/src/content/docs/reference/settings/server.mdx b/site/src/content/docs/reference/settings/server.mdx index cf695023..eaf93934 100644 --- a/site/src/content/docs/reference/settings/server.mdx +++ b/site/src/content/docs/reference/settings/server.mdx @@ -17,10 +17,6 @@ Unknown `FANOUT_` names fail startup; platform-injected service names are tolera Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. Vendor credentials and endpoints keep the vendor's documented names. -Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, -`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are -ignored; only the explicitly selected AI provider's key and endpoint are used. - | Environment variable | Type | Default | |---|---|---| | `FANOUT_ADDR` | string | `127.0.0.1:7520` | diff --git a/site/src/content/docs/reference/settings/smtp.mdx b/site/src/content/docs/reference/settings/smtp.mdx index 248a49f9..ea1d09db 100644 --- a/site/src/content/docs/reference/settings/smtp.mdx +++ b/site/src/content/docs/reference/settings/smtp.mdx @@ -16,10 +16,6 @@ Unknown `FANOUT_` names fail startup; platform-injected service names are tolera Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. Vendor credentials and endpoints keep the vendor's documented names. -Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, -`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are -ignored; only the explicitly selected AI provider's key and endpoint are used. - | Environment variable | Type | Default | |---|---|---| | `FANOUT_SMTP_FROM` | string | — | diff --git a/site/src/content/docs/reference/settings/storage.mdx b/site/src/content/docs/reference/settings/storage.mdx index c84bfabf..b852f70d 100644 --- a/site/src/content/docs/reference/settings/storage.mdx +++ b/site/src/content/docs/reference/settings/storage.mdx @@ -17,10 +17,6 @@ Unknown `FANOUT_` names fail startup; platform-injected service names are tolera Custom names use `FANOUT_` plus a clear name, adding the area only when it clarifies. Vendor credentials and endpoints keep the vendor's documented names. -Fanout also recognizes `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, -`OPENAI_API_KEY`, and `OPENAI_BASE_URL`. Other vendor/tool variables are -ignored; only the explicitly selected AI provider's key and endpoint are used. - | Environment variable | Type | Default | |---|---|---| | `FANOUT_DATA_DIR` | string | `./data` | diff --git a/site/src/content/docs/start/install.mdx b/site/src/content/docs/start/install.mdx index 2720ff39..264f4751 100644 --- a/site/src/content/docs/start/install.mdx +++ b/site/src/content/docs/start/install.mdx @@ -114,24 +114,6 @@ Every tag above also exists under `ghcr.io/labstack/fanout`. Standalone signatures and SBOMs are deliberately not published yet — the reasoning is in the repository's release contract. -## Configuration upgrades - -Apply these deployment changes before restarting with the new binary or image. -Every setting requires a restart. - -| Previous setup | Current setup | -|---|---| -| YAML configuration and `--config` | Configuration comes from built-in defaults and nonempty environment values. Set deployment environment variables for your settings. | -| A mount at `/etc/fanout/fanout.yaml` | The new image silently ignores this file. Transfer its settings to environment variables before upgrading. | -| `FANOUT_AI_API_KEY` and `FANOUT_AI_BASE_URL` | Use the selected provider's `ANTHROPIC_API_KEY` / `ANTHROPIC_BASE_URL` or `OPENAI_API_KEY` / `OPENAI_BASE_URL`. | -| Chat enabled by an inherited key | Set `FANOUT_AI_PROVIDER` to exactly `anthropic` or `openai` and supply its key. Empty disables chat. Model selection remains `FANOUT_AI_MODEL`. | -| OpenAI base without an API version | Include `/v1` in `OPENAI_BASE_URL`; Fanout appends `/responses`. Anthropic still appends `/v1/messages`. Both preserve gateway prefixes. | -| Native listener on all interfaces | Native default is `127.0.0.1:7520`. Set `FANOUT_ADDR=:7520` for remote access. The container image supplies `:7520`. | -| Tracked `.env` defaults | Only generated `.env.example` is tracked, with empty values. Personal `.env` is ignored; `just run` loads it optionally and inherited values win. | - -Secrets also accept `_FILE` paths for managed mounts. Set either the value -or the path, never both nonempty. See [runtime secret delivery](https://github.com/labstack/fanout/blob/main/docs/operations.md#runtime-secret-delivery). - ## What an instance opens | Port or path | Purpose | Native default | @@ -147,8 +129,8 @@ Exporters use the shared port `7520` or the public HTTPS origin. Every OTLP export requires the ingest token. A private collector hostname can be entered on **Connect telemetry** before copying its generated configuration. -Separate interface bindings are also removed. If the dashboard is public but -ingest must remain LAN-only, keep Fanout's backend port private and deny +If the dashboard is public but ingest must remain LAN-only, keep Fanout's +backend port private and deny `/v1`, `/v1/*`, and `/opentelemetry.proto.collector.*` on the public proxy. Allow collectors through the private network or a restricted proxy route, and verify that neither OTLP transport is reachable through the public origin. From e69ab6afd24e6694d0177fa357a5fb9cb5a33f16 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 14 Oct 2026 06:50:14 -0700 Subject: [PATCH 5/6] docs(config): replace remaining YAML setting names with environment names - The authentication guide shows the OIDC community-access example as environment assignments and no longer describes a configuration file. - The storage model, an OAuth comment, and the architecture and persistence diagrams name FANOUT_ROLLUP_SKIP_TO_LATEST, FANOUT_TRUSTED_PROXY_CIDRS, and FANOUT_DATA_DIR. --- docs/authentication.md | 33 ++-- docs/diagrams/architecture.d2 | 6 +- docs/diagrams/architecture.svg | 178 ++++++++--------- docs/diagrams/persistence.d2 | 2 +- docs/diagrams/persistence.svg | 182 +++++++++--------- internal/api/oauth.go | 2 +- site/public/diagrams/architecture.svg | 178 ++++++++--------- site/public/diagrams/persistence.svg | 182 +++++++++--------- .../docs/explanation/storage-model.mdx | 2 +- 9 files changed, 381 insertions(+), 384 deletions(-) diff --git a/docs/authentication.md b/docs/authentication.md index 126315c6..9308ab86 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -1,8 +1,7 @@ # Authentication and access operations -Operator reference for running Fanout's authentication safely. Every setting -below is available as an environment variable and as a key in the layered -configuration file. +Operator reference for running Fanout's authentication safely. Settings are +environment variables; see the [settings reference](https://fanout.run/reference/configuration). ## First run @@ -55,10 +54,10 @@ login link remains available when delivery fails. ### Local self-signup Local accounts are invitation-only by default. To let any verified email -address create an account, enable `auth.self_signup` or -`FANOUT_SELF_SIGNUP=true`. Fanout sends the existing email code and -creates an active `viewer` only after that code is successfully verified. The -role is fixed: self-signup can never create an operator or administrator. +address create an account, set `FANOUT_SELF_SIGNUP=true`. Fanout sends the +existing email code and creates an active `viewer` only after that code is +successfully verified. The role is fixed: self-signup can never create an +operator or administrator. The setting requires complete SMTP configuration and local auth mode. Self-signup remains unavailable until the first administrator completes setup. @@ -75,7 +74,7 @@ and data directory as the server: fanout login-link admin@example.com ``` -For a container using the bundled configuration: +For a running container: ```sh docker exec fanout \ @@ -167,19 +166,17 @@ asserts a verified email. This keeps sessions, conversations, dashboards, and audit attribution isolated without shared credentials or an anonymous authorization bypass: -```yaml -auth: - mode: oidc - oidc: - email_verification: required - auto_provision: true - allowed_domains: "*" - default_role: viewer +```sh +FANOUT_AUTH_MODE=oidc +FANOUT_OIDC_EMAIL_VERIFICATION=required +FANOUT_OIDC_AUTO_PROVISION=true +FANOUT_OIDC_ALLOWED_DOMAINS="*" +FANOUT_OIDC_DEFAULT_ROLE=viewer ``` The wildcard is deliberately explicit and is rejected with -`email_verification: issuer`. Use a specific domain or group allowlist for a -private installation. A wildcard makes the installation available to every +`FANOUT_OIDC_EMAIL_VERIFICATION=issuer`. Use a specific domain or group allowlist +for a private installation. A wildcard makes the installation available to every identity accepted by the configured provider, so rate-limit public traffic and budget AI usage at the deployment edge. diff --git a/docs/diagrams/architecture.d2 b/docs/diagrams/architecture.d2 index 1aa4d40f..0d8c5fbd 100644 --- a/docs/diagrams/architecture.d2 +++ b/docs/diagrams/architecture.d2 @@ -44,9 +44,9 @@ fanout: "fanout — one Go process" { store: Storage { style.fill: transparent - telemetry: "Parquet batches + trace indexes\nstorage.data_dir/telemetry" {shape: cylinder} - qstate: "DuckDB query state\nstorage.data_dir/query" {shape: cylinder} - control: "Control SQLite\nstorage.data_dir/control" {shape: cylinder} + telemetry: "Parquet batches + trace indexes\nFANOUT_DATA_DIR/telemetry" {shape: cylinder} + qstate: "DuckDB query state\nFANOUT_DATA_DIR/query" {shape: cylinder} + control: "Control SQLite\nFANOUT_DATA_DIR/control" {shape: cylinder} } model: "Anthropic or OpenAI" {style.stroke-dash: 3} diff --git a/docs/diagrams/architecture.svg b/docs/diagrams/architecture.svg index 01633b3e..18433e2a 100644 --- a/docs/diagrams/architecture.svg +++ b/docs/diagrams/architecture.svg @@ -1,24 +1,24 @@ -Clientsfanout — one Go processStorageAnthropic or OpenAIOTLP collector or SDKBrowserExternal MCP hostHTTP :7520routes + auth middlewareIngestOTLP gRPC + HTTP :7520Embedded React assetsAgent runtimemodel + tool loopMCP server5 typed + 4 dashboard toolsTyped observability contractTelemetry commit workersatomic Parquet batchesQuery kernelDuckDB + rollupsStorage maintenanceretention + native compactionAlert enginerule evaluation + webhooksParquet batches + trace indexesstorage.data_dir/telemetryDuckDB query statestorage.data_dir/queryControl SQLitestorage.data_dir/control serves embedded assetsAG-UI streamin-memory transporttyped HTTP APIevaluates rollupsOTLP/gRPC or OTLP/HTTPHTTPS/mcp — OAuthindexed traces + SQL scansatomic replacementusers, sessions, settings, dashboards, threadsrules and fired alertsHTTPS - + .d2-37886755 .fill-N1{fill:#0A0F25;} + .d2-37886755 .fill-N2{fill:#676C7E;} + .d2-37886755 .fill-N3{fill:#9499AB;} + .d2-37886755 .fill-N4{fill:#CFD2DD;} + .d2-37886755 .fill-N5{fill:#DEE1EB;} + .d2-37886755 .fill-N6{fill:#EEF1F8;} + .d2-37886755 .fill-N7{fill:#FFFFFF;} + .d2-37886755 .fill-B1{fill:#000536;} + .d2-37886755 .fill-B2{fill:#0F66B7;} + .d2-37886755 .fill-B3{fill:#4393DD;} + .d2-37886755 .fill-B4{fill:#87BFF3;} + .d2-37886755 .fill-B5{fill:#BCDDFB;} + .d2-37886755 .fill-B6{fill:#E5F3FF;} + .d2-37886755 .fill-AA2{fill:#7639C5;} + .d2-37886755 .fill-AA4{fill:#C1A2F3;} + .d2-37886755 .fill-AA5{fill:#DACEFB;} + .d2-37886755 .fill-AB4{fill:#EA99C6;} + .d2-37886755 .fill-AB5{fill:#FFDEF1;} + .d2-37886755 .stroke-N1{stroke:#0A0F25;} + .d2-37886755 .stroke-N2{stroke:#676C7E;} + .d2-37886755 .stroke-N3{stroke:#9499AB;} + .d2-37886755 .stroke-N4{stroke:#CFD2DD;} + .d2-37886755 .stroke-N5{stroke:#DEE1EB;} + .d2-37886755 .stroke-N6{stroke:#EEF1F8;} + .d2-37886755 .stroke-N7{stroke:#FFFFFF;} + .d2-37886755 .stroke-B1{stroke:#000536;} + .d2-37886755 .stroke-B2{stroke:#0F66B7;} + .d2-37886755 .stroke-B3{stroke:#4393DD;} + .d2-37886755 .stroke-B4{stroke:#87BFF3;} + .d2-37886755 .stroke-B5{stroke:#BCDDFB;} + .d2-37886755 .stroke-B6{stroke:#E5F3FF;} + .d2-37886755 .stroke-AA2{stroke:#7639C5;} + .d2-37886755 .stroke-AA4{stroke:#C1A2F3;} + .d2-37886755 .stroke-AA5{stroke:#DACEFB;} + .d2-37886755 .stroke-AB4{stroke:#EA99C6;} + .d2-37886755 .stroke-AB5{stroke:#FFDEF1;} + .d2-37886755 .background-color-N1{background-color:#0A0F25;} + .d2-37886755 .background-color-N2{background-color:#676C7E;} + .d2-37886755 .background-color-N3{background-color:#9499AB;} + .d2-37886755 .background-color-N4{background-color:#CFD2DD;} + .d2-37886755 .background-color-N5{background-color:#DEE1EB;} + .d2-37886755 .background-color-N6{background-color:#EEF1F8;} + .d2-37886755 .background-color-N7{background-color:#FFFFFF;} + .d2-37886755 .background-color-B1{background-color:#000536;} + .d2-37886755 .background-color-B2{background-color:#0F66B7;} + .d2-37886755 .background-color-B3{background-color:#4393DD;} + .d2-37886755 .background-color-B4{background-color:#87BFF3;} + .d2-37886755 .background-color-B5{background-color:#BCDDFB;} + .d2-37886755 .background-color-B6{background-color:#E5F3FF;} + .d2-37886755 .background-color-AA2{background-color:#7639C5;} + .d2-37886755 .background-color-AA4{background-color:#C1A2F3;} + .d2-37886755 .background-color-AA5{background-color:#DACEFB;} + .d2-37886755 .background-color-AB4{background-color:#EA99C6;} + .d2-37886755 .background-color-AB5{background-color:#FFDEF1;} + .d2-37886755 .color-N1{color:#0A0F25;} + .d2-37886755 .color-N2{color:#676C7E;} + .d2-37886755 .color-N3{color:#9499AB;} + .d2-37886755 .color-N4{color:#CFD2DD;} + .d2-37886755 .color-N5{color:#DEE1EB;} + .d2-37886755 .color-N6{color:#EEF1F8;} + .d2-37886755 .color-N7{color:#FFFFFF;} + .d2-37886755 .color-B1{color:#000536;} + .d2-37886755 .color-B2{color:#0F66B7;} + .d2-37886755 .color-B3{color:#4393DD;} + .d2-37886755 .color-B4{color:#87BFF3;} + .d2-37886755 .color-B5{color:#BCDDFB;} + .d2-37886755 .color-B6{color:#E5F3FF;} + .d2-37886755 .color-AA2{color:#7639C5;} + .d2-37886755 .color-AA4{color:#C1A2F3;} + .d2-37886755 .color-AA5{color:#DACEFB;} + .d2-37886755 .color-AB4{color:#EA99C6;} + .d2-37886755 .color-AB5{color:#FFDEF1;}.appendix text.text{fill:#0A0F25}.md{--color-fg-default:#0A0F25;--color-fg-muted:#676C7E;--color-fg-subtle:#9499AB;--color-canvas-default:#FFFFFF;--color-canvas-subtle:#EEF1F8;--color-border-default:#000536;--color-border-muted:#0F66B7;--color-neutral-muted:#EEF1F8;--color-accent-fg:#0F66B7;--color-accent-emphasis:#0F66B7;--color-attention-subtle:#676C7E;--color-danger-fg:red;}.sketch-overlay-B1{fill:url(#streaks-darker-d2-37886755);mix-blend-mode:lighten}.sketch-overlay-B2{fill:url(#streaks-dark-d2-37886755);mix-blend-mode:overlay}.sketch-overlay-B3{fill:url(#streaks-dark-d2-37886755);mix-blend-mode:overlay}.sketch-overlay-B4{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-B5{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-B6{fill:url(#streaks-bright-d2-37886755);mix-blend-mode:darken}.sketch-overlay-AA2{fill:url(#streaks-dark-d2-37886755);mix-blend-mode:overlay}.sketch-overlay-AA4{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-AA5{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-AB4{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-AB5{fill:url(#streaks-bright-d2-37886755);mix-blend-mode:darken}.sketch-overlay-N1{fill:url(#streaks-darker-d2-37886755);mix-blend-mode:lighten}.sketch-overlay-N2{fill:url(#streaks-dark-d2-37886755);mix-blend-mode:overlay}.sketch-overlay-N3{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-N4{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-N5{fill:url(#streaks-bright-d2-37886755);mix-blend-mode:darken}.sketch-overlay-N6{fill:url(#streaks-bright-d2-37886755);mix-blend-mode:darken}.sketch-overlay-N7{fill:url(#streaks-bright-d2-37886755);mix-blend-mode:darken}.light-code{display: block}.dark-code{display: none}]]>Clientsfanout — one Go processStorageAnthropic or OpenAIOTLP collector or SDKBrowserExternal MCP hostHTTP :7520routes + auth middlewareIngestOTLP gRPC + HTTP :7520Embedded React assetsAgent runtimemodel + tool loopMCP server5 typed + 4 dashboard toolsTyped observability contractTelemetry commit workersatomic Parquet batchesQuery kernelDuckDB + rollupsStorage maintenanceretention + native compactionAlert enginerule evaluation + webhooksParquet batches + trace indexesFANOUT_DATA_DIR/telemetryDuckDB query stateFANOUT_DATA_DIR/queryControl SQLiteFANOUT_DATA_DIR/control serves embedded assetsAG-UI streamin-memory transporttyped HTTP APIevaluates rollupsOTLP/gRPC or OTLP/HTTPHTTPS/mcp — OAuthindexed traces + SQL scansatomic replacementusers, sessions, settings, dashboards, threadsrules and fired alertsHTTPS + @@ -113,9 +113,9 @@ - - + + - + diff --git a/docs/diagrams/persistence.d2 b/docs/diagrams/persistence.d2 index d5b508b5..b0d74d6f 100644 --- a/docs/diagrams/persistence.d2 +++ b/docs/diagrams/persistence.d2 @@ -49,7 +49,7 @@ maintenance: "Bounded maintenance" { merge -> swap } -querystate: "Rebuildable query state\nstorage.data_dir/query" { +querystate: "Rebuildable query state\nFANOUT_DATA_DIR/query" { shape: cylinder tooltip: DuckDB rollups and temporary spill. Not authoritative telemetry. } diff --git a/docs/diagrams/persistence.svg b/docs/diagrams/persistence.svg index 50c353fc..13d27eb5 100644 --- a/docs/diagrams/persistence.svg +++ b/docs/diagrams/persistence.svg @@ -1,27 +1,27 @@ -Authenticated OTLP requestsspans, logs, metricsDurable write pathAuthoritative telemetryimmutable *.batch directoriesOrdinary Parquet files are authoritativeReadersBounded maintenanceRebuildable query statestorage.data_dir/queryDuckDB rollups and temporary spill. Not authoritative telemetry.Bounded group admissionconfigured row target + 20 ms windowUp to four commit workersindependent batches encode in parallelStaging batch directorymetadata + signal Parquet + trace.fidx for spansDurable publicationfsync → atomic rename → fsync parentpublished batch directories are discovered directly from the filesystemIndexed trace readbinary-search .fidx → Parquet rowsDuckDB SQLfilters, ordering, broad scansDuckDB rollupsrebuildable analytical cacheRetention + compaction admissionbounded batch and row groupsNative Parquet mergeprepared outside the reader gateCrash-safe namespace swapmarker + bounded renames + fsync acknowledge only after publicationOrdinary Parquet files are authoritative - + .d2-2701310579 .fill-N1{fill:#0A0F25;} + .d2-2701310579 .fill-N2{fill:#676C7E;} + .d2-2701310579 .fill-N3{fill:#9499AB;} + .d2-2701310579 .fill-N4{fill:#CFD2DD;} + .d2-2701310579 .fill-N5{fill:#DEE1EB;} + .d2-2701310579 .fill-N6{fill:#EEF1F8;} + .d2-2701310579 .fill-N7{fill:#FFFFFF;} + .d2-2701310579 .fill-B1{fill:#000536;} + .d2-2701310579 .fill-B2{fill:#0F66B7;} + .d2-2701310579 .fill-B3{fill:#4393DD;} + .d2-2701310579 .fill-B4{fill:#87BFF3;} + .d2-2701310579 .fill-B5{fill:#BCDDFB;} + .d2-2701310579 .fill-B6{fill:#E5F3FF;} + .d2-2701310579 .fill-AA2{fill:#7639C5;} + .d2-2701310579 .fill-AA4{fill:#C1A2F3;} + .d2-2701310579 .fill-AA5{fill:#DACEFB;} + .d2-2701310579 .fill-AB4{fill:#EA99C6;} + .d2-2701310579 .fill-AB5{fill:#FFDEF1;} + .d2-2701310579 .stroke-N1{stroke:#0A0F25;} + .d2-2701310579 .stroke-N2{stroke:#676C7E;} + .d2-2701310579 .stroke-N3{stroke:#9499AB;} + .d2-2701310579 .stroke-N4{stroke:#CFD2DD;} + .d2-2701310579 .stroke-N5{stroke:#DEE1EB;} + .d2-2701310579 .stroke-N6{stroke:#EEF1F8;} + .d2-2701310579 .stroke-N7{stroke:#FFFFFF;} + .d2-2701310579 .stroke-B1{stroke:#000536;} + .d2-2701310579 .stroke-B2{stroke:#0F66B7;} + .d2-2701310579 .stroke-B3{stroke:#4393DD;} + .d2-2701310579 .stroke-B4{stroke:#87BFF3;} + .d2-2701310579 .stroke-B5{stroke:#BCDDFB;} + .d2-2701310579 .stroke-B6{stroke:#E5F3FF;} + .d2-2701310579 .stroke-AA2{stroke:#7639C5;} + .d2-2701310579 .stroke-AA4{stroke:#C1A2F3;} + .d2-2701310579 .stroke-AA5{stroke:#DACEFB;} + .d2-2701310579 .stroke-AB4{stroke:#EA99C6;} + .d2-2701310579 .stroke-AB5{stroke:#FFDEF1;} + .d2-2701310579 .background-color-N1{background-color:#0A0F25;} + .d2-2701310579 .background-color-N2{background-color:#676C7E;} + .d2-2701310579 .background-color-N3{background-color:#9499AB;} + .d2-2701310579 .background-color-N4{background-color:#CFD2DD;} + .d2-2701310579 .background-color-N5{background-color:#DEE1EB;} + .d2-2701310579 .background-color-N6{background-color:#EEF1F8;} + .d2-2701310579 .background-color-N7{background-color:#FFFFFF;} + .d2-2701310579 .background-color-B1{background-color:#000536;} + .d2-2701310579 .background-color-B2{background-color:#0F66B7;} + .d2-2701310579 .background-color-B3{background-color:#4393DD;} + .d2-2701310579 .background-color-B4{background-color:#87BFF3;} + .d2-2701310579 .background-color-B5{background-color:#BCDDFB;} + .d2-2701310579 .background-color-B6{background-color:#E5F3FF;} + .d2-2701310579 .background-color-AA2{background-color:#7639C5;} + .d2-2701310579 .background-color-AA4{background-color:#C1A2F3;} + .d2-2701310579 .background-color-AA5{background-color:#DACEFB;} + .d2-2701310579 .background-color-AB4{background-color:#EA99C6;} + .d2-2701310579 .background-color-AB5{background-color:#FFDEF1;} + .d2-2701310579 .color-N1{color:#0A0F25;} + .d2-2701310579 .color-N2{color:#676C7E;} + .d2-2701310579 .color-N3{color:#9499AB;} + .d2-2701310579 .color-N4{color:#CFD2DD;} + .d2-2701310579 .color-N5{color:#DEE1EB;} + .d2-2701310579 .color-N6{color:#EEF1F8;} + .d2-2701310579 .color-N7{color:#FFFFFF;} + .d2-2701310579 .color-B1{color:#000536;} + .d2-2701310579 .color-B2{color:#0F66B7;} + .d2-2701310579 .color-B3{color:#4393DD;} + .d2-2701310579 .color-B4{color:#87BFF3;} + .d2-2701310579 .color-B5{color:#BCDDFB;} + .d2-2701310579 .color-B6{color:#E5F3FF;} + .d2-2701310579 .color-AA2{color:#7639C5;} + .d2-2701310579 .color-AA4{color:#C1A2F3;} + .d2-2701310579 .color-AA5{color:#DACEFB;} + .d2-2701310579 .color-AB4{color:#EA99C6;} + .d2-2701310579 .color-AB5{color:#FFDEF1;}.appendix text.text{fill:#0A0F25}.md{--color-fg-default:#0A0F25;--color-fg-muted:#676C7E;--color-fg-subtle:#9499AB;--color-canvas-default:#FFFFFF;--color-canvas-subtle:#EEF1F8;--color-border-default:#000536;--color-border-muted:#0F66B7;--color-neutral-muted:#EEF1F8;--color-accent-fg:#0F66B7;--color-accent-emphasis:#0F66B7;--color-attention-subtle:#676C7E;--color-danger-fg:red;}.sketch-overlay-B1{fill:url(#streaks-darker-d2-2701310579);mix-blend-mode:lighten}.sketch-overlay-B2{fill:url(#streaks-dark-d2-2701310579);mix-blend-mode:overlay}.sketch-overlay-B3{fill:url(#streaks-dark-d2-2701310579);mix-blend-mode:overlay}.sketch-overlay-B4{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-B5{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-B6{fill:url(#streaks-bright-d2-2701310579);mix-blend-mode:darken}.sketch-overlay-AA2{fill:url(#streaks-dark-d2-2701310579);mix-blend-mode:overlay}.sketch-overlay-AA4{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-AA5{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-AB4{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-AB5{fill:url(#streaks-bright-d2-2701310579);mix-blend-mode:darken}.sketch-overlay-N1{fill:url(#streaks-darker-d2-2701310579);mix-blend-mode:lighten}.sketch-overlay-N2{fill:url(#streaks-dark-d2-2701310579);mix-blend-mode:overlay}.sketch-overlay-N3{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-N4{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-N5{fill:url(#streaks-bright-d2-2701310579);mix-blend-mode:darken}.sketch-overlay-N6{fill:url(#streaks-bright-d2-2701310579);mix-blend-mode:darken}.sketch-overlay-N7{fill:url(#streaks-bright-d2-2701310579);mix-blend-mode:darken}.light-code{display: block}.dark-code{display: none}]]>Authenticated OTLP requestsspans, logs, metricsDurable write pathAuthoritative telemetryimmutable *.batch directoriesOrdinary Parquet files are authoritativeReadersBounded maintenanceRebuildable query stateFANOUT_DATA_DIR/queryDuckDB rollups and temporary spill. Not authoritative telemetry.Bounded group admissionconfigured row target + 20 ms windowUp to four commit workersindependent batches encode in parallelStaging batch directorymetadata + signal Parquet + trace.fidx for spansDurable publicationfsync → atomic rename → fsync parentpublished batch directories are discovered directly from the filesystemIndexed trace readbinary-search .fidx → Parquet rowsDuckDB SQLfilters, ordering, broad scansDuckDB rollupsrebuildable analytical cacheRetention + compaction admissionbounded batch and row groupsNative Parquet mergeprepared outside the reader gateCrash-safe namespace swapmarker + bounded renames + fsync acknowledge only after publicationOrdinary Parquet files are authoritative + - + -DuckDB rollups and temporary spill. Not authoritative telemetry. - +DuckDB rollups and temporary spill. Not authoritative telemetry. + - + - + diff --git a/internal/api/oauth.go b/internal/api/oauth.go index 4f2b7bc3..eb5909a7 100644 --- a/internal/api/oauth.go +++ b/internal/api/oauth.go @@ -71,7 +71,7 @@ func NewMCPAuthorization(store *appauth.OAuthStore, users *appauth.UserStore, re // registration). Bound how many clients one source can create before // the janitor collects the abandoned ones. The budget is per client // IP, which collapses to one bucket when Fanout sits behind a proxy - // and server.trusted_proxy_cidrs is unset, so it is set well above a + // and FANOUT_TRUSTED_PROXY_CIDRS is unset, so it is set well above a // plausible team rollout rather than at the tightest defensible value. registerLimiter: appauth.NewKeyedLimiter(60, 15*time.Minute), }, nil diff --git a/site/public/diagrams/architecture.svg b/site/public/diagrams/architecture.svg index 01633b3e..18433e2a 100644 --- a/site/public/diagrams/architecture.svg +++ b/site/public/diagrams/architecture.svg @@ -1,24 +1,24 @@ -Clientsfanout — one Go processStorageAnthropic or OpenAIOTLP collector or SDKBrowserExternal MCP hostHTTP :7520routes + auth middlewareIngestOTLP gRPC + HTTP :7520Embedded React assetsAgent runtimemodel + tool loopMCP server5 typed + 4 dashboard toolsTyped observability contractTelemetry commit workersatomic Parquet batchesQuery kernelDuckDB + rollupsStorage maintenanceretention + native compactionAlert enginerule evaluation + webhooksParquet batches + trace indexesstorage.data_dir/telemetryDuckDB query statestorage.data_dir/queryControl SQLitestorage.data_dir/control serves embedded assetsAG-UI streamin-memory transporttyped HTTP APIevaluates rollupsOTLP/gRPC or OTLP/HTTPHTTPS/mcp — OAuthindexed traces + SQL scansatomic replacementusers, sessions, settings, dashboards, threadsrules and fired alertsHTTPS - + .d2-37886755 .fill-N1{fill:#0A0F25;} + .d2-37886755 .fill-N2{fill:#676C7E;} + .d2-37886755 .fill-N3{fill:#9499AB;} + .d2-37886755 .fill-N4{fill:#CFD2DD;} + .d2-37886755 .fill-N5{fill:#DEE1EB;} + .d2-37886755 .fill-N6{fill:#EEF1F8;} + .d2-37886755 .fill-N7{fill:#FFFFFF;} + .d2-37886755 .fill-B1{fill:#000536;} + .d2-37886755 .fill-B2{fill:#0F66B7;} + .d2-37886755 .fill-B3{fill:#4393DD;} + .d2-37886755 .fill-B4{fill:#87BFF3;} + .d2-37886755 .fill-B5{fill:#BCDDFB;} + .d2-37886755 .fill-B6{fill:#E5F3FF;} + .d2-37886755 .fill-AA2{fill:#7639C5;} + .d2-37886755 .fill-AA4{fill:#C1A2F3;} + .d2-37886755 .fill-AA5{fill:#DACEFB;} + .d2-37886755 .fill-AB4{fill:#EA99C6;} + .d2-37886755 .fill-AB5{fill:#FFDEF1;} + .d2-37886755 .stroke-N1{stroke:#0A0F25;} + .d2-37886755 .stroke-N2{stroke:#676C7E;} + .d2-37886755 .stroke-N3{stroke:#9499AB;} + .d2-37886755 .stroke-N4{stroke:#CFD2DD;} + .d2-37886755 .stroke-N5{stroke:#DEE1EB;} + .d2-37886755 .stroke-N6{stroke:#EEF1F8;} + .d2-37886755 .stroke-N7{stroke:#FFFFFF;} + .d2-37886755 .stroke-B1{stroke:#000536;} + .d2-37886755 .stroke-B2{stroke:#0F66B7;} + .d2-37886755 .stroke-B3{stroke:#4393DD;} + .d2-37886755 .stroke-B4{stroke:#87BFF3;} + .d2-37886755 .stroke-B5{stroke:#BCDDFB;} + .d2-37886755 .stroke-B6{stroke:#E5F3FF;} + .d2-37886755 .stroke-AA2{stroke:#7639C5;} + .d2-37886755 .stroke-AA4{stroke:#C1A2F3;} + .d2-37886755 .stroke-AA5{stroke:#DACEFB;} + .d2-37886755 .stroke-AB4{stroke:#EA99C6;} + .d2-37886755 .stroke-AB5{stroke:#FFDEF1;} + .d2-37886755 .background-color-N1{background-color:#0A0F25;} + .d2-37886755 .background-color-N2{background-color:#676C7E;} + .d2-37886755 .background-color-N3{background-color:#9499AB;} + .d2-37886755 .background-color-N4{background-color:#CFD2DD;} + .d2-37886755 .background-color-N5{background-color:#DEE1EB;} + .d2-37886755 .background-color-N6{background-color:#EEF1F8;} + .d2-37886755 .background-color-N7{background-color:#FFFFFF;} + .d2-37886755 .background-color-B1{background-color:#000536;} + .d2-37886755 .background-color-B2{background-color:#0F66B7;} + .d2-37886755 .background-color-B3{background-color:#4393DD;} + .d2-37886755 .background-color-B4{background-color:#87BFF3;} + .d2-37886755 .background-color-B5{background-color:#BCDDFB;} + .d2-37886755 .background-color-B6{background-color:#E5F3FF;} + .d2-37886755 .background-color-AA2{background-color:#7639C5;} + .d2-37886755 .background-color-AA4{background-color:#C1A2F3;} + .d2-37886755 .background-color-AA5{background-color:#DACEFB;} + .d2-37886755 .background-color-AB4{background-color:#EA99C6;} + .d2-37886755 .background-color-AB5{background-color:#FFDEF1;} + .d2-37886755 .color-N1{color:#0A0F25;} + .d2-37886755 .color-N2{color:#676C7E;} + .d2-37886755 .color-N3{color:#9499AB;} + .d2-37886755 .color-N4{color:#CFD2DD;} + .d2-37886755 .color-N5{color:#DEE1EB;} + .d2-37886755 .color-N6{color:#EEF1F8;} + .d2-37886755 .color-N7{color:#FFFFFF;} + .d2-37886755 .color-B1{color:#000536;} + .d2-37886755 .color-B2{color:#0F66B7;} + .d2-37886755 .color-B3{color:#4393DD;} + .d2-37886755 .color-B4{color:#87BFF3;} + .d2-37886755 .color-B5{color:#BCDDFB;} + .d2-37886755 .color-B6{color:#E5F3FF;} + .d2-37886755 .color-AA2{color:#7639C5;} + .d2-37886755 .color-AA4{color:#C1A2F3;} + .d2-37886755 .color-AA5{color:#DACEFB;} + .d2-37886755 .color-AB4{color:#EA99C6;} + .d2-37886755 .color-AB5{color:#FFDEF1;}.appendix text.text{fill:#0A0F25}.md{--color-fg-default:#0A0F25;--color-fg-muted:#676C7E;--color-fg-subtle:#9499AB;--color-canvas-default:#FFFFFF;--color-canvas-subtle:#EEF1F8;--color-border-default:#000536;--color-border-muted:#0F66B7;--color-neutral-muted:#EEF1F8;--color-accent-fg:#0F66B7;--color-accent-emphasis:#0F66B7;--color-attention-subtle:#676C7E;--color-danger-fg:red;}.sketch-overlay-B1{fill:url(#streaks-darker-d2-37886755);mix-blend-mode:lighten}.sketch-overlay-B2{fill:url(#streaks-dark-d2-37886755);mix-blend-mode:overlay}.sketch-overlay-B3{fill:url(#streaks-dark-d2-37886755);mix-blend-mode:overlay}.sketch-overlay-B4{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-B5{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-B6{fill:url(#streaks-bright-d2-37886755);mix-blend-mode:darken}.sketch-overlay-AA2{fill:url(#streaks-dark-d2-37886755);mix-blend-mode:overlay}.sketch-overlay-AA4{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-AA5{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-AB4{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-AB5{fill:url(#streaks-bright-d2-37886755);mix-blend-mode:darken}.sketch-overlay-N1{fill:url(#streaks-darker-d2-37886755);mix-blend-mode:lighten}.sketch-overlay-N2{fill:url(#streaks-dark-d2-37886755);mix-blend-mode:overlay}.sketch-overlay-N3{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-N4{fill:url(#streaks-normal-d2-37886755);mix-blend-mode:color-burn}.sketch-overlay-N5{fill:url(#streaks-bright-d2-37886755);mix-blend-mode:darken}.sketch-overlay-N6{fill:url(#streaks-bright-d2-37886755);mix-blend-mode:darken}.sketch-overlay-N7{fill:url(#streaks-bright-d2-37886755);mix-blend-mode:darken}.light-code{display: block}.dark-code{display: none}]]>Clientsfanout — one Go processStorageAnthropic or OpenAIOTLP collector or SDKBrowserExternal MCP hostHTTP :7520routes + auth middlewareIngestOTLP gRPC + HTTP :7520Embedded React assetsAgent runtimemodel + tool loopMCP server5 typed + 4 dashboard toolsTyped observability contractTelemetry commit workersatomic Parquet batchesQuery kernelDuckDB + rollupsStorage maintenanceretention + native compactionAlert enginerule evaluation + webhooksParquet batches + trace indexesFANOUT_DATA_DIR/telemetryDuckDB query stateFANOUT_DATA_DIR/queryControl SQLiteFANOUT_DATA_DIR/control serves embedded assetsAG-UI streamin-memory transporttyped HTTP APIevaluates rollupsOTLP/gRPC or OTLP/HTTPHTTPS/mcp — OAuthindexed traces + SQL scansatomic replacementusers, sessions, settings, dashboards, threadsrules and fired alertsHTTPS + @@ -113,9 +113,9 @@ - - + + - + diff --git a/site/public/diagrams/persistence.svg b/site/public/diagrams/persistence.svg index 50c353fc..13d27eb5 100644 --- a/site/public/diagrams/persistence.svg +++ b/site/public/diagrams/persistence.svg @@ -1,27 +1,27 @@ -Authenticated OTLP requestsspans, logs, metricsDurable write pathAuthoritative telemetryimmutable *.batch directoriesOrdinary Parquet files are authoritativeReadersBounded maintenanceRebuildable query statestorage.data_dir/queryDuckDB rollups and temporary spill. Not authoritative telemetry.Bounded group admissionconfigured row target + 20 ms windowUp to four commit workersindependent batches encode in parallelStaging batch directorymetadata + signal Parquet + trace.fidx for spansDurable publicationfsync → atomic rename → fsync parentpublished batch directories are discovered directly from the filesystemIndexed trace readbinary-search .fidx → Parquet rowsDuckDB SQLfilters, ordering, broad scansDuckDB rollupsrebuildable analytical cacheRetention + compaction admissionbounded batch and row groupsNative Parquet mergeprepared outside the reader gateCrash-safe namespace swapmarker + bounded renames + fsync acknowledge only after publicationOrdinary Parquet files are authoritative - + .d2-2701310579 .fill-N1{fill:#0A0F25;} + .d2-2701310579 .fill-N2{fill:#676C7E;} + .d2-2701310579 .fill-N3{fill:#9499AB;} + .d2-2701310579 .fill-N4{fill:#CFD2DD;} + .d2-2701310579 .fill-N5{fill:#DEE1EB;} + .d2-2701310579 .fill-N6{fill:#EEF1F8;} + .d2-2701310579 .fill-N7{fill:#FFFFFF;} + .d2-2701310579 .fill-B1{fill:#000536;} + .d2-2701310579 .fill-B2{fill:#0F66B7;} + .d2-2701310579 .fill-B3{fill:#4393DD;} + .d2-2701310579 .fill-B4{fill:#87BFF3;} + .d2-2701310579 .fill-B5{fill:#BCDDFB;} + .d2-2701310579 .fill-B6{fill:#E5F3FF;} + .d2-2701310579 .fill-AA2{fill:#7639C5;} + .d2-2701310579 .fill-AA4{fill:#C1A2F3;} + .d2-2701310579 .fill-AA5{fill:#DACEFB;} + .d2-2701310579 .fill-AB4{fill:#EA99C6;} + .d2-2701310579 .fill-AB5{fill:#FFDEF1;} + .d2-2701310579 .stroke-N1{stroke:#0A0F25;} + .d2-2701310579 .stroke-N2{stroke:#676C7E;} + .d2-2701310579 .stroke-N3{stroke:#9499AB;} + .d2-2701310579 .stroke-N4{stroke:#CFD2DD;} + .d2-2701310579 .stroke-N5{stroke:#DEE1EB;} + .d2-2701310579 .stroke-N6{stroke:#EEF1F8;} + .d2-2701310579 .stroke-N7{stroke:#FFFFFF;} + .d2-2701310579 .stroke-B1{stroke:#000536;} + .d2-2701310579 .stroke-B2{stroke:#0F66B7;} + .d2-2701310579 .stroke-B3{stroke:#4393DD;} + .d2-2701310579 .stroke-B4{stroke:#87BFF3;} + .d2-2701310579 .stroke-B5{stroke:#BCDDFB;} + .d2-2701310579 .stroke-B6{stroke:#E5F3FF;} + .d2-2701310579 .stroke-AA2{stroke:#7639C5;} + .d2-2701310579 .stroke-AA4{stroke:#C1A2F3;} + .d2-2701310579 .stroke-AA5{stroke:#DACEFB;} + .d2-2701310579 .stroke-AB4{stroke:#EA99C6;} + .d2-2701310579 .stroke-AB5{stroke:#FFDEF1;} + .d2-2701310579 .background-color-N1{background-color:#0A0F25;} + .d2-2701310579 .background-color-N2{background-color:#676C7E;} + .d2-2701310579 .background-color-N3{background-color:#9499AB;} + .d2-2701310579 .background-color-N4{background-color:#CFD2DD;} + .d2-2701310579 .background-color-N5{background-color:#DEE1EB;} + .d2-2701310579 .background-color-N6{background-color:#EEF1F8;} + .d2-2701310579 .background-color-N7{background-color:#FFFFFF;} + .d2-2701310579 .background-color-B1{background-color:#000536;} + .d2-2701310579 .background-color-B2{background-color:#0F66B7;} + .d2-2701310579 .background-color-B3{background-color:#4393DD;} + .d2-2701310579 .background-color-B4{background-color:#87BFF3;} + .d2-2701310579 .background-color-B5{background-color:#BCDDFB;} + .d2-2701310579 .background-color-B6{background-color:#E5F3FF;} + .d2-2701310579 .background-color-AA2{background-color:#7639C5;} + .d2-2701310579 .background-color-AA4{background-color:#C1A2F3;} + .d2-2701310579 .background-color-AA5{background-color:#DACEFB;} + .d2-2701310579 .background-color-AB4{background-color:#EA99C6;} + .d2-2701310579 .background-color-AB5{background-color:#FFDEF1;} + .d2-2701310579 .color-N1{color:#0A0F25;} + .d2-2701310579 .color-N2{color:#676C7E;} + .d2-2701310579 .color-N3{color:#9499AB;} + .d2-2701310579 .color-N4{color:#CFD2DD;} + .d2-2701310579 .color-N5{color:#DEE1EB;} + .d2-2701310579 .color-N6{color:#EEF1F8;} + .d2-2701310579 .color-N7{color:#FFFFFF;} + .d2-2701310579 .color-B1{color:#000536;} + .d2-2701310579 .color-B2{color:#0F66B7;} + .d2-2701310579 .color-B3{color:#4393DD;} + .d2-2701310579 .color-B4{color:#87BFF3;} + .d2-2701310579 .color-B5{color:#BCDDFB;} + .d2-2701310579 .color-B6{color:#E5F3FF;} + .d2-2701310579 .color-AA2{color:#7639C5;} + .d2-2701310579 .color-AA4{color:#C1A2F3;} + .d2-2701310579 .color-AA5{color:#DACEFB;} + .d2-2701310579 .color-AB4{color:#EA99C6;} + .d2-2701310579 .color-AB5{color:#FFDEF1;}.appendix text.text{fill:#0A0F25}.md{--color-fg-default:#0A0F25;--color-fg-muted:#676C7E;--color-fg-subtle:#9499AB;--color-canvas-default:#FFFFFF;--color-canvas-subtle:#EEF1F8;--color-border-default:#000536;--color-border-muted:#0F66B7;--color-neutral-muted:#EEF1F8;--color-accent-fg:#0F66B7;--color-accent-emphasis:#0F66B7;--color-attention-subtle:#676C7E;--color-danger-fg:red;}.sketch-overlay-B1{fill:url(#streaks-darker-d2-2701310579);mix-blend-mode:lighten}.sketch-overlay-B2{fill:url(#streaks-dark-d2-2701310579);mix-blend-mode:overlay}.sketch-overlay-B3{fill:url(#streaks-dark-d2-2701310579);mix-blend-mode:overlay}.sketch-overlay-B4{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-B5{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-B6{fill:url(#streaks-bright-d2-2701310579);mix-blend-mode:darken}.sketch-overlay-AA2{fill:url(#streaks-dark-d2-2701310579);mix-blend-mode:overlay}.sketch-overlay-AA4{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-AA5{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-AB4{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-AB5{fill:url(#streaks-bright-d2-2701310579);mix-blend-mode:darken}.sketch-overlay-N1{fill:url(#streaks-darker-d2-2701310579);mix-blend-mode:lighten}.sketch-overlay-N2{fill:url(#streaks-dark-d2-2701310579);mix-blend-mode:overlay}.sketch-overlay-N3{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-N4{fill:url(#streaks-normal-d2-2701310579);mix-blend-mode:color-burn}.sketch-overlay-N5{fill:url(#streaks-bright-d2-2701310579);mix-blend-mode:darken}.sketch-overlay-N6{fill:url(#streaks-bright-d2-2701310579);mix-blend-mode:darken}.sketch-overlay-N7{fill:url(#streaks-bright-d2-2701310579);mix-blend-mode:darken}.light-code{display: block}.dark-code{display: none}]]>Authenticated OTLP requestsspans, logs, metricsDurable write pathAuthoritative telemetryimmutable *.batch directoriesOrdinary Parquet files are authoritativeReadersBounded maintenanceRebuildable query stateFANOUT_DATA_DIR/queryDuckDB rollups and temporary spill. Not authoritative telemetry.Bounded group admissionconfigured row target + 20 ms windowUp to four commit workersindependent batches encode in parallelStaging batch directorymetadata + signal Parquet + trace.fidx for spansDurable publicationfsync → atomic rename → fsync parentpublished batch directories are discovered directly from the filesystemIndexed trace readbinary-search .fidx → Parquet rowsDuckDB SQLfilters, ordering, broad scansDuckDB rollupsrebuildable analytical cacheRetention + compaction admissionbounded batch and row groupsNative Parquet mergeprepared outside the reader gateCrash-safe namespace swapmarker + bounded renames + fsync acknowledge only after publicationOrdinary Parquet files are authoritative + - + -DuckDB rollups and temporary spill. Not authoritative telemetry. - +DuckDB rollups and temporary spill. Not authoritative telemetry. + - + - + diff --git a/site/src/content/docs/explanation/storage-model.mdx b/site/src/content/docs/explanation/storage-model.mdx index 92bcafb5..526d7d7d 100644 --- a/site/src/content/docs/explanation/storage-model.mdx +++ b/site/src/content/docs/explanation/storage-model.mdx @@ -172,7 +172,7 @@ Narrow file selections use explicit lists from footer bounds. Broad selections use a glob with a captured-batch-ID filter, which excludes files published after snapshot capture. Only signals referenced by a query are bound. -`storage.rollup_skip_to_latest` skips historical service and edge aggregation. +`FANOUT_ROLLUP_SKIP_TO_LATEST` skips historical service and edge aggregation. Completed-batch trace caches still rebuild retained history in bounded passes; uncached files remain immediately queryable. From 2b612030be2e5a684bdd4ffc053fda8d4f2e9824 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 14 Oct 2026 07:13:34 -0700 Subject: [PATCH 6/6] fix(healthcheck): verify IP-literal probes against the configured certificate A derived HTTPS probe that dials an IP literal trusted any certificate, because a bind IP is rarely in the certificate's SAN list. It now trusts only the leaf in FANOUT_TLS_CERT_FILE and verifies it under that certificate's first DNS or IP SAN, so the probe confirms the listener presents the configured certificate without disabling verification. The local TLS test asserts tls.CertificateVerificationError, which both Linux (hostname mismatch) and macOS (unknown authority) return. --- cmd/fanout/main.go | 42 ++++++++-- cmd/fanout/main_test.go | 165 ++++++++++++++++++++++++++++++++-------- docs/operations.md | 9 ++- 3 files changed, 173 insertions(+), 43 deletions(-) diff --git a/cmd/fanout/main.go b/cmd/fanout/main.go index fa6257bf..f83950be 100644 --- a/cmd/fanout/main.go +++ b/cmd/fanout/main.go @@ -3,6 +3,8 @@ package main import ( "context" "crypto/tls" + "crypto/x509" + "encoding/pem" "errors" "flag" "fmt" @@ -79,7 +81,7 @@ func main() { } } if healthURL != "" || derivedProbe { - if err := checkHealth(healthURL, derivedProbe); err != nil { + if err := checkHealth(healthURL, derivedProbe, os.Getenv("FANOUT_TLS_CERT_FILE")); err != nil { fmt.Fprintln(os.Stderr, err) os.Exit(1) } @@ -547,11 +549,11 @@ func healthcheckURL(addr, certFile string) (string, error) { return (&url.URL{Scheme: scheme, Host: net.JoinHostPort(host, port), Path: "/healthz"}).String(), nil } -func checkHealth(healthURL string, derivedProbe bool) error { +func checkHealth(healthURL string, derivedProbe bool, certFile string) error { client := &http.Client{Timeout: 3 * time.Second} if derivedProbe { - // Derived probes bypass proxies. IP literals skip certificate verification - // because the certificate may name the public hostname instead of a bind IP. + // Derived probes bypass proxies. HTTPS IP literals trust only the configured + // certificate and verify its first DNS or IP SAN instead of the bind IP. // Hostnames verify normally; explicit URLs retain all default behaviour. probe, err := url.Parse(healthURL) if err != nil { @@ -559,8 +561,36 @@ func checkHealth(healthURL string, derivedProbe bool) error { } transport := http.DefaultTransport.(*http.Transport).Clone() transport.Proxy = nil - if _, err := netip.ParseAddr(probe.Hostname()); err == nil { - transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} + if _, err := netip.ParseAddr(probe.Hostname()); probe.Scheme == "https" && err == nil { + data, err := os.ReadFile(certFile) + if err != nil { + return fmt.Errorf("healthcheck: read certificate %q: %w", certFile, err) + } + var block *pem.Block + for len(data) > 0 { + block, data = pem.Decode(data) + if block == nil || block.Type == "CERTIFICATE" { + break + } + } + if block == nil || block.Type != "CERTIFICATE" { + return fmt.Errorf("healthcheck: no certificate in %q", certFile) + } + leaf, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return fmt.Errorf("healthcheck: parse certificate %q: %w", certFile, err) + } + var serverName string + if len(leaf.DNSNames) > 0 { + serverName = leaf.DNSNames[0] + } else if len(leaf.IPAddresses) > 0 { + serverName = leaf.IPAddresses[0].String() + } else { + return fmt.Errorf("healthcheck: certificate %q has no DNS or IP SAN", certFile) + } + roots := x509.NewCertPool() + roots.AddCert(leaf) + transport.TLSClientConfig = &tls.Config{RootCAs: roots, ServerName: serverName} } defer transport.CloseIdleConnections() client.Transport = transport diff --git a/cmd/fanout/main_test.go b/cmd/fanout/main_test.go index c3a3ed88..ce518c6f 100644 --- a/cmd/fanout/main_test.go +++ b/cmd/fanout/main_test.go @@ -8,6 +8,7 @@ import ( "crypto/rand" "crypto/tls" "crypto/x509" + "encoding/pem" "errors" "flag" "io" @@ -120,7 +121,7 @@ func TestCheckHealth(t *testing.T) { writer.WriteHeader(test.statusCode) })) defer server.Close() - if err := checkHealth(server.URL, false); (err != nil) != test.wantErr { + if err := checkHealth(server.URL, false, ""); (err != nil) != test.wantErr { t.Fatalf("checkHealth() error = %v, wantErr %v", err, test.wantErr) } }) @@ -251,49 +252,151 @@ func TestHealthcheckUsesEnvironmentWithoutValidatingSecrets(t *testing.T) { func TestCheckHealthWithLocalTLS(t *testing.T) { server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) })) defer server.Close() - if err := checkHealth(server.URL+"/healthz", true); err != nil { + certFile := writeHealthcheckCertificate(t, server.Certificate().Raw) + if err := checkHealth(server.URL+"/healthz", true, certFile); err != nil { t.Fatal(err) } - if err := checkHealth(server.URL+"/healthz", false); err == nil { - t.Fatal("explicit URL must verify the certificate") + var verificationError *tls.CertificateVerificationError + if err := checkHealth(server.URL+"/healthz", false, certFile); !errors.As(err, &verificationError) { + t.Fatalf("explicit URL must verify the certificate, got %v", err) + } + other := *server.Certificate() + other.SerialNumber = big.NewInt(2) + other.SignatureAlgorithm = x509.UnknownSignatureAlgorithm + other.PublicKey = nil + otherCertificate := newHealthcheckCertificate(t, &other) + otherFile := writeHealthcheckCertificate(t, otherCertificate.Certificate[0]) + // Only the first certificate is trusted, even when the listener's leaf follows it. + chain := append(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: otherCertificate.Certificate[0]}), + pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})...) + if err := os.WriteFile(otherFile, chain, 0o600); err != nil { + t.Fatal(err) + } + if err := checkHealth(server.URL+"/healthz", true, otherFile); !errors.As(err, &verificationError) { + t.Fatalf("derived IP probe must reject a different certificate, got %v", err) } probe := strings.Replace(server.URL, "127.0.0.1", "localhost", 1) + "/healthz" - var authorityError x509.UnknownAuthorityError - if err := checkHealth(probe, true); !errors.As(err, &authorityError) { - t.Fatalf("derived hostname probe must verify certificate trust, got %v", err) + if err := checkHealth(probe, true, certFile); !errors.As(err, &verificationError) { + t.Fatalf("derived hostname probe must verify the certificate, got %v", err) } } -func TestCheckHealthWithInterfaceTLS(t *testing.T) { +func writeHealthcheckCertificate(t *testing.T, der []byte) string { + t.Helper() + path := filepath.Join(t.TempDir(), "cert.pem") + if err := os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func newHealthcheckCertificate(t *testing.T, certificate *x509.Certificate) tls.Certificate { + t.Helper() key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { t.Fatal(err) } + der, err := x509.CreateCertificate(rand.Reader, certificate, certificate, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + return tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key} +} + +func TestCheckHealthCertificateFileErrors(t *testing.T) { + const contents = "private file contents must not appear in errors" + for _, test := range []struct { + name string + data []byte + }{ + {name: "unreadable"}, + {name: "non-certificate", data: []byte(contents)}, + {name: "other PEM block", data: pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: []byte(contents)})}, + {name: "invalid certificate", data: pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: []byte(contents)})}, + } { + t.Run(test.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "cert.pem") + if test.data != nil { + if err := os.WriteFile(path, test.data, 0o600); err != nil { + t.Fatal(err) + } + } + err := checkHealth("https://127.0.0.1:7520/healthz", true, path) + if err == nil || !strings.HasPrefix(err.Error(), "healthcheck:") || !strings.Contains(err.Error(), path) || strings.Contains(err.Error(), contents) { + t.Fatalf("certificate file error must name its path without contents, got %v", err) + } + }) + } +} + +func TestCheckHealthTLSCertificate(t *testing.T) { + for _, test := range []struct { + name string + dnsNames []string + ips []net.IP + expired bool + clientOnly bool + wantErr bool + }{ + {name: "IP SAN", ips: []net.IP{net.ParseIP("192.0.2.10")}}, + {name: "DNS before IP SAN", dnsNames: []string{"fanout.example.com", "other.example.com"}, ips: []net.IP{net.ParseIP("192.0.2.10")}}, + {name: "no SAN", wantErr: true}, + {name: "expired", dnsNames: []string{"fanout.example.com"}, expired: true, wantErr: true}, + {name: "client usage", dnsNames: []string{"fanout.example.com"}, clientOnly: true, wantErr: true}, + } { + t.Run(test.name, func(t *testing.T) { + certificate := &x509.Certificate{ + SerialNumber: big.NewInt(1), DNSNames: test.dnsNames, IPAddresses: test.ips, + NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + if test.expired { + certificate.NotAfter = time.Now().Add(-time.Minute) + } + if test.clientOnly { + certificate.ExtKeyUsage = []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth} + } + server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { + if len(test.dnsNames) > 0 && request.TLS.ServerName != test.dnsNames[0] { + t.Errorf("TLS server name = %q, want %q", request.TLS.ServerName, test.dnsNames[0]) + } + w.WriteHeader(http.StatusOK) + })) + server.TLS = &tls.Config{Certificates: []tls.Certificate{newHealthcheckCertificate(t, certificate)}} + server.StartTLS() + defer server.Close() + certFile := writeHealthcheckCertificate(t, server.Certificate().Raw) + err := checkHealth(server.URL+"/healthz", true, certFile) + if (err != nil) != test.wantErr { + t.Fatalf("checkHealth() error = %v, wantErr %v", err, test.wantErr) + } + if test.name == "no SAN" && (!strings.HasPrefix(err.Error(), "healthcheck:") || !strings.Contains(err.Error(), certFile)) { + t.Fatalf("missing SAN error must name the certificate path, got %v", err) + } + }) + } +} + +func TestCheckHealthWithInterfaceTLS(t *testing.T) { certificate := &x509.Certificate{ SerialNumber: big.NewInt(1), DNSNames: []string{"fanout.example.com"}, NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour), KeyUsage: x509.KeyUsageDigitalSignature, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, } - der, err := x509.CreateCertificate(rand.Reader, certificate, certificate, &key.PublicKey, key) - if err != nil { - t.Fatal(err) - } server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { if request.URL.Path != "/healthz" { t.Errorf("probe path = %q, want /healthz", request.URL.Path) } w.WriteHeader(http.StatusOK) })) - server.TLS = &tls.Config{Certificates: []tls.Certificate{{Certificate: [][]byte{der}, PrivateKey: key}}} + server.TLS = &tls.Config{Certificates: []tls.Certificate{newHealthcheckCertificate(t, certificate)}} server.StartTLS() defer server.Close() - roots := x509.NewCertPool() - roots.AddCert(server.Certificate()) + certFile := writeHealthcheckCertificate(t, server.Certificate().Raw) for _, addr := range []string{"192.0.2.10:7520", "[2001:db8::10]:7520", "[fe80::1%eth0]:7520", "fanout.example.com:7520", "wrong.example.com:7520"} { t.Run(addr, func(t *testing.T) { original := http.DefaultTransport transport := original.(*http.Transport).Clone() - transport.TLSClientConfig = &tls.Config{RootCAs: roots} // Route binds to a local test listener to exercise real TLS without // requiring those addresses or DNS entries on the test host. transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { @@ -309,16 +412,16 @@ func TestCheckHealthWithInterfaceTLS(t *testing.T) { transport.CloseIdleConnections() }) t.Setenv("FANOUT_ADDR", addr) - t.Setenv("FANOUT_TLS_CERT_FILE", "cert.pem") + t.Setenv("FANOUT_TLS_CERT_FILE", certFile) probe, err := healthcheckURL(os.Getenv("FANOUT_ADDR"), os.Getenv("FANOUT_TLS_CERT_FILE")) if err != nil { t.Fatal(err) } - err = checkHealth(probe, true) - var hostnameError x509.HostnameError - if addr == "wrong.example.com:7520" { - if !errors.As(err, &hostnameError) { - t.Fatalf("derived hostname probe must verify the hostname, got %v", err) + err = checkHealth(probe, true, certFile) + var verificationError *tls.CertificateVerificationError + if strings.HasSuffix(addr, "example.com:7520") { + if !errors.As(err, &verificationError) { + t.Fatalf("derived hostname probe must use system verification, got %v", err) } } else if err != nil { t.Fatalf("derived TLS probe: %v", err) @@ -327,13 +430,9 @@ func TestCheckHealthWithInterfaceTLS(t *testing.T) { if err != nil { t.Fatal(err) } - err = checkHealth(explicitProbe, derivedProbe) - if addr == "fanout.example.com:7520" { - if err != nil { - t.Fatalf("explicit hostname probe: %v", err) - } - } else if !errors.As(err, &hostnameError) { - t.Fatalf("explicit TLS probe must verify the hostname, got %v", err) + err = checkHealth(explicitProbe, derivedProbe, certFile) + if !errors.As(err, &verificationError) { + t.Fatalf("explicit TLS probe must use system verification, got %v", err) } }) } @@ -378,14 +477,14 @@ func TestCheckHealthDerivedIgnoresEnvironmentProxy(t *testing.T) { certFile := "" if scheme == "https" { server.StartTLS() - certFile = "cert.pem" + certFile = writeHealthcheckCertificate(t, server.Certificate().Raw) } else { server.Start() } defer server.Close() original := http.DefaultTransport transport := original.(*http.Transport).Clone() - if scheme == "https" { + if scheme == "https" && addr == "example.com:7520" { roots := x509.NewCertPool() roots.AddCert(server.Certificate()) transport.TLSClientConfig = &tls.Config{RootCAs: roots} @@ -410,10 +509,10 @@ func TestCheckHealthDerivedIgnoresEnvironmentProxy(t *testing.T) { if err != nil { t.Fatal(err) } - if err := checkHealth(probe, true); err != nil { + if err := checkHealth(probe, true, certFile); err != nil { t.Fatalf("derived %s probe with unreachable proxy: %v", scheme, err) } - if err := checkHealth(probe, false); err == nil || !strings.Contains(err.Error(), strings.TrimPrefix(os.Getenv("HTTP_PROXY"), "http://")) { + if err := checkHealth(probe, false, certFile); err == nil || !strings.Contains(err.Error(), strings.TrimPrefix(os.Getenv("HTTP_PROXY"), "http://")) { t.Fatalf("explicit %s probe must use the unreachable proxy, got %v", scheme, err) } }) diff --git a/docs/operations.md b/docs/operations.md index 813cfbdf..55f78a0d 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -176,10 +176,11 @@ original host. `fanout healthcheck` derives `/healthz` from `FANOUT_ADDR` or its built-in default. Empty, `0.0.0.0`, and `::` bind hosts become `127.0.0.1`. `FANOUT_TLS_CERT_FILE` selects HTTPS. Derived probes bypass environment proxies. -IP-literal probes skip certificate verification because the certificate may be -issued for the public hostname; hostname probes verify normally. Healthcheck -reads only these two settings and does not require valid secrets. An explicit -`fanout healthcheck ` uses default proxy and certificate verification behaviour. +HTTPS IP-literal probes trust only the configured certificate and verify its +first DNS or IP SAN, which may differ from the bind IP; hostname probes verify +normally. Healthcheck reads only these two settings and does not require valid +secrets. An explicit `fanout healthcheck ` uses default proxy and +certificate verification behaviour. - `/healthz` is the liveness check. - `/readyz` includes storage readiness and resolved runtime sizing.