diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index f0d9e5f1..5f793e6b 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -82,10 +82,19 @@ The binary is only as fresh as the last UI build.
`just ui-check` fails when the embedded bytes do not match a fresh build, so
CI catches this; it is part of `just check`.
-**Diagrams are generated.** `docs/diagrams/*.svg` is rendered from the `.d2`
-source beside it. Edit the `.d2`, run `just diagrams`, and commit both. The
-committed SVG comes from d2 0.7.1; a different version re-renders every file
-and produces a large diff that is not a real change.
+**Diagrams are generated.** Edit the JSON source in `docs/diagrams/`, run
+`just diagrams`, and commit the JSON and both SVG copies: `docs/diagrams/`
+and `site/public/diagrams/`. Each component cites repository evidence at the
+revision pinned in `meta.repository`. Rendering requires archify v3.1.0 from
+a local install (`ARCHIFY_HOME`, default `$HOME/.agents/skills/archify`), the
+Playwright dependencies in `ui/host/node_modules`, and Chrome for the dual-theme
+SVG export. Archify is never installed through npm. Committed SVGs serve the
+README and docs site, so only diagram editors need the rendering environment.
+The evidence checkout defaults to this repository. If your `origin` points to
+a fork, use `DIAGRAMS_EVIDENCE_ROOT=/path/to/upstream-clone just diagrams`, where
+the clone is labstack/fanout and contains the pinned revision. Archify normalizes
+and verifies repository identity and the pinned revision during `finalize`.
+JSON inputs, SVG outputs, and the site copy still use your contributor checkout.
**The social preview card is generated.** `site/public/social-card.png` is
rendered from `docs/media/social-card.typ` by `just social-card`, which needs Inter on a
@@ -129,7 +138,7 @@ DuckDB maintains its own query schema.
cmd/fanout/ process composition and the single entry point
internal/ ingest, storage, query, agent, MCP, auth, alerts
ui/host/ React browser host and shared MCP app renderer (build-time)
-docs/diagrams/ d2 sources and rendered SVG
+docs/diagrams/ archify JSON sources and exported dual-theme SVGs
```
## Commits and pull requests
diff --git a/README.md b/README.md
index 843e52bf..ee4c2cfd 100644
--- a/README.md
+++ b/README.md
@@ -123,13 +123,13 @@ outgrows one machine's disk and CPU has outgrown Fanout.
One process contains ingest, storage, query, alerts, the agent, MCP, and the React client. Browser
and agent clients share a typed observability contract.
-
+
Parquet holds telemetry, DuckDB query state is rebuildable, and SQLite holds transactional product
state. See [how storage works](https://fanout.run/explanation/storage-model) for durable ingest,
indexed reads, and compaction while reads continue.
-
+
## Configuration
diff --git a/docs/diagrams/architecture.d2 b/docs/diagrams/architecture.d2
deleted file mode 100644
index 0d8c5fbd..00000000
--- a/docs/diagrams/architecture.d2
+++ /dev/null
@@ -1,64 +0,0 @@
-# Fanout runtime architecture: what one process owns, and how a request reaches
-# storage. Render with `just diagrams`; commit the SVG beside this file.
-
-vars: {
- d2-config: {
- layout-engine: dagre
- theme-id: 5
- }
-}
-
-direction: down
-
-clients: Clients {
- style.fill: transparent
- collector: "OTLP collector or SDK" {style.stroke-dash: 3}
- browser: Browser {style.stroke-dash: 3}
- ext: "External MCP host" {style.stroke-dash: 3}
-}
-
-fanout: "fanout — one Go process" {
- style.stroke-width: 2
-
- http: "HTTP :7520\nroutes + auth middleware"
- ingest: "Ingest\nOTLP gRPC + HTTP :7520"
- ui: "Embedded React assets"
- agent: "Agent runtime\nmodel + tool loop"
- mcp: "MCP server\n5 typed + 4 dashboard tools"
- obs: "Typed observability contract"
- commit: "Telemetry commit workers\natomic Parquet batches"
- query: "Query kernel\nDuckDB + rollups"
- maint: "Storage maintenance\nretention + native compaction"
- alert: "Alert engine\nrule evaluation + webhooks"
-
- ingest -> commit
- http -> ui: serves embedded assets
- http -> agent: AG-UI stream
- http -> mcp
- mcp -> agent: in-memory transport
- mcp -> obs
- http -> obs: typed HTTP API
- obs -> query
- alert -> query: evaluates rollups
-}
-
-store: Storage {
- style.fill: transparent
- telemetry: "Parquet batches + trace indexes\nFANOUT_DATA_DIR/telemetry" {shape: cylinder}
- qstate: "DuckDB query state\nFANOUT_DATA_DIR/query" {shape: cylinder}
- control: "Control SQLite\nFANOUT_DATA_DIR/control" {shape: cylinder}
-}
-
-model: "Anthropic or OpenAI" {style.stroke-dash: 3}
-
-clients.collector -> fanout.ingest: "OTLP/gRPC or OTLP/HTTP"
-clients.browser -> fanout.http: HTTPS
-clients.ext -> fanout.http: "/mcp — OAuth"
-
-fanout.commit -> store.telemetry
-fanout.query -> store.telemetry: "indexed traces + SQL scans"
-fanout.query -> store.qstate
-fanout.maint -> store.telemetry: "atomic replacement"
-fanout.http -> store.control: "users, sessions, settings, dashboards, threads"
-fanout.alert -> store.control: "rules and fired alerts"
-fanout.agent -> model: HTTPS
diff --git a/docs/diagrams/architecture.json b/docs/diagrams/architecture.json
new file mode 100644
index 00000000..fd061a64
--- /dev/null
+++ b/docs/diagrams/architecture.json
@@ -0,0 +1,628 @@
+{
+ "schema_version": 1,
+ "diagram_type": "architecture",
+ "meta": {
+ "title": "Fanout runtime architecture",
+ "output": "architecture.html",
+ "quality_profile": "showcase",
+ "locale": "en",
+ "repository": {
+ "url": "https://github.com/labstack/fanout.git",
+ "provider": "github",
+ "link_mode": "web",
+ "revision": "681d46ba8cf0a92ea85764bfa4736a8e2739a8c9"
+ }
+ },
+ "components": [
+ {
+ "id": "collector",
+ "type": "external",
+ "label": "OTLP SDK or Collector",
+ "sublabel": "any OpenTelemetry exporter",
+ "pos": [
+ 40,
+ 40
+ ],
+ "size": [
+ 190,
+ 64
+ ],
+ "sources": [
+ {
+ "path": "internal/ingest/server.go",
+ "line": 54,
+ "end_line": 60
+ }
+ ]
+ },
+ {
+ "id": "browser",
+ "type": "frontend",
+ "label": "Browser",
+ "sublabel": "embedded React UI",
+ "pos": [
+ 330,
+ 40
+ ],
+ "size": [
+ 190,
+ 64
+ ],
+ "sources": [
+ {
+ "path": "internal/ui/ui.go",
+ "line": 18,
+ "end_line": 30
+ },
+ {
+ "path": "cmd/fanout/main.go",
+ "line": 390,
+ "end_line": 393
+ }
+ ]
+ },
+ {
+ "id": "provider",
+ "type": "external",
+ "label": "Anthropic or OpenAI",
+ "sublabel": "selected provider only",
+ "pos": [
+ 620,
+ 40
+ ],
+ "size": [
+ 190,
+ 64
+ ],
+ "sources": [
+ {
+ "path": "internal/agent/provider_http.go",
+ "line": 17,
+ "end_line": 44
+ }
+ ]
+ },
+ {
+ "id": "mcpclient",
+ "type": "external",
+ "label": "External MCP client",
+ "sublabel": "remote agent or IDE",
+ "pos": [
+ 910,
+ 40
+ ],
+ "size": [
+ 190,
+ 64
+ ],
+ "sources": [
+ {
+ "path": "cmd/fanout/main.go",
+ "line": 357,
+ "end_line": 369
+ }
+ ]
+ },
+ {
+ "id": "ingest",
+ "type": "backend",
+ "label": "OTLP ingest",
+ "sublabel": "gRPC + HTTP, token checked",
+ "pos": [
+ 40,
+ 180
+ ],
+ "size": [
+ 190,
+ 64
+ ],
+ "sources": [
+ {
+ "path": "internal/server/http.go",
+ "line": 13,
+ "end_line": 32
+ },
+ {
+ "path": "internal/ingest/auth.go",
+ "line": 51,
+ "end_line": 89
+ },
+ {
+ "path": "cmd/fanout/main.go",
+ "line": 207,
+ "end_line": 209
+ }
+ ]
+ },
+ {
+ "id": "api",
+ "type": "security",
+ "label": "HTTP API + auth",
+ "sublabel": "sessions, OIDC, email codes",
+ "pos": [
+ 330,
+ 180
+ ],
+ "size": [
+ 190,
+ 64
+ ],
+ "sources": [
+ {
+ "path": "cmd/fanout/main.go",
+ "line": 186,
+ "end_line": 190
+ },
+ {
+ "path": "cmd/fanout/main.go",
+ "line": 299,
+ "end_line": 299
+ },
+ {
+ "path": "internal/api/oidc.go",
+ "line": 42,
+ "end_line": 49
+ }
+ ]
+ },
+ {
+ "id": "agent",
+ "type": "backend",
+ "label": "Agent runtime",
+ "sublabel": "AG-UI stream, chat threads",
+ "pos": [
+ 620,
+ 180
+ ],
+ "size": [
+ 190,
+ 64
+ ],
+ "sources": [
+ {
+ "path": "cmd/fanout/main.go",
+ "line": 372,
+ "end_line": 385
+ },
+ {
+ "path": "internal/agent/runtime.go",
+ "line": 60,
+ "end_line": 64
+ }
+ ]
+ },
+ {
+ "id": "mcp",
+ "type": "backend",
+ "label": "MCP server",
+ "sublabel": "20 typed tools",
+ "pos": [
+ 910,
+ 180
+ ],
+ "size": [
+ 190,
+ 64
+ ],
+ "sources": [
+ {
+ "path": "cmd/fanout/main.go",
+ "line": 356,
+ "end_line": 371
+ },
+ {
+ "path": "internal/mcp/server.go",
+ "line": 105,
+ "end_line": 105
+ }
+ ]
+ },
+ {
+ "id": "writer",
+ "type": "backend",
+ "label": "Commit workers",
+ "sublabel": "up to 4, 10 ms group window",
+ "pos": [
+ 40,
+ 320
+ ],
+ "size": [
+ 190,
+ 64
+ ],
+ "sources": [
+ {
+ "path": "internal/telemetry/store/writer.go",
+ "line": 20,
+ "end_line": 28
+ },
+ {
+ "path": "internal/telemetry/store/writer.go",
+ "line": 143,
+ "end_line": 150
+ }
+ ]
+ },
+ {
+ "id": "obs",
+ "type": "backend",
+ "label": "Queries + dashboards",
+ "sublabel": "one contract for API and MCP",
+ "pos": [
+ 330,
+ 320
+ ],
+ "size": [
+ 190,
+ 64
+ ],
+ "sources": [
+ {
+ "path": "cmd/fanout/main.go",
+ "line": 277,
+ "end_line": 284
+ }
+ ]
+ },
+ {
+ "id": "query",
+ "type": "backend",
+ "label": "Query kernel",
+ "sublabel": "DuckDB, rollups, compaction",
+ "pos": [
+ 330,
+ 460
+ ],
+ "size": [
+ 190,
+ 64
+ ],
+ "sources": [
+ {
+ "path": "internal/query/duck.go",
+ "line": 240,
+ "end_line": 244
+ },
+ {
+ "path": "internal/query/duck.go",
+ "line": 505,
+ "end_line": 522
+ },
+ {
+ "path": "internal/query/duck.go",
+ "line": 627,
+ "end_line": 653
+ }
+ ]
+ },
+ {
+ "id": "detector",
+ "type": "backend",
+ "label": "Anomaly detector",
+ "sublabel": "health and z-scores",
+ "pos": [
+ 620,
+ 460
+ ],
+ "size": [
+ 190,
+ 64
+ ],
+ "sources": [
+ {
+ "path": "internal/intelligence/detector.go",
+ "line": 46,
+ "end_line": 55
+ },
+ {
+ "path": "internal/intelligence/detector.go",
+ "line": 240,
+ "end_line": 291
+ },
+ {
+ "path": "cmd/fanout/main.go",
+ "line": 161,
+ "end_line": 162
+ }
+ ]
+ },
+ {
+ "id": "alert",
+ "type": "backend",
+ "label": "Alert engine",
+ "sublabel": "CEL rules, every 30s",
+ "pos": [
+ 910,
+ 460
+ ],
+ "size": [
+ 190,
+ 64
+ ],
+ "sources": [
+ {
+ "path": "cmd/fanout/main.go",
+ "line": 173,
+ "end_line": 183
+ },
+ {
+ "path": "internal/alert/engine.go",
+ "line": 60,
+ "end_line": 60
+ }
+ ]
+ },
+ {
+ "id": "webhooks",
+ "type": "external",
+ "label": "Webhook targets",
+ "sublabel": "per-rule HTTP endpoint",
+ "pos": [
+ 1200,
+ 460
+ ],
+ "size": [
+ 190,
+ 64
+ ],
+ "sources": [
+ {
+ "path": "internal/alert/webhook.go",
+ "line": 50,
+ "end_line": 50
+ },
+ {
+ "path": "internal/alert/engine.go",
+ "line": 360,
+ "end_line": 360
+ }
+ ]
+ },
+ {
+ "id": "parquet",
+ "type": "database",
+ "label": "Parquet batches",
+ "sublabel": "authoritative, trace.fidx",
+ "pos": [
+ 40,
+ 600
+ ],
+ "size": [
+ 190,
+ 64
+ ],
+ "sources": [
+ {
+ "path": "internal/telemetry/parquet.go",
+ "line": 282,
+ "end_line": 282
+ },
+ {
+ "path": "internal/config/config.go",
+ "line": 219,
+ "end_line": 219
+ }
+ ]
+ },
+ {
+ "id": "qstate",
+ "type": "database",
+ "label": "DuckDB query state",
+ "sublabel": "rebuildable cache",
+ "pos": [
+ 330,
+ 600
+ ],
+ "size": [
+ 190,
+ 64
+ ],
+ "sources": [
+ {
+ "path": "internal/config/config.go",
+ "line": 227,
+ "end_line": 232
+ }
+ ]
+ },
+ {
+ "id": "sqlite",
+ "type": "database",
+ "label": "Control SQLite",
+ "sublabel": "users, dashboards, rules, threads",
+ "pos": [
+ 910,
+ 600
+ ],
+ "size": [
+ 190,
+ 64
+ ],
+ "sources": [
+ {
+ "path": "internal/store/store.go",
+ "line": 30,
+ "end_line": 30
+ },
+ {
+ "path": "cmd/fanout/main.go",
+ "line": 165,
+ "end_line": 170
+ }
+ ]
+ }
+ ],
+ "boundaries": [
+ {
+ "kind": "region",
+ "label": "fanout: one Go process, one listener :7520",
+ "wraps": [
+ "ingest",
+ "api",
+ "agent",
+ "mcp",
+ "writer",
+ "obs",
+ "query",
+ "detector",
+ "alert"
+ ]
+ },
+ {
+ "kind": "region",
+ "label": "FANOUT_DATA_DIR",
+ "wraps": [
+ "parquet",
+ "qstate",
+ "sqlite"
+ ]
+ }
+ ],
+ "connections": [
+ {
+ "id": "otlp-in",
+ "from": "collector",
+ "to": "ingest",
+ "label": "OTLP",
+ "variant": "emphasis"
+ },
+ {
+ "id": "browser-in",
+ "from": "browser",
+ "to": "api",
+ "label": "HTTPS",
+ "variant": "emphasis"
+ },
+ {
+ "id": "mcp-in",
+ "from": "mcpclient",
+ "to": "mcp",
+ "label": "/mcp OAuth",
+ "variant": "security"
+ },
+ {
+ "id": "chat-stream",
+ "from": "api",
+ "to": "agent",
+ "label": "AG-UI SSE"
+ },
+ {
+ "id": "model-call",
+ "from": "agent",
+ "to": "provider",
+ "label": "HTTPS stream"
+ },
+ {
+ "id": "tool-calls",
+ "from": "agent",
+ "to": "mcp",
+ "label": "tool calls"
+ },
+ {
+ "id": "ingest-commit",
+ "from": "ingest",
+ "to": "writer",
+ "label": "batch"
+ },
+ {
+ "id": "publish",
+ "from": "writer",
+ "to": "parquet",
+ "label": "atomic publish",
+ "variant": "emphasis"
+ },
+ {
+ "id": "api-queries",
+ "from": "api",
+ "to": "obs",
+ "label": "typed API"
+ },
+ {
+ "id": "mcp-queries",
+ "from": "mcp",
+ "to": "obs",
+ "label": "typed tools",
+ "fromSide": "bottom",
+ "toSide": "right"
+ },
+ {
+ "id": "obs-query",
+ "from": "obs",
+ "to": "query",
+ "label": "SQL"
+ },
+ {
+ "id": "query-cache",
+ "from": "query",
+ "to": "qstate",
+ "label": "rollups"
+ },
+ {
+ "id": "query-parquet",
+ "from": "query",
+ "to": "parquet",
+ "label": "read, compact",
+ "fromSide": "left",
+ "toSide": "right"
+ },
+ {
+ "id": "detector-reads",
+ "from": "detector",
+ "to": "query",
+ "label": "SQL scans"
+ },
+ {
+ "id": "alert-scores",
+ "from": "alert",
+ "to": "detector",
+ "label": "scores"
+ },
+ {
+ "id": "alert-webhook",
+ "from": "alert",
+ "to": "webhooks",
+ "label": "POST"
+ },
+ {
+ "id": "alert-rules",
+ "from": "alert",
+ "to": "sqlite",
+ "label": "rules, history"
+ },
+ {
+ "id": "alert-rollups",
+ "from": "alert",
+ "to": "query",
+ "label": "rollups",
+ "fromSide": "top",
+ "toSide": "top"
+ }
+ ],
+ "cards": [
+ {
+ "dot": "cyan",
+ "title": "One listener",
+ "items": [
+ "OTLP /v1/* and gRPC go to ingest",
+ "Everything else is the Echo app: API, UI, /mcp, /api/agent"
+ ]
+ },
+ {
+ "dot": "violet",
+ "title": "Control SQLite",
+ "items": [
+ "Users, sessions, settings, dashboards, OAuth clients, chat threads, alert rules",
+ "Ingest reads the token hash from it on every export"
+ ]
+ },
+ {
+ "dot": "amber",
+ "title": "Optional",
+ "items": [
+ "Chat only when FANOUT_AI_PROVIDER is set",
+ "OIDC sign-in and SMTP email codes when configured",
+ "/metrics for Prometheus"
+ ]
+ }
+ ]
+}
diff --git a/docs/diagrams/architecture.svg b/docs/diagrams/architecture.svg
index 18433e2a..97858a29 100644
--- a/docs/diagrams/architecture.svg
+++ b/docs/diagrams/architecture.svg
@@ -1,121 +1,1161 @@
-
+
+
\ No newline at end of file
diff --git a/docs/diagrams/persistence.d2 b/docs/diagrams/persistence.d2
deleted file mode 100644
index b0d74d6f..00000000
--- a/docs/diagrams/persistence.d2
+++ /dev/null
@@ -1,63 +0,0 @@
-# How telemetry becomes durable, queryable, and compacted.
-# Render with `just diagrams`; commit the generated SVG beside this file.
-
-direction: down
-
-vars: {
- d2-config: {
- layout-engine: elk
- theme-id: 5
- }
-}
-
-ingest: "Authenticated OTLP requests\nspans, logs, metrics" {
- shape: rectangle
- style.stroke-dash: 3
-}
-
-writers: "Durable write path" {
- style.fill: transparent
- admission: "Bounded group admission\nconfigured row target + 20 ms window" {shape: rectangle}
- workers: "Up to four commit workers\nindependent batches encode in parallel" {shape: rectangle}
- stage: "Staging batch directory\nmetadata + signal Parquet + trace.fidx for spans" {shape: rectangle}
- publish: "Durable publication\nfsync → atomic rename → fsync parent" {shape: rectangle}
-
- admission -> workers
- workers -> stage
- stage -> publish
-}
-
-authority: "Authoritative telemetry\nimmutable *.batch directories" {
- shape: cylinder
- tooltip: Ordinary Parquet files are authoritative; published batch directories are discovered directly from the filesystem.
-}
-
-readers: Readers {
- style.fill: transparent
- trace: "Indexed trace read\nbinary-search .fidx → Parquet rows" {shape: rectangle}
- sql: "DuckDB SQL\nfilters, ordering, broad scans" {shape: rectangle}
- rollups: "DuckDB rollups\nrebuildable analytical cache" {shape: rectangle}
-}
-
-maintenance: "Bounded maintenance" {
- style.fill: transparent
- choose: "Retention + compaction admission\nbounded batch and row groups" {shape: rectangle}
- merge: "Native Parquet merge\nprepared outside the reader gate" {shape: rectangle}
- swap: "Crash-safe namespace swap\nmarker + bounded renames + fsync" {shape: rectangle}
-
- choose -> merge
- merge -> swap
-}
-
-querystate: "Rebuildable query state\nFANOUT_DATA_DIR/query" {
- shape: cylinder
- tooltip: DuckDB rollups and temporary spill. Not authoritative telemetry.
-}
-
-ingest -> writers.admission
-writers.publish -> authority: "acknowledge only after publication"
-authority -> readers.trace
-authority -> readers.sql
-readers.sql -> readers.rollups
-readers.rollups -> querystate
-authority -> maintenance.choose
diff --git a/docs/diagrams/persistence.json b/docs/diagrams/persistence.json
new file mode 100644
index 00000000..b812b1cb
--- /dev/null
+++ b/docs/diagrams/persistence.json
@@ -0,0 +1,290 @@
+{
+ "schema_version": 1,
+ "diagram_type": "dataflow",
+ "meta": {
+ "title": "How telemetry becomes durable and queryable",
+ "output": "persistence.html",
+ "quality_profile": "showcase",
+ "locale": "en",
+ "repository": {
+ "url": "https://github.com/labstack/fanout.git",
+ "provider": "github",
+ "link_mode": "web",
+ "revision": "681d46ba8cf0a92ea85764bfa4736a8e2739a8c9"
+ }
+ },
+ "stages": [
+ {
+ "label": "Accept"
+ },
+ {
+ "label": "Commit"
+ },
+ {
+ "label": "Authority"
+ },
+ {
+ "label": "Read and maintain"
+ },
+ {
+ "label": "Derived"
+ }
+ ],
+ "nodes": [
+ {
+ "id": "otlp",
+ "type": "backend",
+ "label": "OTLP request",
+ "sublabel": "authenticated, decoded",
+ "stage": 0,
+ "row": 1,
+ "tag": "spans, logs, metrics",
+ "sources": [
+ {
+ "path": "internal/ingest/server.go",
+ "line": 54,
+ "end_line": 60
+ }
+ ]
+ },
+ {
+ "id": "admission",
+ "type": "backend",
+ "label": "Group admission",
+ "sublabel": "row target, 10 ms window",
+ "stage": 1,
+ "row": 0,
+ "sources": [
+ {
+ "path": "internal/telemetry/store/writer.go",
+ "line": 20,
+ "end_line": 28
+ }
+ ]
+ },
+ {
+ "id": "workers",
+ "type": "backend",
+ "label": "Commit workers",
+ "sublabel": "up to 4 in parallel",
+ "stage": 1,
+ "row": 1,
+ "sources": [
+ {
+ "path": "internal/telemetry/store/writer.go",
+ "line": 143,
+ "end_line": 150
+ }
+ ]
+ },
+ {
+ "id": "publish",
+ "type": "backend",
+ "label": "Durable publish",
+ "sublabel": "fsync, rename, fsync dir",
+ "stage": 1,
+ "row": 2,
+ "sources": [
+ {
+ "path": "internal/telemetry/parquet.go",
+ "line": 194,
+ "end_line": 217
+ },
+ {
+ "path": "internal/telemetry/parquet.go",
+ "line": 282,
+ "end_line": 282
+ },
+ {
+ "path": "internal/telemetry/parquet.go",
+ "line": 308,
+ "end_line": 345
+ }
+ ]
+ },
+ {
+ "id": "batches",
+ "type": "database",
+ "label": "Parquet batches",
+ "sublabel": "immutable *.batch dirs",
+ "stage": 2,
+ "row": 1,
+ "tag": "authoritative",
+ "sources": [
+ {
+ "path": "internal/telemetry/parquet.go",
+ "line": 26,
+ "end_line": 26
+ },
+ {
+ "path": "internal/config/config.go",
+ "line": 219,
+ "end_line": 219
+ }
+ ]
+ },
+ {
+ "id": "trace",
+ "type": "backend",
+ "label": "Indexed trace read",
+ "sublabel": "trace.fidx lookup",
+ "stage": 3,
+ "row": 0,
+ "sources": [
+ {
+ "path": "internal/telemetry/trace_index.go",
+ "line": 203,
+ "end_line": 235
+ }
+ ]
+ },
+ {
+ "id": "sql",
+ "type": "backend",
+ "label": "DuckDB SQL",
+ "sublabel": "filters and scans",
+ "stage": 3,
+ "row": 1,
+ "sources": [
+ {
+ "path": "internal/query/duck.go",
+ "line": 240,
+ "end_line": 244
+ }
+ ]
+ },
+ {
+ "id": "maintain",
+ "type": "backend",
+ "label": "Maintenance",
+ "sublabel": "retention, compaction",
+ "stage": 3,
+ "row": 2,
+ "sources": [
+ {
+ "path": "internal/query/duck.go",
+ "line": 627,
+ "end_line": 653
+ },
+ {
+ "path": "internal/query/duck.go",
+ "line": 692,
+ "end_line": 693
+ },
+ {
+ "path": "internal/telemetry/store/compaction.go",
+ "line": 132,
+ "end_line": 177
+ }
+ ]
+ },
+ {
+ "id": "rollups",
+ "type": "backend",
+ "label": "Rollups",
+ "sublabel": "service and edge",
+ "stage": 4,
+ "row": 1,
+ "sources": [
+ {
+ "path": "internal/query/duck.go",
+ "line": 505,
+ "end_line": 522
+ },
+ {
+ "path": "internal/query/duck.go",
+ "line": 1437,
+ "end_line": 1437
+ },
+ {
+ "path": "internal/query/duck.go",
+ "line": 1622,
+ "end_line": 1622
+ }
+ ]
+ },
+ {
+ "id": "qstate",
+ "type": "database",
+ "label": "Query state",
+ "sublabel": "DuckDB, rebuildable",
+ "stage": 4,
+ "row": 2,
+ "sources": [
+ {
+ "path": "internal/config/config.go",
+ "line": 227,
+ "end_line": 232
+ }
+ ]
+ }
+ ],
+ "flows": [
+ {
+ "id": "admit",
+ "from": "otlp",
+ "to": "admission",
+ "label": "decoded rows",
+ "variant": "emphasis"
+ },
+ {
+ "id": "group",
+ "from": "admission",
+ "to": "workers",
+ "label": "grouped batch",
+ "variant": "emphasis"
+ },
+ {
+ "id": "encode",
+ "from": "workers",
+ "to": "publish",
+ "label": "Parquet + index",
+ "variant": "emphasis"
+ },
+ {
+ "id": "durable",
+ "from": "publish",
+ "to": "batches",
+ "label": "ack after publish",
+ "variant": "emphasis"
+ },
+ {
+ "id": "trace-read",
+ "from": "batches",
+ "to": "trace",
+ "label": "row ranges"
+ },
+ {
+ "id": "sql-read",
+ "from": "batches",
+ "to": "sql",
+ "label": "Parquet scan"
+ },
+ {
+ "id": "select",
+ "from": "batches",
+ "to": "maintain",
+ "label": "bounded groups",
+ "variant": "dashed"
+ },
+ {
+ "id": "swap",
+ "from": "maintain",
+ "to": "batches",
+ "label": "atomic swap",
+ "variant": "dashed"
+ },
+ {
+ "id": "aggregate",
+ "from": "sql",
+ "to": "rollups",
+ "label": "aggregate"
+ },
+ {
+ "id": "cache",
+ "from": "rollups",
+ "to": "qstate",
+ "label": "cache rows"
+ }
+ ]
+}
diff --git a/docs/diagrams/persistence.svg b/docs/diagrams/persistence.svg
index 13d27eb5..26a3d6e4 100644
--- a/docs/diagrams/persistence.svg
+++ b/docs/diagrams/persistence.svg
@@ -1,138 +1,809 @@
-
+
+
\ No newline at end of file
diff --git a/justfile b/justfile
index 1fb55000..a6de9024 100644
--- a/justfile
+++ b/justfile
@@ -180,23 +180,18 @@ ui-check:
# ── Diagrams ─────────────────────────────────────────────────────────────────
-# Renders docs/diagrams/*.d2 beside each source. Deliberately outside `build`
-# and `check`: the SVG is committed, so only someone editing a diagram needs d2
-# installed at all.
-#
-# D2 output changes between minor versions, so the recipe pins the generator
-# used for committed SVGs instead of depending on a contributor's installed
-# binary.
-
-# Render every d2 diagram to SVG and publish the same assets on the docs site.
+# Archify renders docs/diagrams/*.json and exports one dual-theme SVG per source.
+# Use a local archify 3.1.0 install via ARCHIFY_HOME (default
+# $HOME/.agents/skills/archify) and Chrome; no npm install is needed for archify.
+# The generator version is pinned so committed SVGs remain reproducible.
+# DIAGRAMS_EVIDENCE_ROOT selects an upstream clone containing the pinned revision
+# when origin is a fork (default: this checkout); inputs and SVGs stay here.
+# Deliberately outside `build` and `check`: SVGs are committed, so only diagram
+# editors need archify and the Chrome export environment.
+
+# Export every archify diagram and publish the same SVGs on the docs site.
diagrams:
- #!/usr/bin/env bash
- set -euo pipefail
- for src in docs/diagrams/*.d2; do
- go run oss.terrastruct.com/d2@v0.7.1 "$src" "${src%.d2}.svg"
- done
- mkdir -p site/public/diagrams
- cp docs/diagrams/*.svg site/public/diagrams/
+ node scripts/diagrams.mjs
# Fail when the site serves a diagram other than the committed documentation
# copy. Regeneration stays a deliberate writing step.
@@ -261,7 +256,7 @@ site-build: docs-generate-check diagrams-check site-deps
# missing face does not fail the render; it quietly produces a card set in
# whatever typst found instead. This reads the warning back and fails on it.
#
-# The version is pinned for the same reason d2 is: typst re-lays out text
+# The version is pinned for reproducible output: typst re-lays out text
# between releases, so a different one rewrites every glyph position and
# produces a diff that is not a real change.
#
diff --git a/scripts/diagrams.mjs b/scripts/diagrams.mjs
new file mode 100644
index 00000000..d04e4dc1
--- /dev/null
+++ b/scripts/diagrams.mjs
@@ -0,0 +1,106 @@
+import { spawnSync } from "node:child_process";
+import { access, copyFile, mkdir, mkdtemp, readFile, readdir, rm } from "node:fs/promises";
+import { createRequire } from "node:module";
+import { homedir, tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+
+const ARCHIFY_VERSION = "3.1.0";
+const repo = resolve(dirname(fileURLToPath(import.meta.url)), "..");
+const evidenceRoot = resolve(process.env.DIAGRAMS_EVIDENCE_ROOT || repo);
+const diagrams = join(repo, "docs/diagrams");
+const archify = resolve(process.env.ARCHIFY_HOME || join(homedir(), ".agents/skills/archify"));
+
+async function main() {
+ const sources = [];
+ for (const source of (await readdir(diagrams)).filter(name => name.endsWith(".json")).sort()) {
+ const input = join(diagrams, source);
+ const { diagram_type: type } = JSON.parse(await readFile(input, "utf8"));
+ if (typeof type !== "string" || !type) {
+ throw new Error(`${input} must declare diagram_type.`);
+ }
+ sources.push({ source, input, type });
+ }
+
+ const cli = join(archify, "bin/archify.mjs");
+ try {
+ await access(cli);
+ } catch {
+ throw new Error(`Archify is missing ${cli}; set ARCHIFY_HOME to a local archify ${ARCHIFY_VERSION} install.`);
+ }
+ let release;
+ try {
+ release = JSON.parse(await readFile(join(archify, "skill-release.json"), "utf8"));
+ } catch {
+ throw new Error(`Cannot read archify's skill-release.json in ${archify}; archify ${ARCHIFY_VERSION} is required.`);
+ }
+ if (release?.version !== ARCHIFY_VERSION) {
+ throw new Error(`Archify ${ARCHIFY_VERSION} is required; ${archify} reports version ${JSON.stringify(release?.version)}.`);
+ }
+
+ const require = createRequire(join(repo, "ui/host/package.json"));
+ const { chromium } = require("@playwright/test");
+ let browser;
+ try {
+ for (const { source, input, type } of sources) {
+ const name = source.slice(0, -".json".length);
+ const temporary = await mkdtemp(join(tmpdir(), "fanout-diagram-"));
+ try {
+ const html = join(temporary, `${name}.html`);
+ const result = spawnSync(process.execPath, [cli, "finalize", type, input, html,
+ "--repo-root", evidenceRoot, "--quality", "showcase", "--json"], {
+ cwd: repo,
+ encoding: "utf8",
+ maxBuffer: 16 * 1024 * 1024,
+ });
+ if (result.error || result.status !== 0) {
+ // The JSON receipt contains the failed gate's diagnostics.
+ if (result.stdout) console.error(result.stdout.trim());
+ if (result.stderr) console.error(result.stderr.trim());
+ let receipt;
+ try {
+ receipt = JSON.parse(result.stdout);
+ } catch {
+ // Preserve the original failure when no JSON receipt is available.
+ }
+ if (Array.isArray(receipt?.diagnostics) && receipt.diagnostics.some(
+ diagnostic => typeof diagnostic?.code === "string" && diagnostic.code.startsWith("repository-evidence/"),
+ )) {
+ console.error("Set DIAGRAMS_EVIDENCE_ROOT to a clone of labstack/fanout that contains the pinned revision.");
+ }
+ throw new Error(`Archify finalize failed for ${source}: ${result.error?.message ?? result.signal ?? `exit ${result.status}`}`);
+ }
+ browser ??= await chromium.launch({ channel: "chrome", headless: true });
+ const context = await browser.newContext({ acceptDownloads: true });
+ try {
+ const page = await context.newPage();
+ await page.goto(pathToFileURL(html).href);
+ await page.getByRole("button", { name: "Export diagram", exact: true }).click();
+ const [download] = await Promise.all([
+ page.waitForEvent("download"),
+ page.getByRole("menuitem", { name: /^SVG · Auto/ }).click(),
+ ]);
+ await download.saveAs(join(diagrams, `${name}.svg`));
+ } finally {
+ await context.close();
+ }
+ } finally {
+ await rm(temporary, { recursive: true, force: true });
+ }
+ }
+ const destination = join(repo, "site/public/diagrams");
+ await mkdir(destination, { recursive: true });
+ for (const svg of (await readdir(diagrams)).filter(name => name.endsWith(".svg")).sort()) {
+ await copyFile(join(diagrams, svg), join(destination, svg));
+ }
+ } finally {
+ await browser?.close();
+ }
+}
+
+try {
+ await main();
+} catch (error) {
+ console.error(error instanceof Error ? error.message : error);
+ process.exitCode = 1;
+}
diff --git a/site/public/diagrams/architecture.svg b/site/public/diagrams/architecture.svg
index 18433e2a..97858a29 100644
--- a/site/public/diagrams/architecture.svg
+++ b/site/public/diagrams/architecture.svg
@@ -1,121 +1,1161 @@
-
+
+
\ No newline at end of file
diff --git a/site/public/diagrams/persistence.svg b/site/public/diagrams/persistence.svg
index 13d27eb5..26a3d6e4 100644
--- a/site/public/diagrams/persistence.svg
+++ b/site/public/diagrams/persistence.svg
@@ -1,138 +1,809 @@
-
+
+
\ No newline at end of file
diff --git a/site/src/content/docs/explanation/storage-model.mdx b/site/src/content/docs/explanation/storage-model.mdx
index bf6a8a83..e01155de 100644
--- a/site/src/content/docs/explanation/storage-model.mdx
+++ b/site/src/content/docs/explanation/storage-model.mdx
@@ -25,7 +25,7 @@ history—lives in SQLite.
diff --git a/site/src/content/docs/start/what-fanout-is.mdx b/site/src/content/docs/start/what-fanout-is.mdx
index 0468d837..b5112c60 100644
--- a/site/src/content/docs/start/what-fanout-is.mdx
+++ b/site/src/content/docs/start/what-fanout-is.mdx
@@ -17,7 +17,7 @@ investigator and MCP tools, and hosts the browser client.
