From 18ee7b653fc181a2b7e3f19b4463d4e1e0eaff6a Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 14 Oct 2026 10:37:17 -0700 Subject: [PATCH 1/3] docs(diagrams): author the architecture and persistence diagrams with archify The d2 diagrams had drifted from the code: MCP listed 9 tools instead of 20, the group admission window was 20 ms instead of 10, and the anomaly detector, alert inputs, and webhook targets were missing. Each diagram now has one archify JSON source in docs/diagrams/ whose components cite repository evidence at a pinned revision. `just diagrams` renders it with a local archify 3.1.0 install (ARCHIFY_HOME), exports one dual-theme SVG that follows the reader's color scheme, and copies it to the docs site. The d2 sources and the pinned d2 generator are removed. --- CONTRIBUTING.md | 14 +- README.md | 4 +- docs/diagrams/architecture.d2 | 64 - docs/diagrams/architecture.json | 623 ++++++++ docs/diagrams/architecture.svg | 1282 +++++++++++++++-- docs/diagrams/persistence.d2 | 63 - docs/diagrams/persistence.json | 290 ++++ docs/diagrams/persistence.svg | 947 ++++++++++-- justfile | 27 +- scripts/diagrams.mjs | 90 ++ site/public/diagrams/architecture.svg | 1282 +++++++++++++++-- site/public/diagrams/persistence.svg | 947 ++++++++++-- .../docs/explanation/storage-model.mdx | 2 +- .../src/content/docs/start/what-fanout-is.mdx | 2 +- 14 files changed, 4966 insertions(+), 671 deletions(-) delete mode 100644 docs/diagrams/architecture.d2 create mode 100644 docs/diagrams/architecture.json delete mode 100644 docs/diagrams/persistence.d2 create mode 100644 docs/diagrams/persistence.json create mode 100644 scripts/diagrams.mjs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f0d9e5f1..5ebd06c4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -82,10 +82,14 @@ The binary is only as fresh as the last UI build. `just ui-check` fails when the embedded bytes do not match a fresh build, so CI catches this; it is part of `just check`. -**Diagrams are generated.** `docs/diagrams/*.svg` is rendered from the `.d2` -source beside it. Edit the `.d2`, run `just diagrams`, and commit both. The -committed SVG comes from d2 0.7.1; a different version re-renders every file -and produces a large diff that is not a real change. +**Diagrams are generated.** Edit the JSON source in `docs/diagrams/`, run +`just diagrams`, and commit the JSON and both SVG copies: `docs/diagrams/` +and `site/public/diagrams/`. Each component cites repository evidence at the +revision pinned in `meta.repository`. Rendering requires archify v3.1.0 from +a local install (`ARCHIFY_HOME`, default `$HOME/.agents/skills/archify`), the +Playwright dependencies in `ui/host/node_modules`, and Chrome for the dual-theme +SVG export. Archify is never installed through npm. Committed SVGs serve the +README and docs site, so only diagram editors need the rendering environment. **The social preview card is generated.** `site/public/social-card.png` is rendered from `docs/media/social-card.typ` by `just social-card`, which needs Inter on a @@ -129,7 +133,7 @@ DuckDB maintains its own query schema. cmd/fanout/ process composition and the single entry point internal/ ingest, storage, query, agent, MCP, auth, alerts ui/host/ React browser host and shared MCP app renderer (build-time) -docs/diagrams/ d2 sources and rendered SVG +docs/diagrams/ archify JSON sources and exported dual-theme SVGs ``` ## Commits and pull requests diff --git a/README.md b/README.md index 843e52bf..ee4c2cfd 100644 --- a/README.md +++ b/README.md @@ -123,13 +123,13 @@ outgrows one machine's disk and CPU has outgrown Fanout. One process contains ingest, storage, query, alerts, the agent, MCP, and the React client. Browser and agent clients share a typed observability contract. -![Fanout architecture](docs/diagrams/architecture.svg) +![Fanout runtime architecture: OTLP ingest and commit workers publish Parquet; the browser API, agent runtime, and MCP server share queries and dashboards, with DuckDB query state, SQLite control state, anomaly detection, and webhook alerts.](docs/diagrams/architecture.svg) Parquet holds telemetry, DuckDB query state is rebuildable, and SQLite holds transactional product state. See [how storage works](https://fanout.run/explanation/storage-model) for durable ingest, indexed reads, and compaction while reads continue. -![Fanout persistence](docs/diagrams/persistence.svg) +![Telemetry durability and query flow: authenticated OTLP rows pass through group admission, parallel commit workers, and durable publication into immutable Parquet batches; indexed trace reads, DuckDB SQL, retention, and compaction use those batches, while service and edge rollups populate rebuildable query state.](docs/diagrams/persistence.svg) ## Configuration diff --git a/docs/diagrams/architecture.d2 b/docs/diagrams/architecture.d2 deleted file mode 100644 index 0d8c5fbd..00000000 --- a/docs/diagrams/architecture.d2 +++ /dev/null @@ -1,64 +0,0 @@ -# Fanout runtime architecture: what one process owns, and how a request reaches -# storage. Render with `just diagrams`; commit the SVG beside this file. - -vars: { - d2-config: { - layout-engine: dagre - theme-id: 5 - } -} - -direction: down - -clients: Clients { - style.fill: transparent - collector: "OTLP collector or SDK" {style.stroke-dash: 3} - browser: Browser {style.stroke-dash: 3} - ext: "External MCP host" {style.stroke-dash: 3} -} - -fanout: "fanout — one Go process" { - style.stroke-width: 2 - - http: "HTTP :7520\nroutes + auth middleware" - ingest: "Ingest\nOTLP gRPC + HTTP :7520" - ui: "Embedded React assets" - agent: "Agent runtime\nmodel + tool loop" - mcp: "MCP server\n5 typed + 4 dashboard tools" - obs: "Typed observability contract" - commit: "Telemetry commit workers\natomic Parquet batches" - query: "Query kernel\nDuckDB + rollups" - maint: "Storage maintenance\nretention + native compaction" - alert: "Alert engine\nrule evaluation + webhooks" - - ingest -> commit - http -> ui: serves embedded assets - http -> agent: AG-UI stream - http -> mcp - mcp -> agent: in-memory transport - mcp -> obs - http -> obs: typed HTTP API - obs -> query - alert -> query: evaluates rollups -} - -store: Storage { - style.fill: transparent - telemetry: "Parquet batches + trace indexes\nFANOUT_DATA_DIR/telemetry" {shape: cylinder} - qstate: "DuckDB query state\nFANOUT_DATA_DIR/query" {shape: cylinder} - control: "Control SQLite\nFANOUT_DATA_DIR/control" {shape: cylinder} -} - -model: "Anthropic or OpenAI" {style.stroke-dash: 3} - -clients.collector -> fanout.ingest: "OTLP/gRPC or OTLP/HTTP" -clients.browser -> fanout.http: HTTPS -clients.ext -> fanout.http: "/mcp — OAuth" - -fanout.commit -> store.telemetry -fanout.query -> store.telemetry: "indexed traces + SQL scans" -fanout.query -> store.qstate -fanout.maint -> store.telemetry: "atomic replacement" -fanout.http -> store.control: "users, sessions, settings, dashboards, threads" -fanout.alert -> store.control: "rules and fired alerts" -fanout.agent -> model: HTTPS diff --git a/docs/diagrams/architecture.json b/docs/diagrams/architecture.json new file mode 100644 index 00000000..fc223c02 --- /dev/null +++ b/docs/diagrams/architecture.json @@ -0,0 +1,623 @@ +{ + "schema_version": 1, + "diagram_type": "architecture", + "meta": { + "title": "Fanout runtime architecture", + "output": "architecture.html", + "quality_profile": "showcase", + "locale": "en", + "repository": { + "url": "https://github.com/labstack/fanout.git", + "provider": "github", + "link_mode": "web", + "revision": "681d46ba8cf0a92ea85764bfa4736a8e2739a8c9" + } + }, + "components": [ + { + "id": "collector", + "type": "external", + "label": "OTLP SDK or Collector", + "sublabel": "any OpenTelemetry exporter", + "pos": [ + 40, + 40 + ], + "size": [ + 190, + 64 + ], + "sources": [ + { + "path": "internal/ingest/server.go", + "line": 54, + "end_line": 60 + } + ] + }, + { + "id": "browser", + "type": "frontend", + "label": "Browser", + "sublabel": "embedded React UI", + "pos": [ + 330, + 40 + ], + "size": [ + 190, + 64 + ], + "sources": [ + { + "path": "internal/ui/ui.go", + "line": 18, + "end_line": 30 + }, + { + "path": "cmd/fanout/main.go", + "line": 390, + "end_line": 393 + } + ] + }, + { + "id": "provider", + "type": "external", + "label": "Anthropic or OpenAI", + "sublabel": "selected provider only", + "pos": [ + 620, + 40 + ], + "size": [ + 190, + 64 + ], + "sources": [ + { + "path": "internal/agent/provider_http.go", + "line": 17, + "end_line": 44 + } + ] + }, + { + "id": "mcpclient", + "type": "external", + "label": "External MCP client", + "sublabel": "remote agent or IDE", + "pos": [ + 910, + 40 + ], + "size": [ + 190, + 64 + ], + "sources": [ + { + "path": "cmd/fanout/main.go", + "line": 357, + "end_line": 369 + } + ] + }, + { + "id": "ingest", + "type": "backend", + "label": "OTLP ingest", + "sublabel": "gRPC + HTTP, token checked", + "pos": [ + 40, + 180 + ], + "size": [ + 190, + 64 + ], + "sources": [ + { + "path": "internal/server/http.go", + "line": 13, + "end_line": 32 + }, + { + "path": "internal/ingest/auth.go", + "line": 51, + "end_line": 89 + }, + { + "path": "cmd/fanout/main.go", + "line": 207, + "end_line": 209 + } + ] + }, + { + "id": "api", + "type": "security", + "label": "HTTP API + auth", + "sublabel": "sessions, OIDC, email codes", + "pos": [ + 330, + 180 + ], + "size": [ + 190, + 64 + ], + "sources": [ + { + "path": "cmd/fanout/main.go", + "line": 186, + "end_line": 190 + }, + { + "path": "cmd/fanout/main.go", + "line": 299, + "end_line": 299 + }, + { + "path": "internal/api/oidc.go", + "line": 42, + "end_line": 49 + } + ] + }, + { + "id": "agent", + "type": "backend", + "label": "Agent runtime", + "sublabel": "AG-UI stream, chat threads", + "pos": [ + 620, + 180 + ], + "size": [ + 190, + 64 + ], + "sources": [ + { + "path": "cmd/fanout/main.go", + "line": 372, + "end_line": 385 + }, + { + "path": "internal/agent/runtime.go", + "line": 60, + "end_line": 64 + } + ] + }, + { + "id": "mcp", + "type": "backend", + "label": "MCP server", + "sublabel": "20 typed tools", + "pos": [ + 910, + 180 + ], + "size": [ + 190, + 64 + ], + "sources": [ + { + "path": "cmd/fanout/main.go", + "line": 356, + "end_line": 371 + }, + { + "path": "internal/mcp/server.go", + "line": 105, + "end_line": 105 + } + ] + }, + { + "id": "writer", + "type": "backend", + "label": "Commit workers", + "sublabel": "up to 4, 10 ms group window", + "pos": [ + 40, + 320 + ], + "size": [ + 190, + 64 + ], + "sources": [ + { + "path": "internal/telemetry/store/writer.go", + "line": 20, + "end_line": 28 + }, + { + "path": "internal/telemetry/store/writer.go", + "line": 143, + "end_line": 150 + } + ] + }, + { + "id": "obs", + "type": "backend", + "label": "Queries + dashboards", + "sublabel": "one contract for API and MCP", + "pos": [ + 330, + 320 + ], + "size": [ + 190, + 64 + ], + "sources": [ + { + "path": "cmd/fanout/main.go", + "line": 277, + "end_line": 284 + } + ] + }, + { + "id": "query", + "type": "backend", + "label": "Query kernel", + "sublabel": "DuckDB, rollups, compaction", + "pos": [ + 330, + 460 + ], + "size": [ + 190, + 64 + ], + "sources": [ + { + "path": "internal/query/duck.go", + "line": 240, + "end_line": 244 + }, + { + "path": "internal/query/duck.go", + "line": 505, + "end_line": 522 + }, + { + "path": "internal/query/duck.go", + "line": 627, + "end_line": 653 + } + ] + }, + { + "id": "detector", + "type": "backend", + "label": "Anomaly detector", + "sublabel": "health and z-scores", + "pos": [ + 620, + 460 + ], + "size": [ + 190, + 64 + ], + "sources": [ + { + "path": "internal/intelligence/detector.go", + "line": 46, + "end_line": 55 + }, + { + "path": "cmd/fanout/main.go", + "line": 161, + "end_line": 162 + } + ] + }, + { + "id": "alert", + "type": "backend", + "label": "Alert engine", + "sublabel": "CEL rules, every 30s", + "pos": [ + 910, + 460 + ], + "size": [ + 190, + 64 + ], + "sources": [ + { + "path": "cmd/fanout/main.go", + "line": 173, + "end_line": 183 + }, + { + "path": "internal/alert/engine.go", + "line": 60, + "end_line": 60 + } + ] + }, + { + "id": "webhooks", + "type": "external", + "label": "Webhook targets", + "sublabel": "per-rule HTTP endpoint", + "pos": [ + 1200, + 460 + ], + "size": [ + 190, + 64 + ], + "sources": [ + { + "path": "internal/alert/webhook.go", + "line": 50, + "end_line": 50 + }, + { + "path": "internal/alert/engine.go", + "line": 360, + "end_line": 360 + } + ] + }, + { + "id": "parquet", + "type": "database", + "label": "Parquet batches", + "sublabel": "authoritative, trace.fidx", + "pos": [ + 40, + 600 + ], + "size": [ + 190, + 64 + ], + "sources": [ + { + "path": "internal/telemetry/parquet.go", + "line": 282, + "end_line": 282 + }, + { + "path": "internal/config/config.go", + "line": 219, + "end_line": 219 + } + ] + }, + { + "id": "qstate", + "type": "database", + "label": "DuckDB query state", + "sublabel": "rebuildable cache", + "pos": [ + 330, + 600 + ], + "size": [ + 190, + 64 + ], + "sources": [ + { + "path": "internal/config/config.go", + "line": 227, + "end_line": 232 + } + ] + }, + { + "id": "sqlite", + "type": "database", + "label": "Control SQLite", + "sublabel": "users, dashboards, rules, threads", + "pos": [ + 910, + 600 + ], + "size": [ + 190, + 64 + ], + "sources": [ + { + "path": "internal/store/store.go", + "line": 30, + "end_line": 30 + }, + { + "path": "cmd/fanout/main.go", + "line": 165, + "end_line": 170 + } + ] + } + ], + "boundaries": [ + { + "kind": "region", + "label": "fanout: one Go process, one listener :7520", + "wraps": [ + "ingest", + "api", + "agent", + "mcp", + "writer", + "obs", + "query", + "detector", + "alert" + ] + }, + { + "kind": "region", + "label": "FANOUT_DATA_DIR", + "wraps": [ + "parquet", + "qstate", + "sqlite" + ] + } + ], + "connections": [ + { + "id": "otlp-in", + "from": "collector", + "to": "ingest", + "label": "OTLP", + "variant": "emphasis" + }, + { + "id": "browser-in", + "from": "browser", + "to": "api", + "label": "HTTPS", + "variant": "emphasis" + }, + { + "id": "mcp-in", + "from": "mcpclient", + "to": "mcp", + "label": "/mcp OAuth", + "variant": "security" + }, + { + "id": "chat-stream", + "from": "api", + "to": "agent", + "label": "AG-UI SSE" + }, + { + "id": "model-call", + "from": "agent", + "to": "provider", + "label": "HTTPS stream" + }, + { + "id": "tool-calls", + "from": "agent", + "to": "mcp", + "label": "tool calls" + }, + { + "id": "ingest-commit", + "from": "ingest", + "to": "writer", + "label": "batch" + }, + { + "id": "publish", + "from": "writer", + "to": "parquet", + "label": "atomic publish", + "variant": "emphasis" + }, + { + "id": "api-queries", + "from": "api", + "to": "obs", + "label": "typed API" + }, + { + "id": "mcp-queries", + "from": "mcp", + "to": "obs", + "label": "typed tools", + "fromSide": "bottom", + "toSide": "right" + }, + { + "id": "obs-query", + "from": "obs", + "to": "query", + "label": "SQL" + }, + { + "id": "query-cache", + "from": "query", + "to": "qstate", + "label": "rollups" + }, + { + "id": "query-parquet", + "from": "query", + "to": "parquet", + "label": "read, compact", + "fromSide": "left", + "toSide": "right" + }, + { + "id": "detector-reads", + "from": "detector", + "to": "query", + "label": "rollups" + }, + { + "id": "alert-scores", + "from": "alert", + "to": "detector", + "label": "scores" + }, + { + "id": "alert-webhook", + "from": "alert", + "to": "webhooks", + "label": "POST" + }, + { + "id": "alert-rules", + "from": "alert", + "to": "sqlite", + "label": "rules, history" + }, + { + "id": "alert-rollups", + "from": "alert", + "to": "query", + "label": "rollups", + "fromSide": "top", + "toSide": "top" + } + ], + "cards": [ + { + "dot": "cyan", + "title": "One listener", + "items": [ + "OTLP /v1/* and gRPC go to ingest", + "Everything else is the Echo app: API, UI, /mcp, /api/agent" + ] + }, + { + "dot": "violet", + "title": "Control SQLite", + "items": [ + "Users, sessions, settings, dashboards, OAuth clients, chat threads, alert rules", + "Ingest reads the token hash from it on every export" + ] + }, + { + "dot": "amber", + "title": "Optional", + "items": [ + "Chat only when FANOUT_AI_PROVIDER is set", + "OIDC sign-in and SMTP email codes when configured", + "/metrics for Prometheus" + ] + } + ] +} diff --git a/docs/diagrams/architecture.svg b/docs/diagrams/architecture.svg index 18433e2a..48c11db1 100644 --- a/docs/diagrams/architecture.svg +++ b/docs/diagrams/architecture.svg @@ -1,121 +1,1161 @@ -Clientsfanout — one Go processStorageAnthropic or OpenAIOTLP collector or SDKBrowserExternal MCP hostHTTP :7520routes + auth middlewareIngestOTLP gRPC + HTTP :7520Embedded React assetsAgent runtimemodel + tool loopMCP server5 typed + 4 dashboard toolsTyped observability contractTelemetry commit workersatomic Parquet batchesQuery kernelDuckDB + rollupsStorage maintenanceretention + native compactionAlert enginerule evaluation + webhooksParquet batches + trace indexesFANOUT_DATA_DIR/telemetryDuckDB query stateFANOUT_DATA_DIR/queryControl SQLiteFANOUT_DATA_DIR/control serves embedded assetsAG-UI streamin-memory transporttyped HTTP APIevaluates rollupsOTLP/gRPC or OTLP/HTTPHTTPS/mcp — OAuthindexed traces + SQL scansatomic replacementusers, sessions, settings, dashboards, threadsrules and fired alertsHTTPS - - - - - - - - - - - - - - - + + + Fanout runtime architecture + An architecture diagram generated by Archify. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + OTLP SDK or Collector · any OpenTelemetry exporter · Architecture component + + + + OTLP SDK or Collector + any OpenTelemetry exporter + + + + Browser · embedded React UI · Architecture component + + + + Browser + embedded React UI + + + + Anthropic or OpenAI · selected provider only · Architecture component + + + + Anthropic or OpenAI + selected provider only + + + + External MCP client · remote agent or IDE · Architecture component + + + + External MCP client + remote agent or IDE + + + + OTLP ingest · gRPC + HTTP, token checked · fanout: one Go process, one listener :7520 + + + + OTLP ingest + gRPC + HTTP, token checked + + + + HTTP API + auth · sessions, OIDC, email codes · fanout: one Go process, one listener :7520 + + + + HTTP API + auth + sessions, OIDC, email codes + + + + Agent runtime · AG-UI stream, chat threads · fanout: one Go process, one listener :7520 + + + + Agent runtime + AG-UI stream, chat threads + + + + MCP server · 20 typed tools · fanout: one Go process, one listener :7520 + + + + MCP server + 20 typed tools + + + + Commit workers · up to 4, 10 ms group window · fanout: one Go process, one listener :7520 + + + + Commit workers + up to 4, 10 ms group window + + + + Queries + dashboards · one contract for API and MCP · fanout: one Go process, one listener :7520 + + + + Queries + dashboards + one contract for API and MCP + + + + Query kernel · DuckDB, rollups, compaction · fanout: one Go process, one listener :7520 + + + + Query kernel + DuckDB, rollups, compaction + + + + Anomaly detector · health and z-scores · fanout: one Go process, one listener :7520 + + + + Anomaly detector + health and z-scores + + + + Alert engine · CEL rules, every 30s · fanout: one Go process, one listener :7520 + + + + Alert engine + CEL rules, every 30s + + + + Webhook targets · per-rule HTTP endpoint · Architecture component + + + + Webhook targets + per-rule HTTP endpoint + + + + Parquet batches · authoritative, trace.fidx · FANOUT_DATA_DIR + + + + Parquet batches + authoritative, trace.fidx + + + + DuckDB query state · rebuildable cache · FANOUT_DATA_DIR + + + + DuckDB query state + rebuildable cache + + + + Control SQLite · users, dashboards, rules, threads · FANOUT_DATA_DIR + + + + Control SQLite + users, dashboards, rules, threads + + + + + + OTLP + + + + HTTPS + + + + /mcp OAuth + + + + AG-UI SSE + + + + HTTPS stream + + + + tool calls + + + + batch + + + + atomic publish + + + + typed API + + + + typed tools + + + + SQL + + + + rollups + + + + read, compact + + + + rollups + + + + scores + + + + POST + + + + rules, history + + + + rollups + + + + + + fanout: one Go process, one listener :7520 + + + + + FANOUT_DATA_DIR + + + + + Legend + + + Frontend + + + + Backend + + + + Database + + + + Security + + + + External + + + \ No newline at end of file diff --git a/docs/diagrams/persistence.d2 b/docs/diagrams/persistence.d2 deleted file mode 100644 index b0d74d6f..00000000 --- a/docs/diagrams/persistence.d2 +++ /dev/null @@ -1,63 +0,0 @@ -# How telemetry becomes durable, queryable, and compacted. -# Render with `just diagrams`; commit the generated SVG beside this file. - -direction: down - -vars: { - d2-config: { - layout-engine: elk - theme-id: 5 - } -} - -ingest: "Authenticated OTLP requests\nspans, logs, metrics" { - shape: rectangle - style.stroke-dash: 3 -} - -writers: "Durable write path" { - style.fill: transparent - admission: "Bounded group admission\nconfigured row target + 20 ms window" {shape: rectangle} - workers: "Up to four commit workers\nindependent batches encode in parallel" {shape: rectangle} - stage: "Staging batch directory\nmetadata + signal Parquet + trace.fidx for spans" {shape: rectangle} - publish: "Durable publication\nfsync → atomic rename → fsync parent" {shape: rectangle} - - admission -> workers - workers -> stage - stage -> publish -} - -authority: "Authoritative telemetry\nimmutable *.batch directories" { - shape: cylinder - tooltip: Ordinary Parquet files are authoritative; published batch directories are discovered directly from the filesystem. -} - -readers: Readers { - style.fill: transparent - trace: "Indexed trace read\nbinary-search .fidx → Parquet rows" {shape: rectangle} - sql: "DuckDB SQL\nfilters, ordering, broad scans" {shape: rectangle} - rollups: "DuckDB rollups\nrebuildable analytical cache" {shape: rectangle} -} - -maintenance: "Bounded maintenance" { - style.fill: transparent - choose: "Retention + compaction admission\nbounded batch and row groups" {shape: rectangle} - merge: "Native Parquet merge\nprepared outside the reader gate" {shape: rectangle} - swap: "Crash-safe namespace swap\nmarker + bounded renames + fsync" {shape: rectangle} - - choose -> merge - merge -> swap -} - -querystate: "Rebuildable query state\nFANOUT_DATA_DIR/query" { - shape: cylinder - tooltip: DuckDB rollups and temporary spill. Not authoritative telemetry. -} - -ingest -> writers.admission -writers.publish -> authority: "acknowledge only after publication" -authority -> readers.trace -authority -> readers.sql -readers.sql -> readers.rollups -readers.rollups -> querystate -authority -> maintenance.choose diff --git a/docs/diagrams/persistence.json b/docs/diagrams/persistence.json new file mode 100644 index 00000000..b812b1cb --- /dev/null +++ b/docs/diagrams/persistence.json @@ -0,0 +1,290 @@ +{ + "schema_version": 1, + "diagram_type": "dataflow", + "meta": { + "title": "How telemetry becomes durable and queryable", + "output": "persistence.html", + "quality_profile": "showcase", + "locale": "en", + "repository": { + "url": "https://github.com/labstack/fanout.git", + "provider": "github", + "link_mode": "web", + "revision": "681d46ba8cf0a92ea85764bfa4736a8e2739a8c9" + } + }, + "stages": [ + { + "label": "Accept" + }, + { + "label": "Commit" + }, + { + "label": "Authority" + }, + { + "label": "Read and maintain" + }, + { + "label": "Derived" + } + ], + "nodes": [ + { + "id": "otlp", + "type": "backend", + "label": "OTLP request", + "sublabel": "authenticated, decoded", + "stage": 0, + "row": 1, + "tag": "spans, logs, metrics", + "sources": [ + { + "path": "internal/ingest/server.go", + "line": 54, + "end_line": 60 + } + ] + }, + { + "id": "admission", + "type": "backend", + "label": "Group admission", + "sublabel": "row target, 10 ms window", + "stage": 1, + "row": 0, + "sources": [ + { + "path": "internal/telemetry/store/writer.go", + "line": 20, + "end_line": 28 + } + ] + }, + { + "id": "workers", + "type": "backend", + "label": "Commit workers", + "sublabel": "up to 4 in parallel", + "stage": 1, + "row": 1, + "sources": [ + { + "path": "internal/telemetry/store/writer.go", + "line": 143, + "end_line": 150 + } + ] + }, + { + "id": "publish", + "type": "backend", + "label": "Durable publish", + "sublabel": "fsync, rename, fsync dir", + "stage": 1, + "row": 2, + "sources": [ + { + "path": "internal/telemetry/parquet.go", + "line": 194, + "end_line": 217 + }, + { + "path": "internal/telemetry/parquet.go", + "line": 282, + "end_line": 282 + }, + { + "path": "internal/telemetry/parquet.go", + "line": 308, + "end_line": 345 + } + ] + }, + { + "id": "batches", + "type": "database", + "label": "Parquet batches", + "sublabel": "immutable *.batch dirs", + "stage": 2, + "row": 1, + "tag": "authoritative", + "sources": [ + { + "path": "internal/telemetry/parquet.go", + "line": 26, + "end_line": 26 + }, + { + "path": "internal/config/config.go", + "line": 219, + "end_line": 219 + } + ] + }, + { + "id": "trace", + "type": "backend", + "label": "Indexed trace read", + "sublabel": "trace.fidx lookup", + "stage": 3, + "row": 0, + "sources": [ + { + "path": "internal/telemetry/trace_index.go", + "line": 203, + "end_line": 235 + } + ] + }, + { + "id": "sql", + "type": "backend", + "label": "DuckDB SQL", + "sublabel": "filters and scans", + "stage": 3, + "row": 1, + "sources": [ + { + "path": "internal/query/duck.go", + "line": 240, + "end_line": 244 + } + ] + }, + { + "id": "maintain", + "type": "backend", + "label": "Maintenance", + "sublabel": "retention, compaction", + "stage": 3, + "row": 2, + "sources": [ + { + "path": "internal/query/duck.go", + "line": 627, + "end_line": 653 + }, + { + "path": "internal/query/duck.go", + "line": 692, + "end_line": 693 + }, + { + "path": "internal/telemetry/store/compaction.go", + "line": 132, + "end_line": 177 + } + ] + }, + { + "id": "rollups", + "type": "backend", + "label": "Rollups", + "sublabel": "service and edge", + "stage": 4, + "row": 1, + "sources": [ + { + "path": "internal/query/duck.go", + "line": 505, + "end_line": 522 + }, + { + "path": "internal/query/duck.go", + "line": 1437, + "end_line": 1437 + }, + { + "path": "internal/query/duck.go", + "line": 1622, + "end_line": 1622 + } + ] + }, + { + "id": "qstate", + "type": "database", + "label": "Query state", + "sublabel": "DuckDB, rebuildable", + "stage": 4, + "row": 2, + "sources": [ + { + "path": "internal/config/config.go", + "line": 227, + "end_line": 232 + } + ] + } + ], + "flows": [ + { + "id": "admit", + "from": "otlp", + "to": "admission", + "label": "decoded rows", + "variant": "emphasis" + }, + { + "id": "group", + "from": "admission", + "to": "workers", + "label": "grouped batch", + "variant": "emphasis" + }, + { + "id": "encode", + "from": "workers", + "to": "publish", + "label": "Parquet + index", + "variant": "emphasis" + }, + { + "id": "durable", + "from": "publish", + "to": "batches", + "label": "ack after publish", + "variant": "emphasis" + }, + { + "id": "trace-read", + "from": "batches", + "to": "trace", + "label": "row ranges" + }, + { + "id": "sql-read", + "from": "batches", + "to": "sql", + "label": "Parquet scan" + }, + { + "id": "select", + "from": "batches", + "to": "maintain", + "label": "bounded groups", + "variant": "dashed" + }, + { + "id": "swap", + "from": "maintain", + "to": "batches", + "label": "atomic swap", + "variant": "dashed" + }, + { + "id": "aggregate", + "from": "sql", + "to": "rollups", + "label": "aggregate" + }, + { + "id": "cache", + "from": "rollups", + "to": "qstate", + "label": "cache rows" + } + ] +} diff --git a/docs/diagrams/persistence.svg b/docs/diagrams/persistence.svg index 13d27eb5..26a3d6e4 100644 --- a/docs/diagrams/persistence.svg +++ b/docs/diagrams/persistence.svg @@ -1,138 +1,809 @@ -Authenticated OTLP requestsspans, logs, metricsDurable write pathAuthoritative telemetryimmutable *.batch directoriesOrdinary Parquet files are authoritativeReadersBounded maintenanceRebuildable query stateFANOUT_DATA_DIR/queryDuckDB rollups and temporary spill. Not authoritative telemetry.Bounded group admissionconfigured row target + 20 ms windowUp to four commit workersindependent batches encode in parallelStaging batch directorymetadata + signal Parquet + trace.fidx for spansDurable publicationfsync → atomic rename → fsync parentpublished batch directories are discovered directly from the filesystemIndexed trace readbinary-search .fidx → Parquet rowsDuckDB SQLfilters, ordering, broad scansDuckDB rollupsrebuildable analytical cacheRetention + compaction admissionbounded batch and row groupsNative Parquet mergeprepared outside the reader gateCrash-safe namespace swapmarker + bounded renames + fsync acknowledge only after publicationOrdinary Parquet files are authoritative - - - - - - - - - - - - -DuckDB rollups and temporary spill. Not authoritative telemetry. - - - - - - - - - - - - - - - - + + + How telemetry becomes durable and queryable + A data-flow diagram generated by Archify. + + + + + + + + + + + + + + + + + + + + + + + + + 01 / Accept + + + 02 / Commit + + + 03 / Authority + + + 04 / Read and maintain + + + 05 / Derived + + + + + + + + + + + + + + + + OTLP request · authenticated, decoded · 01 / Accept · spans, logs, metrics + + + + OTLP request + authenticated, decoded + spans, logs, metrics + + + + Group admission · row target, 10 ms window · 02 / Commit + + + + Group admission + row target, 10 ms window + + + + Commit workers · up to 4 in parallel · 02 / Commit + + + + Commit workers + up to 4 in parallel + + + + Durable publish · fsync, rename, fsync dir · 02 / Commit + + + + Durable publish + fsync, rename, fsync dir + + + + Parquet batches · immutable *.batch dirs · 03 / Authority · authoritative + + + + Parquet batches + immutable *.batch dirs + authoritative + + + + Indexed trace read · trace.fidx lookup · 04 / Read and maintain + + + + Indexed trace read + trace.fidx lookup + + + + DuckDB SQL · filters and scans · 04 / Read and maintain + + + + DuckDB SQL + filters and scans + + + + Maintenance · retention, compaction · 04 / Read and maintain + + + + Maintenance + retention, compaction + + + + Rollups · service and edge · 05 / Derived + + + + Rollups + service and edge + + + + Query state · DuckDB, rebuildable · 05 / Derived + + + + Query state + DuckDB, rebuildable + + + + + + decoded rows + + + + grouped batch + + + + Parquet + index + + + + ack after publish + + + + row ranges + + + + Parquet scan + + + + bounded groups + + + + atomic swap + + + + aggregate + + + + cache rows + + + + + Legend + + + primary data + + + + async batch + + + + data store + + + + data flow + + + \ No newline at end of file diff --git a/justfile b/justfile index 1fb55000..36d878d9 100644 --- a/justfile +++ b/justfile @@ -180,23 +180,16 @@ ui-check: # ── Diagrams ───────────────────────────────────────────────────────────────── -# Renders docs/diagrams/*.d2 beside each source. Deliberately outside `build` -# and `check`: the SVG is committed, so only someone editing a diagram needs d2 -# installed at all. -# -# D2 output changes between minor versions, so the recipe pins the generator -# used for committed SVGs instead of depending on a contributor's installed -# binary. - -# Render every d2 diagram to SVG and publish the same assets on the docs site. +# Archify renders docs/diagrams/*.json and exports one dual-theme SVG per source. +# Use a local archify 3.1.0 install via ARCHIFY_HOME (default +# $HOME/.agents/skills/archify) and Chrome; no npm install is needed for archify. +# The generator version is pinned so committed SVGs remain reproducible. +# Deliberately outside `build` and `check`: SVGs are committed, so only diagram +# editors need archify and the Chrome export environment. + +# Export every archify diagram and publish the same SVGs on the docs site. diagrams: - #!/usr/bin/env bash - set -euo pipefail - for src in docs/diagrams/*.d2; do - go run oss.terrastruct.com/d2@v0.7.1 "$src" "${src%.d2}.svg" - done - mkdir -p site/public/diagrams - cp docs/diagrams/*.svg site/public/diagrams/ + node scripts/diagrams.mjs # Fail when the site serves a diagram other than the committed documentation # copy. Regeneration stays a deliberate writing step. @@ -261,7 +254,7 @@ site-build: docs-generate-check diagrams-check site-deps # missing face does not fail the render; it quietly produces a card set in # whatever typst found instead. This reads the warning back and fails on it. # -# The version is pinned for the same reason d2 is: typst re-lays out text +# The version is pinned for reproducible output: typst re-lays out text # between releases, so a different one rewrites every glyph position and # produces a diff that is not a real change. # diff --git a/scripts/diagrams.mjs b/scripts/diagrams.mjs new file mode 100644 index 00000000..e53f8a59 --- /dev/null +++ b/scripts/diagrams.mjs @@ -0,0 +1,90 @@ +import { spawnSync } from "node:child_process"; +import { access, copyFile, mkdir, mkdtemp, readFile, readdir, rm } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { homedir, tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const ARCHIFY_VERSION = "3.1.0"; +const repo = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const diagrams = join(repo, "docs/diagrams"); +const archify = resolve(process.env.ARCHIFY_HOME || join(homedir(), ".agents/skills/archify")); + +async function main() { + const cli = join(archify, "bin/archify.mjs"); + try { + await access(cli); + } catch { + throw new Error(`Archify is missing ${cli}; set ARCHIFY_HOME to a local archify ${ARCHIFY_VERSION} install.`); + } + let release; + try { + release = JSON.parse(await readFile(join(archify, "skill-release.json"), "utf8")); + } catch { + throw new Error(`Cannot read archify's skill-release.json in ${archify}; archify ${ARCHIFY_VERSION} is required.`); + } + if (release?.version !== ARCHIFY_VERSION) { + throw new Error(`Archify ${ARCHIFY_VERSION} is required; ${archify} reports version ${JSON.stringify(release?.version)}.`); + } + + const require = createRequire(join(repo, "ui/host/package.json")); + const { chromium } = require("@playwright/test"); + const sources = (await readdir(diagrams)).filter(name => name.endsWith(".json")).sort(); + let browser; + try { + for (const source of sources) { + const name = source.slice(0, -".json".length); + const input = join(diagrams, source); + const { diagram_type: type } = JSON.parse(await readFile(input, "utf8")); + if (typeof type !== "string" || !type) { + throw new Error(`${input} must declare diagram_type.`); + } + const temporary = await mkdtemp(join(tmpdir(), "fanout-diagram-")); + try { + const html = join(temporary, `${name}.html`); + const result = spawnSync(process.execPath, [cli, "finalize", type, input, html, + "--repo-root", repo, "--quality", "showcase", "--json"], { + cwd: repo, + encoding: "utf8", + maxBuffer: 16 * 1024 * 1024, + }); + if (result.error || result.status !== 0) { + // The JSON receipt contains the failed gate's diagnostics. + if (result.stdout) console.error(result.stdout.trim()); + if (result.stderr) console.error(result.stderr.trim()); + throw new Error(`Archify finalize failed for ${source}: ${result.error?.message ?? result.signal ?? `exit ${result.status}`}`); + } + browser ??= await chromium.launch({ channel: "chrome", headless: true }); + const context = await browser.newContext({ acceptDownloads: true }); + try { + const page = await context.newPage(); + await page.goto(pathToFileURL(html).href); + await page.getByRole("button", { name: "Export diagram", exact: true }).click(); + const [download] = await Promise.all([ + page.waitForEvent("download"), + page.getByRole("menuitem", { name: /^SVG · Auto/ }).click(), + ]); + await download.saveAs(join(diagrams, `${name}.svg`)); + } finally { + await context.close(); + } + } finally { + await rm(temporary, { recursive: true, force: true }); + } + } + const destination = join(repo, "site/public/diagrams"); + await mkdir(destination, { recursive: true }); + for (const svg of (await readdir(diagrams)).filter(name => name.endsWith(".svg")).sort()) { + await copyFile(join(diagrams, svg), join(destination, svg)); + } + } finally { + await browser?.close(); + } +} + +try { + await main(); +} catch (error) { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; +} diff --git a/site/public/diagrams/architecture.svg b/site/public/diagrams/architecture.svg index 18433e2a..48c11db1 100644 --- a/site/public/diagrams/architecture.svg +++ b/site/public/diagrams/architecture.svg @@ -1,121 +1,1161 @@ -Clientsfanout — one Go processStorageAnthropic or OpenAIOTLP collector or SDKBrowserExternal MCP hostHTTP :7520routes + auth middlewareIngestOTLP gRPC + HTTP :7520Embedded React assetsAgent runtimemodel + tool loopMCP server5 typed + 4 dashboard toolsTyped observability contractTelemetry commit workersatomic Parquet batchesQuery kernelDuckDB + rollupsStorage maintenanceretention + native compactionAlert enginerule evaluation + webhooksParquet batches + trace indexesFANOUT_DATA_DIR/telemetryDuckDB query stateFANOUT_DATA_DIR/queryControl SQLiteFANOUT_DATA_DIR/control serves embedded assetsAG-UI streamin-memory transporttyped HTTP APIevaluates rollupsOTLP/gRPC or OTLP/HTTPHTTPS/mcp — OAuthindexed traces + SQL scansatomic replacementusers, sessions, settings, dashboards, threadsrules and fired alertsHTTPS - - - - - - - - - - - - - - - + + + Fanout runtime architecture + An architecture diagram generated by Archify. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + OTLP SDK or Collector · any OpenTelemetry exporter · Architecture component + + + + OTLP SDK or Collector + any OpenTelemetry exporter + + + + Browser · embedded React UI · Architecture component + + + + Browser + embedded React UI + + + + Anthropic or OpenAI · selected provider only · Architecture component + + + + Anthropic or OpenAI + selected provider only + + + + External MCP client · remote agent or IDE · Architecture component + + + + External MCP client + remote agent or IDE + + + + OTLP ingest · gRPC + HTTP, token checked · fanout: one Go process, one listener :7520 + + + + OTLP ingest + gRPC + HTTP, token checked + + + + HTTP API + auth · sessions, OIDC, email codes · fanout: one Go process, one listener :7520 + + + + HTTP API + auth + sessions, OIDC, email codes + + + + Agent runtime · AG-UI stream, chat threads · fanout: one Go process, one listener :7520 + + + + Agent runtime + AG-UI stream, chat threads + + + + MCP server · 20 typed tools · fanout: one Go process, one listener :7520 + + + + MCP server + 20 typed tools + + + + Commit workers · up to 4, 10 ms group window · fanout: one Go process, one listener :7520 + + + + Commit workers + up to 4, 10 ms group window + + + + Queries + dashboards · one contract for API and MCP · fanout: one Go process, one listener :7520 + + + + Queries + dashboards + one contract for API and MCP + + + + Query kernel · DuckDB, rollups, compaction · fanout: one Go process, one listener :7520 + + + + Query kernel + DuckDB, rollups, compaction + + + + Anomaly detector · health and z-scores · fanout: one Go process, one listener :7520 + + + + Anomaly detector + health and z-scores + + + + Alert engine · CEL rules, every 30s · fanout: one Go process, one listener :7520 + + + + Alert engine + CEL rules, every 30s + + + + Webhook targets · per-rule HTTP endpoint · Architecture component + + + + Webhook targets + per-rule HTTP endpoint + + + + Parquet batches · authoritative, trace.fidx · FANOUT_DATA_DIR + + + + Parquet batches + authoritative, trace.fidx + + + + DuckDB query state · rebuildable cache · FANOUT_DATA_DIR + + + + DuckDB query state + rebuildable cache + + + + Control SQLite · users, dashboards, rules, threads · FANOUT_DATA_DIR + + + + Control SQLite + users, dashboards, rules, threads + + + + + + OTLP + + + + HTTPS + + + + /mcp OAuth + + + + AG-UI SSE + + + + HTTPS stream + + + + tool calls + + + + batch + + + + atomic publish + + + + typed API + + + + typed tools + + + + SQL + + + + rollups + + + + read, compact + + + + rollups + + + + scores + + + + POST + + + + rules, history + + + + rollups + + + + + + fanout: one Go process, one listener :7520 + + + + + FANOUT_DATA_DIR + + + + + Legend + + + Frontend + + + + Backend + + + + Database + + + + Security + + + + External + + + \ No newline at end of file diff --git a/site/public/diagrams/persistence.svg b/site/public/diagrams/persistence.svg index 13d27eb5..26a3d6e4 100644 --- a/site/public/diagrams/persistence.svg +++ b/site/public/diagrams/persistence.svg @@ -1,138 +1,809 @@ -Authenticated OTLP requestsspans, logs, metricsDurable write pathAuthoritative telemetryimmutable *.batch directoriesOrdinary Parquet files are authoritativeReadersBounded maintenanceRebuildable query stateFANOUT_DATA_DIR/queryDuckDB rollups and temporary spill. Not authoritative telemetry.Bounded group admissionconfigured row target + 20 ms windowUp to four commit workersindependent batches encode in parallelStaging batch directorymetadata + signal Parquet + trace.fidx for spansDurable publicationfsync → atomic rename → fsync parentpublished batch directories are discovered directly from the filesystemIndexed trace readbinary-search .fidx → Parquet rowsDuckDB SQLfilters, ordering, broad scansDuckDB rollupsrebuildable analytical cacheRetention + compaction admissionbounded batch and row groupsNative Parquet mergeprepared outside the reader gateCrash-safe namespace swapmarker + bounded renames + fsync acknowledge only after publicationOrdinary Parquet files are authoritative - - - - - - - - - - - - -DuckDB rollups and temporary spill. Not authoritative telemetry. - - - - - - - - - - - - - - - - + + + How telemetry becomes durable and queryable + A data-flow diagram generated by Archify. + + + + + + + + + + + + + + + + + + + + + + + + + 01 / Accept + + + 02 / Commit + + + 03 / Authority + + + 04 / Read and maintain + + + 05 / Derived + + + + + + + + + + + + + + + + OTLP request · authenticated, decoded · 01 / Accept · spans, logs, metrics + + + + OTLP request + authenticated, decoded + spans, logs, metrics + + + + Group admission · row target, 10 ms window · 02 / Commit + + + + Group admission + row target, 10 ms window + + + + Commit workers · up to 4 in parallel · 02 / Commit + + + + Commit workers + up to 4 in parallel + + + + Durable publish · fsync, rename, fsync dir · 02 / Commit + + + + Durable publish + fsync, rename, fsync dir + + + + Parquet batches · immutable *.batch dirs · 03 / Authority · authoritative + + + + Parquet batches + immutable *.batch dirs + authoritative + + + + Indexed trace read · trace.fidx lookup · 04 / Read and maintain + + + + Indexed trace read + trace.fidx lookup + + + + DuckDB SQL · filters and scans · 04 / Read and maintain + + + + DuckDB SQL + filters and scans + + + + Maintenance · retention, compaction · 04 / Read and maintain + + + + Maintenance + retention, compaction + + + + Rollups · service and edge · 05 / Derived + + + + Rollups + service and edge + + + + Query state · DuckDB, rebuildable · 05 / Derived + + + + Query state + DuckDB, rebuildable + + + + + + decoded rows + + + + grouped batch + + + + Parquet + index + + + + ack after publish + + + + row ranges + + + + Parquet scan + + + + bounded groups + + + + atomic swap + + + + aggregate + + + + cache rows + + + + + Legend + + + primary data + + + + async batch + + + + data store + + + + data flow + + + \ No newline at end of file diff --git a/site/src/content/docs/explanation/storage-model.mdx b/site/src/content/docs/explanation/storage-model.mdx index bf6a8a83..e01155de 100644 --- a/site/src/content/docs/explanation/storage-model.mdx +++ b/site/src/content/docs/explanation/storage-model.mdx @@ -25,7 +25,7 @@ history—lives in SQLite. Fanout persistence flow: authenticated OTLP requests enter bounded group admission and parallel commit workers, publish immutable Parquet batches atomically, then serve indexed trace reads and DuckDB SQL while native maintenance prepares and swaps compacted replacements. diff --git a/site/src/content/docs/start/what-fanout-is.mdx b/site/src/content/docs/start/what-fanout-is.mdx index 0468d837..b5112c60 100644 --- a/site/src/content/docs/start/what-fanout-is.mdx +++ b/site/src/content/docs/start/what-fanout-is.mdx @@ -17,7 +17,7 @@ investigator and MCP tools, and hosts the browser client. Fanout architecture: collectors, browsers, and MCP hosts connect to one process containing ingest, query, maintenance, alerts, agent, and UI components backed by Parquet, DuckDB query state, and SQLite control state. From 3c7c92424a9a1de71de7fd5722a830bad00dbab9 Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 14 Oct 2026 11:01:48 -0700 Subject: [PATCH 2/3] docs(diagrams): fix the detector edge and support fork checkouts The anomaly detector runs SQL over raw spans and logs, not over rollups; the architecture diagram now says so and cites its queries. Archify verifies that the evidence checkout's origin matches the pinned repository, so a fork clone could not regenerate diagrams. DIAGRAMS_EVIDENCE_ROOT selects a labstack/fanout clone that contains the pinned revision, and the script checks origin and revision up front with one clear error. --- CONTRIBUTING.md | 6 ++++ docs/diagrams/architecture.json | 7 ++++- docs/diagrams/architecture.svg | 8 ++--- justfile | 2 ++ scripts/diagrams.mjs | 45 ++++++++++++++++++++++----- site/public/diagrams/architecture.svg | 8 ++--- 6 files changed, 59 insertions(+), 17 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5ebd06c4..b7ea9b99 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -90,6 +90,12 @@ a local install (`ARCHIFY_HOME`, default `$HOME/.agents/skills/archify`), the Playwright dependencies in `ui/host/node_modules`, and Chrome for the dual-theme SVG export. Archify is never installed through npm. Committed SVGs serve the README and docs site, so only diagram editors need the rendering environment. +The evidence checkout defaults to this repository. If your `origin` points to +a fork, use `DIAGRAMS_EVIDENCE_ROOT=/path/to/upstream-clone just diagrams`, where +the clone's credential-free `origin` matches `meta.repository.url` +(`https://github.com/labstack/fanout.git`) and contains the pinned revision. +The script checks both before rendering. JSON inputs, SVG outputs, and the site +copy still use your contributor checkout. **The social preview card is generated.** `site/public/social-card.png` is rendered from `docs/media/social-card.typ` by `just social-card`, which needs Inter on a diff --git a/docs/diagrams/architecture.json b/docs/diagrams/architecture.json index fc223c02..fd061a64 100644 --- a/docs/diagrams/architecture.json +++ b/docs/diagrams/architecture.json @@ -314,6 +314,11 @@ "line": 46, "end_line": 55 }, + { + "path": "internal/intelligence/detector.go", + "line": 240, + "end_line": 291 + }, { "path": "cmd/fanout/main.go", "line": 161, @@ -564,7 +569,7 @@ "id": "detector-reads", "from": "detector", "to": "query", - "label": "rollups" + "label": "SQL scans" }, { "id": "alert-scores", diff --git a/docs/diagrams/architecture.svg b/docs/diagrams/architecture.svg index 48c11db1..97858a29 100644 --- a/docs/diagrams/architecture.svg +++ b/docs/diagrams/architecture.svg @@ -831,7 +831,7 @@ rect.c-bg-rect { fill: var(--bg); } - + @@ -1102,9 +1102,9 @@ rect.c-bg-rect { fill: var(--bg); } read, compact - - - rollups + + + SQL scans diff --git a/justfile b/justfile index 36d878d9..a6de9024 100644 --- a/justfile +++ b/justfile @@ -184,6 +184,8 @@ ui-check: # Use a local archify 3.1.0 install via ARCHIFY_HOME (default # $HOME/.agents/skills/archify) and Chrome; no npm install is needed for archify. # The generator version is pinned so committed SVGs remain reproducible. +# DIAGRAMS_EVIDENCE_ROOT selects an upstream clone containing the pinned revision +# when origin is a fork (default: this checkout); inputs and SVGs stay here. # Deliberately outside `build` and `check`: SVGs are committed, so only diagram # editors need archify and the Chrome export environment. diff --git a/scripts/diagrams.mjs b/scripts/diagrams.mjs index e53f8a59..be6a6169 100644 --- a/scripts/diagrams.mjs +++ b/scripts/diagrams.mjs @@ -7,10 +7,45 @@ import { fileURLToPath, pathToFileURL } from "node:url"; const ARCHIFY_VERSION = "3.1.0"; const repo = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const evidenceRoot = resolve(process.env.DIAGRAMS_EVIDENCE_ROOT || repo); const diagrams = join(repo, "docs/diagrams"); const archify = resolve(process.env.ARCHIFY_HOME || join(homedir(), ".agents/skills/archify")); +function credentialFreeURL(value) { + return value.replace(/(\w+:\/\/)[^/]*@/, "$1").replace(/^[^/@:]+@(?=[^/:]+:)/, ""); +} + async function main() { + const sources = []; + const remote = spawnSync("git", ["remote", "get-url", "origin"], { + cwd: evidenceRoot, + encoding: "utf8", + }); + const origin = remote.error || remote.status !== 0 + ? "(unavailable)" + : credentialFreeURL(remote.stdout.trim()); + for (const source of (await readdir(diagrams)).filter(name => name.endsWith(".json")).sort()) { + const input = join(diagrams, source); + const { diagram_type: type, meta } = JSON.parse(await readFile(input, "utf8")); + if (typeof type !== "string" || !type) { + throw new Error(`${input} must declare diagram_type.`); + } + const { url, revision } = meta?.repository ?? {}; + if (typeof url !== "string" || !url || typeof revision !== "string" || !revision) { + throw new Error(`${input} must declare meta.repository.url and meta.repository.revision.`); + } + const expected = credentialFreeURL(url); + const commit = spawnSync("git", ["cat-file", "-e", `${revision}^{commit}`], { + cwd: evidenceRoot, + stdio: "ignore", + }); + if (origin !== expected || commit.error || commit.status !== 0) { + const reason = origin !== expected ? "origin does not match" : `pinned revision ${revision} is unavailable`; + throw new Error(`Evidence checkout ${evidenceRoot} has origin ${JSON.stringify(origin)}; ${source} expects ${JSON.stringify(expected)}: ${reason}. Set DIAGRAMS_EVIDENCE_ROOT to a clone of labstack/fanout with the expected origin that contains the pinned revision ${revision}.`); + } + sources.push({ source, input, type }); + } + const cli = join(archify, "bin/archify.mjs"); try { await access(cli); @@ -29,21 +64,15 @@ async function main() { const require = createRequire(join(repo, "ui/host/package.json")); const { chromium } = require("@playwright/test"); - const sources = (await readdir(diagrams)).filter(name => name.endsWith(".json")).sort(); let browser; try { - for (const source of sources) { + for (const { source, input, type } of sources) { const name = source.slice(0, -".json".length); - const input = join(diagrams, source); - const { diagram_type: type } = JSON.parse(await readFile(input, "utf8")); - if (typeof type !== "string" || !type) { - throw new Error(`${input} must declare diagram_type.`); - } const temporary = await mkdtemp(join(tmpdir(), "fanout-diagram-")); try { const html = join(temporary, `${name}.html`); const result = spawnSync(process.execPath, [cli, "finalize", type, input, html, - "--repo-root", repo, "--quality", "showcase", "--json"], { + "--repo-root", evidenceRoot, "--quality", "showcase", "--json"], { cwd: repo, encoding: "utf8", maxBuffer: 16 * 1024 * 1024, diff --git a/site/public/diagrams/architecture.svg b/site/public/diagrams/architecture.svg index 48c11db1..97858a29 100644 --- a/site/public/diagrams/architecture.svg +++ b/site/public/diagrams/architecture.svg @@ -831,7 +831,7 @@ rect.c-bg-rect { fill: var(--bg); } - + @@ -1102,9 +1102,9 @@ rect.c-bg-rect { fill: var(--bg); } read, compact - - - rollups + + + SQL scans From 42c8dc8b8c4854f0450b36ca609f6c4e29f8a81f Mon Sep 17 00:00:00 2001 From: Vishal Rana Date: Wed, 14 Oct 2026 11:26:24 -0700 Subject: [PATCH 3/3] docs(diagrams): let archify verify the evidence repository The script compared origin URLs as strings, rejecting equivalent upstream forms such as SSH, HTTPS without .git, and insteadOf rewrites. Archify already normalizes and verifies repository identity and the pinned revision, so the script defers to it and adds the DIAGRAMS_EVIDENCE_ROOT hint when archify reports a repository-evidence failure. --- CONTRIBUTING.md | 7 +++---- scripts/diagrams.mjs | 37 ++++++++++++------------------------- 2 files changed, 15 insertions(+), 29 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b7ea9b99..5f793e6b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,10 +92,9 @@ SVG export. Archify is never installed through npm. Committed SVGs serve the README and docs site, so only diagram editors need the rendering environment. The evidence checkout defaults to this repository. If your `origin` points to a fork, use `DIAGRAMS_EVIDENCE_ROOT=/path/to/upstream-clone just diagrams`, where -the clone's credential-free `origin` matches `meta.repository.url` -(`https://github.com/labstack/fanout.git`) and contains the pinned revision. -The script checks both before rendering. JSON inputs, SVG outputs, and the site -copy still use your contributor checkout. +the clone is labstack/fanout and contains the pinned revision. Archify normalizes +and verifies repository identity and the pinned revision during `finalize`. +JSON inputs, SVG outputs, and the site copy still use your contributor checkout. **The social preview card is generated.** `site/public/social-card.png` is rendered from `docs/media/social-card.typ` by `just social-card`, which needs Inter on a diff --git a/scripts/diagrams.mjs b/scripts/diagrams.mjs index be6a6169..d04e4dc1 100644 --- a/scripts/diagrams.mjs +++ b/scripts/diagrams.mjs @@ -11,38 +11,14 @@ const evidenceRoot = resolve(process.env.DIAGRAMS_EVIDENCE_ROOT || repo); const diagrams = join(repo, "docs/diagrams"); const archify = resolve(process.env.ARCHIFY_HOME || join(homedir(), ".agents/skills/archify")); -function credentialFreeURL(value) { - return value.replace(/(\w+:\/\/)[^/]*@/, "$1").replace(/^[^/@:]+@(?=[^/:]+:)/, ""); -} - async function main() { const sources = []; - const remote = spawnSync("git", ["remote", "get-url", "origin"], { - cwd: evidenceRoot, - encoding: "utf8", - }); - const origin = remote.error || remote.status !== 0 - ? "(unavailable)" - : credentialFreeURL(remote.stdout.trim()); for (const source of (await readdir(diagrams)).filter(name => name.endsWith(".json")).sort()) { const input = join(diagrams, source); - const { diagram_type: type, meta } = JSON.parse(await readFile(input, "utf8")); + const { diagram_type: type } = JSON.parse(await readFile(input, "utf8")); if (typeof type !== "string" || !type) { throw new Error(`${input} must declare diagram_type.`); } - const { url, revision } = meta?.repository ?? {}; - if (typeof url !== "string" || !url || typeof revision !== "string" || !revision) { - throw new Error(`${input} must declare meta.repository.url and meta.repository.revision.`); - } - const expected = credentialFreeURL(url); - const commit = spawnSync("git", ["cat-file", "-e", `${revision}^{commit}`], { - cwd: evidenceRoot, - stdio: "ignore", - }); - if (origin !== expected || commit.error || commit.status !== 0) { - const reason = origin !== expected ? "origin does not match" : `pinned revision ${revision} is unavailable`; - throw new Error(`Evidence checkout ${evidenceRoot} has origin ${JSON.stringify(origin)}; ${source} expects ${JSON.stringify(expected)}: ${reason}. Set DIAGRAMS_EVIDENCE_ROOT to a clone of labstack/fanout with the expected origin that contains the pinned revision ${revision}.`); - } sources.push({ source, input, type }); } @@ -81,6 +57,17 @@ async function main() { // The JSON receipt contains the failed gate's diagnostics. if (result.stdout) console.error(result.stdout.trim()); if (result.stderr) console.error(result.stderr.trim()); + let receipt; + try { + receipt = JSON.parse(result.stdout); + } catch { + // Preserve the original failure when no JSON receipt is available. + } + if (Array.isArray(receipt?.diagnostics) && receipt.diagnostics.some( + diagnostic => typeof diagnostic?.code === "string" && diagnostic.code.startsWith("repository-evidence/"), + )) { + console.error("Set DIAGRAMS_EVIDENCE_ROOT to a clone of labstack/fanout that contains the pinned revision."); + } throw new Error(`Archify finalize failed for ${source}: ${result.error?.message ?? result.signal ?? `exit ${result.status}`}`); } browser ??= await chromium.launch({ channel: "chrome", headless: true });